From 683e806ded910405f212df09222e1cd6009b4c22 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Wed, 30 Sep 2026 20:54:10 -0700 Subject: [PATCH 01/96] perf: hash and compare plain instances by identity without interpreter round trips A plain instance (no native payload, `object.__hash__`) now hashes by identity directly instead of probing the weakref type and re-entering the interpreter. `LeafProbe` accepts such instances, so `d[obj]` stays in the core loop, and `member_eq` settles two plain instances without resolving `__eq__` four times. Instance-keyed dict reads drop from 2,272 to 918 instructions. --- crates/weavepy-vm/src/lib.rs | 13 ++++++++---- crates/weavepy-vm/src/object.rs | 36 ++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index dcbfecb3..f55ee6d7 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -15303,6 +15303,11 @@ impl Interpreter { let d = d.try_borrow().ok()?; clone_hot(d.get(&probe)?) } + (Object::Dict(d), Object::Instance(_)) if Self::core_droppable(k) => { + let probe = crate::object::LeafProbe::new(k)?; + let d = d.try_borrow().ok()?; + clone_hot(d.get(&probe)?) + } // `seq[a:b:c]` with plain int (or omitted) bounds: the // full handler's own slicing, out of line (a bad step // declines, and the full handler raises). @@ -15316,10 +15321,10 @@ impl Interpreter { _ => return None, }; // SAFETY: both operand slots are initialized. The key is a - // scalar, a string or a slice of ints, and the container a - // shared value - // (`core_droppable`), so neither release runs code; the - // result takes the container's slot. + // scalar, a string, a slice of ints or a droppable instance, + // and the container a shared value (`core_droppable`), so + // neither release runs code; the result takes the + // container's slot. unsafe { drop_hot(base.add(len - 1).read()); drop_hot(base.add(len - 2).read()); diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 22b4bbbe..77bac087 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -3064,12 +3064,19 @@ pub struct LeafProbe<'a> { } impl<'a> LeafProbe<'a> { - /// `None` unless `key` is a `str` or a machine `int`. + /// `None` unless `key` is a `str`, a machine `int`, or a plain + /// instance hashed by identity (its `__hash__` is `object`'s). #[inline] pub fn new(key: &'a Object) -> Option { let hash = match key { Object::Str(s) => SharedStr::hash_cached(s), Object::Int(_) => py_hash_value(key)?, + Object::Instance(i) + if i.native.get().is_none() + && i.class_dunder(crate::types::Dunder::Hash).object_owner() => + { + identity_hash(key) + } _ => return None, }; Some(Self { @@ -3103,6 +3110,9 @@ impl indexmap::Equivalent for LeafProbe<'_> { match (self.key, &key.0) { (Object::Str(a), Object::Str(b)) => a.as_bytes() == b.as_bytes(), (Object::Int(a), Object::Int(b)) => a == b, + // Identity settles an instance probe (CPython compares keys + // with `is` first); any other pairing may need `__eq__`. + (Object::Instance(a), Object::Instance(b)) if Rc::ptr_eq(a, b) => true, _ => { self.foreign.set(true); false @@ -3200,6 +3210,13 @@ pub(crate) fn instance_has_custom_dunder(obj: &Object, dunder: crate::types::Dun inst.native.get().is_none() && !info.object_owner() } +/// A plain instance compared by identity: no native payload, and its +/// `__eq__` is `object`'s. +#[inline] +pub(crate) fn plain_identity_eq(inst: &crate::types::PyInstance) -> bool { + inst.native.get().is_none() && inst.class_dunder(crate::types::Dunder::Eq).object_owner() +} + /// [`instance_has_custom_dunder`] for `__eq__`: membership tests and /// `list.remove`/`index`/`count` ask it for every element they compare. pub(crate) fn instance_has_custom_eq(obj: &Object) -> bool { @@ -3282,6 +3299,14 @@ pub(crate) fn member_eq(a: &Object, b: &Object) -> Result { if a.is_same(b) { return Ok(true); } + // Two distinct plain instances (`object.__eq__` on both sides) are + // unequal: both comparisons return `NotImplemented`. Settled before + // the general path, which resolves each side's `__eq__` twice. + if let (Object::Instance(x), Object::Instance(y)) = (a, b) { + if plain_identity_eq(x) && plain_identity_eq(y) { + return Ok(false); + } + } // Dispatch the full Python comparison whenever an operand can carry // custom `__eq__` semantics: a pure-Python instance with a user dunder, // OR a foreign (C-extension) object whose `tp_richcompare` the native @@ -11967,6 +11992,15 @@ pub(crate) fn py_hash_value(obj: &Object) -> Option { }))) } Object::Instance(inst) => { + // The common case first: a plain instance whose `__hash__` is + // `object`'s hashes by identity, with no interpreter round trip. + if inst.native.get().is_none() + && inst + .class_dunder(crate::types::Dunder::Hash) + .object_owner() + { + return Some(identity_hash(obj)); + } // A `weakref.ref` hashes as its referent (CPython `weakref_hash`), // computed natively and memoised so the hot `DictKey` path never // pays a reentrant `__hash__` dispatch — see `weakref_native_hash`. From f73a4ca2ddaf37937eb3d70619277d4fb0c1127a Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 00:53:48 -0700 Subject: [PATCH 02/96] perf: make the cycle collector and weakrefs non-owning so objects die by refcount The collector's tracked handles and the weakref registry held strong references, so every drop site had to emulate refcount death: suspect maps, prompt-reap cascades, parked drops and copy-resurrected finalizers. Both now hold weak references and objects die when their last reference goes, as in CPython. - `WeakObject` gives every heap variant a non-owning handle; collections upgrade a snapshot, seed `gc_refs` from strong counts and prune dead handles incrementally. - Death hooks in `Drop` clear weakrefs and queue callbacks. Types without a hook are swept at collections. - `Rc`'s last release of an instance or generator that owes a finalizer moves the reference itself into the pending queue, so `__del__` sees the same object (identity preserved, no copy). - A trashcan bounds recursive drops of deep container chains. - The core loop checks for queued work after the releases that can end a binding (stores, pops, container ops, helper calls), at the instruction CPython would run the finalizer. - The JIT constructor fallback no longer finalizes the instance it allocated and abandoned. Removes roughly 5,000 lines of reap machinery. Micro costs drop sharply (instance construction 5,040 to 2,953 instructions, bound method calls 2,055 to 1,068), and deltablue, nbody, fannkuch and pyaes get 6-17% faster. --- crates/weavepy-capi/src/gc_bridge.rs | 4 +- crates/weavepy-capi/src/instance.rs | 30 +- crates/weavepy-capi/src/interp.rs | 9 +- crates/weavepy-capi/src/object.rs | 15 +- crates/weavepy-capi/src/pep741.rs | 4 +- crates/weavepy-jit/src/ir.rs | 10 +- crates/weavepy-vm/src/builtins.rs | 45 +- crates/weavepy-vm/src/capi_watchers.rs | 13 +- crates/weavepy-vm/src/gc_trace.rs | 3487 +++-------------- crates/weavepy-vm/src/hot_filter.rs | 9 +- crates/weavepy-vm/src/hot_gates.rs | 11 +- crates/weavepy-vm/src/inst_dict.rs | 15 +- crates/weavepy-vm/src/lazy_arc.rs | 16 +- crates/weavepy-vm/src/lib.rs | 3420 ++-------------- crates/weavepy-vm/src/object.rs | 89 +- crates/weavepy-vm/src/rc.rs | 155 +- crates/weavepy-vm/src/shared_value.rs | 10 +- crates/weavepy-vm/src/stdlib/ast_build.rs | 21 +- crates/weavepy-vm/src/stdlib/asyncio_mod.rs | 7 +- .../src/stdlib/collections_native.rs | 13 - crates/weavepy-vm/src/stdlib/functools_mod.rs | 13 +- crates/weavepy-vm/src/stdlib/gc_real.rs | 4 +- crates/weavepy-vm/src/stdlib/json_accel.rs | 2 +- crates/weavepy-vm/src/stdlib/sys.rs | 17 +- crates/weavepy-vm/src/stdlib/thread_real.rs | 50 +- crates/weavepy-vm/src/stdlib/warnings_mod.rs | 12 +- crates/weavepy-vm/src/stdlib/weakref_real.rs | 29 +- crates/weavepy-vm/src/tier2.rs | 156 +- crates/weavepy-vm/src/types.rs | 77 +- crates/weavepy-vm/src/vm_singletons.rs | 178 +- crates/weavepy-vm/src/weak_object.rs | 136 + crates/weavepy-vm/src/weakref_registry.rs | 168 +- .../test_finalizer_constructor_fallback.py | 38 + 33 files changed, 1584 insertions(+), 6679 deletions(-) create mode 100644 crates/weavepy-vm/src/weak_object.rs create mode 100644 tests/regrtest/test_finalizer_constructor_fallback.py diff --git a/crates/weavepy-capi/src/gc_bridge.rs b/crates/weavepy-capi/src/gc_bridge.rs index c6b73f9d..e1027162 100644 --- a/crates/weavepy-capi/src/gc_bridge.rs +++ b/crates/weavepy-capi/src/gc_bridge.rs @@ -48,8 +48,8 @@ fn instance_type_ptr(obj: &Object) -> Option<*mut PyTypeObject> { // An instance that never crossed into C (no box, no inline body) // has no C-held edges for `tp_traverse`/`tp_clear` to report or // drop: the VM traverses its own fields. This keeps the bridge — - // a mint, a C call and a free per node — off the prompt-reap - // cascade of every native exception instance. + // a mint, a C call and a free per node — off every collection + // that visits a native exception instance. Object::Instance(i) if i.c_body.get() != 0 => crate::types::type_ptr_for_class(&i.cls()), _ => None, } diff --git a/crates/weavepy-capi/src/instance.rs b/crates/weavepy-capi/src/instance.rs index b58818e7..dc74d22c 100644 --- a/crates/weavepy-capi/src/instance.rs +++ b/crates/weavepy-capi/src/instance.rs @@ -329,30 +329,10 @@ pub unsafe fn release_c_ownership(p: *mut PyObject) { .lock() .ok() .and_then(|mut g| g.as_mut().and_then(|m| m.remove(&(p as usize)))); - if let Some(inst) = &pinned { + if pinned.is_some() { // C just dropped what may be this instance's last program-visible - // reference (CPython would run `tp_dealloc` right here). If the VM - // side still pins it through GC-handle / weakref-registry clones, a - // plain `Rc` drop below cannot reap it — a *tracked* extension - // temporary that dies inside a C call (a Cython generator abandoned - // by `BlockManager.iget`, still holding `self` in its closure) would - // otherwise stay pinned by its own handle until the next full - // collection, keeping everything it references alive with it - // (pandas' `_is_view_after_cow_rules` then reads a stale live - // `Block` weakref). Park it for the eval loop's between-bytecodes - // reap; anything still genuinely alive fails the drain's - // refcount-dead test untouched. - // - // Only a *GC-tracked* instance needs the park: an untracked one has - // no handle pinning it, so the plain `Rc` drop below reclaims it - // immediately (running `tp_dealloc` through the free hook) — and a - // queued clone would instead keep it alive until the next eval-loop - // safe point, which never comes for a drop performed outside - // bytecode execution (an embedding host dropping its last handle). - let obj = weavepy_vm::object::Object::Instance(inst.clone()); - if weavepy_vm::gc_trace::is_tracked(weavepy_vm::weakref_registry::id_of(&obj)) { - weavepy_vm::vm_singletons::queue_cext_dropped(&obj); - } + // reference; the plain `Rc` drop below reclaims it then (running + // `tp_dealloc` through the free hook), as CPython's would. } else if unsafe { crate::mirror::is_orphaned_instance_body(p) } && !body_free_in_flight(p as usize) { @@ -618,8 +598,8 @@ unsafe fn dealloc_and_free_body(p: *mut PyObject) { (*ty).tp_basicsize = orig_basicsize.wrapping_add(8); // A `tp_dealloc` is extension code: its decref chains can // re-enter the VM, and any bytecode that runs beneath it - // must see a live C frame (RFC 0047 — the prompt reaper's - // borrowed-pointer window). + // must see a live C frame (RFC 0047 — the borrowed-pointer + // window that defers parked C drops). let _cext_guard = weavepy_vm::vm_singletons::enter_cext_call(); // Consent protocol: the dealloc must reach a `tp_free` // (absorbed by `PyObject_Free`/`PyObject_GC_Del`, which diff --git a/crates/weavepy-capi/src/interp.rs b/crates/weavepy-capi/src/interp.rs index f05b3401..654d20bd 100644 --- a/crates/weavepy-capi/src/interp.rs +++ b/crates/weavepy-capi/src/interp.rs @@ -145,11 +145,10 @@ pub fn ensure_active(body: impl FnOnce() -> R) -> R { // RFC 0047 (wave 5): mark a C-extension call live on this thread for // the duration of `body`. Every bridged C invocation (dunder shims, // `PyCFunction`s, descriptors, tp_call, foreign hooks) funnels through - // here, so the VM's prompt reaper can tell "plain bytecode, no - // extension frame below" (`depth == 0` — reclaiming a dead escaped - // subgraph is as safe as CPython's own `tp_dealloc`) from "inside a C - // call that may hold borrowed body pointers" (`depth > 0` — defer - // memory reclamation to the tracing collector). + // here, so the VM can tell "plain bytecode, no extension frame below" + // (`depth == 0` — releasing objects C dropped is as safe as CPython's + // own `tp_dealloc`) from "inside a C call that may hold borrowed body + // pointers" (`depth > 0` — keep them parked). let _cext_guard = weavepy_vm::vm_singletons::enter_cext_call(); if current_interpreter_mut().is_some() { if ea_trace_enabled() { diff --git a/crates/weavepy-capi/src/object.rs b/crates/weavepy-capi/src/object.rs index dd2b5290..764a1bb1 100644 --- a/crates/weavepy-capi/src/object.rs +++ b/crates/weavepy-capi/src/object.rs @@ -1659,14 +1659,13 @@ pub(crate) unsafe fn free_box(p: *mut PyObject) { evict_property_box(weavepy_vm::sync::Rc::as_ptr(rc) as usize, p); } // RFC 0047 (wave 5): C is dropping what may be the payload's last - // program-visible reference, from inside an extension call the VM's - // prompt reaper cannot see. Park instance/container payloads for a - // refcount-guarded reap at the next eval-loop safe point so anything - // that died here gets its weakrefs cleared with CPython's timing - // (pandas' `BlockValuesRefs.has_reference` prunes dead block - // weakrefs the very next Python-level call after a Cython-internal - // `self.blocks = ...` rebind). A payload that is still alive - // elsewhere fails the reap's deadness test untouched. + // program-visible reference, from inside an extension call that may + // still hold borrowed body pointers. Park instance/container payloads + // and release them at the next eval-loop safe point outside any C + // call, so anything that dies there gets its weakrefs cleared before + // the next Python-level read (pandas' `BlockValuesRefs.has_reference` + // prunes dead block weakrefs right after a Cython-internal + // `self.blocks = ...` rebind). weavepy_vm::vm_singletons::queue_cext_dropped(&bx.payload.obj); if let Some(d) = bx.payload.destructor { let raw = Box::into_raw(bx); diff --git a/crates/weavepy-capi/src/pep741.rs b/crates/weavepy-capi/src/pep741.rs index 2d5b096e..06d90417 100644 --- a/crates/weavepy-capi/src/pep741.rs +++ b/crates/weavepy-capi/src/pep741.rs @@ -29,8 +29,8 @@ use weavepy_vm::object::Object; use crate::embed::PyInitFn; use crate::initconfig::{ - self, _PyStatus_TYPE_ERROR, _PyStatus_TYPE_EXIT, EmbedConfig, PyConfig, - PyConfig_InitIsolatedConfig, PyStatus, PyWideStringList, + self, EmbedConfig, PyConfig, PyConfig_InitIsolatedConfig, PyStatus, PyWideStringList, + _PyStatus_TYPE_ERROR, _PyStatus_TYPE_EXIT, }; use crate::object::PyObject; diff --git a/crates/weavepy-jit/src/ir.rs b/crates/weavepy-jit/src/ir.rs index 66f7be00..eb1f3ead 100644 --- a/crates/weavepy-jit/src/ir.rs +++ b/crates/weavepy-jit/src/ir.rs @@ -236,7 +236,7 @@ pub enum TOp { /// (staged through `ret_bits`) and write it into closure cell /// `idx` through the registered `wpjit_cell_set` helper. A /// displaced heap value must drop on the interpreter's store path - /// (prompt reap, parked finalizers), so the helper deopts before + /// (which runs any finalizer it queues), so the helper deopts before /// the store in that case and the interpreter re-executes it. CellSet { idx: u32, lane: JitType }, /// RFC 0065 WS5 — `x.append(v)` on a pinned list: pops the value @@ -349,16 +349,16 @@ pub enum TOp { /// interpreted per the trained `val` lane), and calls the /// registered `wpjit_dict_set` helper — the interpreter's own /// `dict_insert` chokepoint, so PEP 509 / watcher discipline is - /// identical. A displaced value that would run the prompt-reap - /// cascade deopts *before* the store (the `wpjit_attr_set` + /// identical. A displaced value whose release could run a + /// finalizer deopts *before* the store (the `wpjit_attr_set` /// discipline), as do active C-API dict watchers and any key-lane /// surprise. DictSet { key: JitType, val: JitType }, /// `del d[k]` on a pinned exact `dict`: pops the key (`key` lane) /// and the dict pin, and calls the registered `wpjit_dict_del` /// helper — the interpreter's own `dict_remove` chokepoint. A - /// missing key, a displaced value that would run the prompt-reap - /// cascade, active C-API dict watchers, or any key-lane surprise + /// missing key, a displaced value whose release could run a + /// finalizer, active C-API dict watchers, or any key-lane surprise /// deopts *before* the delete, and the interpreter re-executes it /// (raising the exact `KeyError`). DictDel { key: JitType }, diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index a71e7de9..716756d9 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -12470,7 +12470,7 @@ fn list_setitem(args: &[Object]) -> Result { } let replaced = crate::apply_slice_assignment(&mut l.borrow_mut(), s, replacement)?; for old in replaced { - queue_removed(old); + drop(old); } return Ok(Object::None); } @@ -12479,7 +12479,7 @@ fn list_setitem(args: &[Object]) -> Result { let n = list_index_arg(l.len(), key, "__setitem__")?; std::mem::replace(&mut l[n], val.clone()) }; - queue_removed(old); + drop(old); Ok(Object::None) } @@ -12493,7 +12493,7 @@ fn list_delitem(args: &[Object]) -> Result { let mut indices = crate::slice_indices(l.len(), s)?; indices.sort_unstable(); for i in indices.into_iter().rev() { - queue_removed(l.remove(i)); + drop(l.remove(i)); } return Ok(Object::None); } @@ -12502,7 +12502,7 @@ fn list_delitem(args: &[Object]) -> Result { let n = list_index_arg(l.len(), key, "__delitem__")?; l.remove(n) }; - queue_removed(removed); + drop(removed); Ok(Object::None) } @@ -12672,7 +12672,7 @@ fn list_remove(args: &[Object]) -> Result { } }; if let Some(removed) = removed { - queue_removed(removed); + drop(removed); } return Ok(Object::None); } @@ -12931,7 +12931,7 @@ fn list_clear(args: &[Object]) -> Result { } let evicted: Vec = std::mem::take(&mut *list_self(args)?.borrow_mut()); for v in evicted { - queue_removed(v); + drop(v); } Ok(Object::None) } @@ -13141,7 +13141,7 @@ fn dict_setitem(args: &[Object]) -> Result { ensure_dict_key(key)?; let old = dict_insert(&d, key.clone(), val.clone())?; if let Some(old) = old { - queue_removed(old); + drop(old); } Ok(Object::None) } @@ -13160,15 +13160,6 @@ fn dict_getitem(args: &[Object]) -> Result { found.ok_or_else(|| key_error_object(key.clone())) } -/// Route a key/value evicted by a container mutator to the prompt-reap -/// queue (see [`crate::vm_singletons::queue_container_removed`]) before -/// dropping our reference. The queue holds a clone; the eval loop reaps it -/// at the next between-bytecodes safe point if the eviction was its last -/// program-visible reference. -pub(crate) fn queue_removed(v: Object) { - crate::vm_singletons::queue_container_removed(&v); -} - fn dict_delitem(args: &[Object]) -> Result { let d = dict_self(args)?; let key = args @@ -13177,8 +13168,8 @@ fn dict_delitem(args: &[Object]) -> Result { ensure_dict_key(key)?; let removed = dict_remove(&d, key)?; if let Some((k, v)) = removed { - queue_removed(k); - queue_removed(v); + drop(k); + drop(v); Ok(Object::None) } else { Err(key_error_object(key.clone())) @@ -13255,7 +13246,7 @@ fn dict_pop(args: &[Object]) -> Result { if let Some((k, v)) = removed { // The *stored* key (equal to, but possibly distinct from, the // lookup key) is evicted too; the value is returned to the caller. - queue_removed(k); + drop(k); Ok(v) } else if let Some(default) = args.get(2).cloned() { Ok(default) @@ -13294,7 +13285,7 @@ fn dict_update(args: &[Object]) -> Result { let src_len = entries.len(); for (k, v) in entries { if let Some(old) = dict_insert(&d, k.0, v)? { - queue_removed(old); + drop(old); } // CPython's `PyDict_Merge` re-checks the source size // *after* every insert: a key `__eq__` that mutates the @@ -13315,7 +13306,7 @@ fn dict_update(args: &[Object]) -> Result { let mut dst = d.borrow_mut(); for (k, v) in entries { if let Some(old) = dst.insert(k, v) { - queue_removed(old); + drop(old); } } } @@ -13337,8 +13328,8 @@ fn dict_clear(args: &[Object]) -> Result { } } for (k, v) in evicted { - queue_removed(k.0); - queue_removed(v); + drop(k.0); + drop(v); } Ok(Object::None) } @@ -13582,7 +13573,7 @@ fn dict_ior(args: &[Object]) -> Result { .collect(); for (k, v) in entries { if let Some(old) = dict_insert(&d, k.0, v)? { - queue_removed(old); + drop(old); } } return Ok(args[0].clone()); @@ -13630,7 +13621,7 @@ fn dict_ior(args: &[Object]) -> Result { let (k, v) = (kv.next().unwrap(), kv.next().unwrap()); ensure_dict_key(&k)?; if let Some(old) = dict_insert(&d, k, v)? { - queue_removed(old); + drop(old); } i += 1; } @@ -13675,12 +13666,12 @@ fn apply_set_inplace_op(s: &mut crate::object::SetData, op: SetInplaceOp) { } SetInplaceOp::Discard(k) => { if let Some(removed) = s.swap_take(&k) { - queue_removed(removed.0); + drop(removed.0); } } SetInplaceOp::Clear => { for k in s.drain(..) { - queue_removed(k.0); + drop(k.0); } } } diff --git a/crates/weavepy-vm/src/capi_watchers.rs b/crates/weavepy-vm/src/capi_watchers.rs index 79d435ab..521225b2 100644 --- a/crates/weavepy-vm/src/capi_watchers.rs +++ b/crates/weavepy-vm/src/capi_watchers.rs @@ -221,11 +221,7 @@ pub fn dict_update_begin(target: &Rc>, target_was_empty: bool) pub fn note_dropped(vm_dropped: &Object) { if let Object::Dict(d) = vm_dropped { if dicts_active() { - let id = crate::weakref_registry::id_of(vm_dropped); - let registry_clones = crate::weakref_registry::strong_clone_count(id); - // The GC registry's `track` handle is a strong clone too. - let gc_clone = usize::from(crate::gc_trace::is_tracked(id)); - if Rc::strong_count(d) <= 1 + registry_clones + gc_clone && dict_mask(d) != 0 { + if Rc::strong_count(d) <= 1 && dict_mask(d) != 0 { dict_event("DEALLOCATED", d, None, None); let ptr = Rc::as_ptr(d) as usize; let mut w = WATCHED_DICTS.lock(); @@ -238,12 +234,7 @@ pub fn note_dropped(vm_dropped: &Object) { } if let Object::Function(f) = vm_dropped { if funcs_active() { - let id = crate::weakref_registry::id_of(vm_dropped); - let registry_clones = crate::weakref_registry::strong_clone_count(id); - // The GC registry holds one strong clone from MAKE_FUNCTION's - // `track`; a dying function is down to our handle + that one. - let gc_clone = usize::from(crate::gc_trace::is_tracked(id)); - if Rc::strong_count(f) <= 1 + registry_clones + gc_clone { + if Rc::strong_count(f) <= 1 { func_event("DESTROY", vm_dropped, &Object::None); } } diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 7dda0326..641f70dc 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -81,6 +81,7 @@ use std::sync::atomic::{ }; use crate::object::Object; +use crate::weak_object::WeakObject; use crate::weakref_registry::{id_of, ObjectId}; /// A set of object ids (addresses), hashed with the address mixer. @@ -131,92 +132,46 @@ pub trait Finalize { fn finalize(&self); } -/// Per-tracked-object metadata. Stored as a `Vec>` -/// inside each [`Generation`] — the `Arc` makes the per-handle -/// state cheaply shared across the candidate snapshot during a -/// collection. +/// A tracked object's entry: a non-owning handle on the object plus the +/// bookkeeping that outlives a collection. The collector never keeps an +/// object alive. An entry whose object died stays in its generation until +/// the next collection or prune of that generation drops it; its weak +/// handle keeps the allocation reserved until then, so the address the +/// index keys it by can't be reused by another object meanwhile. #[allow(missing_debug_implementations)] pub struct TrackedHandle { - /// Strong handle to the tracked object. Holding a strong - /// reference is fine because the GC's job is to *break* - /// cycles by clearing fields, not by dropping the Rc. - pub object: Object, - /// Identity, computed from `id_of(object)` at - /// `track`-time. Cached so the GC's mark phase doesn't - /// have to recompute on every visit. + /// The tracked object, held weakly. + pub object: WeakObject, + /// Identity, computed from `id_of(object)` at `track` time. pub id: ObjectId, - /// Working `gc_refs` field. Reset to a fresh value at the - /// start of every collection cycle. - pub gc_refs: AtomicI64, - /// Tri-color state. Reset to White at cycle start. + /// [`color::Frozen`] while `gc.freeze()` holds the entry, otherwise + /// [`color::White`]. Collections keep their marking state in the + /// candidates they build, not here. pub color: AtomicU8, - /// Generation index (0..N_GENERATIONS). Survivors are - /// promoted by incrementing this. + /// Generation index (0..N_GENERATIONS). Survivors are promoted by + /// incrementing this. pub generation: AtomicU8, /// Position of this handle within its owning `Vec` — /// `generations[generation].handles` normally, or the `frozen` /// list when `color == Frozen`. Maintained by every site that /// pushes, drains, or rebuilds those vectors so that - /// [`GcState::untrack_id`] can `swap_remove` in O(1) instead of - /// scanning every generation (which made drop-heavy, - /// large-heap workloads quadratic — RFC 0039 WS4). + /// [`GcState::untrack_id`] can `swap_remove` in O(1). pub slot: CachedSlot, /// Has this object's `__del__` already *run* to completion? CPython /// guarantees a finaliser runs at most once. pub finalized: AtomicBool, /// Has this object's `__del__` been *queued* by a collection but not yet - /// run? While set, the object is kept tracked and excluded from the - /// `collected` count: its finalizer (drained after `gc.collect()` returns) - /// may resurrect it, and CPython only counts objects that are actually - /// reclaimed. Cleared once the finalizer completes (`finalized` is set). + /// run? While set, the object is excluded from the `collected` count: + /// its finalizer (drained after `gc.collect()` returns) may resurrect + /// it, and CPython only counts objects that are actually reclaimed. pub finalize_queued: AtomicBool, - /// Cached count of callback-bearing weakref clones the registry holds for - /// this object — refreshed by [`GcState::note_weakref_finalizable`]. The - /// prompt-finalization scan uses it as a fast-path liveness filter: an - /// object whose `strong_count` exceeds `1 (our handle) + weak_clones` - /// definitely still has a program reference, so the scan can skip it - /// without the (per-id) registry lookup that computes the exact clone - /// count. The cache is only ever an *upper bound* on the live clone count - /// (weakrefs can clear without notifying us), and an over-estimate only - /// makes the filter admit *more* objects to the precise check — never - /// fewer — so it can never cause a dead object to be missed. - pub weak_clones: AtomicUsize, - /// RFC 0061 (WS1b): set (never cleared) when this handle leaves the - /// GC index (`untrack_id`, the collection rebuild's White purge). The - /// prompt-reap suspect probe reads it instead of a per-entry - /// `is_tracked` registry lookup — that lookup (`GcState::handle_for`) - /// was the hottest non-dispatch symbol on drop-heavy profiles. A - /// re-tracked object gets a *fresh* handle, so a set flag - /// definitively means "this handle is dead". - pub untracked: AtomicBool, - /// RFC 0077 (WS2): set while this finalizable-index entry is *cold*: - /// its strong count sat more than [`FIN_COLD_MARGIN`] above the - /// dead line at the last probe, so it can't die at a single drop - /// safe point without a cascade that frees many holders at once. - /// Cold entries are re-probed every [`FIN_COLD_STRIDE`]-th scan - /// instead of every one; the ABC machinery alone keeps dozens of - /// weakref-callback-enrolled classes live for the process lifetime, - /// and probing them at every reference-dropping opcode was a - /// first-order interpreter cost. Only meaningful for handles in the - /// finalizable index. - pub fin_cold: AtomicBool, - /// Position in `GcState::finalizable_hot` while hot (RFC 0077 WS2); - /// only valid under that vector's lock, fixed up on swap-remove - /// exactly like [`Self::slot`]. - pub fin_hot_slot: CachedSlot, - /// Strong count seen at the previous hot probe and the number of - /// consecutive probes it has held (RFC 0077 WS2); a count unchanged - /// for [`FIN_STABLE_PROBES`] probes demotes the entry to cold. - pub fin_last_sc: AtomicUsize, - pub fin_stable: AtomicU8, } /// A compact vector-position hint, with separate absent and uncached states. /// -/// Positions too large to cache use the existing pointer-search fallback. -/// They must not look absent: a hot finalizer with an uncached position still -/// belongs to the hot set. Neither decoded sentinel can index a live -/// `Vec>`, whose allocation is bounded by `isize::MAX`. +/// Positions too large to cache use the pointer-search fallback. Neither +/// decoded sentinel can index a live `Vec>`, whose +/// allocation is bounded by `isize::MAX`. #[derive(Debug)] pub struct CachedSlot(AtomicU32); @@ -261,31 +216,6 @@ impl CachedSlot { } } -/// RFC 0077 (WS2): a finalizable entry whose strong count exceeds the -/// dead line (`1 + weak clones`) by more than this many references -/// goes cold. Sixteen references dying between two consecutive drop -/// safe points takes a cascade, and the cold stride bounds even that -/// case's latency. -const FIN_COLD_MARGIN: usize = 16; -/// Cold finalizable entries are re-probed on every N-th scan. -const FIN_COLD_STRIDE: u64 = 32; -/// A hot entry whose strong count is unchanged across this many -/// consecutive probes goes cold regardless of the margin. The margin -/// alone left deltablue's one finalizable (a callback-weakref'd class, -/// three references above its dead line for the whole run) hot for -/// 2.6M probes; a fixed set of holders is the signature of a long-lived -/// object, and the cold stride still finds its eventual death within -/// `FIN_COLD_STRIDE` drop safe points — the direct paths -/// (`prompt_reap_dropped` and the cascade) catch the common shapes -/// sooner regardless. -const FIN_STABLE_PROBES: u8 = 64; - -/// `WEAVEPY_NO_FIN_COLD=1` pins every finalizable entry hot (bisection). -fn fin_no_cold() -> bool { - static NO_FIN_COLD: std::sync::OnceLock = std::sync::OnceLock::new(); - *NO_FIN_COLD.get_or_init(|| std::env::var_os("WEAVEPY_NO_FIN_COLD").is_some()) -} - #[allow(non_upper_case_globals)] pub mod color { pub const White: u8 = 0; @@ -295,24 +225,19 @@ pub mod color { } impl TrackedHandle { - pub fn new(object: Object, generation: usize) -> Self { + /// A generation-`generation` entry for `object`, or `None` for a value + /// with no heap allocation of its own. + pub fn new(object: &Object, generation: usize) -> Option { assert!(generation < N_GENERATIONS, "invalid collector generation"); - Self { - id: id_of(&object), - object, - gc_refs: AtomicI64::new(0), + Some(Self { + id: id_of(object), + object: WeakObject::new(object)?, color: AtomicU8::new(color::White), generation: AtomicU8::new(generation as u8), slot: CachedSlot::new(0), finalized: AtomicBool::new(false), finalize_queued: AtomicBool::new(false), - weak_clones: AtomicUsize::new(0), - untracked: AtomicBool::new(false), - fin_cold: AtomicBool::new(false), - fin_hot_slot: CachedSlot::new(usize::MAX), - fin_last_sc: AtomicUsize::new(0), - fin_stable: AtomicU8::new(0), - } + }) } } @@ -366,46 +291,75 @@ pub struct GcStats { pub uncollectable: u64, } +/// One object under examination by a collection: a strong reference for +/// the collection's duration, its entry (absent for a temporary candidate +/// discovered through an untracked container), and the marking state. +struct Cand { + obj: Object, + id: ObjectId, + entry: Option>, + gc_refs: std::cell::Cell, + color: std::cell::Cell, +} + +impl Cand { + fn new(obj: Object, id: ObjectId, entry: Option>) -> Self { + Self { + obj, + id, + entry, + gc_refs: std::cell::Cell::new(0), + color: std::cell::Cell::new(color::White), + } + } + + fn is_white(&self) -> bool { + self.color.get() == color::White + } + + /// Whether the object still owes a `__del__` (or a generator close). + fn pending_finalizer(&self) -> bool { + has_finalizer(&self.obj) + && !self + .entry + .as_ref() + .is_some_and(|e| e.finalized.load(Ordering::Acquire)) + } +} + +/// How many entries of the older generations each young collection or +/// prune examines for dead objects, so a dead old object's allocation is +/// released within a bounded number of young collections rather than at +/// the next full one. +const OLD_PRUNE_BUDGET: usize = 512; + /// Public state of the cycle GC. /// /// A single instance lives in a process-global `LazyLock` (see /// [`with_state`]) and is shared by every OS thread, mirroring /// CPython's one-collector-per-interpreter model. This is required -/// for correctness: post-RFC-0025 the heap is `Arc`-rooted and a -/// cycle's links can be allocated on different threads, so only a -/// shared tracked-set can ever observe and break such a cycle. All -/// fields are `Sync` (interior `GilCell`s + atomics), so concurrent -/// access is memory-safe; the GIL additionally serializes mutators. +/// for correctness: the heap is `Arc`-rooted and a cycle's links can +/// be allocated on different threads, so only a shared tracked-set can +/// ever observe and break such a cycle. All fields are `Sync` (interior +/// `GilCell`s + atomics), so concurrent access is memory-safe; the GIL +/// additionally serializes mutators. +/// +/// The collector owns nothing: its entries are weak, so objects die by +/// reference count exactly when CPython's would, and collections only +/// have to find and break the cycles. #[allow(missing_debug_implementations)] pub struct GcState { generations: RefCell<[Generation; N_GENERATIONS]>, - /// RFC 0065 (WS4): insert-only miss-filter over `index`, maintained - /// at [`Self::track_now`] and consulted by the usually-miss - /// `is_tracked`/`handle_for` probes on the drop paths. Rebuilt from - /// `index` once most of its bits name objects long gone. - /// - /// One per state, not one per process: a rebuild takes *this* - /// state's index as the truth, so a shared filter let any state's - /// full collection erase every other state's bits, and their - /// `handle_for` then answered `None` for live objects. Production - /// runs one state, which hid it; the test binary runs many. + /// Insert-only miss-filter over `index`, maintained at + /// [`Self::track_now`] and consulted by the usually-miss `is_tracked` + /// probes. Rebuilt from `index` once most of its bits name objects + /// long gone. tracked_filter: crate::hot_filter::RebuildableBloom, - /// Id → handle index over every tracked object (all generations - /// plus the frozen set). Keeps `track` dedupe, `find_handle`, and - /// `is_tracked` O(1) — the linear scans they replace made - /// allocation-heavy workloads quadratic once the tracked - /// population grew past a few thousand. Keyed by object *address*, - /// so the internal fast hasher applies (consulted on every object - /// drop via the prompt reaper — SipHash here was a top-ten CPU - /// consumer under pandas). + /// Id → handle index over every tracked object (all generations plus + /// the frozen set): keeps `track` dedupe and `is_tracked` O(1). index: RefCell, - /// Re-entrancy guard: a collection can indirectly allocate (e.g. - /// queued finalizers running Python at the next safe point may - /// re-enter `track`), and a nested collection would see torn - /// generation lists. An `AtomicBool` (rather than a `Cell`) so the - /// whole `GcState` is `Sync` and can live in a process-global - /// `LazyLock` — the cycle collector is shared across every OS - /// thread, matching the `Arc`-rooted shared heap (RFC 0039 WS4). + /// Re-entrancy guard: a nested collection would see torn generation + /// lists. Atomic so the whole `GcState` is `Sync`. collecting: AtomicBool, /// Per-generation thresholds. Gen 0's threshold is /// "allocations since last gen 0 collection"; gens 1 and 2 @@ -417,24 +371,15 @@ pub struct GcState { /// generation. counts: RefCell<[usize; N_GENERATIONS]>, /// Lock-free mirror of `counts[0]` and `thresholds[0]`, packed as - /// `count << 32 | threshold`. `auto_collect_due` is polled by the - /// burst loop's allocation opcodes, where two `RefCell` borrows and - /// their guards cost more than the test they guard; both halves are - /// small enough that a `u64` holds them without loss on the - /// allocation counts any real program reaches. + /// `count << 32 | threshold`, for the allocation sites' due check. gen0_gauge: AtomicU64, /// Containers born holding only atomic values (see /// [`container_can_cycle`]). They cannot take part in a cycle while /// that holds, so they are not registered with the collector — only - /// weakly remembered here, so that a collection can re-examine them - /// and promote any that has since acquired a non-atomic element. - /// A weak reference never keeps its container alive, so a deferred - /// container still dies by refcount exactly when CPython's would. + /// remembered here, so that a collection can re-examine them and + /// promote any that has since acquired a non-atomic element. deferred: RefCell>, /// Length at which [`GcState::sweep_deferred`] compacts `deferred`. - /// Re-armed to twice the surviving population after each sweep, so - /// the O(live) scan stays amortized O(1) per deferred birth even for - /// a program holding millions of scalar lists. deferred_limit: AtomicUsize, /// Frozen handles. `gc.freeze()` moves all tracked objects /// here; they are skipped by future collections until @@ -454,55 +399,15 @@ pub struct GcState { /// Bumped on every change to the tracked-object set so /// callers can know when to invalidate caches. pub tracked_version: AtomicUsize, - /// Total tracked-object population (live count). Useful - /// for `gc.get_count` and for the threshold check. + /// Total tracked-object population (entries, live or not yet pruned). pub tracked_count: AtomicUsize, /// Ids whose `__del__` has been run (or queued) by a finalizing /// collection or teardown. Persists past the point where the handle /// leaves the tracked set so `gc.is_finalized()` still answers `True` /// for an object its finalizer resurrected (PEP 442 / `test_is_finalized`). finalized_ids: RefCell>, - /// Dedicated index over just the *finalizable* tracked objects — - /// instances whose class defines `__del__` and unfinished - /// generator-family objects. CPython runs `__del__` the instant an - /// object's refcount reaches zero; our tracing handle pins it until a - /// collection, so [`Self::reap_dead_finalizable`] emulates the prompt - /// path by scanning *this* small set (not the whole tracked - /// population) at the interpreter's reference-drop safe points. Keyed - /// by id like `index`; an object is in both while finalizable. - finalizable: RefCell>>, - /// Live population of [`Self::finalizable`]. A relaxed load of this - /// atomic is the gate the interpreter checks before every prompt- - /// finalization sweep: when it is zero (the overwhelmingly common - /// case — most code never defines `__del__`) the sweep is skipped - /// entirely, so the feature costs one atomic load per safe point. - finalizable_count: AtomicUsize, - /// Rotating scan position for [`Self::reap_dead_finalizable_locked`] - /// when the finalizable index outgrows its per-safe-point scan budget - /// (70k callback-weakrefs from a `WeakKeyDictionary` stress test must - /// not turn every reference-dropping opcode into a full index walk — - /// test_weakref's threaded-copy tests went quadratic). Stores the id - /// the next bounded scan resumes from. - fin_scan_cursor: std::sync::atomic::AtomicU64, - /// Safe-point call counter paired with the cursor: when the index is - /// over budget, only every [`FIN_SCAN_STRIDE`]-th safe point pays for - /// a window scan, keeping the steady-state per-opcode cost a counter - /// bump instead of 256 atomic strong-count loads. - fin_scan_tick: std::sync::atomic::AtomicU64, - /// RFC 0077 (WS2): the *hot* (not `fin_cold`) subset of - /// [`Self::finalizable`], so the per-drop scan walks only entries - /// that could plausibly die this safe point. Membership is mirrored - /// by each handle's `fin_cold` flag and `fin_hot_slot` index. - finalizable_hot: RefCell>>, - /// `finalizable_hot.len()`, published for the lock-free gate: when - /// zero, a scan that isn't on the cold stride returns without - /// borrowing anything — the steady state of a program whose - /// finalizables are all long-lived classes. - fin_hot_count: AtomicUsize, - /// Rotating index into `finalizable_hot` for over-budget hot scans. - fin_hot_cursor: AtomicUsize, - /// Scan counter for the cold stride (RFC 0077 WS2). - fin_cold_tick: std::sync::atomic::AtomicU64, + /// Where the next prune of the older generations resumes. + old_prune_cursor: AtomicUsize, } impl Default for GcState { @@ -511,51 +416,6 @@ impl Default for GcState { } } -impl Drop for GcState { - fn drop(&mut self) { - // Thread teardown: the tracked set can hold long generator / - // container chains whose recursive field-drops overflow the - // native stack (each `Arc` link is one `drop_in_place` frame). - // Clear every tracked object's container fields *iteratively* - // first so the chains are already severed when the handle - // vectors drop. Safe at this point: the thread is exiting, no - // Python code will observe the cleared objects. - let mut handles: Vec> = Vec::new(); - if let Ok(gens) = self.generations.try_borrow() { - for g in gens.iter() { - handles.extend(g.handles.iter().cloned()); - } - } - if let Ok(frozen) = self.frozen.try_borrow() { - handles.extend(frozen.iter().cloned()); - } - // Only clear objects whose sole remaining strong reference is the - // registry handle itself. Anything with extra references escaped - // into shared state that outlives this thread — e.g. Flag - // pseudo-members a worker published into the enum class's - // `_value2member_map_` — and other threads *will* observe it, so - // wiping its fields would corrupt live objects. Iterate to a - // fixpoint: each cleared object releases its referents, which can - // drop a chained object's count to 1 and make it clearable on the - // next pass — severing long chains without recursive drops. - loop { - let mut progress = false; - handles.retain(|h| { - if strong_count_for(&h.object) <= 1 { - clear_object_fields(&h.object); - progress = true; - false - } else { - true - } - }); - if !progress { - break; - } - } - } -} - impl GcState { pub fn new() -> Self { Self { @@ -577,117 +437,19 @@ impl GcState { tracked_version: AtomicUsize::new(0), tracked_count: AtomicUsize::new(0), finalized_ids: RefCell::new(crate::fasthash::FxHashSet::default()), - finalizable: RefCell::new(std::collections::BTreeMap::new()), - finalizable_count: AtomicUsize::new(0), - fin_scan_cursor: std::sync::atomic::AtomicU64::new(0), - fin_scan_tick: std::sync::atomic::AtomicU64::new(0), - finalizable_hot: RefCell::new(Vec::new()), - fin_hot_count: AtomicUsize::new(0), - fin_hot_cursor: AtomicUsize::new(0), - fin_cold_tick: std::sync::atomic::AtomicU64::new(0), - } - } - - /// Insert `h` into the finalizable index (RFC 0077 WS2: one place - /// for the population and hot-set bookkeeping). New entries start - /// hot; the next scan grades them. Returns whether it was new. - fn fin_insert(&self, id: ObjectId, h: HandleRc) -> bool { - let mut fin = self.finalizable.borrow_mut(); - if fin.contains_key(&id) { - return false; - } - fin.insert(id, h.clone()); - drop(fin); - self.fin_make_hot(&h); - if self.finalizable_count.fetch_add(1, Ordering::AcqRel) == 0 { - // RFC 0065 (WS1): 0→1 population transition — kick - // dispatch loops off the quiet path. - crate::hot_gates::bump_loop_gen(); - } - true - } - - /// Remove `id` from the finalizable index, if present, keeping the - /// population and hot set exact. - fn fin_remove(&self, id: ObjectId) -> bool { - // The usual case: nothing finalizable is tracked at all. (Inserts - // count after they land, and the same thread removes an id it - // inserted.) - if self.finalizable_count.load(Ordering::Acquire) == 0 { - return false; - } - let removed = self.finalizable.borrow_mut().remove(&id); - let Some(h) = removed else { - return false; - }; - self.fin_remove_bookkeeping(&h); - true - } - - /// The hot-set and population half of [`Self::fin_remove`], for the - /// one caller that already holds the finalizable lock. - fn fin_remove_bookkeeping(&self, h: &HandleRc) { - self.fin_make_cold(h); - if self.finalizable_count.fetch_sub(1, Ordering::AcqRel) == 1 { - // RFC 0065 (WS1): population reached zero — dispatch loops - // may re-enter the quiet path. - crate::hot_gates::bump_loop_gen(); - } - } - - /// Move `h` into the hot set (no-op if already hot). - fn fin_make_hot(&self, h: &HandleRc) { - let mut hot = self.finalizable_hot.borrow_mut(); - if !h.fin_cold.swap(false, Ordering::AcqRel) - && h.fin_hot_slot.load(Ordering::Relaxed) != usize::MAX - { - return; - } - h.fin_hot_slot.store(hot.len(), Ordering::Relaxed); - h.fin_stable.store(0, Ordering::Relaxed); - hot.push(h.clone()); - self.fin_hot_count.store(hot.len(), Ordering::Release); - } - - /// Move `h` out of the hot set (no-op if already cold). O(1) via - /// the handle's cached slot, with the swap-remove fixup. - fn fin_make_cold(&self, h: &HandleRc) { - let mut hot = self.finalizable_hot.borrow_mut(); - h.fin_cold.store(true, Ordering::Release); - let slot = h.fin_hot_slot.swap(usize::MAX, Ordering::AcqRel); - if slot == usize::MAX { - return; - } - if slot < hot.len() && HandleRc::ptr_eq(&hot[slot], h) { - hot.swap_remove(slot); - if let Some(moved) = hot.get(slot) { - moved.fin_hot_slot.store(slot, Ordering::Relaxed); - } - } else if let Some(pos) = hot.iter().position(|x| HandleRc::ptr_eq(x, h)) { - hot.swap_remove(pos); - if let Some(moved) = hot.get(pos) { - moved.fin_hot_slot.store(pos, Ordering::Relaxed); - } + old_prune_cursor: AtomicUsize::new(0), } - self.fin_hot_count.store(hot.len(), Ordering::Release); } /// Reinitialise the collector's locks in a `fork(2)` child, preserving the - /// inherited tracked-object state. The collector is process-global and its - /// `GilCell` fields are normally serialised by the GIL, but an `Object` - /// whose last `Arc` is released on a peer thread can drop — and run through - /// the collector's prompt-reap bookkeeping — without that thread holding - /// the GIL. If such a peer vanishes mid-`borrow` in the fork, the inherited + /// inherited tracked-object state. An `Object` whose last `Arc` is + /// released on a peer thread can drop without that thread holding the + /// GIL; if such a peer vanishes mid-`borrow` in the fork, the inherited /// `parking_lot` lock would wedge the child's very first allocation - /// (`test_threading.test_reinit_tls_after_fork` forks from 16 threads; the - /// child deadlocks in `threading._after_fork`'s `set(_enumerate())`). - /// Rebuild every field's lock in place and clear the re-entrancy guard — - /// CPython's `PyOS_AfterFork_Child` reinitialises the runtime's locks for - /// the same reason. - /// - /// Takes a raw `*mut Self` so each field's lock rebuild is driven from a - /// laundered raw pointer rather than a `&self` cast (see - /// [`crate::sync::GilCell::reinit_lock_after_fork`]). + /// (`test_threading.test_reinit_tls_after_fork`). Rebuild every field's + /// lock in place and clear the re-entrancy guard — CPython's + /// `PyOS_AfterFork_Child` reinitialises the runtime's locks for the same + /// reason. /// /// # Safety /// @@ -701,13 +463,12 @@ impl GcState { RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).index)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).thresholds)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).counts)); + RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).deferred)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).frozen)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).garbage)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).callbacks)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).stats)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).finalized_ids)); - RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).finalizable)); - RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).finalizable_hot)); // A peer may have vanished mid-collection with this set. (*this).collecting.store(false, Ordering::Release); } @@ -743,10 +504,9 @@ impl GcState { /// is already tracked, this is a no-op. pub fn track(&self, obj: Object) { // A container holding only atomic values cannot anchor a cycle - // yet, so it stays off the GC's books (and out of the prompt-reap - // cascade that every tracked death pays) until it holds something - // that could close one. `defer_container` keeps a weak reference - // so the next collection can promote it first. + // yet, so it stays off the GC's books until it holds something + // that could close one. `defer_container` remembers it so the + // next collection can promote it first. if !container_can_cycle(&obj) && self.defer_container(&obj) { return; } @@ -754,16 +514,13 @@ impl GcState { } /// Remember `obj` weakly instead of tracking it. Returns `false` when - /// the kind has no weak form (the caller tracks it as before). + /// the kind has no deferred form (the caller tracks it as before). /// /// A deferred birth does not advance the gen-0 allocation counter. /// That counter paces automatic collections against the rate cyclic /// garbage can appear, and nothing deferred here can be part of a - /// cycle. Counting the births and repaying them at the next sweep - /// instead made the counter sawtooth: a `[i, i]` loop under the JIT, - /// whose drops the native driver releases in batches, went from 1 - /// automatic collection to 59. A population that genuinely - /// accumulates is paced by `DEFERRED_CAP` instead. + /// cycle. A population that genuinely accumulates is paced by + /// `DEFERRED_CAP` instead. fn defer_container(&self, obj: &Object) -> bool { let Some(weak) = DeferredContainer::new(obj) else { return false; @@ -850,45 +607,29 @@ impl GcState { let new_id = id_of(&obj); { let mut index = self.index.borrow_mut(); - // One probe decides the dedupe and the insert (this runs for - // every function, list and dict born). + // One probe decides the dedupe and the insert. An entry whose + // object died can't sit at this address: its weak handle keeps + // the allocation reserved until the entry is pruned. let entry = match index.entry(new_id) { std::collections::hash_map::Entry::Occupied(_) => return, std::collections::hash_map::Entry::Vacant(e) => e, }; - // RFC 0065 (WS4): publish to the miss-filter *before* the - // insert becomes observable (we hold the index borrow, so - // no prober can race past a fresh registration). + let Some(handle) = TrackedHandle::new(&obj, 0) else { + return; + }; + let handle = HandleRc::new(handle); + // Publish to the miss-filter *before* the insert becomes + // observable (we hold the index borrow). self.tracked_filter.insert(new_id); - let handle = HandleRc::new(TrackedHandle::new(obj, 0)); entry.insert(handle.clone()); - // Enroll finalizable objects in the dedicated prompt-finalization - // index so the per-safe-point sweep scans only them, not the whole - // tracked population. - if has_finalizer(&handle.object) { - // Seed the clone-count cache from the registry: weakrefs - // created *before* tracking (asyncio's WeakValueDictionary - // registers a transport before its first cycle-suspect - // mutation tracks it) would otherwise make the fast-path - // filter read the object as permanently live. - handle.weak_clones.store( - crate::weakref_registry::strong_clone_count(new_id), - Ordering::Release, - ); - self.fin_insert(new_id, handle.clone()); - } let mut gens = self.generations.borrow_mut(); handle.slot.store(gens[0].handles.len(), Ordering::Release); gens[0].handles.push(handle); } // `finalized_ids` is keyed by object id (a pointer), which the // allocator recycles. A freshly tracked object at a recycled address - // must start *un*-finalized, so drop any stale entry — otherwise - // `gc.is_finalized(new_obj)` would inherit the previous tenant's - // finalized flag (`test_is_finalized`). - // Almost always empty (only `__del__`-bearing objects ever land - // there), and the borrow plus hash would otherwise be paid by - // every tracked birth. + // must start *un*-finalized (`test_is_finalized`). Almost always + // empty (only `__del__`-bearing objects ever land there). if !self.finalized_ids.borrow().is_empty() { self.finalized_ids.borrow_mut().remove(&new_id); } @@ -897,58 +638,20 @@ impl GcState { self.note_gen0_alloc(); } - /// Stop tracking `obj`. Used by the cycle-clearing path - /// after an object is reclaimed, and by the explicit - /// `gc._untrack(obj)` extension. + /// Stop tracking `obj`. Backs the explicit `gc._untrack(obj)` extension + /// and the C-API `PyObject_GC_UnTrack`. pub fn untrack_id(&self, id: ObjectId) { let Some(handle) = self.index.borrow_mut().remove(&id) else { return; }; - // RFC 0061 (WS1b): let suspect probes see the removal without a - // registry lookup. - handle.untracked.store(true, Ordering::Release); - // Purge any suspect-list clone of this handle in lock-step: the - // suspect entry shares the same `Arc`, so dropping - // the index's Arc alone would leave the handle's strong `object` - // reference alive in the suspect list (see `remove_suspect`). - remove_suspect(id); - // CPython's `PyObject_GC_Del` decrements the gen-0 allocation - // counter for every GC-tracked object freed, whatever its - // generation — so churn workloads whose young objects die by - // refcount (asyncio's per-connection task/future/handle webs) - // barely advance toward `threshold0` and automatic collections - // stay rare. Without the decrement, our prompt-reap untracks kept - // the gross allocation count, firing young collections an order - // of magnitude more often than CPython — frequently enough to - // land inside the window where a pending asyncio task is - // reachable only through its own await cycle. Collecting there is - // *correct* per CPython semantics (the docs tell users to hold - // task references; test_log_destroyed_pending_task relies on it), - // but CPython's cadence means its suite never observes it in e.g. - // test_streams.test_start_server — match the cadence, not just - // the semantics (RFC 0054). { let mut counts = self.counts.borrow_mut(); counts[0] = counts[0].saturating_sub(1); self.sync_gen0_gauge(counts[0], None); } - // Drop the finalizable-index entry in lock-step with the main index so - // the cheap prompt-finalization scan never sees a reclaimed object. - self.fin_remove(id); - // O(1) removal via the handle's cached `slot`. The index is the - // dedupe authority, so exactly one handle existed for `id`, and - // its `slot`/`generation`/`color` pinpoint its position without a - // per-generation scan (which made drop-heavy large heaps - // quadratic — RFC 0039 WS4). - // - // The cached `slot`/`generation` are *only* valid while the owning - // generation/frozen lock is held: a `swap_remove` elsewhere updates a - // moved handle's `slot` under that same lock. So we must acquire the - // vector lock *before* reading the cached position, and — because the - // GC is process-global and shared across OS threads — fall back to a - // pointer search if the cached slot is stale, rather than corrupting - // the vector with a wrong `swap_remove` (the bug behind the - // "generation slot index out of sync" panic under threaded GC). + // O(1) removal via the handle's cached `slot`. The cached position is + // only valid under the owning vector's lock; fall back to a pointer + // search if it's stale. if handle.color.load(Ordering::Acquire) == color::Frozen { let mut frozen = self.frozen.borrow_mut(); let slot = handle.slot.load(Ordering::Acquire); @@ -976,9 +679,6 @@ impl GcState { { swap_remove_handle(&mut gens[g].handles, slot); } else if !remove_handle_by_ptr(&mut gens[g].handles, &handle) { - // Declared generation was wrong too (e.g. a concurrent - // promotion landed between the `generation` and `slot` - // reads). Search the rest before giving up. for gg in 0..N_GENERATIONS { if gg != g && remove_handle_by_ptr(&mut gens[gg].handles, &handle) { break; @@ -990,511 +690,118 @@ impl GcState { serial_add(&self.tracked_version, 1); } - /// Reclaim every tracked object on this thread whose only remaining - /// strong reference is the cycle collector's own handle — dead - /// *acyclic* garbage that CPython's refcounting frees the instant - /// its last binding drops, but which our per-thread strong handle - /// pins until a collection. Skips finalizable objects (their - /// `__del__` must be ordered by a finalizing collection) and - /// weakref-watched objects (clearing their weakrefs runs user - /// callbacks). Because every survivor of these filters runs no - /// Python on the way out, this is safe to call from any GIL-holding - /// safe point — notably a `Thread.join` return, where a worker has - /// just dropped the last *program* reference to objects this thread - /// allocated (RFC 0039 WS4: cross-thread prompt reclamation across - /// the per-thread-heap boundary). Iterates to a fixpoint so freeing - /// one object reclaims any acyclic chain it anchored. Returns the - /// number of objects reclaimed. - pub fn reap_dead_acyclic(&self) -> usize { - // A collection already walks the same set; never re-enter it. - // Atomic claim (see `collect_impl`): overlapping reaps/collections - // from two threads corrupt each other's refcount math. - if self - .collecting - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_err() - { - return 0; - } - let n = self.reap_dead_acyclic_locked(); - self.collecting.store(false, Ordering::Release); - n - } - - /// [`Self::reap_dead_acyclic`] body, called with the `collecting` claim - /// already held (by `reap_dead_acyclic` itself or by `collect_impl`). - fn reap_dead_acyclic_locked(&self) -> usize { - let mut reclaimed = 0usize; - // A worklist of ids that *may* have just died. Freeing an object - // drops its fields, so the only way a tracked object can become - // dead during this pass is by being a (transitive) child of one we - // freed; seeding the worklist from those children makes a long - // acyclic chain (`UserList([UserList([...])])` 200k deep, - // `test_userlist.test_repr_deep`) O(n) instead of one full index - // rescan per link. The outer loop's full scan remains the fixpoint - // authority: it seeds the first batch and confirms the last. - let mut work: Vec = Vec::new(); - loop { - let dead: Vec = { - let index = self.index.borrow(); - index - .iter() - .filter(|(id, h)| { - strong_count_for(&h.object) <= 1 - && !has_finalizer(&h.object) - && crate::weakref_registry::count_for(**id) == 0 - }) - .map(|(id, _)| *id) - .collect() - }; - if dead.is_empty() { - break; - } - work.extend(dead); - while let Some(id) = work.pop() { - // Re-validate under a fresh borrow: a free earlier in this - // batch may have already reclaimed `id` as a child, and a - // worklist candidate may simply still be alive (counts - // only fall here, so nothing is ever revived). - let handle = self.index.borrow().get(&id).cloned(); - let Some(h) = handle else { continue }; - let still_dead = strong_count_for(&h.object) <= 1 - && !has_finalizer(&h.object) - && crate::weakref_registry::count_for(id) == 0; - if !still_dead { - continue; - } - if crate::hot_gates::env_flags::reap_trace() { - eprintln!("[ACYCLIC-REAP] {}", h.object.type_name_owned()); + /// Drop the entries of objects that have died from `handles`, and from + /// the index; `budget` caps how many entries are examined, starting at + /// `start` (wrapping). Returns how many were dropped and where the + /// examination stopped. + fn prune_dead_in( + index: &mut GcIndex, + handles: &mut Vec>, + start: usize, + budget: usize, + ) -> (usize, usize) { + let mut removed = 0; + let mut i = if start < handles.len() { start } else { 0 }; + let mut examined = 0; + while examined < budget && i < handles.len() { + examined += 1; + if handles[i].object.is_dead() { + let h = handles.swap_remove(i); + index.remove(&h.id); + if let Some(moved) = handles.get(i) { + moved.slot.store(i, Ordering::Release); } - // Enqueue the tracked objects this one holds, looking - // through untracked intermediaries (a tuple or frozenset - // holding the next link) which die with their parent. - self.collect_child_candidates(&h.object, &mut work); - self.untrack_id(id); - // `h` was cloned out of the index above; drop that clone so - // the untrack's field drops actually happen now (and the - // children's counts fall before they are re-examined). - drop(h); - reclaimed += 1; - } - } - reclaimed - } - - /// Push the ids of the tracked objects directly held by `obj` onto - /// `work`, descending through *untracked* children (immutable - /// containers and iterators are not persistently tracked) so a tracked - /// grandchild behind a tuple is still found. Bounded by an explicit - /// stack and a visited set so deep or shared untracked webs neither - /// recurse on the C stack nor blow up the candidate list. - fn collect_child_candidates(&self, obj: &Object, work: &mut Vec) { - let mut pending: Vec = Vec::new(); - traverse_object(obj, &mut |child| pending.push(child.clone())); - let mut seen: IdSet = IdSet::default(); - while let Some(child) = pending.pop() { - let id = id_of(&child); - if !seen.insert(id) { - continue; - } - if self.index.borrow().contains_key(&id) { - work.push(id); - } else if strong_count_for(&child) <= 2 { - // Untracked and about to die with its parent (our `child` - // clone plus the parent's own slot are the only counts): its - // fields fall too, so look through it. - traverse_object(&child, &mut |grandchild| pending.push(grandchild.clone())); - } - } - } - - /// True iff at least one finalizable object is currently tracked. A single - /// relaxed atomic load — the gate the interpreter checks at every - /// reference-drop safe point before deciding whether a prompt-finalization - /// sweep is even worth attempting. - #[inline] - pub fn has_any_finalizable(&self) -> bool { - self.finalizable_count.load(Ordering::Relaxed) > 0 - } - - /// Enroll a tracked object in the prompt-finalization index because a - /// weakref *with a callback* now watches it (`weakref.ref(obj, cb)`, - /// `weakref.finalize`, `multiprocessing.util.Finalize`). CPython fires - /// such a callback the instant the referent's last strong reference drops; - /// without this the callback would wait for the next cyclic collection. - /// No-op when the object isn't tracked (untracked weakref targets — - /// plain functions, bound methods — are handled by the collection-time - /// [`sweep_weakref_only_targets`] sweep instead). - pub fn note_weakref_finalizable(&self, id: ObjectId) { - let handle = self.index.borrow().get(&id).cloned(); - if let Some(h) = handle { - // Refresh the cached clone count used by the prompt-finalization - // fast-path filter (see `TrackedHandle::weak_clones`). - let clones = crate::weakref_registry::strong_clone_count(id); - h.weak_clones.store(clones, Ordering::Release); - // A fresh callback-weakref means somebody expects prompt - // notification: re-grade an already-enrolled cold entry. - if !self.fin_insert(id, h.clone()) { - self.fin_make_hot(&h); + removed += 1; + } else { + i += 1; } } + (removed, i) } - /// Drive one prompt-finalization pass over the dedicated finalizable index: - /// for every tracked finalizable object whose last *program* reference just - /// dropped, run its `__del__` and/or fire its weakref callbacks and reclaim - /// it. CPython does this by refcount the instant the count hits zero; our - /// tracing handle pins the object until a collection, so this emulates the - /// prompt path between bytecodes. - /// - /// "Dead" means the effective program refcount is zero: - /// `strong_count - 1 (our own GC handle) - (registry weakref clones)`. - /// A weakref slot keeps one strong clone of its target alive - /// ([`crate::weakref_registry::WeakRefSlot::target`]); discounting those is - /// what lets a `util.Finalize`-watched object — reachable now only through - /// the GC handle and the finalizer's own weakref — collapse to zero. - /// - /// Per dead object: - /// * If a `__del__` is still pending, queue it (set `finalize_queued`) and - /// leave the object tracked — its finalizer might resurrect it, and any - /// weakref callbacks must fire *after* `__del__` (CPython order). The - /// driver's next pass re-checks death and, if it stuck, fires the - /// weakrefs and reclaims. - /// * Otherwise (no `__del__`, or it already ran) fire the weakref callbacks - /// (`notify_clear` → queued for the interpreter) and untrack the object — - /// dropping the handle frees it and cascades the refcount into referents, - /// exposing the next layer of dead finalizables on the following pass. - /// - /// Returns the number of objects that made progress (queued a finalizer or - /// were reclaimed); the driver loops until this is zero. Cheap: scans only - /// the finalizable index, which holds just the `__del__`/callback-weakref - /// objects (typically a handful). - pub fn reap_dead_finalizable(&self) -> usize { - // RFC 0077 (WS2): the hot/cold gate runs *before* the claim below. - // With every finalizable cold (the steady state: the ABC - // machinery's callback-weakref'd type, a long-lived `__del__` - // object) and no cold tick due, the whole scan is two relaxed - // loads and a counter bump, not a CAS on the shared claim word. - // `WEAVEPY_NO_FIN_COLD=1` pins every entry hot for bisection. - let cold_tick = fin_no_cold() - || take_force_cold_tick() - || self - .fin_cold_tick - .fetch_add(1, Ordering::Relaxed) - .is_multiple_of(FIN_COLD_STRIDE); - if !cold_tick && self.fin_hot_count.load(Ordering::Relaxed) == 0 { - return 0; - } - // Atomic claim (see `collect_impl`): the untrack path below mutates - // the shared index/generations, which must not overlap a - // collection's mark walk on another thread. - if self - .collecting - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_err() - { - return 0; - } - let n = self.reap_dead_finalizable_locked(cold_tick); - self.collecting.store(false, Ordering::Release); - n - } - - /// [`Self::reap_dead_finalizable`] body, called with the `collecting` - /// claim held. - fn reap_dead_finalizable_locked(&self, cold_tick: bool) -> usize { - // Borrow-only scan: collect just the dead handles. The common case — - // all finalizables still reachable — allocates nothing (an empty - // `Vec::new()` doesn't heap-allocate) and pays only a cheap - // `strong_count` atomic load per object, skipping the per-id registry - // lookup via the `weak_clones` fast-path filter. - static FIN_TRACE: std::sync::OnceLock = std::sync::OnceLock::new(); - let fin_trace = *FIN_TRACE.get_or_init(|| std::env::var_os("WEAVEPY_FIN_TRACE").is_some()); - // Per-safe-point scan budget. Below this size the whole index is - // scanned (full CPython-like promptness — the overwhelmingly common - // shape); above it, a rotating window bounds the cost so a huge - // population of callback-weakrefs (70k `WeakKeyDictionary` keys in - // test_weakref's threaded-copy stress) doesn't make every - // reference-dropping opcode O(index) — quadratic over the run. - // Deaths are still detected within index_len/budget safe points. - const FIN_SCAN_BUDGET: usize = 256; - /// When over budget, additionally scan only every N-th safe point: - /// with tens of thousands of live callback-weakrefs even a bounded - /// window per drop opcode dominates the run; deaths are batched, - /// so probing less often loses nothing but a little latency. - const FIN_SCAN_STRIDE: u64 = 8; - // RFC 0077 (WS2): the hot/cold gate (decided by the caller, see - // `reap_dead_finalizable`). Cold entries — strong count far above - // the dead line, or unchanged across `FIN_STABLE_PROBES` - // consecutive probes — are re-probed only on the cold stride. - let no_cold = fin_no_cold(); - // The over-budget stride applies to whichever population this - // scan walks: the hot set on an ordinary scan, the whole index - // on a cold tick. - let walk_len = if cold_tick { - self.finalizable.borrow().len() - } else { - self.fin_hot_count.load(Ordering::Relaxed) - }; - if walk_len > FIN_SCAN_BUDGET { - let tick = self.fin_scan_tick.fetch_add(1, Ordering::Relaxed); - if !tick.is_multiple_of(FIN_SCAN_STRIDE) { - return 0; - } - } - floor_stats::bump(&floor_stats::FIN_SCANS, 1); - let mut probed = 0u64; - /// Outcome of one probe: keep hot, go cold, or dead. - enum Grade { - Hot, - Cold, - } - let mut out: Vec> = Vec::new(); - let mut probe = |h: &HandleRc, - out: &mut Vec>| - -> Grade { - probed += 1; - let sc = strong_count_for(&h.object); - let cached = h.weak_clones.load(Ordering::Acquire); - if fin_trace { - let tn = h.object.type_name_owned(); - if tn.contains("Transport") || tn.contains("SSLContext") || tn == "SSLProtocol" { - eprintln!( - "[FIN-SCAN] {tn} sc={sc} cached={cached} clones={}", - crate::weakref_registry::strong_clone_count(h.id) - ); - } - } - // Fast reject: more strong refs than our handle plus all of its - // (cached, upper-bound) weakref clones ⇒ a program reference is - // still live. Skip without touching the registry. Far above the - // dead line grades cold; so does a count that hasn't moved in - // `FIN_STABLE_PROBES` consecutive probes (a long-lived object - // with a fixed set of holders — the shape that otherwise pays - // a probe at every drop safe point for the process lifetime). - // Near the dead line and changing stays hot. - if sc > 1 + cached { - if no_cold { - return Grade::Hot; - } - if sc > 1 + cached + FIN_COLD_MARGIN { - return Grade::Cold; - } - // Already cold and still above the dead line: stay cold. - // Only reaching the borderline (handled below) re-arms it, - // so a long-lived object whose count merely jitters - // doesn't oscillate between the sets. - if h.fin_cold.load(Ordering::Relaxed) { - return Grade::Cold; - } - if h.fin_last_sc.swap(sc, Ordering::Relaxed) == sc { - let stable = h.fin_stable.fetch_add(1, Ordering::Relaxed) + 1; - if stable >= FIN_STABLE_PROBES { - h.fin_stable.store(0, Ordering::Relaxed); - return Grade::Cold; - } - } else { - h.fin_stable.store(0, Ordering::Relaxed); - } - return Grade::Hot; - } - // Borderline: compute the exact live clone count and test for - // an effective program refcount of zero. - let clones = crate::weakref_registry::strong_clone_count(h.id); - // Refresh the cached bound with the exact value. A cleared or - // died weakref leaves the cache stale-high, and a stale-high - // bound keeps a live object "borderline" — paying this - // registry lookup again at *every* reference-dropping safe - // point. This scan runs per drop opcode while any finalizable - // is live, so a single stale entry costs a whole test run - // (statistics.kde's hot sum-loop spent ~40% of its time - // here). New weakrefs refresh the cache upward via - // `note_weakref_finalizable`, so tightening it is safe. - h.weak_clones.store(clones, Ordering::Release); - if sc.saturating_sub(1).saturating_sub(clones) == 0 { - out.push(h.clone()); - } - Grade::Hot - }; - if cold_tick { - // Walk the whole index (or a rotating window of it) and - // re-grade every entry; collect the transitions and apply - // them after the index borrow ends. - let mut to_cold: Vec> = Vec::new(); - let mut to_hot: Vec> = Vec::new(); - { - let fin = self.finalizable.borrow(); - let mut grade = |h: &HandleRc| { - let was_cold = h.fin_cold.load(Ordering::Relaxed); - match probe(h, &mut out) { - Grade::Cold if !was_cold => to_cold.push(h.clone()), - Grade::Hot if was_cold => to_hot.push(h.clone()), - _ => {} - } - }; - if fin.len() <= FIN_SCAN_BUDGET { - for h in fin.values() { - grade(h); - } - } else { - let start = self.fin_scan_cursor.load(Ordering::Relaxed); - let mut next_cursor = start; - for (scanned, (id, h)) in - fin.range(start..).chain(fin.range(..start)).enumerate() - { - if scanned == FIN_SCAN_BUDGET { - next_cursor = *id; - break; - } - grade(h); - } - // budget < len guarantees the break above ran and set the - // resume point to the first unscanned id. - self.fin_scan_cursor.store(next_cursor, Ordering::Relaxed); - } - } - for h in &to_cold { - self.fin_make_cold(h); - } - for h in &to_hot { - self.fin_make_hot(h); - } - } else { - // Ordinary scan: only the hot set, demoting in place. Over - // budget, a rotating window (the cursor is an index here) - // keeps every hot entry visited within len/budget scans. - let mut hot = self.finalizable_hot.borrow_mut(); - let len = hot.len(); - let budget = len.min(FIN_SCAN_BUDGET); - let mut i = if len > FIN_SCAN_BUDGET { - self.fin_hot_cursor.load(Ordering::Relaxed) % len + /// Drop the dead young entries, plus a bounded slice of the older + /// generations'. Returns the young generation's live population. + fn prune(&self) -> usize { + let (removed, live) = { + let mut index = self.index.borrow_mut(); + let mut gens = self.generations.borrow_mut(); + let (mut removed, _) = + Self::prune_dead_in(&mut index, &mut gens[0].handles, 0, usize::MAX); + // The older generations, a slice at a time: the cursor runs + // over gen 1 then gen 2 as one sequence. + let cursor = self.old_prune_cursor.load(Ordering::Relaxed); + let n1 = gens[1].handles.len(); + let (g, start) = if cursor < n1 { + (1, cursor) } else { - 0 + (2, cursor - n1) }; - let mut scanned = 0usize; - while scanned < budget && !hot.is_empty() { - if i >= hot.len() { - i = 0; - } - scanned += 1; - let h = hot[i].clone(); - match probe(&h, &mut out) { - Grade::Cold => { - h.fin_cold.store(true, Ordering::Release); - h.fin_hot_slot.store(usize::MAX, Ordering::Relaxed); - hot.swap_remove(i); - if let Some(moved) = hot.get(i) { - moved.fin_hot_slot.store(i, Ordering::Relaxed); - } - } - Grade::Hot => i += 1, - } - } - if len > FIN_SCAN_BUDGET { - self.fin_hot_cursor.store(i, Ordering::Relaxed); - } - self.fin_hot_count.store(hot.len(), Ordering::Release); - } - let dead = out; - floor_stats::bump(&floor_stats::FIN_PROBED, probed); - if dead.is_empty() { - return 0; - } - floor_stats::bump(&floor_stats::FIN_DEAD, dead.len() as u64); - let mut progressed = 0; - for h in dead { - // Re-validate under fresh counts: an earlier finalizer in this batch - // may have resurrected `h`. - let clones = crate::weakref_registry::strong_clone_count(h.id); - let effective = strong_count_for(&h.object) - .saturating_sub(1) - .saturating_sub(clones); - if effective != 0 { - continue; // resurrected between scan and now - } - let del_pending = has_finalizer(&h.object) && !h.finalized.load(Ordering::Acquire); - if del_pending { - // Queue `__del__`; defer weakref callbacks + reclamation to a - // later pass (after the finalizer has run and either resurrected - // the object or left it dead). - if !h.finalize_queued.swap(true, Ordering::AcqRel) { - run_finalizer(&h.object); - progressed += 1; + let (r, stop) = + Self::prune_dead_in(&mut index, &mut gens[g].handles, start, OLD_PRUNE_BUDGET); + removed += r; + let next = if stop >= gens[g].handles.len() { + if g == 1 { + gens[1].handles.len() + } else { + 0 } + } else if g == 1 { + stop } else { - // No pending `__del__`: fire the (callback) weakrefs and - // reclaim. Weakref callbacks receive the weakref wrapper, not - // the target, so they cannot resurrect it — the object is - // definitively dead once they're queued. - crate::weakref_registry::queue_callbacks(crate::weakref_registry::notify_clear( - h.id, - )); - self.untrack_id(h.id); - progressed += 1; - } + gens[1].handles.len() + stop + }; + self.old_prune_cursor.store(next, Ordering::Relaxed); + (removed, gens[0].handles.len()) + }; + if removed > 0 { + self.tracked_count.fetch_sub( + removed.min(self.tracked_count.load(Ordering::Acquire)), + Ordering::AcqRel, + ); + serial_add(&self.tracked_version, 1); } - progressed + live } pub fn is_tracked(&self, id: ObjectId) -> bool { - // RFC 0065 (WS4): usually-miss probe — two relaxed loads - // instead of the index borrow. A stale filter bit (untracked - // id) just takes the precise path. + // Usually-miss probe: two relaxed loads instead of the index + // borrow. A stale filter bit just takes the precise path. if !self.tracked_filter.may_contain(id) { return false; } self.index.borrow().contains_key(&id) } - /// Copy only weakref targets not already owned by the cycle collector. - /// Hold the index before the registry, matching `track_now`'s lock order. - /// Both borrows end before the sweep clears slots or queues callbacks. - fn untracked_weakref_targets( - &self, - registry: &crate::weakref_registry::WeakRefRegistry, - ) -> Vec<(ObjectId, Object)> { - let index = self.index.borrow(); - registry.targets_matching(|id| !index.contains_key(&id)) - } - /// O(1) handle lookup by object id (any generation or frozen). pub fn handle_for(&self, id: ObjectId) -> Option> { - // RFC 0065 (WS4): see `is_tracked`. if !self.tracked_filter.may_contain(id) { return None; } self.index.borrow().get(&id).cloned() } - /// Snapshot every tracked object that still carries an unrun + /// Snapshot every live tracked object that still carries an unrun /// `__del__`. The interpreter's shutdown pass walks this list to - /// finalize objects that are still alive at exit — CPython runs - /// finalizers for everything during interpreter teardown, not just - /// for cyclic garbage. The per-handle `finalized` flag (shared with - /// the cycle collector) guarantees each `__del__` runs at most once. - pub fn finalization_candidates(&self) -> Vec> { + /// finalize objects that are still alive at exit. The per-handle + /// `finalized` flag guarantees each `__del__` runs at most once. + pub fn finalization_candidates(&self) -> Vec<(HandleRc, Object)> { let mut out = Vec::new(); - let pending = |h: &HandleRc| { - !h.finalized.load(Ordering::Acquire) - // A finalizer already queued by a collection (but not yet - // drained) must not be listed again — the pending queue owns - // it, and running it twice would double-fire `__del__`. - && !h.finalize_queued.load(Ordering::Acquire) - && has_finalizer(&h.object) - }; - let gens = self.generations.borrow(); - for gen in gens.iter() { - for h in &gen.handles { - if pending(h) { - out.push(h.clone()); - } + let mut consider = |h: &HandleRc| { + // A finalizer already queued by a collection (but not yet + // drained) must not be listed again. + if h.finalized.load(Ordering::Acquire) || h.finalize_queued.load(Ordering::Acquire) { + return; } - } - for h in self.frozen.borrow().iter() { - if pending(h) { - out.push(h.clone()); + if let Some(obj) = h.object.upgrade() { + if has_finalizer(&obj) { + out.push((h.clone(), obj)); + } } + }; + for gen in self.generations.borrow().iter() { + gen.handles.iter().for_each(&mut consider); } + self.frozen.borrow().iter().for_each(&mut consider); out } @@ -1565,14 +872,7 @@ impl GcState { .store(((count as u64) << 32) | th, Ordering::Relaxed); } - /// Threshold-driven automatic collection (CPython's `gc_alloc` - /// path): when the gen-0 allocation counter passes `threshold0`, - /// collect the *oldest* generation whose own counter has also - /// passed its threshold. Returns the number of objects reclaimed. - /// Callers must be at a safe point (no outstanding container - /// borrows); the interpreter invokes this from its allocation - /// sites. - /// Whether the next [`Self::maybe_auto_collect`] would run a + /// Whether the next [`Self::maybe_auto_collect`] would consider a /// collection (the young-generation counter has reached its /// threshold). Lets an allocation site that cannot run finalizers /// hand the allocation to one that can. @@ -1585,11 +885,23 @@ impl GcState { threshold != 0 && (gauge >> 32) >= threshold } + /// Threshold-driven automatic collection (CPython's `gc_alloc` + /// path): when the gen-0 counter passes `threshold0`, collect the + /// *oldest* generation whose own counter has also passed its + /// threshold. Returns whether a collection ran. Callers must be at a + /// safe point (no outstanding container borrows). + /// + /// CPython's counter is allocations *minus deallocations* of tracked + /// objects. Deaths aren't counted as they happen here, so the counter + /// is corrected when it trips: the dead young entries are pruned and + /// the count restarts from the survivors, unless they alone make up + /// half the threshold (which also bounds the pruning work at two + /// entries per allocation). pub fn maybe_auto_collect(&self) -> bool { if !self.is_enabled() || self.collecting.load(Ordering::Acquire) { return false; } - let (count0, eligible) = { + let (due, eligible, threshold0) = { let counts = self.counts.borrow(); let thresholds = self.thresholds.borrow(); if thresholds[0] == 0 { @@ -1602,16 +914,20 @@ impl GcState { gen = 2; } } - (counts[0] >= thresholds[0], gen) + (counts[0] >= thresholds[0], gen, thresholds[0]) }; - if !count0 { + if !due { return false; } - // Automatic young collection: single incremental pass, no whole-index - // acyclic reap (see `collect_impl`'s `exact` discussion). Report that a - // collection ran (regardless of how many objects it reclaimed) so the - // caller drains any `__del__` finalizers it deferred — without paying a - // pending-queue probe on every allocation. + let live = self.prune(); + if live < threshold0 / 2 { + let mut counts = self.counts.borrow_mut(); + counts[0] = live; + self.sync_gen0_gauge(live, None); + return false; + } + // Automatic young collection: a single pass (see `collect_impl`'s + // `exact` discussion). self.collect_impl(eligible, false); true } @@ -1626,7 +942,7 @@ impl GcState { n + self.frozen.borrow().len() } - /// Snapshot all tracked objects. Used by + /// Snapshot all live tracked objects. Used by /// `gc.get_objects(generation=...)`. pub fn snapshot(&self, generation: Option) -> Vec { // `gc.get_objects()` enumerates every container CPython tracks, @@ -1634,24 +950,15 @@ impl GcState { self.promote_all_deferred(); let gens = self.generations.borrow(); let mut out = Vec::new(); + let mut push = |h: &HandleRc| out.extend(h.object.upgrade()); match generation { - Some(g) if g < N_GENERATIONS => { - for h in &gens[g].handles { - out.push(h.object.clone()); - } - } - _ => { - for g in gens.iter() { - for h in &g.handles { - out.push(h.object.clone()); - } - } - } + Some(g) if g < N_GENERATIONS => gens[g].handles.iter().for_each(&mut push), + _ => gens + .iter() + .for_each(|g| g.handles.iter().for_each(&mut push)), } if generation.is_none() { - for h in self.frozen.borrow().iter() { - out.push(h.object.clone()); - } + self.frozen.borrow().iter().for_each(&mut push); } out } @@ -1674,9 +981,7 @@ impl GcState { /// `gc.unfreeze()` — move every frozen object back to /// generation 0. pub fn unfreeze_all(&self) { - // Lock order: generations before frozen, matching `freeze_all` - // (consistent ordering avoids a cross-cell deadlock now that the - // GC is process-global — RFC 0039 WS4). + // Lock order: generations before frozen, matching `freeze_all`. let mut gens = self.generations.borrow_mut(); let mut frozen = self.frozen.borrow_mut(); for h in frozen.drain(..) { @@ -1698,30 +1003,18 @@ impl GcState { /// Runs regardless of `gc.isenabled()`: CPython's `gc.disable()` only /// suppresses the *automatic*, threshold-driven collections (see /// [`Self::maybe_auto_collect`]); an explicit `gc.collect()` always runs a - /// full sweep. (`test_gc` disables the collector module-wide via - /// `setUpModule` and then asserts that explicit collections still reclaim - /// cycles.) The re-entrancy guard still applies — a collection triggered - /// from inside a collection (e.g. an allocating finalizer) is a no-op. + /// full sweep. The re-entrancy guard still applies — a collection + /// triggered from inside a collection is a no-op. pub fn collect(&self, upto: usize) -> usize { - // An explicit `gc.collect()` is "exact": it reaps acyclic dead and - // iterates to a fixpoint so the returned count matches CPython. self.collect_impl(upto, true) } /// Run the cycle collector's mark phase across all generations and fire /// the weakref callbacks of every unreachable, non-finalizable object, /// *without* the destructive teardown of a real collection. See - /// [`Self::collect_generation`]'s `weakref_only` discussion. The - /// re-entrancy guard applies, so this is a no-op inside a collection. + /// [`Self::collect_generation`]'s `weakref_only` discussion. pub fn fire_dead_weakrefs(&self) { - // Promote any deferred container that can now anchor a cycle, so - // the mark phase below sees the whole candidate population - // (see `sweep_deferred`). Before the re-entrancy claim: promotion - // calls `track_now`, which a collection in progress must not see. self.sweep_deferred(false); - // Atomic claim (see `collect_impl`): even a mark-only pass mutates - // the shared `gc_refs` counters, so it must not overlap a real - // collection on another thread. if self .collecting .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -1736,31 +1029,20 @@ impl GcState { /// Shared collection body. `exact` selects between the two cost/precision /// profiles: /// - /// * `true` — an explicit `gc.collect()`. Reap acyclic dead up front (so - /// they stay out of the cyclic count and `DEBUG_SAVEALL`) and iterate the - /// mark-sweep to a fixpoint, reproducing CPython's "one call reclaims all - /// current cyclic garbage" guarantee that `test_gc`'s exact-count - /// assertions depend on. - /// * `false` — a threshold-driven *automatic* young collection. CPython's - /// auto path is a single incremental pass (leftover garbage waits for the - /// next trigger or an explicit collect), so we skip both the whole-index - /// acyclic reap and the fixpoint loop. That keeps the per-allocation cost - /// flat: with the reap+fixpoint on every auto-collect, an allocation-heavy - /// suite (`test_set`'s mutation stress) re-scanned the entire accumulated - /// tracked set several times per trigger and blew the time budget. + /// * `true` — an explicit `gc.collect()`: iterate the mark-sweep to a + /// fixpoint, reproducing CPython's "one call reclaims all current + /// cyclic garbage" guarantee that `test_gc`'s exact-count assertions + /// depend on. + /// * `false` — a threshold-driven automatic young collection: a single + /// pass (leftover garbage waits for the next trigger), keeping the + /// per-allocation cost flat. fn collect_impl(&self, upto: usize, exact: bool) -> usize { // Promote any deferred container that can now anchor a cycle, so - // the mark phase below sees the whole candidate population - // (see `sweep_deferred`). Before the re-entrancy claim: promotion - // calls `track_now`, which a collection in progress must not see. + // the mark phase sees the whole candidate population. Before the + // re-entrancy claim: promotion calls `track_now`. self.sweep_deferred(false); - // Atomic claim — a plain load-then-store gate let two threads both - // observe `false` and run *overlapping* collections over the shared - // heap. Each phase-3 walk then subtracted the same internal edges - // from the same `gc_refs` counters, so a live object with one - // external root went negative and was swept as garbage — observed - // as a peer thread's suspended generator being closed mid-`for` - // loop (test_threading.test_foreign_thread wait_threads_exit). + // Atomic claim: overlapping collections over the shared heap would + // subtract the same internal edges twice. if self .collecting .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -1768,53 +1050,20 @@ impl GcState { { return 0; } - // Drop this thread's parked C-dropped clones (RFC 0047, wave 5) - // before seeding reachability. Each queue entry is a strong - // `Object` clone awaiting the eval loop's prompt-reap safe point — - // but when C code runs without returning to the eval loop (a - // C-driven embedding, or the collector's own traverse/clear boxes - // freed mid-pass) the queue never drains, and every entry inflates - // its object's `Rc::strong_count`. The mark phase seeds `gc_refs` - // from exactly that count, so a queued clone makes its object — - // and everything reachable from it — look externally rooted, - // pinning dead cycles forever. A collection is itself a safe - // point: anything that dies when these clones drop is either - // reaped below (acyclic) or found unreachable by the mark phase, - // with weakrefs/finalizers handled by the normal collection path. - // - // Guarded on no extension frame being live on this thread: while - // one is (`gc.collect()` invoked *from* C), a queued clone may be - // the last count backing a body pointer that C still borrows - // across its call, and dropping it here would free the body under - // C's feet — the exact UAF the queue exists to prevent. The clones - // then simply wait for the eval-loop drain, as designed. + // Drop this thread's parked C-dropped clones before seeding + // reachability: each inflates its object's strong count, which the + // mark phase reads as an external root. Not while an extension frame + // is live on this thread: a queued clone may back a body pointer C + // still borrows. if !crate::vm_singletons::cext_call_active() { drop(crate::vm_singletons::drain_pending_cext_drops()); } - if exact { - // Reap dead *acyclic* garbage first. CPython frees these by refcount - // the instant their last binding drops, so they never reach the - // cycle collector; we pin them on the registry handle until now. - // Doing it up front keeps them out of the cyclic `collected` count - // *and* out of `gc.garbage` under `DEBUG_SAVEALL` (`test_saveall` - // asserts only the genuine cycle is saved, not an incidental dead - // `[]` temporary). - self.reap_dead_acyclic_locked(); - } let gen = upto.min(N_GENERATIONS - 1); - // Iterate the mark-sweep to a fixpoint (exact only). Reachability is - // seeded from an *approximate* outer refcount (`Rc::strong_count`), so a - // transient Rust-side reference (an operand-stack slot not yet - // overwritten, an in-flight clone) can make a dead object — and - // everything reachable only through it — look live for a single pass. - // CPython's collector is refcount-exact and reclaims *all* current - // cyclic garbage in one `gc.collect()`; the count tests in `test_gc` - // (`gc.collect()` returns exactly the cycle size) depend on that - // completeness. Repeating until a pass collects nothing reproduces it: - // each pass re-seeds from a fresh refcount snapshot, so a reference that - // was transient last pass no longer pins its subgraph. Passes collect a - // strictly shrinking set, so this converges quickly; the cap is a guard - // against pathological churn. + // Reachability is seeded from strong counts, so a transient + // Rust-side reference can make a dead object — and everything + // reachable only through it — look live for a single pass. + // Repeating until a pass collects nothing (exact only) reclaims + // all current cyclic garbage, as CPython's one call does. let passes = if exact { MAX_COLLECT_PASSES } else { 1 }; let mut collected = 0usize; for _ in 0..passes { @@ -1830,10 +1079,9 @@ impl GcState { stats[gen].collected = stats[gen].collected.saturating_add(collected as u64); } { - // CPython resets the counters of every collected - // generation and credits one "tick" to the next older - // one — that tick is what eventually promotes a gen-1 / - // gen-2 collection in `maybe_auto_collect`. + // CPython resets the counters of every collected generation and + // credits one "tick" to the next older one — that tick is what + // eventually promotes a gen-1 / gen-2 collection. let mut counts = self.counts.borrow_mut(); for c in counts.iter_mut().take(gen + 1) { *c = 0; @@ -1844,20 +1092,21 @@ impl GcState { self.sync_gen0_gauge(counts[0], None); } // The tracked-id filter only ever gains bits; once most of them - // name objects long gone, rebuild it from the live index (a full - // collection has just walked everything anyway). + // name objects long gone, rebuild it from the live index. let live = self.index.borrow().len(); let stale = self.tracked_filter.inserts_since_rebuild(); if gen == N_GENERATIONS - 1 || stale > 4096.max(live.saturating_mul(4)) { self.rebuild_tracked_filter(); } self.collecting.store(false, Ordering::Release); + // Referents without a death hook (see `weakref_registry`) are + // noticed here. + crate::weakref_registry::sweep_dead_targets(); collected } - /// Rebuild the tracked-id miss filter from the live index (see - /// [`crate::hot_filter::RebuildableBloom`]). Holding the index borrow - /// serializes it against every `track`. + /// Rebuild the tracked-id miss filter from the live index. Holding the + /// index borrow serializes it against every `track`. pub fn rebuild_tracked_filter(&self) { let index = self.index.borrow(); self.tracked_filter.rebuild(index.keys().copied()); @@ -1867,763 +1116,300 @@ impl GcState { /// /// `weakref_only` runs the identical mark phase but stops once the /// unreachable set is known: it fires the weakref callbacks of the dead, - /// non-finalizable objects (flipping `weakref.ref(obj)()` to `None`) and - /// returns *without* running finalizers, clearing fields, untracking, or - /// rebuilding generations. It is used from a blocking `Thread.join` to - /// fire a reference-count-dead `ThreadPoolExecutor`'s `weakref_cb` (which - /// signals its idle workers to exit) without the destructive teardown of a - /// full collection — which, run while a worker holds an in-flight - /// `_WorkItem` in a frame the collector can't see as a root, would clear - /// that live work item mid-use (RFC 0040: `test_shutdown`). Because it - /// never mutates object contents, such a misclassification is harmless - /// here (a `_WorkItem` has no weakref, so its `notify_clear` is a no-op). + /// non-finalizable objects and returns *without* running finalizers, + /// clearing fields or rebuilding generations. It is used from a + /// blocking `Thread.join` to fire a dead `ThreadPoolExecutor`'s + /// `weakref_cb` without the destructive teardown of a full collection + /// (RFC 0040: `test_shutdown`). fn collect_generation(&self, gen: usize, weakref_only: bool) -> usize { - // Phase 1: snapshot the handles in this generation, plus - // any younger ones (collecting gen N also collects all - // gens 0..N). We treat gens 0..=gen as the candidate set. - let candidate_set = self.snapshot_for_collection(gen); - let cs_len = candidate_set.len(); - if cs_len == 0 { + // Phase 1: the live objects of this generation and the younger ones, + // each held strongly for the collection's duration. Entries whose + // object died are dropped on the way. + let mut cands = self.snapshot_for_collection(gen); + let n_real = cands.len(); + if n_real == 0 { return 0; } - - // Phase 2: initialise gc_refs from the *outer* refcount. - // For Rc-wrapped objects we approximate by - // `Rc::strong_count - 1` (the candidate set holds one - // reference itself, in `TrackedHandle::object`). - for handle in &candidate_set { - // A weak reference must not keep its referent reachable, but - // each live slot holds a strong `Object` clone of the target - // (the registry's drop-driven clear model). Discount those - // clones here so an object reachable *only* through weakrefs - // collapses to `gc_refs == 0` and is collected — which fires - // `notify_clear` and flips `weakref.ref(obj)()` to `None`. - let weak_clones = crate::weakref_registry::strong_clone_count(handle.id) as i64; - let outer = strong_count_for(&handle.object) - .saturating_sub(1) - .saturating_sub(weak_clones as usize) as i64; - handle.gc_refs.store(outer, Ordering::Release); - handle.color.store(color::White, Ordering::Release); - } - - // Index the candidate set by id so the per-child lookups in - // phases 3 and 4 are O(1) — a linear `find` here makes the - // whole collection quadratic, which generator-heavy programs - // (itertools pipelines) hit hard. - // Use the same address-mixing hasher as the persistent index. This - // temporary map only serves identity lookups, not collection order. - let mut by_id: GcIndex = candidate_set.iter().map(|h| (h.id, h.clone())).collect(); - - // Phase 2b: promote untracked iterators reachable from the candidate - // set to *temporary* candidates for this pass only. CPython GC-tracks - // its `*_iterator` objects, so an iterator-mediated cycle (bug #3680: - // `obj.x = iter(set_containing_obj)`) is collectible: the iterator's - // single internal ref to the container has to be subtracted off the - // container's `gc_refs`, otherwise the container looks externally - // reachable and pins the whole cycle. We keep transient *loop* - // iterators untracked for speed (enrolling every `for`-loop iterator - // in a generation regressed allocation-heavy suites by triggering far - // more young collections); instead we discover only the iterators that - // are actually reachable from already-tracked objects, here, while a - // collection is already in flight. The temporary handles take part in - // the subtract/mark walk (so their edges are accounted) but never enter - // a generation, are never cleared/finalized, never touch the index, and - // are not counted as collected — they're dropped when this pass ends, - // and the underlying iterator is freed by refcount once the real - // objects in its (dead) cycle are cleared. - let mut temp_handles: Vec> = Vec::new(); - { - // Scan the existing candidate list, then the growing temporary - // list, preserving discovery order without copying either list. - // Borrow real candidates; only a current temporary needs an - // owning handle while discovery extends its list. These are - // `Arc` handles, not extra `Object` - // clones: a discovered object gains only the strong reference - // inside its temporary handle. Newly discovered objects join the - // scan, including private buffers and their iterators. - // We promote untracked iterators and untracked - // `list` buffers: a snapshot iterator (`frozenset`/`dict.values()`/ - // file) hands back a fresh, untracked `Object::List` for its - // buffer, whose `-> elements` edges have to be accounted for the - // cycle to collapse. An `iter(list)` shares the live list's buffer, - // which is already a real candidate and is found by id below. - let mut scanned = 0usize; - while scanned < cs_len + temp_handles.len() { - let current_temporary = if scanned >= cs_len { - Some(temp_handles[scanned - cs_len].clone()) - } else { - None - }; - let h = match ¤t_temporary { - Some(handle) => handle, - None => &candidate_set[scanned], - }; - scanned += 1; - // Immutable containers (tuple/frozenset) and iterators are not - // persistently GC-tracked — pinning them in a generation would - // hold transient `(type, value, tb)` triples and loop iterators - // alive past the point CPython frees them by refcount - // (`test_traceback`'s `getrefcount` asserts, the loop-iterator - // churn that regressed allocation-heavy suites). But a cycle can - // still *route through* one (`l=[]; t=(l,); l.append(t)`; - // `obj.x = iter(set_containing_obj)`), so we discover the ones - // reachable from the (mutable, tracked) candidate set here and - // promote them to temporary candidates: their internal edges are - // accounted, the dead ones are counted, and the handles are - // dropped when the pass ends (no persistent pinning). - // - // Lists are tracked at creation, so an *untracked* list is only - // ever an iterator's private snapshot buffer (`frozenset`/ - // `dict.values()`/file iterators); promote those only when - // reached directly through an iterator, so we never re-scan the - // whole (already tracked) list population. - let parent_is_iter = matches!(&h.object, Object::Iter(_)); - let parent_is_frame = matches!(&h.object, Object::Frame(_)); - traverse_object(&h.object, &mut |child| { - let promote = match child { - // Dict views join iterators here: they're not - // persistently tracked, but a cycle can route - // through one (`obj.v = container.keys()` with - // `container = {obj: 1}` — test_container_iterator). - // Bound methods likewise (CPython GC-tracks - // `method`): a stored `self.cb` closes the classic - // callback cycle — asyncio's `future._callbacks → - // task.__wakeup → task → future` (RFC 0054, - // test_tasks.test_log_destroyed_pending_task). - // Closure cells likewise (CPython GC-tracks `cell`): - // a nested function that calls itself - // (`def inner(): ... inner()`) closes the cycle - // `function → cell → function`, and the cell's edge - // must be subtracted or the function always looks - // externally reachable (RFC 0054: asyncio's - // `iter_one`-style recursive callbacks pin the async - // generator they iterate, test_base_events' - // asyncgen-finalization-by-gc tests). - // Tracebacks and frames likewise (CPython GC-tracks - // both): an exception object owns `__traceback__ → - // frame → f_locals`, and a local that references the - // exception again (`except* E as excs` materialises - // the handled group in the frame's locals mirror) - // closes a cycle whose edges live entirely in these - // untracked node types (RFC 0054, - // test_taskgroups.test_exception_refcycles_*). - // Descriptor wrappers likewise (CPython GC-tracks - // staticmethod/classmethod/property): a user - // `__new__` is stored in the class dict behind a - // staticmethod wrapper, so the wrapper's edge to - // the function must be subtracted or a dead - // `namespace -> class -> __new__ -> __globals__` - // exec cycle keeps the function externally - // reachable forever - // (test_module.test_clear_dict_in_ref_cycle). - Object::Iter(_) - | Object::Tuple(_) - | Object::FrozenSet(_) - | Object::DictView(_) - | Object::Slice(_) - | Object::Cell(_) - | Object::Traceback(_) - | Object::Frame(_) - | Object::BoundMethod(_) - | Object::StaticMethod(_) - | Object::ClassMethod(_) - | Object::Property(_) => true, - Object::List(_) => parent_is_iter, - // An *exception* instance is untracked until a - // mutation marks it a cycle suspect, yet `raise X - // from …` inside an `except` builds the classic - // `group → __context__ exc → __traceback__ → frame → - // f_locals → group` loop where the chained exception - // is the only instance node. Promote untracked - // exceptions so their `__context__`/`__cause__`/ - // traceback edges are subtracted (RFC 0054, - // test_taskgroups.test_exception_refcycles_*). - // An instance whose tracking is still deferred - // (`track_deferred_owner`) holds only atomic values, - // but its edge to its class can still close a cycle - // (`A.a = A(); del A`): promote it so that edge is - // subtracted, like CPython's always-tracked instance. - Object::Instance(i) => i.cls().flags.is_exception || i.is_gc_deferred(), - // A frame's `f_locals` cache is an internal, - // untracked dict; it carries the frame's only - // object-graph edges to the locals (the `eg` in the - // cycle above), so it joins the walk when reached - // through its frame. - Object::Dict(_) => parent_is_frame, - _ => false, - }; - if !promote { - return; - } - let cid = id_of(child); - if by_id.contains_key(&cid) { - return; - } - let handle = HandleRc::new(TrackedHandle::new(child.clone(), 0)); - by_id.insert(cid, handle.clone()); - temp_handles.push(handle); - }); - } - // Seed `gc_refs` *after* discovery: an iterator synthesises a fresh - // `Object::List`/`Object::Set` wrapper for its buffer on each - // traverse, and that wrapper is alive only for the duration of the - // `visit` call above. Computing the outer refcount here — once - // every such transient clone has been dropped — keeps the seed - // exact (referrers + the one clone the handle itself holds). - for handle in &temp_handles { - let weak_clones = crate::weakref_registry::strong_clone_count(handle.id) as i64; - let outer = strong_count_for(&handle.object) - .saturating_sub(1) - .saturating_sub(weak_clones as usize) as i64; - handle.gc_refs.store(outer, Ordering::Release); - handle.color.store(color::White, Ordering::Release); + let mut by_id: GcIdMap = cands.iter().enumerate().map(|(i, c)| (c.id, i)).collect(); + + // Phase 2: promote untracked nodes reachable from the candidates to + // temporary candidates for this pass only. CPython GC-tracks + // iterators, tuples, frames, tracebacks, cells, bound methods and + // descriptor wrappers; we keep them off the books for speed and + // discover the ones a cycle actually routes through here, so their + // internal edges are accounted. Temporaries take part in the + // subtract/mark walk but are never reclaimed or tracked. + let mut scanned = 0usize; + while scanned < cands.len() { + let parent_is_iter = matches!(cands[scanned].obj, Object::Iter(_)); + let parent_is_frame = matches!(cands[scanned].obj, Object::Frame(_)); + let mut found: Vec = Vec::new(); + traverse_object(&cands[scanned].obj, &mut |child| { + if promotes_temporarily(child, parent_is_iter, parent_is_frame) + && !by_id.contains_key(&id_of(child)) + { + found.push(child.clone()); + } + }); + for child in found { + let cid = id_of(&child); + if let std::collections::hash_map::Entry::Vacant(e) = by_id.entry(cid) { + e.insert(cands.len()); + cands.push(Cand::new(child, cid, None)); + } } + scanned += 1; } - // Real candidates plus the temporary iterator candidates take part in - // the subtract/mark walk; only the real ones are reclaimed below. - // Cloning this iterator copies positions, not handles. The two lists - // retain ownership through finalizers and the resurrection re-mark. - let scan_all = candidate_set.iter().chain(temp_handles.iter()); - - // Phase 3: subtract internal refs by walking each - // tracked object's children. Self-references count too — - // a `self.self = self` instance has one internal ref to - // itself which must be subtracted off so a pure self-cycle - // collapses to gc_refs == 0. - for handle in scan_all.clone() { - traverse_object(&handle.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - target.gc_refs.fetch_sub(1, Ordering::AcqRel); + // Phase 3: seed gc_refs from the outer refcount, after discovery (an + // iterator synthesises a fresh wrapper for its buffer on each + // traverse, alive only during the visit). Each candidate holds one + // reference of its own. + for c in &cands { + c.gc_refs.set(strong_count_for(&c.obj) as i64 - 1); + } + // Subtract internal references, self-references included. + for c in &cands { + traverse_object(&c.obj, &mut |child| { + if let Some(&i) = by_id.get(&id_of(child)) { + let t = &cands[i]; + t.gc_refs.set(t.gc_refs.get() - 1); } }); } - // Phase 4: anything with gc_refs > 0 is reachable from - // outside; mark it black and propagate. - // The candidate lists and immutable identity index retain ownership - // throughout marking, so work stacks can borrow their handles. - let mut grey: Vec<&TrackedHandle> = Vec::new(); - for handle in scan_all.clone() { - if handle.gc_refs.load(Ordering::Acquire) > 0 { - handle.color.store(color::Grey, Ordering::Release); - grey.push(handle.as_ref()); + // Phase 4: anything with gc_refs > 0 is reachable from outside; mark + // it black and propagate. + let mut grey: Vec = Vec::new(); + for (i, c) in cands.iter().enumerate() { + if c.gc_refs.get() > 0 { + c.color.set(color::Grey); + grey.push(i); } } if std::env::var_os("WP_ROOT_DBG").is_some() { - for h in &grey { - let d = match &h.object { - Object::Module(m) => format!("module {}", m.name), - Object::Function(f) => format!("function {}", f.name), - Object::Type(t) => format!("class {}", t.name), - Object::Instance(i) => format!("instance {}", i.cls().name), - Object::Dict(d) => format!( - "dict len={} first={:?}", - d.borrow().len(), - d.borrow().iter().next().map(|(k, _)| k.0.repr()) - ), - Object::List(l) => format!("list len={}", l.borrow().len()), - o => o.type_name_owned(), - }; + for &i in &grey { + let c = &cands[i]; eprintln!( "[root] id={} gc_refs={} sc={} {}", - h.id, - h.gc_refs.load(Ordering::Acquire), - strong_count_for(&h.object), - d + c.id, + c.gc_refs.get(), + strong_count_for(&c.obj), + c.obj.type_name_owned() ); } } - while let Some(h) = grey.pop() { - h.color.store(color::Black, Ordering::Release); - traverse_object(&h.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - if target.color.load(Ordering::Acquire) == color::White { - target.color.store(color::Grey, Ordering::Release); - grey.push(target.as_ref()); - } - } - }); - } + Self::propagate_black(&cands, &by_id, &mut grey); - // Phase 5: white objects are unreachable cyclic garbage. - let unreachable: Vec> = candidate_set - .iter() - .filter(|h| h.color.load(Ordering::Acquire) == color::White) - .cloned() - .collect(); + // Phase 5: white real candidates are unreachable cyclic garbage. + let unreachable: Vec = (0..n_real).filter(|&i| cands[i].is_white()).collect(); - if crate::hot_gates::env_flags::reap_dbg() { - let dbg_class = std::env::var("WP_REAP_DBG_CLASS").unwrap_or("Executor".into()); - for h in &candidate_set { - let matches_dbg = match &h.object { - Object::Instance(i) => i.cls().name.contains(&dbg_class), - Object::Type(t) => t.name.contains(&dbg_class), - _ => false, - }; - { - if matches_dbg { - let exec_id = h.id; - let mut referrers: Vec = Vec::new(); - for c in scan_all.clone() { - if c.id == exec_id { - continue; - } - let mut hit = false; - traverse_object(&c.object, &mut |child| { - if id_of(child) == exec_id { - hit = true; - } - }); - if hit { - let nm = match &c.object { - Object::Instance(ci) => format!("Instance({})", ci.cls().name), - other => other.type_name().to_string(), - }; - referrers.push(nm); - } - } - eprintln!( - "[mark wronly={}] {} sc={} clones={} gc_refs={} white={} tracked_referrers={:?}", - weakref_only, - h.object.type_name_owned(), - strong_count_for(&h.object), - crate::weakref_registry::strong_clone_count(h.id), - h.gc_refs.load(Ordering::Acquire), - h.color.load(Ordering::Acquire) == color::White, - referrers, - ); - } - } - } - } - - // Weakref-only pass: fire the dead objects' weakref callbacks and - // stop. We deliberately skip everything destructive below (finalizer - // execution, field clearing, untracking, generation rebuild) so a - // frame-rooted live object the mark mis-coloured White is left fully - // intact — only its (absent) weakrefs would be touched. A genuinely - // dead, weakref-watched object (the `del`'d `ThreadPoolExecutor`) gets - // its `weakref_cb` queued, which is all a blocking `join` needs to - // unblock its idle workers. Finalizable objects are left for a real - // collection so `tp_finalize` ordering is preserved. // CPython's `handle_weakrefs`: a weakref that is *itself* part of the // cyclic trash has its callback cleared without invocation — only // weakrefs rooted outside the dying subgraph observe the deaths - // (test_callbacks_on_callback: `c.wr`/`d.wr` stay silent while the - // external `safe_callback` fires). Snapshot the trash ids so the - // queue loops below can drop callbacks belonging to trash wrappers. - let mut trash_ids: IdSet = unreachable.iter().map(|h| h.id).collect(); - let wrapper_is_trash = |slot: &crate::sync::Rc, - trash: &IdSet| { + // (test_callbacks_on_callback). + let trash_ids: IdSet = unreachable.iter().map(|&i| cands[i].id).collect(); + let wrapper_is_trash = |slot: &crate::sync::Rc| { slot.py_ref .borrow() .as_ref() .and_then(crate::sync::Weak::upgrade) .is_none_or(|inst| { - trash.contains(&(crate::sync::Rc::as_ptr(&inst) as usize as u64)) + trash_ids.contains(&(crate::sync::Rc::as_ptr(&inst) as usize as u64)) }) }; if weakref_only { let mut weakref_callbacks = Vec::new(); - for h in &unreachable { - if has_finalizer(&h.object) && !h.finalized.load(Ordering::Acquire) { + for &i in &unreachable { + if cands[i].pending_finalizer() { continue; } - for (slot, cb) in crate::weakref_registry::notify_clear(h.id) { + for (slot, cb) in crate::weakref_registry::notify_clear(cands[i].id) { if let Some(cb) = cb { - if wrapper_is_trash(&slot, &trash_ids) { - continue; + if !wrapper_is_trash(&slot) { + weakref_callbacks.push((slot, cb)); } - weakref_callbacks.push((slot, cb)); } } } - for (slot, cb) in weakref_callbacks { - let wr = slot - .py_ref - .borrow() - .as_ref() - .and_then(crate::sync::Weak::upgrade) - .map(crate::object::Object::Instance); - if let Some(wr) = wr { - crate::vm_singletons::push_pending_weakref_callback(cb, wr); - } - } + queue_weakref_callbacks(weakref_callbacks, |_| false); return 0; } // CPython clears weakrefs to the *entire* unreachable set - // (`handle_weakrefs`) BEFORE running any finalizer (`finalize_garbage`) - // and before resurrection handling. So a weakref watching an object a - // finalizer later revives stays cleared even though the object itself - // survives — e.g. `_pyio.FileIO.__del__` recording a - // `ResourceWarning(source=self)` into a live `catch_warnings` log - // resurrects the file, yet `weakref.ref(f)()` must read `None` - // (`test_io.test_garbage_collection`). Mirror that ordering: clear and - // queue callbacks for every unreachable object now, regardless of - // whether the resurrection re-mark or a finalizer keeps it alive below. - // (Counting still tracks only objects actually reclaimed — `dead` — so - // a resurrected object is uncounted but its weakref is gone, exactly as - // in CPython.) + // (`handle_weakrefs`) BEFORE running any finalizer, so a weakref + // watching an object a finalizer later revives stays cleared + // (`test_io.test_garbage_collection`). let mut weakref_callbacks = Vec::new(); - for h in &unreachable { - for (slot, cb) in crate::weakref_registry::notify_clear(h.id) { + for &i in &unreachable { + for (slot, cb) in crate::weakref_registry::notify_clear(cands[i].id) { if let Some(cb) = cb { - if wrapper_is_trash(&slot, &trash_ids) { - continue; + if !wrapper_is_trash(&slot) { + weakref_callbacks.push((slot, cb)); } - weakref_callbacks.push((slot, cb)); } } } - // Split the unreachable set into objects whose `__del__` hasn't run - // yet ("deferred") and the rest. A deferred object is queued for - // finalization and kept tracked: its finalizer (drained right after - // `gc.collect()` returns control to the interpreter) might resurrect - // it, and CPython only counts objects it actually reclaims - // (`test_resurrection_*`). The interpreter then collects again — by - // which point the finalizer has set `finalized`, so a survivor that - // wasn't resurrected falls into `dead` and is reclaimed (its weakrefs - // cleared in that second pass, so single-`collect()` weakref tests - // still observe `ref() is None`). - let mut deferred: Vec> = Vec::new(); - let mut maybe_dead: Vec> = Vec::new(); - for h in &unreachable { - let pending_finalizer = - has_finalizer(&h.object) && !h.finalized.load(Ordering::Acquire); - if pending_finalizer { - deferred.push(h.clone()); - } else { - maybe_dead.push(h.clone()); - } - } - - // Run each deferred object's finalizer (once). A finalizer is arbitrary - // Python: it can execute bytecode, hit a `periodic_gil_checkpoint`, and - // hand the GIL to another OS thread — which may then *resurrect* an - // object the mark phase just classified unreachable (store it somewhere - // reachable, or, in the threaded queue reproducers, pull it off a buffer - // into a live frame local). Every mark color computed above predates - // these finalizers, so it is stale the instant any finalizer runs. - for h in &deferred { - if !h.finalize_queued.swap(true, Ordering::AcqRel) { - run_finalizer(&h.object); - } - } - - // CPython's `handle_resurrected_objects`: after `finalize_garbage` runs - // every `tp_finalize`, it re-derives reachability and moves any object - // that came back to life out of the to-be-cleared set. Mirror that — but - // only when a finalizer actually ran, since that is the sole point in - // this routine where the GIL can be released and the object graph can - // change underneath us. Re-seed `gc_refs` from a *fresh* strong-count - // snapshot (so a reference a concurrent thread or a finalizer added is - // counted), re-subtract internal edges, and re-propagate reachability. - // Without this, a live object reachable only through an untraversed root - // (a running thread's frame locals) that a finalizer's GIL hand-off - // revived is cleared mid-use — emptying its `__dict__` while another - // thread pickles it (RFC 0040: `ProcessPoolExecutor` / multiprocessing - // `Queue` feeder dropping a `_CallItem` into a worker's pipe). - if !deferred.is_empty() { - for handle in scan_all.clone() { - let weak_clones = crate::weakref_registry::strong_clone_count(handle.id) as i64; - let outer = strong_count_for(&handle.object) - .saturating_sub(1) - .saturating_sub(weak_clones as usize) as i64; - handle.gc_refs.store(outer, Ordering::Release); - handle.color.store(color::White, Ordering::Release); - } - for handle in scan_all.clone() { - traverse_object(&handle.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - target.gc_refs.fetch_sub(1, Ordering::AcqRel); - } - }); - } - let mut grey: Vec<&TrackedHandle> = Vec::new(); - for handle in scan_all.clone() { - if handle.gc_refs.load(Ordering::Acquire) > 0 { - handle.color.store(color::Grey, Ordering::Release); - grey.push(handle.as_ref()); + // Objects whose `__del__` hasn't run yet are queued for + // finalization and survive this pass: the finalizer (drained after + // the collection) may resurrect them, and CPython only counts + // objects it actually reclaims. The next collection, by which point + // the finalizer has set `finalized`, reclaims the ones it didn't. + let (deferred, maybe_dead): (Vec, Vec) = unreachable + .iter() + .partition(|&&i| cands[i].pending_finalizer()); + for &i in &deferred { + if let Some(e) = &cands[i].entry { + if !e.finalize_queued.swap(true, Ordering::AcqRel) { + run_finalizer(&cands[i].obj); } } - while let Some(h) = grey.pop() { - h.color.store(color::Black, Ordering::Release); - traverse_object(&h.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - if target.color.load(Ordering::Acquire) == color::White { - target.color.store(color::Grey, Ordering::Release); - grey.push(target.as_ref()); - } - } - }); - } - } - - // Recolor the deferred roots Black and protect their whole reachable - // subgraph. CPython runs `finalize_garbage` *before* `delete_garbage`, so - // a pending finalizer always sees its own class, closure cells, and - // referents intact — even when those are themselves unreachable cyclic - // garbage (a locally-defined class whose only instance is dying, the - // `__del__` function closing over the cycle, …). Those objects are - // reclaimed by a later pass once the owning finalizer has run and they, - // too, are plain garbage. Re-applied here so it survives the resurrection - // re-mark above (which reset every color from the fresh refcounts). - let mut protect_stack: Vec<&TrackedHandle> = Vec::new(); - for h in &deferred { - h.color.store(color::Black, Ordering::Release); - protect_stack.push(h.as_ref()); } - while let Some(h) = protect_stack.pop() { - traverse_object(&h.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - if target.color.load(Ordering::Acquire) == color::White { - target.color.store(color::Black, Ordering::Release); - protect_stack.push(target.as_ref()); - } - } - }); + // CPython runs `finalize_garbage` before `delete_garbage`, so a + // pending finalizer always sees its own class, closure cells, and + // referents intact: protect the deferred objects' whole subgraphs. + let mut protect: Vec = Vec::new(); + for &i in &deferred { + cands[i].color.set(color::Grey); + protect.push(i); } + Self::propagate_black(&cands, &by_id, &mut protect); - // Whatever stayed White after the resurrection re-mark and the finalizer - // subgraph protection is genuinely dead this pass. - let dead: Vec> = maybe_dead + let dead: Vec = maybe_dead .into_iter() - .filter(|h| h.color.load(Ordering::Acquire) == color::White) + .filter(|&i| cands[i].is_white()) .collect(); let collected = dead.len(); + // Temporarily promoted iterators and immutable containers that ended + // up white are cyclic garbage too, freed by refcount once the cycle's + // mutable anchor is cleared below. CPython counts each (`test_tuple` + // asserts the closing tuple is counted alongside its list). + let reported = collected + + cands[n_real..] + .iter() + .filter(|c| { + c.is_white() + && matches!( + c.obj, + Object::Iter(_) | Object::Tuple(_) | Object::FrozenSet(_) + ) + }) + .count(); - // Temporarily-promoted iterators / immutable containers (tuple, - // frozenset) that ended up White are genuine cyclic garbage: they'll be - // freed by refcount the moment the mutable anchor in their cycle is - // cleared just below. CPython counts each in the `gc.collect()` total - // (`test_tuple` asserts the closing tuple is counted alongside its - // list), so fold the dead real-object temporaries into the *reported* - // count. The private list buffers an iterator snapshots have no CPython - // counterpart, so they don't count; and none of these were ever in - // `tracked_count`, so that bookkeeping uses `collected` (real) below. - let mut reported = collected; - for h in &temp_handles { - if h.color.load(Ordering::Acquire) == color::White - && matches!( - h.object, - Object::Iter(_) | Object::Tuple(_) | Object::FrozenSet(_) - ) - { - reported += 1; - } - } - - // 5a: weakrefs for the unreachable set were already cleared above (the - // CPython `handle_weakrefs`-before-`finalize_garbage` ordering), so the - // `dead` objects' weakrefs are gone and their callbacks are already - // queued in `weakref_callbacks` for invocation in 5d. - - // 5b (RFC 0039 WS5): before tearing the dead objects down, record the - // children they referenced *outside* this collection's candidate set. - // These seed the older-generation refcount cascade in 5c2; they must - // be captured here, while the dead objects' fields are still intact. - let saveall = self.debug.load(Ordering::Acquire) & DEBUG_SAVEALL != 0; - let mut cascade_seed: Vec = Vec::new(); - if !saveall { - for h in &dead { - traverse_object(&h.object, &mut |child| { - cascade_seed.push(id_of(child)); - }); - } - } - - // 5c: break cycles by clearing the reclaimed objects' fields — or, - // under `gc.DEBUG_SAVEALL`, park them in `gc.garbage` intact for - // inspection instead of tearing them down. - if saveall { + // Break the cycles by clearing the reclaimed objects' fields — or, + // under `gc.DEBUG_SAVEALL`, park them in `gc.garbage` intact. + if self.debug.load(Ordering::Acquire) & DEBUG_SAVEALL != 0 { let mut garbage = self.garbage.borrow_mut(); - for h in &dead { - garbage.push(h.object.clone()); + for &i in &dead { + garbage.push(cands[i].obj.clone()); } } else { - // Instances whose `__dict__` is shared are deferred; once - // every other dead object has released its references, a - // dict held only by dead holders is down to one owner and - // the retry clears it (a live holder keeps it intact). - let mut shared_dict_holders: Vec<&HandleRc> = Vec::new(); - for h in &dead { - if !clear_object_fields(&h.object) { - shared_dict_holders.push(h); + // Instances whose `__dict__` is shared are deferred; once every + // other dead object has released its references, a dict held + // only by dead holders is down to one owner and the retry + // clears it (a live holder keeps it intact). + let mut shared_dict_holders: Vec = Vec::new(); + for &i in &dead { + if !clear_object_fields(&cands[i].obj) { + shared_dict_holders.push(i); } } - for h in shared_dict_holders { - clear_object_fields(&h.object); + for i in shared_dict_holders { + clear_object_fields(&cands[i].obj); } } - // 5c2 (RFC 0039 WS5): cascade refcount-reclamation into *older* - // generations the current pass didn't scan. CPython frees an object - // the instant its refcount hits zero, regardless of generation: - // clearing a young cyclic-garbage object (`c1`) drops the last - // reference to an old object (`c0`) it pointed at, which frees `c0` - // and fires `c0`'s weakref callback — even during a young-only - // collection (`test_gc` `test_bug1055820c`). Our tracked handle pins - // such an object, so the refcount never reaches zero on its own; - // emulate the cascade explicitly. Starting from the children the now - // cleared dead objects referenced (captured in 5b), reap any tracked - // object that (a) isn't part of this collection's candidate set (those - // are handled by the normal mark/rebuild) and (b) is now reachable only - // through its own tracked handle and weakref slots, firing its weakref - // callbacks and recursing into its children. Finalizable orphans are - // left for a finalizing collection so `__del__` ordering is preserved. - if !saveall { - let dead_ids: IdSet = dead.iter().map(|h| h.id).collect(); - let mut worklist = cascade_seed; - let mut seen: IdSet = IdSet::default(); - while let Some(cid) = worklist.pop() { - if dead_ids.contains(&cid) || by_id.contains_key(&cid) || !seen.insert(cid) { - // Dead (already reaped), a candidate this collection owns, - // or already visited — skip. - continue; - } - let Some(h) = self.index.borrow().get(&cid).cloned() else { - continue; - }; - // Leave finalizable objects to a finalizing collection. - if has_finalizer(&h.object) { - continue; - } - let weak_clones = crate::weakref_registry::strong_clone_count(cid); - let effective = strong_count_for(&h.object) - .saturating_sub(1) - .saturating_sub(weak_clones); - if effective != 0 { - // Still reachable from a survivor — keep it. But the - // extra references may be Rust-side transients rather - // than heap edges: an *automatic* collection can land - // mid-teardown (asyncio cancellation), while the task - // machinery still holds in-flight clones of a dying - // child. The collection consumes the dead subgraph, so - // the eval-loop cascade never walks it and the child is - // pinned by its own handle forever once the transients - // drop. Mirror that cascade's slack: enroll borderline - // survivors for the suspect re-probe so they still die - // at CPython's refcount timing (test_ssl - // test_handshake_timeout_handler_leak with a mid-run - // gen-0 collection). - if effective <= 3 { - note_suspect(h.clone()); - } - continue; - } - // Orphaned: fire its weakref callbacks (queued in 5d below), - // capture its children for the cascade, tear it down, and drop - // it from the tracked set. The orphan joins the trash set - // first so a weakref *wrapper* dying in this cascade never - // fires its own callback (CPython `handle_weakrefs` parity). - trash_ids.insert(cid); - for (slot, cb) in crate::weakref_registry::notify_clear(cid) { - if let Some(cb) = cb { - weakref_callbacks.push((slot, cb)); - } - } - traverse_object(&h.object, &mut |child| { - worklist.push(id_of(child)); - }); - clear_object_fields(&h.object); - self.untrack_id(cid); - } - } - - // 5d: queue weakref callbacks (after finalisers and cyclic - // clears, matching CPython's order). The interpreter drains - // the queue at its next safe point — the GC layer can't call - // Python itself. Wrappers that turned out to be trash (including - // cascade orphans discovered after their callbacks were queued) - // are dropped here. - for (slot, cb) in weakref_callbacks { - if wrapper_is_trash(&slot, &trash_ids) { - continue; - } - let wr = slot - .py_ref + // Queue the weakref callbacks (after finalisers and cyclic clears, + // matching CPython's order) for the interpreter's next safe point. + queue_weakref_callbacks(weakref_callbacks, |slot| { + slot.py_ref .borrow() .as_ref() .and_then(crate::sync::Weak::upgrade) - .map(crate::object::Object::Instance); - if let Some(wr) = wr { - crate::vm_singletons::push_pending_weakref_callback(cb, wr); - } - } - - // Phase 6: rebuild the generation lists. Survivors of - // generation `g` (color != White) move to generation - // min(g+1, N_GENERATIONS-1). - self.rebuild_generations(gen, &candidate_set); - - // The rebuild dropped the dead objects' index handles *directly* - // (not via `untrack_id`), so purge any suspect-list clones in - // lock-step here — after the index borrow is released, matching - // `untrack_id`'s ordering. A stale suspect entry shares the dead - // handle's `Arc`, whose strong `object` reference would otherwise - // keep the just-collected object alive until the suspect probe - // ages it out: `test_descr.test_remove_subclass` observed a - // collected class still listed in `Parent.__subclasses__()` - // right after an explicit `gc.collect()`. - for h in &dead { - remove_suspect(h.id); - } + .is_none_or(|inst| { + trash_ids.contains(&(crate::sync::Rc::as_ptr(&inst) as usize as u64)) + }) + }); - // Adjust the population counter. + // Phase 6: rebuild the generation lists. Survivors of generation + // `g` move to generation min(g+1, N_GENERATIONS-1); the dead leave + // the index. Dropping `cands` then frees the dead by refcount. + self.rebuild_generations(gen, &cands[..n_real]); self.tracked_count.fetch_sub( collected.min(self.tracked_count.load(Ordering::Acquire)), Ordering::AcqRel, ); self.tracked_version.fetch_add(1, Ordering::AcqRel); - + drop(cands); reported } - fn snapshot_for_collection(&self, upto: usize) -> Vec> { - let gens = self.generations.borrow(); - let selected = &gens[..=upto.min(N_GENERATIONS - 1)]; - let mut out = Vec::with_capacity(selected.iter().map(|g| g.handles.len()).sum()); - for g in selected { - out.extend_from_slice(&g.handles); + /// Blacken everything reachable from the grey candidates in `grey`. + fn propagate_black(cands: &[Cand], by_id: &GcIdMap, grey: &mut Vec) { + while let Some(i) = grey.pop() { + cands[i].color.set(color::Black); + traverse_object(&cands[i].obj, &mut |child| { + if let Some(&j) = by_id.get(&id_of(child)) { + if cands[j].is_white() { + cands[j].color.set(color::Grey); + grey.push(j); + } + } + }); + } + } + + /// The live objects of generations `0..=upto` as collection candidates; + /// entries whose object died are dropped from those generations and the + /// index. + fn snapshot_for_collection(&self, upto: usize) -> Vec { + let mut index = self.index.borrow_mut(); + let mut gens = self.generations.borrow_mut(); + let mut out = Vec::new(); + let mut pruned = 0usize; + for g in gens.iter_mut().take(upto.min(N_GENERATIONS - 1) + 1) { + g.handles.retain(|h| match h.object.upgrade() { + Some(obj) => { + out.push(Cand::new(obj, h.id, Some(h.clone()))); + true + } + None => { + index.remove(&h.id); + pruned += 1; + false + } + }); + for (i, h) in g.handles.iter().enumerate() { + h.slot.store(i, Ordering::Release); + } + } + if pruned > 0 { + self.tracked_count.fetch_sub( + pruned.min(self.tracked_count.load(Ordering::Acquire)), + Ordering::AcqRel, + ); } out } - fn rebuild_generations(&self, upto: usize, candidates: &[HandleRc]) { - // Lock order MUST match `track` (index before generations): the - // collector and a mutator thread can both reach the GC under the - // shared, process-global state, and acquiring these two cells in - // opposite orders is a textbook deadlock (observed under - // `test_weakref`'s background-collector loop — RFC 0039 WS4). + fn rebuild_generations(&self, upto: usize, cands: &[Cand]) { + // Lock order MUST match `track` (index before generations). let mut index = self.index.borrow_mut(); - let mut fin = self.finalizable.borrow_mut(); let mut gens = self.generations.borrow_mut(); for g in 0..=upto.min(N_GENERATIONS - 1) { gens[g].handles.clear(); } - for h in candidates { - let color = h.color.load(Ordering::Acquire); - if color == color::White { + for c in cands { + let Some(h) = &c.entry else { continue }; + if c.is_white() { index.remove(&h.id); - // RFC 0061 (WS1b): mirror `untrack_id`'s flag so a stale - // suspect entry self-identifies as reclaimed. - h.untracked.store(true, Ordering::Release); - // Same bookkeeping as `fin_remove`, under the locks - // already held (lock order: index, finalizable, - // generations — as in `track`). - if let Some(fh) = fin.remove(&h.id) { - self.fin_remove_bookkeeping(&fh); - } continue; } let g = h.generation.load(Ordering::Acquire); let new_g = g.saturating_add(1).min(MAX_GENERATION); h.generation.store(new_g, Ordering::Release); - h.color.store(color::White, Ordering::Release); let new_g = usize::from(new_g); h.slot.store(gens[new_g].handles.len(), Ordering::Release); gens[new_g].handles.push(h.clone()); @@ -2631,6 +1417,71 @@ impl GcState { } } +/// Id → candidate position, for one collection. +type GcIdMap = std::collections::HashMap>; + +/// Whether a collection promotes `child`, an untracked node reached from a +/// candidate, to a temporary candidate (see `collect_generation`'s phase 2). +fn promotes_temporarily(child: &Object, parent_is_iter: bool, parent_is_frame: bool) -> bool { + match child { + // Iterators, immutable containers, dict views, slices, closure + // cells, tracebacks, frames, bound methods and descriptor wrappers + // are all GC-tracked by CPython, and a cycle can route through any + // of them (`obj.x = iter(set_containing_obj)`, `l = []; t = (l,); + // l.append(t)`, a recursive closure's `function -> cell -> + // function`, an exception's `__traceback__ -> frame -> f_locals`). + Object::Iter(_) + | Object::Tuple(_) + | Object::FrozenSet(_) + | Object::DictView(_) + | Object::Slice(_) + | Object::Cell(_) + | Object::Traceback(_) + | Object::Frame(_) + | Object::BoundMethod(_) + | Object::StaticMethod(_) + | Object::ClassMethod(_) + | Object::Property(_) => true, + // Lists are tracked at creation, so an *untracked* list is only an + // iterator's private snapshot buffer. + Object::List(_) => parent_is_iter, + // An untracked exception (the `group -> __context__ -> __traceback__ + // -> frame -> f_locals -> group` loop), or an instance whose + // tracking is still deferred, whose edge to its class can close a + // cycle (`A.a = A(); del A`). + Object::Instance(i) => i.cls().flags.is_exception || i.is_gc_deferred(), + // A frame's `f_locals` cache carries the frame's only edges to its + // locals. + Object::Dict(_) => parent_is_frame, + _ => false, + } +} + +/// Queue the collected weakref callbacks for the interpreter's next safe +/// point, skipping those whose wrapper `is_trash`. +fn queue_weakref_callbacks( + callbacks: Vec<( + crate::sync::Rc, + Object, + )>, + is_trash: impl Fn(&crate::sync::Rc) -> bool, +) { + for (slot, cb) in callbacks { + if is_trash(&slot) { + continue; + } + let wr = slot + .py_ref + .borrow() + .as_ref() + .and_then(crate::sync::Weak::upgrade) + .map(crate::object::Object::Instance); + if let Some(wr) = wr { + crate::vm_singletons::push_pending_weakref_callback(cb, wr); + } + } +} + /// Process-global hook counting `Object` clones held *only* by the C-API /// layer's pin caches (parked argument-pinned identity boxes, dead /// scalar/tuple pins) — infrastructure references `sys.getrefcount` must @@ -2711,6 +1562,11 @@ pub fn strong_count_for(obj: &Object) -> usize { Object::File(f) => Rc::strong_count(f), // Promoted transiently when a cycle routes through one. Object::Slice(s) => Rc::strong_count(s), + Object::Property(p) => Rc::strong_count(p), + Object::StaticMethod(m) | Object::ClassMethod(m) => Rc::strong_count(m), + Object::SlotDescriptor(d) => Rc::strong_count(d), + Object::LazyIter(l) => Rc::strong_count(l), + Object::AsyncGenAwait(a) => Rc::strong_count(a), // Leaf types — no internal refs to trace. _ => 1, } @@ -3044,8 +1900,8 @@ fn run_external_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) { /// A small append-only registry of `(matches, hook)` function-pointer /// pairs with lock-free reads: hooks are registered at interpreter -/// init and polled on every prompt-reap cascade and collection walk, -/// so readers must not take a lock (nor allocate) per object. +/// init and polled on every collection walk, so readers must not take a +/// lock (nor allocate) per object. const HOOK_TABLE_CAP: usize = 16; struct HookTable { @@ -3262,25 +2118,10 @@ fn has_finalizer(obj: &Object) -> bool { // The class's cached `__del__` verdict (reset whenever `__del__` // or the MRO changes), not an MRO walk per tracked instance. Object::Instance(inst) => inst.cls().instances_need_finalize(), - // RFC 0065 (WS4, item 3) tried gating this on "close can run - // user code" (empty exception table ⇒ skip enrollment) so - // `yield`-loop workloads could reach the fully-quiet dispatch - // path. Measured: a decisive regression — long-lived suspended - // generators that stop enrolling *here* enroll as **active - // suspects** instead, and an active suspect vetoes the quiet - // path outright (finalizable population only demotes it to the - // still-fast fin-quiet mode). jitloop lost its entire quiet- - // path win (236ms → 314ms). Left as-is per the RFC's fallback - // clause; the split would need an active-suspect exemption - // co-designed with the suspects machinery first. // A generator that has never been started has nothing to clean - // up: `close()` on it runs no Python, and neither does its drop. - // Enrolling it cost the prompt-finalization index a pair of - // inserts and removals per generator born, which is most of what - // a short-lived generator costs. A *coroutine* still enrolls + // up: `close()` on it runs no Python. A *coroutine* still counts // unstarted — finalizing one that was never awaited emits the - // RuntimeWarning, which is exactly the observable cleanup this - // index exists to run promptly. + // RuntimeWarning. Object::Generator(g) => !g.is_finished() && !g.is_unstarted(), Object::Coroutine(g) | Object::AsyncGenerator(g) => !g.is_finished(), _ => false, @@ -3319,93 +2160,6 @@ pub fn with_state(f: impl FnOnce(&GcState) -> R) -> R { f(&GC_STATE) } -/// References to `target` held by *zombie* tracked memoryviews — views -/// whose only remaining strong reference is the registry's own handle -/// (plus weakref-slot clones). Under CPython refcounting such a view is -/// already freed, so `sys.getrefcount` must not let its exporter edge -/// inflate the exporter's count (test_memoryview's getitem/setitem tests -/// assert `getrefcount(b)` returns to baseline after a short-lived view -/// over `b` is dropped). Chains (a zombie view of a zombie view) resolve -/// iteratively. Restricted to memoryviews to keep the scan O(#views), -/// well away from `getrefcount`-hot paths like pandas'. -pub fn zombie_memoryview_refs_to(target: ObjectId) -> usize { - with_state(|s| { - let mut handles: Vec> = Vec::new(); - { - let Ok(gens) = s.generations.try_borrow() else { - return 0; - }; - for gen in gens.iter() { - for h in &gen.handles { - if matches!(h.object, Object::MemoryView(_)) { - handles.push(h.clone()); - } - } - } - } - if let Ok(frozen) = s.frozen.try_borrow() { - for h in frozen.iter() { - if matches!(h.object, Object::MemoryView(_)) { - handles.push(h.clone()); - } - } - } - if handles.is_empty() { - return 0; - } - let mut zombies: IdSet = IdSet::default(); - loop { - // Inbound references each candidate receives from the current - // zombie set (a dropped chain of sub-views keeps inner views' - // counts up via exporter edges). - let mut inbound: std::collections::HashMap< - ObjectId, - usize, - BuildHasherDefault, - > = std::collections::HashMap::default(); - for h in &handles { - if zombies.contains(&h.id) { - traverse_object(&h.object, &mut |c| { - *inbound.entry(id_of(c)).or_insert(0) += 1; - }); - } - } - let mut changed = false; - for h in &handles { - if zombies.contains(&h.id) { - continue; - } - let strong = strong_count_for(&h.object); - let weak = crate::weakref_registry::strong_clone_count(h.id); - let from_zombies = inbound.get(&h.id).copied().unwrap_or(0); - if strong - .saturating_sub(1) // the registry handle itself - .saturating_sub(weak) - .saturating_sub(from_zombies) - == 0 - { - zombies.insert(h.id); - changed = true; - } - } - if !changed { - break; - } - } - let mut n = 0usize; - for h in &handles { - if zombies.contains(&h.id) { - traverse_object(&h.object, &mut |c| { - if id_of(c) == target { - n += 1; - } - }); - } - } - n - }) -} - /// Convenience: track `obj` in the shared, process-global GC. pub fn track(obj: Object) { if let Object::Instance(inst) = &obj { @@ -3455,19 +2209,17 @@ fn serial_add(counter: &AtomicUsize, n: usize) { /// Deferred instance tracking. /// /// CPython tracks every instance of a Python-defined class at -/// allocation. A tracked object here costs an index entry, a strong -/// handle and, on its death, the prompt-reap cascade — several hundred -/// nanoseconds for a temporary whose dict only ever holds ints. An -/// instance that holds only atomic values ([`Object::is_gc_atomic`]) +/// allocation. A tracked object here costs an index entry and a weak +/// handle, which a temporary whose dict only ever holds ints doesn't +/// need. An instance that holds only atomic values ([`Object::is_gc_atomic`]) /// cannot take part in a cycle, so a fresh plain instance starts /// *untracked* and its `__dict__` records the instance address as its /// deferred owner. The first store that could put a non-atomic value /// into the instance — any generic dict mutation (`DictData`'s /// `DerefMut`), a slot store of a non-atomic value, a `__dict__` /// replacement — tracks it through this function; so do `gc.is_tracked` -/// and weakref creation (the registry's strong clone would otherwise -/// keep it alive with no collector to reap it). An instance whose owner -/// record is still set when it dies is freed by its last `Arc` drop. +/// and weakref creation. An instance whose owner record is still set when +/// it dies is freed by its last `Arc` drop. /// /// `owner` is the address recorded by [`crate::object::DictData`]; it is /// 0 when the record was already taken. The instance is alive: it clears @@ -3492,26 +2244,6 @@ pub fn track_deferred_owner(owner: usize) { track(Object::Instance(inst)); } -/// Track `obj` *and* enroll it in the prompt-finalization index even though -/// it has no `__del__`/weakref callback. For handle-pinned glue objects whose -/// CPython counterpart dies by refcount with observable timing — e.g. -/// `_asyncio.FutureIter`, whose strong ref on its Future would otherwise keep -/// a finished Task (and its coroutine frame, and every local in it) alive -/// until the next cyclic collection (test_ssl's weakref-based leak tests). -/// The prompt sweep sees the object with no pending finalizer and simply -/// untracks it the moment its last program reference drops, cascading the -/// frees exactly like CPython's refcounting. -pub fn track_prompt_reclaim(obj: Object) { - let id = crate::weakref_registry::id_of(&obj); - with_state(|s| { - s.track(obj); - let handle = s.index.borrow().get(&id).cloned(); - if let Some(h) = handle { - s.fin_insert(id, h); - } - }); -} - /// True while a collection (mark/sweep or weakref-only pass) is in /// flight on the process-global collector. Used by the capi boundary to /// suppress side-effectful bookkeeping (e.g. the C-drop reap queue) for @@ -3719,10 +2451,6 @@ pub fn track_if_cyclic(obj: &Object) -> bool { } } -/// Convenience: threshold-driven automatic collection on the current -/// thread's GC (see [`GcState::maybe_auto_collect`]). Returns the -/// number of objects reclaimed; the caller should drain pending -/// finalizers when this is non-zero. /// Whether `id` may be tracked: the miss-filter probe (no false /// negatives — every tracked object's bits are set at track time; a bit /// left by an untracked object is a harmless false positive). @@ -3737,12 +2465,11 @@ pub fn auto_collect_due() -> bool { with_state(GcState::auto_collect_due) } +/// Convenience: threshold-driven automatic collection on the shared GC +/// (see [`GcState::maybe_auto_collect`]). Returns whether a collection +/// ran; the caller should then drain pending finalizers. pub fn maybe_auto_collect() -> bool { - let ran = with_state(GcState::maybe_auto_collect); - if ran { - sweep_weakref_only_targets(); - } - ran + with_state(GcState::maybe_auto_collect) } /// Convenience: find a tracked handle by object id (O(1) via the @@ -3751,9 +2478,7 @@ pub fn find_handle(id: ObjectId) -> Option> { with_state(|s| s.handle_for(id)) } -/// Convenience: is `id` currently tracked by the cycle GC? Used -/// by refcount-emulation paths to discount the registry's own -/// strong handle. +/// Convenience: is `id` currently tracked by the cycle GC? pub fn is_tracked(id: ObjectId) -> bool { with_state(|s| s.is_tracked(id)) } @@ -3781,555 +2506,29 @@ pub fn complete_finalizer(id: ObjectId) { with_state(|s| s.complete_finalizer(id)); } -/// Convenience: snapshot all tracked objects with an unrun `__del__` -/// in the shared GC (see [`GcState::finalization_candidates`]). -pub fn finalization_candidates() -> Vec> { +/// Convenience: the live tracked objects with an unrun `__del__` (see +/// [`GcState::finalization_candidates`]). +pub fn finalization_candidates() -> Vec<(HandleRc, Object)> { with_state(|s| s.finalization_candidates()) } -/// Convenience: refcount-reclaim dead acyclic garbage in the shared -/// GC (see [`GcState::reap_dead_acyclic`]). -pub fn reap_dead_acyclic() -> usize { - with_state(|s| s.reap_dead_acyclic()) -} - -/// [`reap_dead_acyclic`] for the prompt-finalization drain's hot path: -/// the full-index cascade scan is O(tracked) and the drain runs it once -/// per pass that freed a finalizable. With a huge tracked population -/// shedding finalizables continuously (70k `WeakKeyDictionary` keys -/// dying one pop at a time in test_weakref's threaded-copy stress) that -/// multiplies into minutes, so over a size threshold only every N-th -/// drain pays for the scan — the skipped cascades are plain containers -/// whose reclamation the next scan (or any collection) picks up. Small -/// heaps keep CPython-like promptness (asyncio's SSL leak chains). -pub fn reap_dead_acyclic_amortized() -> usize { - const TRACKED_THRESHOLD: usize = 8192; - static STRIDE: std::sync::OnceLock = std::sync::OnceLock::new(); - let stride = *STRIDE.get_or_init(|| { - std::env::var("WEAVEPY_ACYCLIC_STRIDE") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(64) - }); - static TICK: AtomicU64 = AtomicU64::new(0); - with_state(|s| { - if s.tracked_count.load(Ordering::Relaxed) > TRACKED_THRESHOLD { - let tick = TICK.fetch_add(1, Ordering::Relaxed); - if !tick.is_multiple_of(stride) { - return 0; - } - } - s.reap_dead_acyclic() - }) -} - -/// Convenience: is any finalizable object currently tracked in the shared GC? -/// The interpreter's prompt-finalization gate (see -/// [`GcState::has_any_finalizable`]). -#[inline] -pub fn has_any_finalizable() -> bool { - with_state(GcState::has_any_finalizable) -} - /// Rebuild the tracked-id miss filter from the live tracked set (see /// [`GcState::rebuild_tracked_filter`]). pub fn rebuild_tracked_filter() { with_state(GcState::rebuild_tracked_filter); } -/// Prompt-reap *suspects* (RFC 0054): tracked, non-finalizable objects a -/// [`Interpreter::reap_dead_subgraph`] cascade visited but had to skip -/// because something still referenced them — typically a Rust-side -/// transient (an in-flight `PyException` clone, a native call's argument) -/// that dies a few opcodes later, *between* safe points, leaving the object -/// pinned by its own collector handle until the next full collection. -/// CPython's refcounting frees such objects the instant the transient dies; -/// re-probing the (tiny) suspect list at drop safe points recovers that -/// timing. Each entry carries a probe budget so a genuinely long-lived -/// skipped object (the event loop itself) stops costing anything after a -/// bounded number of checks. -/// -/// The canonical chain: asyncio's `wait_for` timeout leaves a -/// `CancelledError` (skipped mid-cascade while the task machinery still -/// held a clone) whose traceback pins the cancelled `create_connection` -/// frames — and through their locals the SSL transport, protocol, and -/// `SSLContext` that test_ssl's leak tests watch via weakref. -const SUSPECT_CAP: usize = 256; -/// RFC 0077 (WS2): active probes an entry gets before it goes dormant. -/// Was 64; the census showed the per-drop `retain` walk over the -/// active population as a first-order interpreter cost on object-heavy -/// fixtures. A suspect pinned by a Rust-side transient that "dies a few -/// opcodes later" is caught within 16 drop safe points; one that -/// outlives them (a live object the cascade merely couldn't prove -/// dead, or a transient held across a long native call) moves to the -/// dormant stride as before, so nothing is forgotten — it just stops -/// being re-probed at every drop. (A "reset the budget when the count -/// moves" variant was measured and rejected: live objects churn their -/// counts constantly, which kept ~65 entries permanently active on -/// `deltablue`.) -const SUSPECT_BUDGET: u8 = 16; -/// Aged-out (budget-exhausted) suspects turn *dormant* rather than being -/// forgotten: they are re-probed only every `DORMANT_STRIDE`-th sweep, so -/// a long-lived skipped object costs two atomic loads per stride instead -/// of per safe point — but an object whose last external reference dies -/// *after* its budget ran out is still recovered. (test_ssl's -/// handshake-timeout leak tests: the CancelledError web enrolls at -/// cancellation, the event loop burns >64 drop safe points before the -/// Task machinery lets go, and with eager eviction the web stayed pinned -/// by its collector handle until a full `gc.collect()`.) -const DORMANT_STRIDE: u64 = 64; -/// RFC 0077 (WS2): a *dormant* entry probed at more than this many strong -/// references above its dead line is forgotten outright. The dormant -/// stride exists for the transient-pinned shape — an object one or two -/// Rust-side clones away from death that no Python-visible drop will -/// re-notice. An entry that has outlived its active budget *and* still -/// carries several extra references is simply a live object the cascade -/// couldn't prove dead; its eventual death is re-noticed the ordinary way -/// (the holder's own `prompt_reap_dropped` cascade walks to it), so -/// keeping it enrolled only made every stride sweep pay for the whole -/// live population. `deltablue` filled the map with ~200 such entries -/// and re-probed all of them 40K times per run (the census's ~9.4M -/// suspect probes). -const SUSPECT_LIVE_MARGIN: usize = 3; -/// RFC 0077 (WS2): stride probes a dormant entry survives before it is -/// forgotten regardless of its count. The margin above can't tell "one -/// live holder forever" (deltablue's `Variable`s: one list reference -/// each, 191 of them filling the map to its cap) from "one transient -/// about to die", so dormancy is bounded in time instead: 16 active -/// probes then 16 stride probes gives every suspect ~1000 drop safe -/// points to have its transient released — an order of magnitude past -/// the test_ssl cancellation chain that motivated dormancy — after which -/// its death is left to the cascade through its holders (or a full -/// collection). Cost per enrollment is now bounded at 32 probes. -const SUSPECT_DORMANT_PROBES: u8 = 16; -/// RFC 0061 (WS1b): keyed by [`ObjectId`] so `remove_suspect` (called in -/// lock-step with every `untrack_id`) and enrollment dedup are O(1) -/// map hits instead of linear scans of the list — `remove_suspect`'s -/// `retain` was a measurable share of drop-heavy profiles (`list_ops`). -/// One enrolled suspect: its handle, remaining active probe budget, and -/// (once dormant) the number of stride probes it has survived. -struct Suspect { - handle: HandleRc, - budget: u8, - dormant_probes: u8, -} -type SuspectMap = indexmap::IndexMap>; -/// The enrolled suspects, split by phase: entries with probe budget left -/// (re-probed at every sweep) and aged-out dormant ones (re-probed only on -/// the stride), so an ordinary sweep never walks the dormant population. -#[derive(Default)] -struct Suspects { - /// Every entry has budget remaining. - active: SuspectMap, - /// Every entry's budget is spent. - dormant: SuspectMap, -} -impl Suspects { - fn len(&self) -> usize { - self.active.len() + self.dormant.len() - } - fn contains_key(&self, id: ObjectId) -> bool { - self.active.contains_key(&id) || self.dormant.contains_key(&id) - } - fn values(&self) -> impl Iterator { - self.active.values().chain(self.dormant.values()) - } -} -static SUSPECTS: std::sync::LazyLock> = - std::sync::LazyLock::new(|| parking_lot::Mutex::new(Suspects::default())); -static SUSPECT_COUNT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); -/// Entries with probe budget remaining. When only dormant entries are -/// left, [`has_suspects`] admits a sweep every [`DORMANT_STRIDE`]-th -/// safe point instead of every one. -static SUSPECT_ACTIVE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); -static SUSPECT_TICK: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); -/// RFC 0065 (WS4): insert-only miss-filter over ever-enrolled suspect -/// ids, so the per-`untrack_id` [`remove_suspect`] probe skips the map -/// lock for the overwhelming never-enrolled majority. -static SUSPECT_FILTER: crate::hot_filter::AtomicBloom = crate::hot_filter::AtomicBloom::new(); - -/// RFC 0077 (WS1): floor counters for the `WEAVEPY_VM_STATS` shutdown -/// print — how often the drop-path probes ran and what they found. -pub mod floor_stats { - use std::sync::atomic::{AtomicU64, Ordering}; - pub static SUSPECT_ENROLLED: AtomicU64 = AtomicU64::new(0); - pub static SUSPECT_SWEEPS: AtomicU64 = AtomicU64::new(0); - pub static SUSPECT_PROBED: AtomicU64 = AtomicU64::new(0); - pub static SUSPECT_DEAD: AtomicU64 = AtomicU64::new(0); - pub static SUSPECT_FORGOTTEN: AtomicU64 = AtomicU64::new(0); - pub static FIN_SCANS: AtomicU64 = AtomicU64::new(0); - pub static FIN_PROBED: AtomicU64 = AtomicU64::new(0); - pub static FIN_DEAD: AtomicU64 = AtomicU64::new(0); - pub static DROP_NOTES: AtomicU64 = AtomicU64::new(0); - pub static DROP_NOTES_MARKED: AtomicU64 = AtomicU64::new(0); - - #[inline] - pub fn bump(c: &AtomicU64, n: u64) { - if crate::specialize::stats_enabled() { - c.fetch_add(n, Ordering::Relaxed); - } - } - - /// Coarse "operand stack shrank ⇒ maybe dead" marks by opcode name - /// (stats-only; the eval loop calls this under `stats_enabled`). - static COARSE_MARKS: std::sync::LazyLock< - parking_lot::Mutex>, - > = std::sync::LazyLock::new(Default::default); - - #[cold] - pub fn coarse_mark(op: &'static str) { - *COARSE_MARKS.lock().entry(op).or_insert(0) += 1; - } - - /// Markdown block for the shutdown stats print; `None` when nothing - /// was counted. - pub fn markdown() -> Option { - let rows = [ - ( - "drop notes (audited handlers)", - DROP_NOTES.load(Ordering::Relaxed), - ), - ( - "drop notes that marked maybe-dead", - DROP_NOTES_MARKED.load(Ordering::Relaxed), - ), - ( - "suspects enrolled", - SUSPECT_ENROLLED.load(Ordering::Relaxed), - ), - ("suspect sweeps", SUSPECT_SWEEPS.load(Ordering::Relaxed)), - ( - "suspect entries probed", - SUSPECT_PROBED.load(Ordering::Relaxed), - ), - ("suspects found dead", SUSPECT_DEAD.load(Ordering::Relaxed)), - ( - "dormant suspects forgotten (live)", - SUSPECT_FORGOTTEN.load(Ordering::Relaxed), - ), - ("finalizable scans", FIN_SCANS.load(Ordering::Relaxed)), - ( - "finalizable entries probed", - FIN_PROBED.load(Ordering::Relaxed), - ), - ("finalizables found dead", FIN_DEAD.load(Ordering::Relaxed)), - ]; - if rows.iter().all(|(_, v)| *v == 0) { - return None; - } - let mut out = - String::from("\n## Drop-path floor (RFC 0077 WS2)\n\n| counter | value |\n|---|---|\n"); - for (name, v) in rows { - out.push_str(&format!("| {name} | {v} |\n")); - } - // Residual populations at exit: what the per-drop scans were - // actually walking. Type name, strong count above the dead line. - let mut census = |title: &str, rows: Vec<(String, usize)>| { - if rows.is_empty() { - return; - } - out.push_str(&format!("\n{title} at exit ({}):\n", rows.len())); - let mut grouped: std::collections::BTreeMap> = - std::collections::BTreeMap::new(); - for (tn, excess) in rows { - grouped.entry(tn).or_default().push(excess); - } - for (tn, mut ex) in grouped { - ex.sort_unstable(); - let shown: Vec = ex.iter().take(8).map(|e| e.to_string()).collect(); - out.push_str(&format!( - "- {tn} x{} (excess refs: {}{})\n", - ex.len(), - shown.join(","), - if ex.len() > 8 { ",…" } else { "" } - )); - } - }; - census("hot finalizables", super::residual_hot_finalizables()); - census("enrolled suspects", super::residual_suspects()); - let marks = COARSE_MARKS.lock(); - if !marks.is_empty() { - let mut by_count: Vec<(&&str, &u64)> = marks.iter().collect(); - by_count.sort_by(|a, b| b.1.cmp(a.1)); - out.push_str("\ncoarse maybe-dead marks by opcode:\n"); - for (op, n) in by_count.into_iter().take(12) { - out.push_str(&format!("- {op}: {n}\n")); - } - } - Some(out) - } -} - -/// Stats-only snapshot of the hot finalizable set: `(type name, strong -/// refs above the dead line)` per entry. -fn residual_hot_finalizables() -> Vec<(String, usize)> { - with_state(|gc| { - gc.finalizable_hot - .borrow() - .iter() - .map(|h| { - let sc = strong_count_for(&h.object); - let cached = h.weak_clones.load(Ordering::Relaxed); - (h.object.type_name_owned(), sc.saturating_sub(1 + cached)) - }) - .collect() - }) -} - -/// Stats-only snapshot of the suspect map, same shape. -fn residual_suspects() -> Vec<(String, usize)> { - SUSPECTS - .lock() - .values() - .map(|e| { - let h = &e.handle; - let sc = strong_count_for(&h.object); - let cached = h.weak_clones.load(Ordering::Relaxed); - (h.object.type_name_owned(), sc.saturating_sub(1 + cached)) - }) - .collect() -} - -/// Publish the count gates after the locked suspect maps changed. -fn publish_suspect_counts(s: &Suspects) { - let active = s.active.len(); - let was_active = SUSPECT_ACTIVE.swap(active, Ordering::AcqRel) > 0; - let was_present = SUSPECT_COUNT.swap(s.len(), Ordering::AcqRel) > 0; - // RFC 0065 (WS1): the dispatch loops' quiet-path snapshots consult - // `active_suspects_present` / the population gates, so a - // transition of either invalidates them. Same-state churn (one - // active suspect replacing another) doesn't. - if was_active != (active > 0) || was_present != (s.len() > 0) { - crate::hot_gates::bump_loop_gen(); - } -} - -/// Evict the entry that has had the most chances to die: the oldest -/// dormant one, or with none, the first lowest-budget active one. The -/// caller holds the maps' lock and republishes the counts. -fn evict_lowest_budget_suspect(s: &mut Suspects) -> bool { - if !s.dormant.is_empty() { - s.dormant.shift_remove_index(0); - return true; - } - let Some(victim) = s - .active - .values() - .enumerate() - .min_by_key(|(_, entry)| entry.budget) - .map(|(index, _)| index) - else { - return false; - }; - s.active.swap_remove_index(victim); - true -} - -/// RFC 0065 (WS1): *active*-population probe for the dispatch loop's -/// quiet predicate, with no dormant-stride tick side effect. While an -/// active (budget-remaining) suspect is enrolled the loop must stay -/// on the full prologue so the per-drop re-probe cadence holds; -/// dormant (aged-out) entries are instead re-probed from the quiet -/// loop's GIL-checkpoint cadence via [`has_suspects`] + -/// [`take_dead_suspects`]. -#[inline] -pub fn active_suspects_present() -> bool { - SUSPECT_ACTIVE.load(Ordering::Relaxed) > 0 -} - -/// Enroll a cascade-skipped tracked object for later deadness re-probes. -/// Deduplicated; silently dropped when the list is full (the next full -/// collection reclaims it instead). -pub fn note_suspect(h: HandleRc) { - static NO_SUSPECTS: std::sync::OnceLock = std::sync::OnceLock::new(); - if *NO_SUSPECTS.get_or_init(|| std::env::var_os("WEAVEPY_NO_SUSPECTS").is_some()) { - return; - } - let mut s = SUSPECTS.lock(); - if s.contains_key(h.id) { - return; - } - // RFC 0065 (WS4): publish to the miss-filter before the enrollment - // becomes observable (we hold the map lock). - SUSPECT_FILTER.insert(h.id); - // Refresh the handle's cached weakref-clone upper bound once at - // enrollment so the re-probe loop can fast-reject live suspects with - // two atomic loads instead of a registry lookup per safe point (see - // `take_dead_suspects` — the probe runs at *every* reference-dropping - // opcode, and a hot loop that keeps re-enrolling a long-lived object - // like `statistics.kde`'s sample list would otherwise spend ~40% of - // its time in `strong_clone_count`). - h.weak_clones.store( - crate::weakref_registry::strong_clone_count(h.id), - Ordering::Release, - ); - if s.len() >= SUSPECT_CAP { - // Full: evict the most-probed entry (lowest remaining budget, - // dormant first) — it has had the most chances to die and is - // the closest to aging out anyway. Silently dropping the *new* - // suspect instead loses the one object whose last real - // reference just died (asyncio's wait_for cancellation chain - // arrived after ~200 module-teardown stragglers and was never - // re-probed, pinning the Timeout→Task→frame web the test_ssl - // leak tests watch). - if !evict_lowest_budget_suspect(&mut s) { - return; - } - } - floor_stats::bump(&floor_stats::SUSPECT_ENROLLED, 1); - s.active.insert( - h.id, - Suspect { - handle: h, - budget: SUSPECT_BUDGET, - dormant_probes: 0, - }, - ); - publish_suspect_counts(&s); -} - -/// Cheap gate for the eval loop's safe point: always sweep while an -/// *active* suspect is enrolled; with only dormant (aged-out) entries -/// left, admit every [`DORMANT_STRIDE`]-th safe point so long-lived -/// suspects cost two atomic loads per stride, not per drop. -#[inline] -pub fn has_suspects() -> bool { - if SUSPECT_ACTIVE.load(Ordering::Relaxed) > 0 { - return true; - } - if SUSPECT_COUNT.load(Ordering::Relaxed) == 0 { - return false; - } - SUSPECT_TICK - .fetch_add(1, Ordering::Relaxed) - .is_multiple_of(DORMANT_STRIDE) -} - -/// Drop the suspect entry for `id`, if any. Called in lock-step with -/// `untrack_id`: a suspect's `Arc` is a *clone of the -/// index's handle*, so removing the object from the index alone leaves -/// the handle — and its strong `object` reference — alive in the suspect -/// list until the next re-probe. That stale strong clone pins an object -/// the prompt-reap cascade just untracked and expected to free by `Rc` -/// drop, deferring everything it anchors (`unittest`'s -/// `_AssertRaisesContext` → stored exception → `AttributeError.obj` io -/// temporary whose `close()` must fire at `with`-exit — -/// `test_io.test_error_through_destructor`) to a later safe point. -pub fn remove_suspect(id: ObjectId) { - if SUSPECT_COUNT.load(Ordering::Relaxed) == 0 { - return; - } - // RFC 0065 (WS4): with a resident (typically dormant) suspect - // population this is called for *every* `untrack_id` — probe the - // insert-only filter before taking the map lock; a never-enrolled - // id answers with two relaxed loads. - if !SUSPECT_FILTER.may_contain(id) { - return; - } - let mut s = SUSPECTS.lock(); - let removed = s.active.swap_remove(&id).is_some() || s.dormant.swap_remove(&id).is_some(); - if removed { - publish_suspect_counts(&s); - } -} - -/// Re-probe the suspect list: return the objects that are now dead in -/// refcount terms (nothing beyond the GC handle and weakref strong clones -/// holds them) for the caller to run through the prompt-reap cascade, and -/// decay the probe budget of the rest. -pub fn take_dead_suspects() -> Vec { - let mut out = Vec::new(); - let mut s = SUSPECTS.lock(); - // With only dormant entries left, `has_suspects` already stride-gated - // this sweep; with actives present the stride ticks here instead. - let probe_dormant = s.active.is_empty() - || SUSPECT_TICK - .fetch_add(1, Ordering::Relaxed) - .is_multiple_of(DORMANT_STRIDE); - floor_stats::bump(&floor_stats::SUSPECT_SWEEPS, 1); - let mut probed = 0u64; - // A suspect whose last reference is gone (beyond the handle and any - // weakref strong clones) goes to `out`. Returns whether it stays. - let mut dead = |e: &Suspect, out: &mut Vec| -> Option { - probed += 1; - let h = &e.handle; - // RFC 0061 (WS1b): the handle self-identifies as reclaimed (set - // in lock-step with every index removal) — no registry lookup. - if h.untracked.load(Ordering::Acquire) { - return None; // already reclaimed elsewhere - } - // Fast reject via the cached weakref-clone upper bound (refreshed - // at enrollment): more strong refs than the handle plus every - // possible weakref clone ⇒ a program reference is still live, no - // registry lookup needed. Only borderline counts pay for the - // exact `strong_clone_count`. - let sc = strong_count_for(&h.object); - let cached = h.weak_clones.load(Ordering::Acquire); - if sc <= 1 + cached { - let weak = crate::weakref_registry::strong_clone_count(h.id); - if sc <= 1 + weak { - out.push(h.object.clone()); - return None; - } - } - Some(sc.saturating_sub(1 + cached)) - }; - // Active entries spend a probe; one whose budget runs out turns - // dormant after this sweep (it isn't stride-probed in the same one). - let mut aged = Vec::new(); - s.active.retain(|&id, e| { - if dead(e, &mut out).is_none() { - return false; - } - e.budget -= 1; - if e.budget == 0 { - aged.push(( - id, - Suspect { - handle: e.handle.clone(), - budget: 0, - dormant_probes: e.dormant_probes, - }, - )); - return false; - } - true - }); - if probe_dormant { - // A dormant entry still well above its dead line, or one that has - // sat through its dormant allowance, is plainly alive: drop it - // from the map (see `SUSPECT_LIVE_MARGIN` / `SUSPECT_DORMANT_PROBES`). - s.dormant.retain(|_, e| { - let Some(excess) = dead(e, &mut out) else { - return false; - }; - e.dormant_probes = e.dormant_probes.saturating_add(1); - if excess > SUSPECT_LIVE_MARGIN || e.dormant_probes > SUSPECT_DORMANT_PROBES { - floor_stats::bump(&floor_stats::SUSPECT_FORGOTTEN, 1); - return false; - } - true - }); - } - s.dormant.extend(aged); - floor_stats::bump(&floor_stats::SUSPECT_PROBED, probed); - floor_stats::bump(&floor_stats::SUSPECT_DEAD, out.len() as u64); - publish_suspect_counts(&s); - out -} - thread_local! { - /// Set whenever the interpreter executes an opcode that may have dropped - /// the last reference to an object (a `POP_*`/`STORE_*`/`DELETE_*`, a - /// frame teardown, …). The eval loop only runs a prompt-finalization sweep - /// when this is set *and* a finalizable object is live, so a hot loop that - /// never drops a reference pays nothing beyond the gate's atomic load. - /// Thread-local because each thread reclaims the objects whose last - /// reference *it* dropped, matching CPython's per-thread `tp_dealloc`. + /// Set when work for the interpreter's next safe point was queued on + /// this thread: a finalizer resurrected by a dying object's `Drop`, a + /// weakref callback, an unclosed resource's warning. The dispatch + /// loops poll it between instructions, so the work runs before the + /// instruction after the one that dropped the object — CPython runs + /// it inside that instruction's `Py_DECREF`. static MAYBE_DEAD: std::cell::Cell = const { std::cell::Cell::new(false) }; } -/// A pointer to the calling thread's maybe-dead flag, for a dispatch +/// A pointer to the calling thread's pending-work flag, for a dispatch /// loop that polls it after every instruction: one thread-local lookup /// per loop entry instead of one per poll. Valid for the life of the /// calling thread; only ever dereferenced on that thread. @@ -4338,341 +2537,38 @@ pub(crate) fn maybe_dead_flag() -> *const std::cell::Cell { MAYBE_DEAD.with(std::ptr::from_ref) } -/// Note that the current thread just executed a reference-dropping opcode, so a -/// finalizable object may now be dead. Cheap (a thread-local `Cell` store); the -/// actual sweep is deferred to the next eval-loop safe point. +/// Note that the current thread queued work for its next safe point. +/// Teardown-safe: a thread whose locals are gone has no next safe point. #[inline] pub fn mark_maybe_dead() { - MAYBE_DEAD.with(|c| c.set(true)); -} - -/// RFC 0077 (WS2): set by [`mark_bulk_drop`], consumed by the next -/// finalizable scan, which then re-grades the *whole* index (a forced -/// cold tick) instead of only the hot set. -static FORCE_COLD_TICK: AtomicBool = AtomicBool::new(false); - -/// Note that the current thread just released many references at once -/// without grading them individually: a frame torn down by a -/// propagating exception. Its locals and operands drop wholesale, and -/// any of them may have been the last holder of a finalizable that the -/// hot/cold gate had already demoted to cold (a coroutine created, -/// passed to a call that raised, and never awaited: its count was -/// stable across the exception's construction). The ordinary hot-set -/// scan would find it only on the cold stride, up to `FIN_COLD_STRIDE` -/// safe points later, which is past the `with assertWarns(...)` block -/// that expects the "never awaited" warning (test_asyncio's -/// `test_run_until_complete_nesting`). Exceptional unwinds are rare -/// relative to drop safe points, so forcing a full re-grade here keeps -/// promptness without giving back the steady-state gate. -#[inline] -pub fn mark_bulk_drop() { - MAYBE_DEAD.with(|c| c.set(true)); - FORCE_COLD_TICK.store(true, Ordering::Relaxed); -} - -/// Consume the forced-cold-tick request (see [`mark_bulk_drop`]). -#[inline] -pub(crate) fn take_force_cold_tick() -> bool { - FORCE_COLD_TICK.load(Ordering::Relaxed) && FORCE_COLD_TICK.swap(false, Ordering::Relaxed) -} - -/// Precise per-value drop note (RFC 0059 WS1b). Called by the audited -/// opcode handlers (`POP_TOP`, `STORE_FAST`, the generic `BINARY_OP` / -/// `COMPARE_OP` paths) with each value they discard; the eval loop -/// exempts those opcodes from its coarse "operand stack shrank ⇒ maybe -/// dead" heuristic in return. -/// -/// A **pure leaf** — a variant that owns no [`Object`]s and whose -/// instances can never carry a finalizer (subclass instances live in -/// `Object::Instance`, never in the scalar variants) — cannot make -/// anything finalizable unreachable when dropped, so it schedules no -/// sweep. This is what keeps an int-accumulating hot loop -/// (`total = total + i`, the RFC 0059 `nested_loops` profile) from -/// paying a `reap_dead_finalizable` scan per iteration whenever *some* -/// `__del__`-bearing object is alive anywhere in the process. -/// Everything else is graded by its strong count (RFC 0077 WS2). The -/// drop about to happen decrements `obj`'s count by one and, only if -/// that was the last reference, frees `obj` and cascades into whatever -/// it held. So a tracked finalizable or suspect can become dead here in -/// exactly two ways: -/// -/// * `obj` is about to be freed (`strong_count == 1`): the cascade may -/// release the last program reference to anything — mark. -/// * `obj` itself is tracked and this drop takes it to handle-only -/// (`strong_count == 2`), or `obj` is tracked and weakref clones make -/// the dead line unknowable cheaply — mark. -/// -/// A drop of an untracked object that survives it (`strong_count >= 2`, -/// e.g. `self` rebound in a local, a shared list popped off the stack) -/// changes nothing observable and schedules no sweep. The filters are -/// insert-only Bloom bits (false positives take the conservative -/// path), and the sweep itself refcount-checks, so a false positive -/// costs one scan, never correctness. -#[inline] -pub fn note_dropped(obj: &crate::object::Object) { - if note_dropped_marks(obj) { - mark_maybe_dead(); - } -} - -/// The grading half of [`note_dropped`]: would dropping `obj` right now -/// schedule a prompt-finalization sweep? Split out so the leaf burst -/// (`Interpreter::leaf_burst`) can end its run at exactly the -/// instruction that marks, handing the very next safe point back to the -/// full eval-loop prologue. -#[inline] -pub fn note_dropped_marks(obj: &crate::object::Object) -> bool { - use crate::object::Object as O; - match obj { - O::None - | O::Unbound - | O::Bool(_) - | O::Int(_) - | O::Long(_) - | O::Float(_) - | O::Complex(_) - | O::Str(_) - | O::WStr(_) - | O::Bytes(_) - | O::Range(_) - | O::Code(_) => false, - // The common heap operands, with their count and identity read - // inline (no variant dispatch). - // A natively served class's instance (never tracked, no - // finalizer, only ints and a `tzinfo` inside) cannot start a - // finalization when it dies. - O::Instance(i) if i.cls_raw().native_kind.get() != 0 => false, - O::Instance(i) => note_dropped_counted( - crate::sync::Rc::strong_count(i), - crate::sync::Rc::as_ptr(i) as usize as u64, - ), - O::List(l) => { - let (sc, id) = ( - crate::sync::Rc::strong_count(l), - crate::sync::Rc::as_ptr(l) as usize as u64, - ); - if inert_death(sc, id, || { - l.try_borrow() - .is_ok_and(|v| v.len() <= INERT_SCAN_CAP && v.iter().all(is_atomic)) - }) { - return false; - } - note_dropped_counted(sc, id) - } - O::Dict(d) => { - let (sc, id) = ( - crate::sync::Rc::strong_count(d), - crate::sync::Rc::as_ptr(d) as usize as u64, - ); - if inert_death(sc, id, || { - d.try_borrow().is_ok_and(|m| { - m.len() <= INERT_SCAN_CAP - && m.iter().all(|(k, v)| is_atomic(&k.0) && is_atomic(v)) - }) - }) { - return false; - } - note_dropped_counted(sc, id) - } - O::Function(f) => note_dropped_counted( - crate::sync::Rc::strong_count(f), - crate::sync::Rc::as_ptr(f) as usize as u64, - ), - O::Type(t) => note_dropped_counted( - crate::sync::Rc::strong_count(t), - crate::sync::Rc::as_ptr(t) as usize as u64, - ), - O::Tuple(t) => note_dropped_counted( - ThinArc::strong_count(t), - ThinArc::as_ptr(t).cast::<()>() as usize as u64, - ), - _ => note_dropped_marks_other(obj), - } -} - -/// Does dropping this reference to `obj` leave an instance that others -/// still hold, with no weakref watching it? Past two owners only a -/// weakref-watched object can be at its dead line (see -/// [`note_dropped_counted`]), so such a drop needs neither a prompt reap -/// nor a mark: the cheap test ahead of the full grades. -#[inline(always)] -pub fn drop_survives_plainly(obj: &crate::object::Object) -> bool { - match obj { - crate::object::Object::Instance(i) => { - crate::sync::Rc::strong_count(i) > 2 - && !crate::weakref_registry::may_have_weakrefs( - crate::sync::Rc::as_ptr(i) as usize as u64 - ) - } - _ => false, - } -} - -/// How many elements a dying container may hold and still be graded by -/// inspection (see [`inert_death`]). Capped so the grade stays O(1) on a -/// path that runs for every discarded heap value. -const INERT_SCAN_CAP: usize = 32; - -/// Does this death need no prompt handling at all? A container dying at -/// its last reference, with no collector handle and no weakref, whose -/// every element is atomic, frees nothing that could finalize — so the -/// burst need not end to run a sweep that would find nothing. Grading it -/// `true` is what keeps a `x = [i, i]` loop inside the burst: the store -/// that displaces the previous list ended it on every iteration. -#[inline] -fn inert_death(sc: usize, id: ObjectId, all_atomic: impl FnOnce() -> bool) -> bool { - sc <= 1 - && !process_tracked_filter().may_contain(id) - && !crate::weakref_registry::may_have_weakrefs(id) - && all_atomic() -} - -/// [`inert_death`] for a value the caller still holds the last reference -/// to: dropping it frees nothing that could finalize, clear a weakref or -/// need untracking, so the plain `Rc` drop is its whole teardown and no -/// prompt-reap cascade has to run. -pub fn dies_inert(obj: &crate::object::Object) -> bool { - use crate::object::Object as O; - match obj { - O::List(l) => inert_death( - crate::sync::Rc::strong_count(l), - crate::sync::Rc::as_ptr(l) as usize as u64, - || { - l.try_borrow() - .is_ok_and(|v| v.len() <= INERT_SCAN_CAP && v.iter().all(is_atomic)) - }, - ), - O::Dict(d) => inert_death( - crate::sync::Rc::strong_count(d), - crate::sync::Rc::as_ptr(d) as usize as u64, - || { - d.try_borrow().is_ok_and(|m| { - m.len() <= INERT_SCAN_CAP - && m.iter().all(|(k, v)| is_atomic(&k.0) && is_atomic(v)) - }) - }, - ), - _ => false, - } -} - -/// The grade for an object with strong count `sc` and identity `id` -/// (the identity is the one [`crate::weakref_registry::id_of`] yields). -#[inline(always)] -fn note_dropped_counted(sc: usize, id: ObjectId) -> bool { - floor_stats::bump(&floor_stats::DROP_NOTES, 1); - // A tracked object dies here when only its collector handle (and any - // registry-held weakref clones) remain beside the dropped reference. - // Probed cheapest-first: past two references only a weakref-watched - // object can be at its dead line, and almost none are. - let marked = if sc <= 1 { - true - } else if sc == 2 { - process_tracked_filter().may_contain(id) - } else { - crate::weakref_registry::may_have_weakrefs(id) - && process_tracked_filter().may_contain(id) - && sc <= 2 + crate::weakref_registry::strong_clone_count(id) - }; - if marked { - floor_stats::bump(&floor_stats::DROP_NOTES_MARKED, 1); - } - marked -} - -#[inline(never)] -fn note_dropped_marks_other(obj: &crate::object::Object) -> bool { - use crate::object::Object as O; - match obj { - // A dying bound method (the `obj.m(...)` call temporary, freed at - // every method call) owns exactly one object: its receiver. Grade - // that instead of marking on the method's own count of one. - O::BoundMethod(bm) if crate::sync::Rc::strong_count(bm) <= 1 => { - note_dropped_marks(&bm.receiver) - } - _ => { - let sc = strong_count_for(obj); - let id = crate::weakref_registry::id_of(obj); - note_dropped_counted(sc, id) - } - } + let _ = MAYBE_DEAD.try_with(|c| c.set(true)); } -/// Consume the "a reference may have dropped" flag, returning whether it was -/// set. The eval loop calls this to decide whether a prompt-finalization sweep -/// is warranted this instruction. +/// Consume the pending-work flag, returning whether it was set. #[inline] pub fn take_maybe_dead() -> bool { MAYBE_DEAD.with(|c| c.replace(false)) } -/// Convenience: drive one prompt-finalization pass over refcount-dead -/// finalizable objects in the shared GC (see -/// [`GcState::reap_dead_finalizable`]). -pub fn reap_dead_finalizable() -> usize { - with_state(|s| s.reap_dead_finalizable()) -} - -/// Convenience: enroll a tracked object in the prompt-finalization index -/// because a callback-bearing weakref now watches it (see -/// [`GcState::note_weakref_finalizable`]). -pub fn note_weakref_finalizable(id: ObjectId) { - with_state(|s| s.note_weakref_finalizable(id)); -} - /// Convenience: run a full collection on the shared GC. Returns the /// number of objects collected. pub fn collect_all() -> usize { - let n = with_state(|s| s.collect(N_GENERATIONS - 1)); - sweep_weakref_only_targets(); - n + with_state(|s| s.collect(N_GENERATIONS - 1)) } /// Convenience: run a partial collection of generations /// `0..=upto`. pub fn collect_upto(upto: usize) -> usize { - let n = with_state(|s| s.collect(upto)); - sweep_weakref_only_targets(); - n + with_state(|s| s.collect(upto)) } /// Convenience: fire dead objects' weakref callbacks via a non-destructive -/// mark pass on the shared GC (see [`GcState::fire_dead_weakrefs`]), then -/// sweep the untracked weakref-only targets. Used from a blocking -/// `Thread.join` to unblock idle `ThreadPoolExecutor` workers without the -/// teardown risk of a full collection. +/// mark pass on the shared GC (see [`GcState::fire_dead_weakrefs`]). Used +/// from a blocking `Thread.join` to unblock idle `ThreadPoolExecutor` +/// workers without the teardown risk of a full collection. pub fn fire_dead_weakrefs() { with_state(|s| s.fire_dead_weakrefs()); - sweep_weakref_only_targets(); -} - -/// Clear weakrefs whose referent isn't in the tracked set and whose -/// only remaining strong references are the weakref slots' own -/// clones. Covers weakref-able objects the cycle collector never -/// sees — plain functions, bound methods, types — so -/// `del f; gc.collect()` flips `weakref.ref(f)()` to `None` exactly -/// like CPython's refcount-driven `tp_dealloc` would. -pub fn sweep_weakref_only_targets() -> usize { - let targets = - with_state(|s| crate::weakref_registry::with_registry(|r| s.untracked_weakref_targets(r))); - let mut cleared = 0; - for (id, target) in targets { - if is_tracked(id) { - // Tracked objects belong to the cycle pass (their handle - // holds an extra strong ref this arithmetic doesn't model). - continue; - } - let clones = crate::weakref_registry::strong_clone_count(id); - // `target` itself is one clone we hold for the probe. - if strong_count_for(&target) <= clones + 1 { - crate::weakref_registry::queue_callbacks(crate::weakref_registry::notify_clear(id)); - cleared += 1; - } - } - if cleared > 0 { - crate::weakref_registry::with_registry(|r| r.shrink()); - } - cleared + crate::weakref_registry::sweep_dead_targets(); } #[cfg(test)] @@ -4683,104 +2579,6 @@ mod tests { use crate::object::DictData; - #[test] - fn suspect_eviction_takes_dormant_then_lowest_budget_and_releases() { - let mut suspects = Suspects::default(); - let mut handles = Vec::new(); - for index in 0..8usize { - let object = Object::List(Rc::new(RefCell::new(Vec::new()))); - let handle = HandleRc::new(TrackedHandle::new(object, 0)); - handles.push((handle.id, HandleRc::downgrade(&handle))); - let budget = [0, 5, 0, 3, 9, 3, 0, 1][index]; - let entry = Suspect { - handle, - budget, - dormant_probes: 0, - }; - if budget == 0 { - suspects.dormant.insert(handles[index].0, entry); - } else { - suspects.active.insert(handles[index].0, entry); - } - } - // Dormant entries go first, oldest first. - for expected in [0, 2, 6] { - assert!(evict_lowest_budget_suspect(&mut suspects)); - assert!(!suspects.contains_key(handles[expected].0)); - assert!(handles[expected].1.upgrade().is_none()); - } - // Then active ones by budget: 1, the first 3, the other 3, 5, 9. - for expected in [7, 3, 5, 1, 4] { - assert!(evict_lowest_budget_suspect(&mut suspects)); - assert!(!suspects.contains_key(handles[expected].0)); - assert!(handles[expected].1.upgrade().is_none()); - } - assert_eq!(suspects.len(), 0); - assert!(!evict_lowest_budget_suspect(&mut suspects)); - } - - #[test] - fn weakref_snapshot_skips_tracked_payloads_and_observes_transitions() { - use crate::weakref_registry::{kind, WeakRefRegistry, WeakRefSlot}; - - let state = GcState::new(); - let registry = WeakRefRegistry::new(); - let roots: Vec<_> = (0..3) - .map(|_| Object::List(Rc::new(RefCell::new(Vec::new())))) - .collect(); - let slots: Vec<_> = roots - .iter() - .map(|target| { - let slot = Rc::new(WeakRefSlot::new( - id_of(target), - target.clone(), - false, - kind::REF, - )); - registry.register(slot.clone()); - slot - }) - .collect(); - state.track_now(roots[0].clone()); - // The excluded payload cannot even be borrowed. This catches a - // regression to cloning every target and filtering afterwards. - let excluded = slots[0].target.borrow_mut(); - let before = strong_count_for(&roots[0]); - let snapshot = state.untracked_weakref_targets(®istry); - assert_eq!(snapshot.len(), 2); - assert_eq!(strong_count_for(&roots[0]), before); - for (id, target) in &snapshot { - assert_eq!(*id, id_of(target)); - assert!(!target.is_same(&roots[0])); - } - drop(snapshot); - drop(excluded); - - state.track_now(roots[1].clone()); - let snapshot = state.untracked_weakref_targets(®istry); - assert_eq!(snapshot.len(), 1); - assert!(snapshot[0].1.is_same(&roots[2])); - drop(snapshot); - state.untrack_id(id_of(&roots[0])); - state.untrack_id(id_of(&roots[1])); - assert_eq!(state.untracked_weakref_targets(®istry).len(), 3); - - slots[1].clear(); - let snapshot = state.untracked_weakref_targets(®istry); - assert_eq!(snapshot.len(), 2); - assert!(snapshot - .iter() - .all(|(_, target)| !target.is_same(&roots[1]))); - // A returned snapshot owns its targets independently of the slots. - for slot in &slots { - slot.clear(); - } - assert!(state.untracked_weakref_targets(®istry).is_empty()); - assert!(snapshot.iter().all(|(id, target)| *id == id_of(target))); - drop(slots); - assert!(registry.targets().is_empty()); - } - #[test] fn compact_positions_keep_absent_and_uncached_states_distinct() { for value in [0, 1, (u32::MAX - 2) as usize, usize::MAX] { @@ -4801,7 +2599,7 @@ mod tests { { let hint = CachedSlot::new(u32::MAX as usize + 100); assert_eq!(hint.load(Ordering::Acquire), usize::MAX - 1); - assert_eq!(std::mem::size_of::(), 64); + assert_eq!(std::mem::size_of::(), 40); } } @@ -4837,105 +2635,6 @@ mod tests { } } - #[test] - fn uncacheable_hot_positions_preserve_membership_and_move_fixups() { - let state = GcState::new(); - let roots: Vec<_> = (0..3) - .map(|_| Object::Dict(Rc::new(RefCell::new(DictData::default())))) - .collect(); - let handles: Vec<_> = roots - .iter() - .map(|root| { - state.track_now(root.clone()); - state.handle_for(id_of(root)).unwrap() - }) - .collect(); - for handle in &handles { - assert!(state.fin_insert(handle.id, handle.clone())); - } - let middle = &handles[1]; - middle - .fin_hot_slot - .store((u32::MAX - 1) as usize, Ordering::Release); - state.fin_make_hot(middle); - assert_eq!(state.finalizable_hot.borrow().len(), 3); - state.fin_make_cold(middle); - assert_eq!(state.finalizable_hot.borrow().len(), 2); - assert_eq!(handles[2].fin_hot_slot.load(Ordering::Acquire), 1); - assert!(middle.fin_cold.load(Ordering::Acquire)); - assert_eq!(middle.fin_hot_slot.load(Ordering::Acquire), usize::MAX); - state.fin_make_hot(middle); - assert_eq!(middle.fin_hot_slot.load(Ordering::Acquire), 2); - handles[0] - .fin_hot_slot - .store((u32::MAX - 1) as usize, Ordering::Release); - assert!(state.fin_remove(handles[0].id)); - assert!(!state.fin_remove(handles[0].id)); - assert_eq!(middle.fin_hot_slot.load(Ordering::Acquire), 0); - for handle in &handles { - state.untrack_id(handle.id); - } - assert!(state.finalizable_hot.borrow().is_empty()); - assert_eq!(state.finalizable_count.load(Ordering::Acquire), 0); - assert_eq!(state.fin_hot_count.load(Ordering::Acquire), 0); - } - - #[test] - fn deferred_instance_drop_ignores_stale_gc_filter() { - const CHILD: &str = "WEAVEPY_DEFERRED_DROP_TEST_CHILD"; - if std::env::var_os(CHILD).is_none() { - let status = std::process::Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "gc_trace::tests::deferred_instance_drop_ignores_stale_gc_filter", - "--nocapture", - ]) - .env(CHILD, "1") - .status() - .expect("spawn deferred-drop test"); - assert!(status.success(), "deferred-drop child: {status}"); - return; - } - let _gil = crate::gil::global_gil().acquire(); - let class = - crate::types::TypeObject::new_user("DeferredDrop", Vec::new(), DictData::default()) - .unwrap(); - let owner = crate::types::PyInstance::new_deferred(class); - let operand = Object::Instance(owner.clone()); - let id = id_of(&operand); - assert!(owner.is_gc_deferred()); - assert!(!is_tracked(id)); - assert_eq!(Rc::strong_count(&owner), 2); - - // Model stale bits from an earlier allocation at this address. - // This makes the reported CI failure deterministic without - // relying on allocator placement or weakening path coverage. - process_tracked_filter().insert(id); - assert!(maybe_tracked(id)); - assert!(!is_tracked(id)); - assert!(note_dropped_marks(&operand)); - assert!(crate::Interpreter::core_droppable(&operand)); - - // A final program owner still requires ordinary teardown. - drop(owner); - assert!(!crate::Interpreter::core_droppable(&operand)); - - // A non-atomic field revokes the deferral proof. After releasing - // the extra program owner, the collector and operand are the - // only owners, so this release must mark prompt cleanup. - let Object::Instance(owner) = &operand else { - unreachable!() - }; - owner.note_slot_store(&Object::List(Rc::new(RefCell::new(Vec::new())))); - assert!(!owner.is_gc_deferred()); - assert!(is_tracked(id)); - assert_eq!(Rc::strong_count(owner), 2); - assert!(note_dropped_marks(&operand)); - assert!(!crate::Interpreter::core_droppable(&operand)); - with_state(|state| state.untrack_id(id)); - assert!(!is_tracked(id)); - } - #[test] fn gc_index_distributes_aligned_object_addresses() { use std::hash::BuildHasher; @@ -5002,7 +2701,7 @@ mod tests { #[test] fn generation_constructor_rejects_out_of_range_indices() { for generation in [N_GENERATIONS, u8::MAX as usize + 1, usize::MAX] { - let result = std::panic::catch_unwind(|| TrackedHandle::new(Object::None, generation)); + let result = std::panic::catch_unwind(|| TrackedHandle::new(&Object::None, generation)); assert!(result.is_err()); } } @@ -5025,22 +2724,36 @@ mod tests { let s = GcState::new(); let dict = Rc::new(RefCell::new(DictData::default())); let outer = Object::Dict(dict.clone()); - s.track(outer.clone()); + s.track_now(outer.clone()); // The dict references itself: a 1-cycle. dict.borrow_mut().insert( crate::object::DictKey(Object::from_static("self")), outer.clone(), ); - // Drop the local strong ref; only the cycle + the GC's - // tracked handle keep it alive. + let weak = Rc::downgrade(&dict); drop(outer); - let collected = s.collect(2); - // We expect the cyclic dict to be discovered (the cycle's - // gc_refs is balanced by the self-pointer). The actual - // assertion is loose — the GC may or may not collect on - // the first pass depending on Rust-side stash counts. - let _ = collected; - // What we *do* assert: the GC didn't crash. + drop(dict); + // Only the cycle keeps the dict alive; the collector holds nothing. + assert!(weak.upgrade().is_some()); + assert_eq!(s.collect(2), 1); + assert!(weak.upgrade().is_none()); + assert_eq!(s.population(), 0); + } + + #[test] + fn tracked_objects_die_by_refcount() { + let s = GcState::new(); + let dict = Rc::new(RefCell::new(DictData::default())); + let obj = Object::Dict(dict.clone()); + s.track_now(obj.clone()); + let weak = Rc::downgrade(&dict); + drop(obj); + drop(dict); + // No collection ran: the entry is stale but owns nothing. + assert!(weak.upgrade().is_none()); + assert_eq!(s.population(), 1); + assert_eq!(s.collect(0), 0); + assert_eq!(s.population(), 0); } #[test] diff --git a/crates/weavepy-vm/src/hot_filter.rs b/crates/weavepy-vm/src/hot_filter.rs index 8f11da51..12af799a 100644 --- a/crates/weavepy-vm/src/hot_filter.rs +++ b/crates/weavepy-vm/src/hot_filter.rs @@ -1,10 +1,9 @@ //! Insert-only atomic bloom pre-filters — RFC 0065 (WS4). //! -//! Three registries (the weakref registry, the GC tracked-object -//! index, the prompt-reap suspect map) sit behind `GilCell`/mutex -//! borrows but are consulted on *usually-miss* probes from the -//! interpreter's drop paths: "is anything watching this object?", -//! "is this object tracked?", "is this id enrolled as a suspect?". +//! Registries (the weakref registry, the GC tracked-object index) sit +//! behind `GilCell`/mutex borrows but are consulted on *usually-miss* +//! probes from the interpreter's drop paths: "is anything watching this +//! object?", "is this object tracked?". //! RFC 0061 measured the borrow machinery of those misses at ~5% //! of drop-heavy fixtures. An [`AtomicBloom`] answers the miss with //! two relaxed loads, no lock, no TLS. diff --git a/crates/weavepy-vm/src/hot_gates.rs b/crates/weavepy-vm/src/hot_gates.rs index c3a4f070..396e54fd 100644 --- a/crates/weavepy-vm/src/hot_gates.rs +++ b/crates/weavepy-vm/src/hot_gates.rs @@ -65,13 +65,11 @@ static HOT: AtomicU32 = AtomicU32::new(0); /// RFC 0065 (WS1): the dispatch loop's *generation* word. Bumped by /// every mutation that can change the loop prologue's decisions — /// hot-gate bits ([`set`]/[`clear`] below), observer registration -/// (`trace::bump_observer_gen`), the GC finalizable/suspect -/// population transitions (`gc_trace`), and frame materialization +/// (`trace::bump_observer_gen`), and frame materialization /// (`FrameShell::materialize`). The dispatch loop caches a snapshot /// of the prologue's inputs keyed by this generation: while the /// generation is unchanged *and* the snapshot says "quiet" (no -/// pending work, no finalizables, no suspects, no observers, no -/// materialized frame), the loop runs one relaxed load + compare per +/// pending work, no observers, no materialized frame), the loop runs one relaxed load + compare per /// instruction instead of the full ten-plus-probe prologue. /// /// Same producer/consumer discipline as [`HOT`]: a spurious bump is @@ -138,11 +136,6 @@ pub mod env_flags { } }; } - once_flag!( - /// `WEAVEPY_REAP_TRACE`: prompt-reap tracing. - reap_trace, - "WEAVEPY_REAP_TRACE" - ); once_flag!( /// `WEAVEPY_NO_QUIET`: pin the dispatch loop to its full prologue. no_quiet, diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs index a7f7e8a3..57451d59 100644 --- a/crates/weavepy-vm/src/inst_dict.rs +++ b/crates/weavepy-vm/src/inst_dict.rs @@ -1128,10 +1128,17 @@ impl crate::types::PyInstance { /// layout, no materializing). pub fn attr_position_str(&self, name: &str) -> Option { let i = match self.dict.published() { - Some(d) => d - .try_borrow() - .ok()? - .get_index_of(&crate::object::StrKey(name))?, + Some(d) => { + // Never runs Python: a stored key only a user `__eq__` + // could equate leaves the position unknown. + let probe = + crate::object::LeafNameProbe::new(name, crate::object::py_str_hash(name)); + let i = d.try_borrow().ok()?.get_index_of(&probe); + if probe.saw_exotic() { + return None; + } + i? + } None => self .dict .split_cell() diff --git a/crates/weavepy-vm/src/lazy_arc.rs b/crates/weavepy-vm/src/lazy_arc.rs index 728983c2..afa5ea57 100644 --- a/crates/weavepy-vm/src/lazy_arc.rs +++ b/crates/weavepy-vm/src/lazy_arc.rs @@ -10,14 +10,14 @@ use std::mem::ManuallyDrop; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; -pub struct LazyArc { +pub struct LazyArc { pointer: AtomicPtr, // AtomicPtr alone would allow Send/Sync regardless of T. Match Arc's // ownership, drop checking, and thread-safety requirements instead. owner: PhantomData>, } -impl LazyArc { +impl LazyArc { pub const fn new() -> Self { Self { pointer: AtomicPtr::new(ptr::null_mut()), @@ -101,7 +101,7 @@ impl LazyArc { } } -impl LazyArc { +impl LazyArc { #[inline] pub fn share(&self) -> Arc { let pointer = self.get_or_init_ptr(|| Arc::new(T::default())); @@ -113,13 +113,13 @@ impl LazyArc { } } -impl Default for LazyArc { +impl Default for LazyArc { fn default() -> Self { Self::new() } } -impl From> for LazyArc { +impl From> for LazyArc { fn from(value: Arc) -> Self { Self { pointer: AtomicPtr::new(Arc::into_raw(value).cast_mut()), @@ -135,7 +135,7 @@ impl From> for LazyArc { // (`PyInstance::dict_cell` for the instance dictionary), so every // creator is visible at the call site. -impl Clone for LazyArc { +impl Clone for LazyArc { fn clone(&self) -> Self { // A shallow instance clone must share even a previously empty dict: // subsequent writes through either owner must reach the same object. @@ -143,13 +143,13 @@ impl Clone for LazyArc { } } -impl fmt::Debug for LazyArc { +impl fmt::Debug for LazyArc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("LazyArc").field(&self.get()).finish() } } -impl Drop for LazyArc { +impl Drop for LazyArc { fn drop(&mut self) { let pointer = *self.pointer.get_mut(); if !pointer.is_null() { diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index f55ee6d7..8ad1530b 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -90,6 +90,7 @@ pub mod types; pub mod unicode_case; pub mod unicode_numeric; pub mod vm_singletons; +pub mod weak_object; pub mod weakref_registry; use crate::builtin_types::{builtin_types, instance_is_subclass, make_exception_with_class}; @@ -491,63 +492,6 @@ fn py_traceback_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) { } } -/// Snapshot the objects in a generator's suspended frame that will -/// need finalization when the frame is dropped (instances with a -/// `__del__`). Captured *before* `close()` tears the frame down so -/// the caller can emulate CPython's prompt refcount-driven -/// finalization afterwards. -fn frame_reapables(g: &Rc) -> Vec { - let Ok(state) = g.state.try_borrow() else { - return Vec::new(); - }; - let boxed = match &*state { - GeneratorState::Created(b) | GeneratorState::Suspended(b) => b, - _ => return Vec::new(), - }; - let frame: &Frame = boxed; - // CPython clears a generator's frame the instant it finishes/closes - // (`gen_send`/`gen_close` → `_PyFrame_ClearExceptCode`), so every - // local and value-stack entry is decref'd promptly. WeavePy mirrors - // that by running each prompt-reapable entry through the cascade - // after the frame is torn down — not just `__del__`-bearing - // instances but anything that anchors a finalizable acyclic - // subgraph *or* whose only surviving reference (once the frame is - // gone) is a weakref slot's strong clone in our registry. The - // latter is what lets `weakref.ref(obj)` zero out by refcount the - // way CPython's does: without it a suspended generator that - // referenced `obj` would keep `obj` pinned through the registry's - // clone until the next cyclic collection (RFC 0040: - // `test_concurrent_futures.test_init` leaked semaphores because the - // `_assert_logged`/`sleeping_retry` generators pinned the test - // instance → its `multiprocessing.Queue` → SemLocks). - let mut out: Vec = Vec::new(); - let mut seen: std::collections::HashSet = - std::collections::HashSet::new(); - let locals = frame.locals.borrow(); - for o in locals.iter().chain(frame.stack.iter()) { - if (Interpreter::local_needs_prompt_reap(o) - || crate::weakref_registry::count_for(crate::weakref_registry::id_of(o)) > 0) - && seen.insert(crate::weakref_registry::id_of(o)) - { - out.push(o.clone()); - } - } - // RFC 0073 WS4 — a parked native activation's pin table holds the - // live values (the locals above are the stale park-time snapshot). - #[cfg(feature = "jit")] - if let Some(act) = &frame.parked_native { - for o in act.pinned_objects() { - if (Interpreter::local_needs_prompt_reap(&o) - || crate::weakref_registry::count_for(crate::weakref_registry::id_of(&o)) > 0) - && seen.insert(crate::weakref_registry::id_of(&o)) - { - out.push(o); - } - } - } - out -} - /// Warm the JIT's code generator on a background thread when the JIT /// is on (see `tier2::prewarm_codegen`): a program's first compile then /// doesn't pay the generator's cold start. @@ -2451,16 +2395,13 @@ impl Interpreter { result } - /// Emulate CPython's prompt refcount-driven death for an object - /// whose binding was just dropped (a `del` statement, or a frame - /// torn down by `close()`). If `dropped` holds the last - /// program-visible reference — everything else is the GC - /// registry's handle and weakref slots' strong clones — run its - /// finalizer now and clear its weakrefs, exactly as CPython's - /// `tp_dealloc` would. Conservative: any extra reference anywhere - /// (containers, caches, other bindings) skips the reap and leaves - /// the object to the cycle collector. - fn prompt_reap_dropped(&mut self, dropped: Object) { + /// Release `dropped`: CPython's `Py_DECREF` at the end of a binding. + /// Objects die by reference count (the collector and the weakref + /// registry hold no strong references), so this is a plain drop, plus + /// the deallocation side effects a `Drop` impl can't produce itself, + /// plus the freelists, which take a dead plain instance's or tuple's + /// allocation for the next one. + fn release(&self, dropped: Object) { // C-API watcher probes (test_capi.test_watchers): a watched dict // losing its last binding fires PyDict_EVENT_DEALLOCATED; a dying // function fires PyFunction_EVENT_DESTROY. Both gates are relaxed @@ -2474,1256 +2415,39 @@ impl Interpreter { if crate::stdlib::testinternalcapi_mod::reftrace_print_active() { println!("DESTROY {}", dropped.type_name_owned()); } - // A discarded never-driven `asend`/`athrow`/`aclose` awaitable - // warns at finalization (gh-113753; CPython's - // `async_gen_asend_finalize` family). - if let Object::AsyncGenAwait(a) = &dropped { - if !a.started.get() && !a.consumed.get() && Rc::strong_count(a) <= 1 { - a.consumed.set(true); - let method = match a.kind { - crate::object::AgenAwaitKind::Send => "asend", - crate::object::AgenAwaitKind::Throw => "athrow", - crate::object::AgenAwaitKind::Close => "aclose", - }; - let qualname = match &a.agen { - Object::AsyncGenerator(g) => g.qualname.borrow().to_str(), - other => other.type_name_owned(), - }; - let _ = self.emit_runtime_warning(format!( - "coroutine method '{method}' of '{qualname}' was never awaited" - )); - } - return; - } - // A `BoundMethod` is itself untracked and carries no finalizer, so - // the fast-path early-return below would merely `Rc`-drop it — - // reclaiming the method object but leaving a *tracked* receiver - // pinned by its own GC handle (the early-return only accounts for - // *untracked* children a value anchors). Release the method's hold - // and run the receiver through the cascade, exactly as - // `reap_call_receiver` does for call temporaries. CPython decrefs a - // `method`'s `__self__` the instant the method object dies; this is - // what reaps a stored bound `__exit__` — the compiler's `.with_exit` - // synthetic local, dropped by `DeleteFast` on every `with` exit path - // — when the context manager is a plain instance that anchors - // resources. `unittest._AssertRaisesContext` holds the test case, - // which in `test_concurrent_futures.test_init` pinned the log - // `Queue` and its SemLocks until the next cyclic collection - // (RFC 0040: resource-tracker exited nonzero on leaked semaphores). - if matches!(&dropped, Object::BoundMethod(_)) { - // A bound method that escaped into an instance attribute - // (`self.cb = self.handler`) was GC-tracked at store time (see - // `generic_setattr_instance`); merely dropping our clone would - // leave it pinned by its own collector handle — and, through - // `__self__`, its receiver too. Route it through the cascade, - // which untracks and frees it. pickle's `_Unpickler` is the - // motivating case: it stores `self.append = self.stack.append` - // and rebinds it on every `load`, and the displaced method - // otherwise pinned the previous stack list and every - // freshly-unpickled object on it until the next cyclic - // collection (test_concurrent_futures - // `test_ressources_gced_in_workers`). - // - // Receivers can themselves be bound methods, arbitrarily deep - // (`f = f.__call__` wrapper towers — test_descr - // test_wrapper_segfault builds a million of them), so the walk - // is a loop, not recursion, and stops as soon as a link is - // still referenced elsewhere (its receiver isn't dead). - let mut cur = dropped; - loop { - let (tracked, alive, recv) = { - let Object::BoundMethod(bm) = &cur else { break }; - let id = crate::weakref_registry::id_of(&cur); - // `cur` is our only handle when nothing else holds the - // Rc (modulo registry strong clones); a method still - // aliased — e.g. as an outer wrapper's `__self__` — - // keeps its receiver alive, so the cascade must not - // descend into it. - let alive = - Rc::strong_count(bm) > 1 + crate::weakref_registry::strong_clone_count(id); - (gc_trace::is_tracked(id), alive, bm.receiver.clone()) - }; - if tracked { - self.reap_dead_subgraph(cur); - return; + match dropped { + // A discarded never-driven `asend`/`athrow`/`aclose` awaitable + // warns at finalization (gh-113753; CPython's + // `async_gen_asend_finalize` family), at the next safe point. + Object::AsyncGenAwait(a) => { + if !a.started.get() && !a.consumed.get() && Rc::strong_count(&a) <= 1 { + a.consumed.set(true); + crate::vm_singletons::push_pending_finalizer(Object::AsyncGenAwait(a)); } - if alive { - return; - } - drop(cur); - if !(Self::local_needs_prompt_reap(&recv) && Self::looks_reapable_temporary(&recv)) - { - return; - } - cur = recv; } - self.prompt_reap_dropped(cur); - return; - } - // A tuple of scalars anchors nothing, is never GC-tracked at - // birth (the filter has no false negatives) and can't be weakly - // referenced: the freelist (or the plain drop) is its teardown. - if let Object::Tuple(t) = &dropped { - if t.iter().all(Object::is_gc_atomic) - && !gc_trace::maybe_tracked(crate::weakref_registry::id_of(&dropped)) - { - self.maybe_donate_tuple(dropped); - return; + // A dead plain tuple's allocation is recycled (CPython's tuple + // freelist; see `maybe_donate_tuple`). + dropped @ Object::Tuple(_) => self.maybe_donate_tuple(dropped), + // Likewise a dead instance's (see `PyInstance::try_recycle`). + Object::Instance(i) if Rc::strong_count(&i) == 1 && i.dies_by_plain_drop() => { + PyInstance::try_recycle(i); } + dropped => drop(dropped), } - // A deferred-tracking instance holds only atomic values, has no - // weakref (creating one tracks it) and no collector handle: the - // plain `Rc` drop is its whole teardown. - if let Object::Instance(i) = &dropped { - if i.dies_by_plain_drop() { - if let Object::Instance(i) = dropped { - PyInstance::try_recycle(i); - } - return; - } - } - let id0 = crate::weakref_registry::id_of(&dropped); - // Nothing parked on this object needs prompt handling: no - // finalizer to run, no weakref to clear, and no cycle-GC strong - // handle pinning it. Ordinary `Rc` drop reclaims it (and any - // *untracked* children it anchors, recursively, by refcount). - // - // An instance that escaped into a C extension (`c_body != 0`) is - // excluded from this fast path even when untracked/unwatched: its - // `tp_dealloc` releases references held in *C struct fields* - // (invisible to the plain `Rc` drop's recursion), and a *tracked* - // child freed that way is left pinned by its own GC handle unless - // the cascade below walks to it and reaps it in turn. pandas' - // chained `df[cols].iloc[rows]` is the motivating case: the - // discarded `_iLocIndexer` temporary holds the intermediate - // DataFrame only through Cython's `self->obj` field, and the - // intermediate's weakref (`subset._is_copy`) must clear the - // instant the indexer dies, as it does on CPython. - if !Self::object_is_finalizable(&dropped) - && crate::weakref_registry::count_for(id0) == 0 - && !gc_trace::is_tracked(id0) - && !Self::instance_escaped_to_c(&dropped) - { - // One more container-shaped exception to the fast path: an - // *untracked* container can be the sole holder of a *tracked* - // child, and the plain `Rc` drop below would leave that child - // refcount-dead but pinned by its own collector handle until - // the next cyclic collection. Tuples are the canonical case — - // they are never GC-tracked at `BuildTuple` time — and - // `multiprocessing`'s wire format is exactly this shape: - // `conn.recv()` unpickles `('#ERROR', exc)`, the manager proxy - // re-raises `exc`, and the handled exception must die at - // handler exit even with `gc.disable()` in force - // (test_multiprocessing_* `TestManagerExceptions.test_queue_get`, - // gh-106558). Route such containers through the cascade, which - // recurses through untracked containers to reap the tracked - // descendants. - // - // Untracked *instances* are the same shape (RFC 0076 WS3): the - // canonical case is a dying handled exception. `except E as e:` - // compiles to POP_EXCEPT (while `e` still binds the exception — - // the reap there sees it alive) followed by `e = None; del e`, - // so the actual death site is the STORE displacing `e` — this - // path. The exception is untracked and non-finalizable, but - // through `__traceback__` → frame → locals it is routinely the - // *sole* holder of tracked instances from the unwound frames. - // Pillow's `Image.open` is the motivating case: each failed - // plugin candidate (`ImImageFile(fp)` raising SyntaxError in - // `__init__`) lives only in the constructor frame the traceback - // pins; skipping the cascade here left the candidates handle- - // pinned holding `fp`, so the "unclosed file" ResourceWarning - // fired at the next amortized acyclic sweep instead of at - // handler exit (mpo/psd/spider/tiff `test_unclosed_file`). - // The deadness pre-check keeps the common alive-rebind - // (`self.x = ...` displacing a shared value) at two cheap - // lookups; `anchors_tracked_child` is node-capped. - if !(matches!( - dropped, - Object::Tuple(_) - | Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::FrozenSet(_) - | Object::Instance(_) - | Object::Frame(_) - ) - // A container of nothing but scalars anchors no tracked - // child by inspection, so it can skip both the deadness - // test and the traversal — the shape most temporaries - // have, and (since tracking such a container is deferred) - // the one that reaches here most. - && !Self::is_scalar_leaf_container(&dropped) - && Self::is_refcount_dead(&dropped, 1) - && Self::anchors_tracked_child(&dropped, 6)) - { - // A dead plain tuple's allocation is recycled (CPython's - // tuple freelist; see `maybe_donate_tuple`). - if matches!(dropped, Object::Tuple(_)) && Self::is_refcount_dead(&dropped, 1) { - self.maybe_donate_tuple(dropped); - } - return; - } - } - // Is `dropped` actually dead — i.e. is this its last - // program-visible reference? The caller still holds `dropped` - // (one reference), the GC may hold a handle, and weakrefs may - // hold strong clones; anything beyond that is a live binding and - // we must leave the object to the cycle collector. - if !Self::is_refcount_dead(&dropped, 1) { - // Still referenced — but possibly only by Rust-side transients - // that die between safe points (the same shape the cascade's - // child check handles via `note_suspect`, but at the *root*: a - // `del obj` racing another thread's in-flight teardown clones). - // A weakly-referenced tracked root that misses its reap here is - // user-visible — the weakref never clears - // (test_threading.test_no_refcycle_through_target under load) — - // so enroll borderline cases for the eval loop's re-probe. - // Finalizable objects stay excluded, as everywhere on the - // suspect path: their `__del__` must not fire from a probe. - return; - } - self.reap_dead_subgraph(dropped); } - /// The cascade's commonest root, settled without it: a refcount-dead - /// plain instance (no `__del__`, no weakref, no C body or native - /// value, no `__slots__` values) whose every attribute is a scalar - /// leaf, a value that outlives it, or a scalar-leaf container only it - /// holds. Nothing in that subgraph finalizes, clears a weakref or - /// anchors a tracked object the drop would strand, so untracking the - /// instance and those containers (the cascade's next links, which it - /// would retire the same way) and dropping it is the whole teardown. - /// A tracked attribute value near its dead line (the cascade enrolls - /// those as suspects) or anything else the cascade walks into hands - /// `dropped` back in `Err`. - fn reap_plain_instance(dropped: Object) -> Result<(), Object> { - let Some((leaves, n)) = Self::plain_instance_leaves(&dropped) else { - return Err(dropped); - }; - for cid in &leaves[..n] { - gc_trace::with_state(|s| s.untrack_id(*cid)); - } - let id = crate::weakref_registry::id_of(&dropped); - if gc_trace::is_tracked(id) { - gc_trace::with_state(|s| s.untrack_id(id)); - } - drop(dropped); - Ok(()) - } - - /// A dead function (a rebound lambda or closure — CPython frees it by - /// refcount on the spot) whose death strands nothing: no weakref, and - /// every edge it releases — namespaces, defaults, closure cells, its - /// `__dict__` and getset slots — either outlives it or holds only - /// scalars. Untracking and dropping it is then the whole teardown. - fn plain_function_dies_alone(obj: &Object) -> bool { - let Object::Function(f) = obj else { - return false; - }; - if crate::vm_singletons::cext_call_active() - || crate::vm_singletons::has_pending_cext_drops() - || crate::weakref_registry::may_have_weakrefs(crate::weakref_registry::id_of(obj)) - { - return false; - } - // A value this function held: fine when it holds nothing a cascade - // would visit, or when another reference keeps it alive past the - // suspect band (a collector handle is one of the references). - fn released_ok(v: &Object) -> bool { - if gc_trace::is_atomic(v) { - return true; - } - let id = crate::weakref_registry::id_of(v); - if crate::weakref_registry::may_have_weakrefs(id) { - return false; - } - let sc = gc_trace::strong_count_for(v); - if gc_trace::is_tracked(id) { - sc > 4 - } else { - sc >= 2 - } - } - // The namespaces are module-owned in every ordinary case. - let namespaces = [&f.globals, &f.builtins]; - if namespaces.iter().any(|d| Rc::strong_count(d) < 3) { - return false; - } - if !f.defaults.iter().all(released_ok) || !f.kw_defaults.iter().all(|(_, v)| released_ok(v)) - { - return false; - } - // A closure cell only this function holds dies with it. - if !f.closure.iter().all(|c| match c { - Object::Cell(cell) => { - Rc::strong_count(cell) >= 2 - || cell.try_borrow().is_ok_and(|v| gc_trace::is_atomic(&v)) - } - other => released_ok(other), - }) { - return false; - } - let Ok(attrs) = f.attrs.try_borrow() else { - return false; - }; - if let Some(attrs) = attrs.as_ref() { - if Rc::strong_count(attrs) == 1 && !attrs.try_borrow().is_ok_and(|d| d.is_empty()) { - return false; - } + /// Release a finished call's arguments now, before the caller handles + /// the result (CPython's `CALL` decrefs its operands first): an + /// argument only the call held dies here, finalizers and all. + fn reap_call_args(&mut self, args: &mut [Object]) { + for a in args { + drop(std::mem::replace(a, Object::Unbound)); } - let Ok(slots) = f.slots.try_borrow() else { - return false; - }; - slots - .iter() - .all(|(k, v)| gc_trace::is_atomic(&k.0) && released_ok(v)) } - /// [`Self::reap_plain_instance`]'s test: the tracked scalar-leaf - /// containers `obj` alone holds (ids, count), or `None` when the - /// general cascade must run. - fn plain_instance_leaves( - obj: &Object, - ) -> Option<([crate::weakref_registry::ObjectId; 4], usize)> { - let Object::Instance(inst) = obj else { - return None; - }; - if inst.c_body.get() != 0 - || inst.native.get().is_some() - || crate::vm_singletons::cext_call_active() - || crate::vm_singletons::has_pending_cext_drops() - || crate::weakref_registry::may_have_weakrefs(crate::weakref_registry::id_of(obj)) - { - return None; - } - match inst.class.try_borrow() { - Ok(cls) if cls.native_kind.get() == 0 && !cls.instances_need_finalize() => {} - _ => return None, - } - if !inst - .slots - .try_borrow() - .is_ok_and(|s| s.iter().next().is_none()) - { - return None; - } - let mut leaves = [0; 4]; - let mut n = 0; - // A dict shared through `vars(obj)` outlives the instance, and - // with it every value. - let split; - let d; - let attrs: &mut dyn Iterator = match inst.dict.published() { - Some(dict) if inst.dict.strong_count() == 1 => { - d = dict.try_borrow().ok()?; - &mut d.iter() - } - Some(_) => return Some((leaves, n)), - None => { - split = inst.dict.split_cell().try_borrow().ok()?; - &mut split.iter() - } - }; - for (k, v) in attrs { - if !gc_trace::is_atomic(&k.0) { - return None; - } - if gc_trace::is_atomic(v) { - continue; - } - let cid = crate::weakref_registry::id_of(v); - if crate::weakref_registry::may_have_weakrefs(cid) { - return None; - } - let sc = gc_trace::strong_count_for(v); - if gc_trace::is_tracked(cid) { - // Past the cascade's suspect band: stays alive. - if sc > 4 { - continue; - } - // Its handle and this dict: it dies here, and a container - // of scalars is retired by untracking. - if sc == 2 && n < leaves.len() && Self::is_scalar_leaf_container(v) { - leaves[n] = cid; - n += 1; - continue; - } - return None; - } - // Untracked: another holder keeps it alive, or it dies by - // plain drop holding nothing but scalars. - if sc >= 2 - || Self::is_scalar_leaf_container(v) - || matches!(v, Object::Tuple(t) if t.iter().all(gc_trace::is_atomic)) - { - continue; - } - return None; - } - Some((leaves, n)) - } - - /// The dead-acyclic-subgraph cascade — the shared tail of - /// [`Self::prompt_reap_dropped`], split out so a call site that has - /// already established `dropped` is refcount-dead and anchors - /// finalizable/tracked children, yet is itself an untracked, - /// non-finalizable *leaf* the fast-path early-return in - /// `prompt_reap_dropped` would skip, can drive the cascade directly. - /// The motivating case is a handled exception cleared at `POP_EXCEPT`: - /// an `AttributeError` carries no `__del__` and is not GC-tracked, but - /// it anchors `AttributeError.obj` (and values reachable only through - /// its saved `__traceback__` frames) whose `__del__` must run at - /// handler exit, matching CPython's refcount timing. Every node is - /// independently refcount-guarded, so anything that stays externally - /// reachable is left to the cycle collector. - fn reap_dead_subgraph(&mut self, dropped: Object) { - if !Self::is_refcount_dead(&dropped, 1) { - return; - } - let Err(dropped) = Self::reap_plain_instance(dropped) else { - return; - }; - if Self::plain_function_dies_alone(&dropped) { - let id = crate::weakref_registry::id_of(&dropped); - if gc_trace::is_tracked(id) { - gc_trace::with_state(|s| s.untrack_id(id)); - } - drop(dropped); - return; - } - // RFC 0045 (wave 5): if the dead subgraph contains any instance - // that has escaped into a C extension (owns a faithful inline - // body), reclaiming it is only safe when no extension frame is - // live on this thread. The reaper's deadness test is - // `Rc`-refcount only and cannot see a *borrowed* C reference — a - // raw body pointer an extension holds across a re-entrant call - // (pandas caches `Index._engine` and reads it back through - // `PyDict_GetItem` as a borrowed ref; an `Index`/`DataFrame` holds - // its `ndarray`/`BlockManager` the same way). Freeing a body such - // a borrow still points at hands the slot to the next `ndarray` - // and the extension's pointer becomes a type-confused - // use-after-free (`'ndarray' has no attribute 'is_unique'` in - // `Index.unique` during `merge`). - // - // A borrowed pointer, however, can only be *in flight* while the - // extension's own stack frame is: C-owned references are always - // visible to the deadness test (an inline body's C refcount pins - // the instance through the capi `STRONG` map; an identity box - // pins through its strong payload), so a refcount-dead object has - // no C *owner*, and with no extension frame below us no borrow - // can outlive its owner either (the one C idiom that stashes a - // raw pointer past refcount zero — Cython's `@cython.freelist` — - // is neutralised at the dealloc bridge). `cext_call_active()` - // distinguishes the two: `weavepy-capi` marks every bridged C - // invocation for its duration, including bytecode the extension - // re-enters. At depth 0 — plain bytecode, the overwhelmingly - // common site of a `del`/rebind/frame-exit reap — the full - // cascade below is exactly CPython's refcount-driven - // `tp_dealloc` chain, and pandas *depends* on that timing: a - // chained `df[cols][rows]` stamps the subset with - // `_is_copy = weakref.ref(intermediate)` and the indexer - // temporary holds the intermediate in a Cython C field, so the - // intermediate only dies (and its weakref only reads `None`, - // which is what `_check_setitem_copy` accepts) if the indexer's - // `tp_dealloc` runs promptly and releases that C field. - // - // Inside a live C call the memory must wait for the tracing - // cycle collector — but the object is still *semantically* dead, - // and CPython runs `tp_finalize` and clears weakrefs the instant - // the refcount hits zero regardless. So run the root's finalizer - // and clear its weakrefs now, leaving the GC handle (and - // therefore the allocation) in place. Inert for pure-Python - // programs (no instance ever owns a body). - if crate::vm_singletons::cext_call_active() && Self::subgraph_contains_escaped(&dropped) { - let id = crate::weakref_registry::id_of(&dropped); - if Self::object_is_finalizable(&dropped) { - let claimed = if gc_trace::is_tracked(id) { - gc_trace::mark_finalized(id) - } else { - true - }; - if claimed { - crate::vm_singletons::push_pending_finalizer(dropped.clone()); - self.run_pending_finalizers(); - if !Self::is_refcount_dead(&dropped, 1) { - return; // resurrected by its own __del__ - } - } - } - crate::weakref_registry::queue_callbacks(crate::weakref_registry::notify_clear(id)); - self.run_pending_finalizers(); - // The allocation itself is parked (GC handle / this queue clone - // keep it alive while C may still borrow into its subgraph), but - // CPython would have freed the whole chain inside the extension - // call. Requeue it for the between-bytecodes safe point — the - // first moment with no extension frame live — where the drain - // re-runs this cascade and the full teardown proceeds. Without - // this, a temporary that died *inside* a C call (pandas' - // `mgr.iget` dropping its transient wrappers) pins everything it - // anchors (the manager, its `blocks` tuple, the `Block`s and - // their weakrefs) until the next full collection — and pandas' - // `_is_view_after_cow_rules` reads a stale live Block weakref - // within the very statement that dropped the intermediate. - // - // The *unfiltered* queue entry: the park must accept every - // object kind the reaper handles. A closure function (the - // compiler's ``/`` temporary) is GC-tracked - // and holds `cell(self)`, but fell through the C-drop kind - // filter — the discarded park left it handle-pinned, leaking - // one `self` reference per C-initiated call of any method - // whose parameter is promoted to a cell (pandas' - // `BlockManager.shape` via Cython's `PyObject_GetAttr`). - crate::vm_singletons::queue_parked_drop(&dropped); - return; - } - // RFC 0039 (WS4): cascade through the dead *acyclic* subgraph, - // emulating CPython's refcount-driven `tp_dealloc` chain. Freeing - // `dropped` drops the last reference to its tracked children, - // which become dead in turn; each is finalized (in `tp_finalize` - // order — `__del__` before the object's contents are released) - // and untracked here instead of lingering in the GC's tracked set - // pinned by its own handle. Without this, a deep acyclic chain of - // finalizable/tracked objects (`test_gc.test_trashcan_threads`, - // `test_weakref`'s len-race) is only drained one layer per - // O(heap) collection — quadratic, and easily never finished for a - // quiescing interpreter. A reference *cycle* never enters the - // cascade: its members reference one another, so the refcount - // test below keeps `strong` above the dead threshold and leaves - // the cycle to the tracing collector. - let mut work = vec![dropped]; - // Each node's scratch, emptied per node and allocated once per - // cascade (a dead graph of many small containers paid four - // allocations and a hash set per node). - let mut children: Vec> = Vec::new(); - let mut weakref_candidates: Vec = Vec::new(); - let mut pool_candidates: Vec = Vec::new(); - let mut scan_through: Vec = Vec::new(); - let mut scanned: std::collections::HashSet< - crate::weakref_registry::ObjectId, - std::hash::BuildHasherDefault, - > = std::collections::HashSet::default(); - loop { - let Some(obj) = work.pop() else { - // Freeing the objects above may have parked C-side dealloc - // products in the pending-drop queue: `free_mirror` queues a - // container a C `tp_dealloc` just released (a Cython - // `BlockManager`'s `blocks` tuple), and identity boxes minted - // transiently by this very cascade (`run_external_traverse`) - // queue their payloads. CPython runs those dealloc chains - // inside the same instruction, so pull them into this cascade - // rather than leaving them for the next between-bytecodes - // safe point — pandas' `_check_setitem_copy` / - // `_is_view_after_cow_rules` read the dropped intermediate's - // `Block` weakref *within the statement* that dropped it - // (`zed["eyes"]["right"].fillna(..., inplace=True)`), one - // bytecode too early for the deferred drain. Deadness-guarded - // exactly like the safe-point drain: anything still alive - // just sheds the queue's clone. Skipped while an extension - // frame is live — the safe-point path owns that case (the - // escaped-subgraph park above). - if crate::hot_gates::env_flags::reap_trace() { - eprintln!( - "[DRAIN] cext_active={} pending={}", - crate::vm_singletons::cext_call_active(), - crate::vm_singletons::has_pending_cext_drops() - ); - } - if crate::vm_singletons::cext_call_active() - || !crate::vm_singletons::has_pending_cext_drops() - { - break; - } - for d in crate::vm_singletons::drain_pending_cext_drops() { - if crate::hot_gates::env_flags::reap_trace() { - eprintln!( - "[DRAIN-ITEM] {} sc={} dead={}", - d.type_name_owned(), - gc_trace::strong_count_for(&d), - Self::is_refcount_dead(&d, 1) - ); - } - if Self::is_refcount_dead(&d, 1) { - work.push(d); - } - } - continue; - }; - let id = crate::weakref_registry::id_of(&obj); - // RFC 0077 (WS2): a dead exact `list`/`dict`/`set` holding only - // scalar leaves is CPython's `list_dealloc` of a list of ints: - // untrack and free, nothing else. Every step of the cascade - // body below is a no-op for it — no `__del__` (only instances - // and unfinished generators finalize), no weakrefs (the - // registry filter says so), no child that could be tracked or - // anchor one (scalars never are), and no C crossing to guard - // with the cascade marker — yet the body still paid a - // cascade-set insert, a weakref clear, two finalizer drains - // and a per-element `id_of`/`handle_for` traversal. Since - // every list is GC-tracked, that overhead was the dominant - // cost of *dying* for the commonest containers in the language - // (`list(range(n))` / `[...]` temporaries measured at ~35% of - // the fannkuch fixture once `list(it)` results were tracked). - // `try_borrow` failing (a re-entrant mutation in flight) just - // takes the general path. - if Self::is_scalar_leaf_container(&obj) && crate::weakref_registry::count_for(id) == 0 { - if gc_trace::is_tracked(id) { - gc_trace::with_state(|s| s.untrack_id(id)); - } - drop(obj); - continue; - } - // Mark `obj` as mid-cascade for the rest of this iteration: the - // teardown below crosses it back into C (the GC bridge's - // `tp_traverse` mints and releases a transient self box), and - // that release must not re-queue the object into the pending - // C-drop queue — the queued clone would make the deadness - // re-checks see it as alive, abort the teardown, and restart - // the cascade from the drain: a livelock (RFC 0047, wave 5). - let _cascade_guard = crate::vm_singletons::enter_cascade(id); - let finalizable = Self::object_is_finalizable(&obj); - // Run `__del__` once, while `obj` and its children are still - // intact (CPython runs `tp_finalize` before `tp_clear`). A - // tracked object claims its finalizer via the GC handle's - // one-shot flag; an untracked finalizable object (a generator - // the GC never saw) runs directly. - if finalizable { - let claimed = if gc_trace::is_tracked(id) { - gc_trace::mark_finalized(id) - } else { - true - }; - if claimed { - crate::vm_singletons::push_pending_finalizer(obj.clone()); - self.run_pending_finalizers(); - // The finalizer ran arbitrary Python and may have - // resurrected `obj` (stashed it somewhere reachable). - // If so it is alive again: leave it tracked and stop - // cascading through it. - if !Self::is_refcount_dead(&obj, 1) { - continue; - } - } - } - // Clear weakrefs and queue their callbacks (CPython clears - // weakrefs after `tp_finalize`). - crate::weakref_registry::queue_callbacks(crate::weakref_registry::notify_clear(id)); - self.run_pending_finalizers(); - if !Self::is_refcount_dead(&obj, 1) { - continue; - } - // Snapshot the tracked children before `obj` is freed: those - // are the only ones a strong GC handle would otherwise pin - // (untracked children are reclaimed by ordinary `Rc` drop). - // - // Crucially, recurse *through* untracked containers to reach the - // tracked descendants hidden behind them. A container is left - // GC-untracked either because it was built all-scalar - // (`BuildTuple`/`BuildSet`) or — the subtler case — because it was - // created *empty* (so `container_can_cycle` saw nothing to track) - // and only later populated with objects: a dict literal `{}` grown - // by `d[k] = obj` is the canonical example, and - // `pickle._Unpickler.memo` (`self.memo = {}`, then memoized objects - // stuffed in) is exactly that. When such a container is the sole - // holder of a tracked, finalizable object, freeing the container by - // refcount drops that object to "GC handle only" — refcount-dead, - // but pinned by its own handle and so invisible to the next link of - // this cascade unless we walk into the container here. RFC 0040: - // `test_concurrent_futures.test_ressources_gced_in_workers` submits - // an object whose `__del__` signals a `Manager` event; the worker - // unpickles it (memoized in `_Unpickler.memo`), runs the job, and - // `del`s the `_CallItem`, expecting the argument's `__del__` to have - // fired — which only happens if dropping the call item (and the - // unpickler temporary holding the memo) cascades through the - // untracked `memo` dict to the tracked argument. The refcount guard - // below still filters anything that stays externally reachable. - debug_assert!(children.is_empty()); - // Untracked descendants with live weakrefs: CPython clears an - // object's weakrefs at refcount zero whether or not the GC ever - // tracked it, but this cascade's "next link" set is (otherwise) - // tracked-only, and a plain `Rc` drop cannot clear weakref - // slots — the registry's strong slot clones would keep the - // referent alive and the weakref reading it forever. The - // canonical victims are C-minted Cython instances (RFC 0047): - // `PyType_GenericAlloc` bodies are not GC-tracked on the VM - // side, and pandas' `Block`s — weakly referenced by every - // `BlockValuesRefs` — die exactly this way when a setter drops - // the old `blocks` tuple (`test_consolidate` asserts - // `has_reference()` flips to False). Collect them during the - // scan and run the dead ones through the cascade so their - // weakrefs clear at death, like any tracked link. - debug_assert!(weakref_candidates.is_empty()); - // Plain-tuple children about to die with `obj` are recycled - // through the tuple pool (see `maybe_donate_tuple`): the plain - // `Rc` drop below frees them invisibly, and this cascade is - // where the common `f(*args)`-shaped temporaries actually die - // (the argument tuple is anchored by a dead list/frame local). - debug_assert!(pool_candidates.is_empty()); - scan_through.push(obj.clone()); - scanned.clear(); - while let Some(parent) = scan_through.pop() { - gc_trace::traverse_object(&parent, &mut |c| { - // Scalars are never tracked, weakly referenced, or - // anchors of anything; a deferred-tracking instance - // holds only scalars and has no collector handle. - // Neither can lead the cascade anywhere, which matters - // for the commonest large death: a list of plain - // objects falling out of scope. - if gc_trace::is_atomic(c) { - return; - } - if let Object::Instance(i) = c { - if i.is_gc_deferred() && i.c_body.get() == 0 { - let cid = crate::weakref_registry::id_of(c); - if crate::weakref_registry::count_for(cid) > 0 && scanned.insert(cid) { - weakref_candidates.push(c.clone()); - } - return; - } - } - let cid = crate::weakref_registry::id_of(c); - if let Object::Tuple(t) = c { - if !t.is_empty() - && t.len() <= 16 - && !gc_trace::is_tracked(cid) - && crate::weakref_registry::count_for(cid) == 0 - { - pool_candidates.push(c.clone()); - } - } - if let Some(h) = gc_trace::find_handle(cid) { - children.push(h); - } else if matches!( - c, - Object::Tuple(_) - | Object::FrozenSet(_) - | Object::Dict(_) - | Object::List(_) - | Object::Set(_) - | Object::MappingProxy(_) - | Object::SimpleNamespace(_) - // An *untracked instance* (and `Cell`/`BoundMethod` - // hop) can sit between `obj` and a tracked - // finalizable just like an untracked container does. - // The canonical case: a caught exception stashed on - // an instance attribute — `unittest`'s - // `_AssertRaisesContext.exception` — where the - // `AttributeError` is itself untracked yet anchors - // `AttributeError.obj` (a finalizable io temporary). - // When the context manager is the sole holder, - // dropping it frees the exception by `Rc`, which - // drops the temporary to "GC handle only" — - // refcount-dead but pinned by its own handle, and - // invisible to the next cascade link unless we walk - // into the untracked instance here - // (`test_io.test_error_through_destructor`). The - // refcount guard below still filters anything that - // stays externally reachable. - | Object::Instance(_) - | Object::Cell(_) - | Object::BoundMethod(_) - // A dying exception owns its traceback, and the - // traceback's frames own the unwound activations' - // locals — where a cancelled asyncio call chain - // keeps its transports and SSL contexts. CPython - // frees all of it by refcount the moment the - // exception dies; walk through so the tracked - // descendants become the cascade's next links - // (test_ssl.test_handshake_timeout_handler_leak). - // Live frames (still on the stack / owned by a - // suspended generator) survive the per-child - // refcount guard below unharmed. - | Object::Traceback(_) - | Object::Frame(_) - ) && scanned.insert(cid) - { - if crate::weakref_registry::count_for(cid) > 0 { - weakref_candidates.push(c.clone()); - } - scan_through.push(c.clone()); - } - }); - } - // Drop the GC's strong handle, then our own reference: `obj` - // is reclaimed by `Rc` here and its children lose a reference. - if gc_trace::is_tracked(id) { - gc_trace::with_state(|s| s.untrack_id(id)); - } - drop(obj); - // A tuple child whose last holder was `obj` is now ours alone - // (strong count 1 = the scan's clone); park its allocation. - // Anything still referenced elsewhere just sheds the clone. - for cand in pool_candidates.drain(..) { - if matches!(&cand, Object::Tuple(t) if ThinArc::strong_count(t) == 1) { - self.maybe_donate_tuple(cand); - } - } - // Any tracked child that just lost its last program reference - // is the next link in the chain. - for h in children.drain(..) { - if !h.untracked.load(std::sync::atomic::Ordering::Acquire) { - let weak = crate::weakref_registry::strong_clone_count(h.id); - // `h.object` is the GC's own strong reference; the - // child is dead iff nothing beyond that handle and its - // weakref clones still points at it. - let sc = gc_trace::strong_count_for(&h.object); - if sc <= 1 + weak { - work.push(h.object.clone()); - } else if sc <= 4 + weak && !Self::object_is_finalizable(&h.object) { - // Still referenced — but only just. The extra refs are - // often Rust-side transients (an in-flight `PyException` - // clone in the task machinery, a native call's argument) - // that die between safe points, where no opcode-level - // reap can see the drop. Enroll for the eval loop's - // suspect re-probe so the object still dies at CPython's - // refcount timing (asyncio cancellation chains: - // test_ssl.test_handshake_timeout_handler_leak). - gc_trace::note_suspect(h.clone()); - } - } - } - // An untracked, weakly-referenced descendant whose last - // program reference just died: push it through the cascade so - // its weakrefs clear now (the refcount guard at the top of the - // loop re-checks liveness — our `weakref_candidates` clone is - // accounted for by the cascade's `extra` slack of 1). - for cand in weakref_candidates.drain(..) { - if Self::is_refcount_dead(&cand, 1) { - work.push(cand); - } - } - } - } - - /// CPython finalizes a frame's locals by refcount the instant the - /// frame is torn down: a local holding the *last* reference to a - /// finalizable object runs its `__del__` / weakref callback promptly - /// on `return`, not at the next cycle collection. WeavePy otherwise - /// only reaps on an explicit rebind (`STORE_FAST` over a live slot, - /// `del`), so a function that lets a finalizable local fall out of - /// scope by *returning* (`tempfile.NamedTemporaryFile` with - /// `delete=True`, `test_tempfile.test_del_by_finalizer`) leaked the - /// reclamation to the tracing GC. Move each interesting local out and - /// run it through the prompt-reap cascade; the refcount guard inside - /// leaves anything still reachable (including the return value, which - /// the caller still holds) untouched. - /// - /// Gated to the object kinds that can carry a finalizer (or anchor a - /// finalizable acyclic subgraph) so the common all-scalars frame pays - /// nothing on the hot return path. - fn reap_frame_locals_on_exit(&mut self, frame: &mut Frame, retval: Option<&Object>) { - // Class/module bodies keep their names in a namespace dict, not in - // fast locals, so this is a no-op for them; only real function - // activations (incl. comprehensions/lambdas) carry fast locals. - // Operands still on the stack at `RETURN_VALUE` (an iterator the - // compiler unwinds past, a `with` exit pair) are dropped with the - // frame too; grade them like the slots below. - for v in &frame.stack { - gc_trace::note_dropped(v); - } - let nlocals = frame.locals.borrow().len(); - if nlocals == 0 && frame.cells.is_empty() { - return; - } - let _ = retval; - // Fast path: no local can be at its dead line. A candidate the - // cascade below could reap has nothing but this frame's slots, its - // collector handle, and weakref clones holding it, so its strong - // count is at most `nlocals + 1` when no weakref watches it. - // Every candidate above that (the usual `self`, an argument the - // caller still holds) is plainly escaped, which the passes below - // would conclude too, one allocation and two scans later. Grading - // each slot is all that is left to do. - // (An escaped `PyFrame` sharing the locals storage keeps the - // slot-clearing passes below.) - if frame.code.cellvars.is_empty() && Rc::strong_count(&frame.locals) == 1 { - // SAFETY: RFC 0065 (WS3) — see `load_fast_value`; nothing in - // this block runs Python code or touches the locals storage - // through another path. - let locals: &Vec = unsafe { &*frame.locals.as_ptr() }; - let escaped = |v: &Object| { - let sc = gc_trace::strong_count_for(v); - if sc <= 1 { - return false; - } - let id = crate::weakref_registry::id_of(v); - // Held past every slot and a collector handle: no count. - if sc > nlocals + 1 { - return !crate::weakref_registry::may_have_weakrefs(id); - } - // How many of this frame's slots hold `v`: exact for - // small frames (the usual `self` is held once here and - // once by the caller), the slot count otherwise. - let held_here = if nlocals <= 8 { - locals - .iter() - .filter(|w| { - std::mem::discriminant(*w) == std::mem::discriminant(v) - && crate::weakref_registry::id_of(w) == id - }) - .count() - } else { - nlocals - }; - // A tracked object's collector handle is one more holder - // (the miss-filter answer: a false positive only sends - // the value down the careful path below). - let handle = usize::from(gc_trace::maybe_tracked(id)); - sc > held_here + handle && !crate::weakref_registry::may_have_weakrefs(id) - }; - if locals.iter().all(|v| { - !(Self::local_needs_prompt_reap(v) || matches!(v, Object::Function(_))) - || escaped(v) - || matches!(v, Object::Instance(i) if i.dies_by_plain_drop()) - || Self::atomic_container_dies_plainly(v) - || Self::tracked_container_dies_inertly(v) - }) { - for v in locals { - // A dying tracked container sheds its collector handle - // now (the cascade would find nothing else to do). - if Self::tracked_container_dies_inertly(v) { - gc_trace::untrack_id(crate::weakref_registry::id_of(v)); - } else { - gc_trace::note_dropped(v); - } - } - return; - } - } - // Pass 1: move every candidate local out of its slot. A *closure - // function* local is a candidate too: it is GC-tracked - // (`MakeFunction`), so merely dropping the frame's `Rc` leaves it - // pinned by its own GC handle — and still holding its captured - // cells — until the next cyclic collection. Routing a - // uniquely-held (non-escaped) function through the cascade - // untracks and frees it now, releasing its share of this frame's - // cells so the cellvar sweep below can reclaim a captured-but- - // unescaped finalizable (`test_tempfile.test_writelines_rollover`). - // - // Two slots naming the *same* object (`x = C(); for y in [x]: …`, - // or test_interpreters' `second = create(); for interp in - // list_all(): …`) each see the other's reference and would both - // fail the escaped-value pre-filter below, so the object would - // fall to the plain `Rc` drop — whose `Drop` net queues `__del__` - // *after* the weakrefs have been cleared (CPython's - // `subtype_dealloc` finalizes first). Dedupe by identity here so - // the frame contributes exactly one reference per object. - // - // The locals borrow is released before any cascade runs: the - // cascade can execute arbitrary `__del__` code, which may - // re-enter this very frame's shared locals storage (e.g. via a - // traceback's `f_locals`). - let mut candidates: Vec = Vec::new(); - { - let mut locals = frame.locals.borrow_mut(); - for i in 0..nlocals { - let needs = Self::local_needs_prompt_reap(&locals[i]) - || matches!(locals[i], Object::Function(_)); - if !needs { - // RFC 0077 (WS2): `RETURN_VALUE` is exempt from the - // eval loop's coarse stack-shrink mark; the frame's - // slots are the references it actually releases, so - // grade each one here (no-op for scalar locals). - gc_trace::note_dropped(&locals[i]); - continue; - } - let v = std::mem::replace(&mut locals[i], Object::Unbound); - if candidates.iter().any(|c| c.is_same(&v)) { - // Duplicate slot: releasing this clone is the frame - // letting go of its second reference. - drop(v); - } else { - candidates.push(v); - } - } - } - // Pass 2: cheap pre-filter, then cascade. A candidate holding the - // *last* program reference has `strong_count == our clone (1) + GC - // handle (<=1)`, plus any weakref slots' strong clones (which the - // cascade discounts, so an object reachable *only* through - // `weakref.ref(obj)` is still dead). Anything larger means the - // value escaped (it was returned, stored on an instance, captured - // by a closure, …), so leave it alone without paying for the full - // refcount/weakref test or a cascade. Keeps the hot return path of - // scalar-only frames free. - for v in candidates { - if Self::looks_reapable_temporary(&v) { - self.prompt_reap_dropped(v); - } else { - gc_trace::note_dropped(&v); - } - } - // A value captured by a nested closure does not live in a fast local - // — it lives in one of this frame's *cell variables*, and that cell is - // shared with the closure object(s) that closed over it. CPython - // decrefs the frame's cells together with those closures when the - // frame is torn down, so a captured-but-unescaped finalizable dies - // promptly. The locals sweep above has already reaped the dying - // closures (functions, and generators/coroutines via - // `local_needs_prompt_reap`), dropping their share of the cells; now - // reap any *own* cellvar (never a freevar, which belongs to an - // enclosing frame) that is left uniquely held by this frame and whose - // contents are a dead finalizable. - let ncellvars = frame.code.cellvars.len(); - if ncellvars == 0 || ncellvars > frame.cells.len() { - return; - } - for idx in 0..ncellvars { - let cell = frame.cells[idx].clone(); - // Only a cell uniquely held by this frame can have its contents - // die with the frame; one still shared with an escaped closure - // (returned/stored) keeps `strong_count > 1` (our local `cell` - // clone is the +1 we account for). - if Rc::strong_count(&cell) > 2 { - continue; - } - let reapable = { - let guard = cell.borrow(); - Self::local_needs_prompt_reap(&guard) && Self::looks_reapable_temporary(&guard) - }; - if reapable { - let v = std::mem::replace(&mut *cell.borrow_mut(), Object::Unbound); - self.prompt_reap_dropped(v); - } - } - } - - /// Reap a generator/coroutine frame the instant it *finishes* (a - /// normal return, an exhausting `next()`, or an exception escaping - /// the body). CPython clears the frame on completion, so a local or - /// value-stack entry holding the last reference to a finalizable - /// object dies promptly — and, in WeavePy's `Rc`-rooted model, an - /// object whose only other reference is a weakref slot's strong - /// clone in the registry becomes refcount-dead, which clears its - /// weakref (so `weakref.ref(obj)()` zeroes out without waiting for a - /// cyclic collection). The reapables are moved *out* of the frame - /// first so the frame no longer counts toward their refcount when - /// the cascade's deadness test runs; the frame is dropped (now - /// holding `Unbound` placeholders) by the caller. Mirrors - /// [`Self::reap_frame_locals_on_exit`] for the suspended-frame world. - fn reap_dead_frame(&mut self, frame: &mut Frame) { - let mut reap: Vec = Vec::new(); - // Same-object slots collapse to one entry (see - // `reap_frame_locals_on_exit`): the deadness test must see the - // frame as a single reference. - let take = |slot: &mut Object, reap: &mut Vec| { - if Self::local_needs_prompt_reap(slot) - || crate::weakref_registry::count_for(crate::weakref_registry::id_of(slot)) > 0 - { - let v = std::mem::replace(slot, Object::Unbound); - if !reap.iter().any(|c| c.is_same(&v)) { - reap.push(v); - } - } else { - gc_trace::note_dropped(slot); - } - }; - { - let mut locals = frame.locals.borrow_mut(); - for slot in locals.iter_mut() { - take(slot, &mut reap); - } - } - for slot in &mut frame.stack { - take(slot, &mut reap); - } - for v in reap { - self.prompt_reap_dropped(v); - } - } - - /// After a bound-method call returns, give the receiver a prompt-reap - /// chance. While the call executed, the `BoundMethod` object held a - /// strong reference to its receiver, so the callee's frame-exit reaping - /// of the `self` local saw the receiver as still-referenced and left it - /// to the cycle collector. Now that the method object is being - /// discarded, a receiver that was only a *temporary* — `Maker().make()`, - /// or `_Unpickler(buf).load()` inside `pickle.loads` — becomes - /// refcount-dead and must run its (or its referents') finalizers - /// immediately, mirroring CPython decref'ing a `method`'s `__self__` the - /// instant the call frame is torn down. RFC 0040: - /// `test_concurrent_futures.test_ressources_gced_in_workers` submits an - /// object whose `__del__` signals a `Manager` event; the worker unpickles - /// its `_CallItem`, runs it, and `del`s it expecting the argument's - /// `__del__` to have fired — but the unpickler's `memo` (pinned by the - /// unpickler temporary, in turn pinned only by the GC handle once the - /// `loads` bound-method call returns) kept the argument alive until the - /// next cyclic collection. - /// - /// Cheap on the hot path: only `BoundMethod` callables pay anything, and - /// a receiver still bound elsewhere (the overwhelmingly common `self`) - /// fails the `strong_count` pre-check without touching the GC's maps. - fn reap_call_receiver(&mut self, callable: Object) { - match &callable { - Object::BoundMethod(bm) => { - // A native method can release references the interpreter - // never sees (`lst.clear()`, `d.pop(k)`, `sys.setprofile(f)` - // dropping the previous hook — test_sys_setprofile's - // `test_reentrancy` expects the displaced hook's `__del__` - // to have run by the next statement). `CALL` no longer - // takes the coarse stack-shrink mark (RFC 0077 WS2), so - // native callees keep it explicitly; the mark is a - // thread-local flag and the scan it schedules is the - // hot-set-only pass. - if matches!(bm.function, Object::Builtin(_)) { - gc_trace::mark_maybe_dead(); - } - let recv = bm.receiver.clone(); - // End the borrow and release the BoundMethod's hold on the - // receiver so `strong_count` reflects only real bindings + the - // GC handle. - drop(callable); - // `recv` (1) + the GC's tracking handle (<=1) + any weakref - // clones is the fingerprint of a dead temporary; anything - // larger means a live binding still holds it. The - // authoritative refcount/weakref test lives in - // `prompt_reap_dropped`; this is just a cheap pre-filter. - if Self::local_needs_prompt_reap(&recv) && Self::looks_reapable_temporary(&recv) { - self.prompt_reap_dropped(recv); - } else { - gc_trace::note_dropped(&recv); - } - } - // A *function* built as a call temporary — the canonical case is a - // comprehension's anonymous ``/``/``/ - // ``, emitted by `MakeFunction` and consumed by the very - // next `Call` — is GC-tracked when it captures cells (its closure). - // A plain `Rc` drop then leaves it pinned by its own GC handle, - // still holding those cells and, through them, the enclosing locals - // it closed over (e.g. `self` in `[self.submit(x) for x in it]`, - // promoted to a cell). That defers the finalization of anything - // uniquely reachable through the capture until the next cyclic - // collection. RFC 0040: `ThreadPoolExecutor.map`'s `result_iterator` - // closes over the list comprehension's `self`, so each `map` call - // leaked one reference to the executor — `del executor` then never - // dropped its refcount to zero, so the weakref callback that wakes - // idle workers never fired and `test_concurrent_futures` - // `test_del_shutdown` (and the other prompt-finalization shutdown - // cases) hung. Route a uniquely-held (non-escaped) function through - // the same cascade the frame-exit locals sweep uses for - // closure-function *locals*, so its cells — and any finalizable they - // uniquely hold — are released now, matching CPython's refcount - // timing. A function still bound elsewhere (a global `foo` in - // `foo()`, an escaped/returned closure) keeps `strong_count` above - // the dead threshold and fails the cheap pre-filter untouched, so - // the hot path of ordinary named-function calls pays only a single - // `strong_count` read. `prompt_reap_dropped` early-returns for an - // untracked function (no cells to leak), so it is reclaimed by the - // ordinary `Rc` drop of `callable` here. - Object::Function(_) => { - if Self::looks_reapable_temporary(&callable) { - self.prompt_reap_dropped(callable); - } else { - gc_trace::note_dropped(&callable); - } - } - // A callable *instance* temporary — `C()()` dispatching - // `__call__` — is GC-tracked and would otherwise stay pinned - // by its own collector handle until the next cyclic - // collection, exactly like the bound-method receiver above. - // CPython decrefs the callee operand as the CALL completes. - Object::Instance(_) => { - if Self::local_needs_prompt_reap(&callable) - && Self::looks_reapable_temporary(&callable) - { - self.prompt_reap_dropped(callable); - } else { - gc_trace::note_dropped(&callable); - } - } - // Native function: same reasoning as the builtin-method arm - // above (`sys.setprofile(f)` is a module-level `Builtin`). - Object::Builtin(_) => gc_trace::mark_maybe_dead(), - // RFC 0077 (WS2): `CALL` is exempt from the eval loop's coarse - // stack-shrink mark (see `note_dropped`); every operand it - // discards is graded here or in `reap_call_args` instead. - _ => gc_trace::note_dropped(&callable), - } - } - - /// After a call returns, give each *argument temporary* a prompt-reap - /// chance. While the call executed, the caller's argument vector held a - /// strong reference to every argument, so the callee's frame-exit reaping - /// of the corresponding parameter local saw the value as still-referenced - /// and left it to the cycle collector. Now that the caller is dropping its - /// argument vector, an argument that was only a *temporary* — `f(Make())`, - /// `runner.run(test_class('m'))` — becomes refcount-dead and must run its - /// (or its referents') finalizers immediately, mirroring CPython clearing - /// a call's stack slots the instant the call returns. RFC 0040: - /// `test_concurrent_futures.test_init` runs a child test inline as - /// `runner.run(test_class(...))`; the test case is never bound to a name, - /// so without reaping it here its `ProcessPoolExecutor` (and the log - /// `Queue`'s SemLocks) lingered to the next cyclic collection and the - /// resource tracker reported leaked semaphores. Moves each dead-looking - /// argument *out* of the vector before reaping so the vector slot no - /// longer inflates its refcount. - fn reap_call_args(&mut self, args: &mut [Object]) { - for a in args { - // `Function` mirrors `reap_call_receiver`'s comprehension-function - // arm, for a function consumed as an *argument*: the canonical - // case is `__build_class__(, name, *bases)` — the - // body function is GC-tracked when it captures cells (any `class` - // in a closure-bearing scope) and CPython frees it the instant the - // call returns, while a plain drop left it pinned by its own - // collector handle. test_descr `test_slots` observes exactly that: - // `len(gc.get_objects())` taken before/after a compare loop - // differed by the stale `G` body function dying in between. - // `prompt_reap_dropped` early-returns for untracked functions. - if (Self::local_needs_prompt_reap(a) || matches!(a, Object::Function(_))) - && Self::looks_reapable_temporary(a) - { - let v = std::mem::replace(a, Object::Unbound); - self.prompt_reap_dropped(v); - } else { - gc_trace::note_dropped(a); - } - } - } - - /// Prompt-reap the value an assignment just displaced. CPython decrefs - /// the previous attribute/subscript value the instant a store - /// overwrites it (`self.x = y` decrefs the old `self.x`); a value whose - /// last program-visible binding was that slot dies immediately, running - /// its `__del__`/weakref callbacks rather than waiting for the next - /// cyclic collection. The crucial case is a slot holding the only - /// strong reference to an object that *itself* anchors a weakref false - /// cycle: `ExecutorMixin.tearDown` does `self.executor = None` after - /// `shutdown()`, and the `ProcessPoolExecutor`↔`_ExecutorManagerThread` - /// pair is only kept reachable by the thread's `weakref.ref(executor)` - /// (a registry strong-clone in WeavePy's `Rc` model). Reaping the - /// displaced executor clears that weakref, breaks the cycle, and - /// cascades through the manager thread into the call/result queues so - /// their SemLocks unlink before the test stops the resource tracker - /// (RFC 0040: `test_concurrent_futures.test_init` - /// `FailingInitializerResourcesTest`). The cheap `local_needs_prompt_reap` - /// gate keeps scalar stores (`self.n = i` in a loop) off this path. - fn maybe_prompt_reap_replaced(&mut self, old: Object) { - static NO_STORE_REAP: std::sync::OnceLock = std::sync::OnceLock::new(); - let reap = !*NO_STORE_REAP.get_or_init(|| std::env::var_os("WP_NO_STORE_REAP").is_some()); - if reap && Self::local_needs_prompt_reap(&old) && Self::looks_reapable_temporary(&old) { - self.prompt_reap_dropped(old); - } else { - // RFC 0077 (WS2): the specialized `STORE_ATTR` hits are exempt - // from the coarse stack-shrink mark; the displaced value is - // the reference they release. - gc_trace::note_dropped(&old); - } - } - - /// Cheap "looks like a dead temporary" pre-filter shared by the - /// prompt-reap call sites. An object holding the single program reference - /// about to be dropped has `strong_count == that reference (1) + the GC's - /// tracking handle (<=1)`. A live weakref additionally keeps one strong - /// *clone* per slot in the registry (see [`crate::weakref_registry`]); - /// the cascade discounts those (an object reachable only through - /// `weakref.ref(obj)` is dead in CPython too), so they must not veto the - /// reap here. Anything beyond `2 + weak_clones` is a live binding. The - /// registry lookup is skipped on the common `strong_count <= 2` fast path. /// Park a dead tuple's allocation in [`Self::tuple_pool`] (CPython's - /// tuple freelist analogue). Only called from the prompt reaper's - /// fast path, which has already established the tuple carries no - /// finalizer, no weakref slot, no GC handle, and anchors no tracked - /// child; the `strong_count == 1` check here (we own the only - /// handle) plus `Rc::get_mut` (which additionally refuses if weak + /// tuple freelist analogue). The `strong_count == 1` check (we own the + /// only handle) plus `Rc::get_mut` (which additionally refuses if weak /// references exist) make the park equivalent to a free. Elements /// are cleared in place so their refcounts fall now, exactly as a /// plain drop would. @@ -3799,493 +2523,6 @@ impl Interpreter { Object::new_tuple_array(items) } - /// Is an exhausted iterator's backing list refcount-dead — held only by - /// the iterator itself (plus the cycle collector's tracking handle, - /// which `BuildList` installs on every list, and any weakref slots' - /// strong clones)? True exactly when the loop iterated a temporary — - /// `for x in f():` — so the elements die with the iterator, and CPython - /// would free them by refcount right now. - fn iter_backing_list_dead(items: &Rc>>) -> bool { - let probe = Object::List(items.clone()); - let id = crate::weakref_registry::id_of(&probe); - // 2 = the iterator's own handle + our `probe` clone. - let spare = 2 - + usize::from(gc_trace::is_tracked(id)) - + crate::weakref_registry::strong_clone_count(id); - Rc::strong_count(items) <= spare - } - - /// Prompt-reap the elements of an exhausted `FOR_ITER` iterator's - /// backing list, harvested at exhaustion because the iterator held the - /// last reference (the loop iterated a call-expression temporary). - /// CPython frees these by refcount the moment the iterator dies; our - /// detach-and-`Rc`-drop path skipped `__del__`s. Each element is - /// independently refcount-guarded by the cascade, so one still bound - /// (e.g. the final loop variable) is left alone. - fn reap_iter_backing(&mut self, items: Vec) { - for v in items { - if !(Self::local_needs_prompt_reap(&v) && Self::looks_reapable_temporary(&v)) { - continue; - } - if Self::exc_has_finalizable(&v, 4) { - // A finalizable behind an untracked container (a tuple of - // instances) would be skipped by `prompt_reap_dropped`'s - // fast path; drive the cascade directly. - self.reap_dead_subgraph(v); - } else { - self.prompt_reap_dropped(v); - } - } - } - - /// Whether the caller's reference to `o` may be its last program - /// reference: nothing holds it beyond the caller, its collector - /// handle (when tracked), and weakref-registry clones. - #[inline] - fn looks_reapable_temporary(o: &Object) -> bool { - let sc = gc_trace::strong_count_for(o); - if sc <= 1 { - return true; - } - let id = crate::weakref_registry::id_of(o); - let handle = usize::from(gc_trace::maybe_tracked(id)); - if sc <= 1 + handle { - return true; - } - crate::weakref_registry::may_have_weakrefs(id) - && sc <= 1 + handle + crate::weakref_registry::strong_clone_count(id) - } - - /// Whether a frame local is worth running through the prompt-reap - /// cascade on frame exit. Mirrors the `PopTop` discard gate: instances - /// (possible `__del__` / weakref finalizer), and live - /// generators/coroutines/async-generators (their `close()` delivers - /// `GeneratorExit`). Kept deliberately narrow so the hot return path of - /// scalar-only frames pays a single cheap `matches!` per local. - /// A small untracked list, tuple or dict of atomic values held only - /// here: its release frees nothing a reap would visit (no finalizer, - /// weakref or collector handle can hang off it or its values). - fn atomic_container_dies_plainly(o: &Object) -> bool { - const MAX: usize = 16; - fn atomic(o: &Object) -> bool { - matches!( - o, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None | Object::Str(_) - ) - } - let (sc, id) = match o { - Object::List(l) => (Rc::strong_count(l), Rc::as_ptr(l) as usize as u64), - Object::Dict(d) => (Rc::strong_count(d), Rc::as_ptr(d) as usize as u64), - Object::Tuple(t) => ( - ThinArc::strong_count(t), - ThinArc::as_ptr(t).cast::<()>() as usize as u64, - ), - _ => return false, - }; - if sc != 1 || gc_trace::maybe_tracked(id) || crate::weakref_registry::may_have_weakrefs(id) - { - return false; - } - match o { - Object::List(l) => l - .try_borrow() - .is_ok_and(|v| v.len() <= MAX && v.iter().all(atomic)), - Object::Dict(d) => d - .try_borrow() - .is_ok_and(|d| d.len() <= MAX && d.iter().all(|(k, v)| atomic(&k.0) && atomic(v))), - Object::Tuple(t) => t.len() <= MAX && t.iter().all(atomic), - _ => false, - } - } - - /// A collector-tracked list or dict whose only holders are the caller's - /// reference and its collector handle, watched by no weakref, holding - /// only values its death can't finalize (atomic ones, or instances - /// others still hold): untracking it and dropping the reference is its - /// whole teardown, as in CPython's `list_dealloc`. - fn tracked_container_dies_inertly(o: &Object) -> bool { - const MAX: usize = 16; - fn inert(v: &Object) -> bool { - matches!( - v, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None | Object::Str(_) - ) || gc_trace::drop_survives_plainly(v) - } - let (sc, id) = match o { - Object::List(l) => (Rc::strong_count(l), Rc::as_ptr(l) as usize as u64), - Object::Dict(d) => (Rc::strong_count(d), Rc::as_ptr(d) as usize as u64), - _ => return false, - }; - if sc != 2 || crate::weakref_registry::may_have_weakrefs(id) || !gc_trace::is_tracked(id) { - return false; - } - match o { - Object::List(l) => l - .try_borrow() - .is_ok_and(|v| v.len() <= MAX && v.iter().all(inert)), - Object::Dict(d) => d - .try_borrow() - .is_ok_and(|d| d.len() <= MAX && d.iter().all(|(k, v)| inert(&k.0) && inert(v))), - _ => false, - } - } - - fn local_needs_prompt_reap(o: &Object) -> bool { - matches!( - o, - Object::Instance(_) - | Object::Generator(_) - | Object::Coroutine(_) - | Object::AsyncGenerator(_) - | Object::AsyncGenAwait(_) - // A container local can anchor a finalizable acyclic subgraph - // that dies with the frame: pickle's `_dict_op` builds - // `items = [key, value]` via `_pop_to_mark`, stuffs it into a - // dict, and lets `items` fall out of scope at return — leaving - // the only non-GC reference to a freshly-unpickled, finalizable - // `value` on a dead list local. CPython frees it by refcount on - // return; reap it here so its `__del__` runs promptly instead of - // lingering to the next cyclic collection (RFC 0040: - // `test_concurrent_futures.test_ressources_gced_in_workers`). - // The `strong_count` pre-filter at the call site keeps escaped - // containers off the cascade. - | Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::Tuple(_) - | Object::FrozenSet(_) - // A bound method anchors its receiver (`__self__`); a dead - // one must release it promptly — and a *stored* one - // (GC-tracked at attribute-store time) must shed its own - // collector handle. See the `BoundMethod` arm of - // `prompt_reap_dropped`. - | Object::BoundMethod(_) - // A function is GC-tracked: a rebound lambda or closure must - // shed its collector handle (and whatever it alone holds) - // now, as CPython frees it by refcount. - | Object::Function(_) - // An escaped frame owns locals after its activation exits. - // Releasing the last frame alias must retire those locals, - // including tracked containers and weakly held instances. - | Object::Frame(_) - ) - } - - /// Does `obj` carry a finalizer that prompt reclamation must run - /// before freeing it? Instances with `__del__`, and unfinished - /// generators/coroutines/async-generators (their `close()` delivers - /// `GeneratorExit`). - fn object_is_finalizable(obj: &Object) -> bool { - match obj { - // Cached per class (`TypeObject::has_del`), invalidated on - // `__del__` assignment / MRO changes — the uncached - // `lookup("__del__")` walk clones the MRO on every reaped - // instance and dominated the prompt-reap path. - Object::Instance(i) => i.cls().instances_need_finalize(), - // An unstarted generator has nothing to finalize: closing it - // runs no Python (see `gc_trace::has_finalizer`). A coroutine - // still does — the never-awaited RuntimeWarning. - Object::Generator(g) => !g.is_finished() && !g.is_unstarted(), - Object::Coroutine(g) | Object::AsyncGenerator(g) => !g.is_finished(), - _ => false, - } - } - - /// An exact builtin `list`/`dict`/`set` whose every element (and key) - /// is a scalar leaf ([`gc_trace::is_atomic`]): a container that can - /// neither finalize nor anchor anything the prompt reaper would have - /// to cascade into. Subclass instances are `Object::Instance` and never - /// match. A container that is currently borrowed reports `false` so - /// the caller takes the general path. - fn is_scalar_leaf_container(obj: &Object) -> bool { - // Capped like `anchors_tracked_child`'s node budget: the answer is - // an optimisation, never a correctness input, and an unbounded - // scan would make dropping a large container O(len) on a path that - // runs for every displaced binding. - const SCAN_CAP: usize = 64; - match obj { - Object::List(l) => l - .try_borrow() - .map(|v| v.len() <= SCAN_CAP && v.iter().all(gc_trace::is_atomic)) - .unwrap_or(false), - Object::Set(s) => s - .try_borrow() - .map(|m| m.len() <= SCAN_CAP && m.iter().all(|k| gc_trace::is_atomic(&k.0))) - .unwrap_or(false), - Object::Dict(d) => d - .try_borrow() - .map(|m| { - m.len() <= SCAN_CAP - && m.iter() - .all(|(k, v)| gc_trace::is_atomic(&k.0) && gc_trace::is_atomic(v)) - }) - .unwrap_or(false), - _ => false, - } - } - - /// The common handled exception, recognized without the reap walks: a - /// class with no finalizer, no instance dict, and slots holding only - /// atomic values, a tuple of them (`args`), or a traceback whose frames - /// are all still executing. Neither `exc_has_finalizable` nor - /// `anchors_tracked_child` can find anything in it. - fn exc_plainly_inert(obj: &Object) -> bool { - let Object::Instance(inst) = obj else { - return false; - }; - if inst.cls().instances_need_finalize() - || inst - .dict - .get() - .is_some_and(|d| d.try_borrow().map_or(true, |d| !d.is_empty())) - { - return false; - } - let Ok(slots) = inst.slots.try_borrow() else { - return false; - }; - // (Bound, not returned directly: the iterator borrows `slots`.) - #[allow(clippy::let_and_return)] - let inert = slots.iter().all(|(_, v)| match v { - Object::Tuple(t) => t.iter().all(Object::is_gc_atomic), - Object::Traceback(tb) => { - let mut cur = Some(tb.clone()); - while let Some(node) = cur { - if node.frame.on_stack.get() == 0 { - return false; - } - cur = node.next.borrow().clone(); - } - true - } - other => other.is_gc_atomic(), - }); - inert - } - - /// Cheap, allocation-free pre-check for the `POP_EXCEPT` reap: does the - /// acyclic subgraph rooted at `obj` contain a finalizable object - /// (`__del__` / an unfinished generator) reachable through value - /// containers and user instances, up to `depth` links? The - /// overwhelmingly common handled exception — `ValueError(i)`, - /// `KeyError(k)`, scalar/`str` args, no `__del__` anywhere — returns - /// `false` here in pure recursion (no heap traffic, no weakref-registry - /// work), so the hot handler-exit path stays at baseline cost and only - /// an exception that actually anchors a destructor pays for the full - /// [`Self::reap_dead_subgraph`] cascade. Tracebacks/frames are - /// deliberately *not* traversed: walking the captured call stack on - /// every caught exception would dominate the path, and a finalizable - /// pinned only by a traceback frame is left to the cycle collector - /// (CPython's own traceback-clearing is likewise lazy here). - fn exc_has_finalizable(obj: &Object, depth: u8) -> bool { - if Self::object_is_finalizable(obj) { - return true; - } - if depth == 0 { - return false; - } - if matches!( - obj, - Object::Tuple(_) - | Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::FrozenSet(_) - | Object::Instance(_) - | Object::Cell(_) - | Object::MappingProxy(_) - | Object::SimpleNamespace(_) - ) { - let mut found = false; - crate::gc_trace::traverse_object(obj, &mut |c| { - if !found && Self::exc_has_finalizable(c, depth - 1) { - found = true; - } - }); - return found; - } - false - } - - /// Does the container subgraph rooted at `root` contain a *GC-tracked* - /// child within `depth` links? Used by [`Self::prompt_reap_dropped`] to - /// decide whether an untracked, unwatched, non-finalizable container is - /// still worth the cascade: a plain `Rc` drop frees the container but - /// leaves a tracked child it solely anchored pinned by its own - /// collector handle (weakrefs unclear, `__del__` deferred to the next - /// cyclic collection). Bounded — both in depth and in total nodes - /// visited — so a huge scalar container (a million-int list dying at - /// frame exit) costs a few dozen `is_tracked` lookups at most; hitting - /// the cap conservatively reports `false` (skip the prompt reap; the - /// tracing collector reclaims later). - fn anchors_tracked_child(root: &Object, depth: u8) -> bool { - const NODE_CAP: usize = 64; - fn walk(obj: &Object, depth: u8, budget: &mut usize) -> bool { - if depth == 0 || *budget == 0 { - return false; - } - let mut found = false; - crate::gc_trace::traverse_object(obj, &mut |c| { - // Scalars are never tracked and hold nothing; a frame still - // on the stack keeps its locals whatever dies here (a - // traceback leads to the very frame that is handling it). - if found - || *budget == 0 - || c.is_gc_atomic() - || matches!(c, Object::Frame(f) if f.on_stack.get() > 0) - || matches!(c, Object::Instance(i) if i.is_gc_deferred() && i.c_body.get() == 0) - { - return; - } - *budget -= 1; - if crate::gc_trace::is_tracked(crate::weakref_registry::id_of(c)) { - found = true; - return; - } - if matches!( - c, - Object::Tuple(_) - | Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::FrozenSet(_) - | Object::Instance(_) - | Object::Cell(_) - // Follow `__traceback__` → frame → locals: a dying - // exception is routinely the last holder of tracked - // objects that live only in the frames it unwound. - | Object::Traceback(_) - | Object::Frame(_) - ) && walk(c, depth - 1, budget) - { - found = true; - } - }); - found - } - let mut budget = NODE_CAP; - walk(root, depth, &mut budget) - } - - /// Is `obj` dead in refcount terms — does the caller hold its last - /// program-visible reference? `local_refs` is the number of strong - /// references the caller itself holds (the binding being dropped, or - /// a work-list entry). The GC's own handle and weakref strong clones - /// are discounted: neither is a program-visible binding. - fn is_refcount_dead(obj: &Object, local_refs: usize) -> bool { - let id = crate::weakref_registry::id_of(obj); - let registry_holds = usize::from(gc_trace::is_tracked(id)); - let weak_clones = crate::weakref_registry::strong_clone_count(id); - gc_trace::strong_count_for(obj) <= local_refs + registry_holds + weak_clones - } - - /// Has `obj` escaped into a C extension — does it own a faithful - /// inline instance body (RFC 0045, `c_body != 0`)? Such an instance's - /// pointer may be held *borrowed* (uncounted) by a C extension across - /// a re-entrant call into the VM, so the prompt reaper — whose - /// deadness test sees only VM `Rc` references — must not reclaim it - /// (that would free the body and let the allocator reuse it under the - /// extension's live pointer). The tracing cycle collector reclaims it - /// later, once C's transient borrows are gone. A single `Cell` read; - /// `false` for every pure-Python instance and non-instance object. - #[inline] - fn instance_escaped_to_c(obj: &Object) -> bool { - matches!(obj, Object::Instance(inst) if inst.c_body.get() != 0) - } - - /// Does the acyclic subgraph rooted at `root` contain any instance - /// that has escaped into a C extension (see [`instance_escaped_to_c`])? - /// Used by the prompt reaper to leave such subgraphs to the tracing - /// cycle collector (a C extension may hold a borrowed pointer into one - /// of the bodies). A bounded breadth-first walk over the same edges - /// `gc_trace::traverse_object` exposes, deduplicated by object id and - /// capped so a pathological structure can never make frame exit - /// quadratic; hitting the cap conservatively reports `true` (skip the - /// reap — always safe). Returns immediately for the overwhelmingly - /// common all-scalar / pure-Python subgraph. - fn subgraph_contains_escaped(root: &Object) -> bool { - // Fast path: a root that can hold no references (scalar, string, - // …) trivially has no escaped instance in its subgraph. This - // covers the overwhelming majority of reaped temporaries at zero - // allocation cost. - if Self::instance_escaped_to_c(root) { - return true; - } - if !Self::object_can_hold_refs(root) { - return false; - } - // Bounded DFS reusing a thread-local scratch stack + visited set so - // the hot reap path allocates nothing. Cap chosen well above - // pandas' shallow `_cache`/`__dict__` fan-out; on overflow we - // proceed with the reap (return `false`) — the only structures big - // enough to hit it are large escaped-free graphs, and a pandas - // escaped body is always shallow. - const VISIT_CAP: usize = 2048; - thread_local! { - static SCRATCH: std::cell::RefCell<( - Vec, - std::collections::HashSet, - )> = std::cell::RefCell::new((Vec::new(), std::collections::HashSet::new())); - } - SCRATCH.with(|cell| { - let Ok(mut guard) = cell.try_borrow_mut() else { - // Re-entrant (a finalizer walked here): fall back to a - // conservative "skip the reap" rather than risk aliasing - // the scratch buffers. - return true; - }; - let (stack, seen) = &mut *guard; - stack.clear(); - seen.clear(); - gc_trace::traverse_object(root, &mut |child| stack.push(child.clone())); - let mut visited = 0usize; - let mut found = false; - while let Some(obj) = stack.pop() { - if !seen.insert(crate::weakref_registry::id_of(&obj)) { - continue; - } - if Self::instance_escaped_to_c(&obj) { - found = true; - break; - } - visited += 1; - if visited > VISIT_CAP { - break; - } - if Self::object_can_hold_refs(&obj) { - gc_trace::traverse_object(&obj, &mut |child| stack.push(child.clone())); - } - } - stack.clear(); - seen.clear(); - found - }) - } - - /// Cheap discriminator: can `obj` hold references to other objects - /// (and therefore possibly reach an escaped instance)? `false` for - /// scalars/strings/bytes and other leaves, letting - /// [`subgraph_contains_escaped`] skip the walk entirely. - #[inline] - fn object_can_hold_refs(obj: &Object) -> bool { - matches!( - obj, - Object::List(_) - | Object::Tuple(_) - | Object::Dict(_) - | Object::Set(_) - | Object::FrozenSet(_) - | Object::MappingProxy(_) - | Object::SimpleNamespace(_) - | Object::Instance(_) - | Object::Cell(_) - | Object::BoundMethod(_) - | Object::Function(_) - | Object::Type(_) - | Object::Frame(_) - ) - } - /// Emulate CPython deleting a terminating thread's thread-state /// dict: drop `ident`'s slot from every `_threading_local.local`. /// Run from the native worker teardown (`spawn_python_worker`) the @@ -4338,63 +2575,17 @@ impl Interpreter { self.run_pending_finalizers(); } - /// Invoke any `__del__` finalizers queued by the cycle GC. - /// Each finalizer runs at most once. Exceptions from a - /// finalizer are routed through `sys.unraisablehook` (the - /// default hook prints `Exception ignored in: …` to stderr, - /// exactly like CPython) so they don't propagate. - /// Prompt (deterministic) finalization driver — the interpreter half of - /// RFC 0040's GC arc. CPython runs an object's `__del__` the instant its - /// refcount reaches zero; weavepy's tracing collector pins every tracked - /// object on a strong handle, so a refcount-dead finalizable object would - /// otherwise linger until the next cyclic collection. This closes that gap - /// by, at each reference-drop safe point, running `__del__` for objects - /// whose last *program* reference just dropped (`strong_count <= 1`). - /// - /// Drives a small fixpoint loop: queue the dead finalizables' `__del__`, - /// run them, reclaim the (non-resurrected) corpses — which cascades the - /// refcount drop into their referents and can expose the next layer of - /// now-dead finalizables — and repeat until quiescent. Bounded by - /// `MAX_COLLECT_PASSES` against a pathological `__del__` that keeps - /// resurrecting and re-killing objects. - /// - /// Gated by [`gc_trace::has_any_finalizable`] at every call site, so when - /// no `__del__`-bearing object is live (the common case) it costs one - /// relaxed atomic load. Re-entrancy is suppressed via a thread-local: a - /// `__del__` body re-enters the eval loop, whose own safe points would - /// otherwise recurse here. + /// Run the work queued for this safe point: finalizers a dying + /// object's `Drop` resurrected, and weakref callbacks. pub fn drain_prompt_finalizers(&mut self) { - thread_local! { - static IN_PROMPT_FINALIZE: std::cell::Cell = const { std::cell::Cell::new(false) }; - } - if IN_PROMPT_FINALIZE.with(std::cell::Cell::get) { - return; - } - IN_PROMPT_FINALIZE.with(|c| c.set(true)); - for _ in 0..gc_trace::MAX_COLLECT_PASSES { - let progressed = gc_trace::reap_dead_finalizable(); - if progressed == 0 { - break; - } - // Run the `__del__`s and weakref callbacks just queued. They may - // drop the last reference to — or resurrect — other finalizables, - // which the next pass re-evaluates. - self.run_pending_finalizers(); - // Cascade through *non*-finalizable tracked objects too: freeing - // a `__del__`-bearing object often exposes a chain of plain - // containers (transport → protocol → context dicts) that CPython's - // refcounting would free in the same instant, and whose survival - // would keep further finalizables/weakref targets alive past this - // drain (asyncio's SSL leak tests watch such a chain). Only runs - // on the rare "a finalizable just died" path, so the O(tracked) - // scan stays off the hot loop; amortized further when the - // tracked population itself is huge (see - // `reap_dead_acyclic_amortized`). - gc_trace::reap_dead_acyclic_amortized(); - } - IN_PROMPT_FINALIZE.with(|c| c.set(false)); + self.run_pending_finalizers(); } + /// Invoke any `__del__` finalizers and weakref callbacks queued by a + /// dying object's `Drop` or by the cycle collector. Each finalizer runs + /// at most once. Exceptions from a finalizer are routed through + /// `sys.unraisablehook` (the default hook prints `Exception ignored + /// in: …` to stderr, exactly like CPython) so they don't propagate. pub fn run_pending_finalizers(&mut self) -> usize { // RFC 0077 (WS2): the prompt reaper drains after every node it // tears down, and for the overwhelming majority nothing was @@ -4753,19 +2944,20 @@ impl Interpreter { Some(Object::Module(m)) => m.dict.borrow().iter().map(|(_k, v)| v.clone()).collect(), _ => Vec::new(), }; - let mut sys_deferred: Vec> = Vec::new(); + let mut sys_deferred: Vec<(crate::sync::Rc, Object)> = + Vec::new(); for _ in 0..8 { let candidates = crate::gc_trace::finalization_candidates(); if candidates.is_empty() { break; } - for handle in candidates { - if sys_values.iter().any(|v| v.is_same(&handle.object)) { + for (handle, obj) in candidates { + if sys_values.iter().any(|v| v.is_same(&obj)) { if !sys_deferred .iter() - .any(|h| crate::sync::Rc::ptr_eq(h, &handle)) + .any(|(h, _)| crate::sync::Rc::ptr_eq(h, &handle)) { - sys_deferred.push(handle); + sys_deferred.push((handle, obj)); } continue; } @@ -4777,21 +2969,21 @@ impl Interpreter { { continue; } - self.invoke_finalizer(&handle.object); + self.invoke_finalizer(&obj); } // A finalizer may have queued cyclic finalizers of its own. self.run_pending_finalizers(); } if !sys_deferred.is_empty() { crate::vm_singletons::set_unraisable_silenced(true); - for handle in sys_deferred { + for (handle, obj) in sys_deferred { if handle .finalized .swap(true, std::sync::atomic::Ordering::AcqRel) { continue; } - self.invoke_finalizer(&handle.object); + self.invoke_finalizer(&obj); } crate::vm_singletons::set_unraisable_silenced(false); } @@ -4826,9 +3018,6 @@ impl Interpreter { if std::env::var_os("WEAVEPY_VM_STATS").is_some() { let snap = crate::specialize::snapshot(); eprint!("{}", crate::specialize::format_stats_markdown(&snap)); - if let Some(md) = gc_trace::floor_stats::markdown() { - eprint!("{md}"); - } if let Some(md) = jit_stats_markdown() { eprint!("{md}"); } @@ -4864,6 +3053,23 @@ impl Interpreter { /// prompt, collector, and shutdown requests may refer to the same object. /// Only the winning request may publish finalizer completion. fn invoke_finalizer(&mut self, obj: &Object) -> bool { + // A discarded never-driven `asend`/`athrow`/`aclose` awaitable (see + // `release`). + if let Object::AsyncGenAwait(a) = obj { + let method = match a.kind { + crate::object::AgenAwaitKind::Send => "asend", + crate::object::AgenAwaitKind::Throw => "athrow", + crate::object::AgenAwaitKind::Close => "aclose", + }; + let qualname = match &a.agen { + Object::AsyncGenerator(g) => g.qualname.borrow().to_str(), + other => other.type_name_owned(), + }; + let _ = self.emit_runtime_warning(format!( + "coroutine method '{method}' of '{qualname}' was never awaited" + )); + return true; + } let finalize_ran = match obj { Object::Instance(inst) => &inst.finalize_ran, Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { @@ -5731,7 +3937,6 @@ impl Interpreter { fn run_frame(&mut self, frame: &mut Frame) -> Result { match self.run_until_yield_or_return(frame, None)? { FrameOutcome::Returned(v) => { - self.reap_frame_locals_on_exit(frame, Some(&v)); self.recycle_frame_allocs(frame); Ok(v) } @@ -6176,12 +4381,6 @@ impl Interpreter { let quiet_off = self.quiet_off; let mut loop_snap_gen: u64 = 0; // stale → first iteration derives let mut loop_quiet = false; - let mut loop_fin_quiet = false; - // Within fin-quiet mode: is a finalizable object live (the coarse - // stack-shrink drop heuristic applies), or only active suspects - // (which the full prologue probes at marked safe points without - // it)? - let mut loop_coarse = false; // The loops are off only because a frame object is observed (see // `frame_object_observed`): re-derive once nothing outside the // activation holds it any more (an `except` clause dropped the @@ -6197,21 +4396,6 @@ impl Interpreter { self.gil_countdown = self.gil_countdown.wrapping_sub(1); if self.gil_countdown == 0 { self.gil_countdown = crate::gil::GIL_CHECK_INTERVAL; - // RFC 0065 (WS1): the quiet paths skip the per- - // instruction suspect probes, so aged-out (dormant) - // suspects are instead re-probed on this ~128- - // instruction cadence. `has_suspects` keeps its - // dormant-stride admission, so a dormant-only - // population pays two relaxed loads here and a real - // probe only every DORMANT_STRIDE-th checkpoint. - if (loop_quiet || loop_fin_quiet) - && !gc_trace::active_suspects_present() - && gc_trace::has_suspects() - { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } crate::gil::yield_checkpoint(); } // RFC 0065 (WS1): re-derive the prologue snapshot only when @@ -6236,27 +4420,7 @@ impl Interpreter { let frame_observed = Self::frame_object_observed(&shell, py_frame_slot.as_ref(), frame); frame_blocks_quiet = unobserved && frame_observed; - let base_quiet = unobserved && !frame_observed; - // Graded: fully quiet when no finalizable object exists - // anywhere; "fin-quiet" — everything cold except the - // CPython-timing-bearing prompt-finalization probe — - // when the only live input is the finalizable - // population (in practice ≥ 1 in any real program: the - // ABC machinery keeps a weakref-callback-enrolled type - // alive for the process lifetime). - let fin_live = gc_trace::has_any_finalizable(); - // An *active* suspect (one still inside its probe budget) - // is the same kind of input: it is re-probed at every - // marked drop safe point, which is exactly the fin-quiet - // path's `take_maybe_dead` gate. Pinning the loop to the - // full prologue for it instead made every instruction of - // a drop-free hot loop pay that prologue for as long as - // the suspect's budget lasted, since nothing there probes - // it. - let suspects_live = gc_trace::active_suspects_present(); - loop_quiet = base_quiet && !fin_live && !suspects_live; - loop_fin_quiet = base_quiet && (fin_live || suspects_live); - loop_coarse = fin_live; + loop_quiet = unobserved && !frame_observed; } // The leaf burst: while the loop is quiet, run the straight // run of leaf instructions at `frame.pc` in one tight loop @@ -6268,43 +4432,24 @@ impl Interpreter { && !crate::gil::free_threading_enabled(); let cur_pc: usize; let mut instruction_ran = false; - let watch_drops: bool; - let stack_before: usize; - let op_before: Option; let stepped = if let Some((stepped, at)) = pending.take() { // An outcome produced before this loop took over (see the // method docs). Its producer ran the per-instruction - // protocol, drop heuristic included. - watch_drops = false; - stack_before = 0; - op_before = None; + // protocol. cur_pc = at; instruction_ran = true; stepped - } else if loop_quiet || loop_fin_quiet { - // The quiet paths: nothing is pending, nobody is - // watching. Semantically identical to the full prologue - // with the corresponding gates cold — each gate's - // producer bumps the loop generation before its gate - // can read hot, so the full prologue resumes on the - // very next instruction. - // The one live subsystem in fin-quiet mode: prompt - // finalization. Same probe order and cadence as the full - // prologue's finalizable arm (`take_maybe_dead` gate, then - // the finalizer drain, then the stride-gated suspect - // re-probe), and the same drop-watch bookkeeping in the - // shared tail below. - let drained = loop_fin_quiet && self.drain_if_maybe_dead(); - // Nothing pending: hand the frame to the quiet loop proper. - // (After a drain the finalizers may have changed any loop - // input, so that iteration single-steps.) - let quiet_exit = if burst_ok && !drained { + } else if loop_quiet { + // The quiet path: nothing is pending, nobody is watching. + // Semantically identical to the full prologue with the + // corresponding gates cold — each gate's producer bumps the + // loop generation before its gate can read hot, so the full + // prologue resumes on the very next instruction. + let quiet_exit = if burst_ok { Some(self.quiet_run( frame, QuietShell::Ready(&shell), loop_snap_gen, - loop_fin_quiet, - loop_coarse, &mut prev_pc, )) } else { @@ -6335,25 +4480,11 @@ impl Interpreter { stepped, cur_pc: at, }) => { - // `quiet_run` already applied the drop heuristic. - watch_drops = false; - stack_before = 0; - op_before = None; cur_pc = at; instruction_ran = true; stepped } None => { - if loop_fin_quiet && loop_coarse { - watch_drops = true; - stack_before = frame.stack.len(); - op_before = - frame.code.instructions.get(frame.pc as usize).map(|i| i.op); - } else { - watch_drops = false; - stack_before = 0; - op_before = None; - } cur_pc = frame.pc as usize; shell .lasti @@ -6379,42 +4510,13 @@ impl Interpreter { // Each bit-guarded arm below re-checks its subsystem's // precise gate, so a stale-set bit is harmless. let hot = crate::hot_gates::load(); - // RFC 0040 (GC arc): prompt finalization. Between bytecodes — a - // safe point with no outstanding container borrows — run `__del__` - // and fire callback-weakrefs for any object whose last reference - // dropped during the previous instruction, matching CPython's - // refcount-driven `tp_dealloc` timing. Two gates keep this free in - // the common case: a finalizable object must be live *and* the - // previous instruction must have dropped a reference (`take_maybe_ - // _dead`, made variant-precise by RFC 0059 WS1b), so a hot loop - // that neither allocates `__del__` objects nor drops heap-graph - // references pays only the relaxed loads. - let any_finalizable = gc_trace::has_any_finalizable(); - if (any_finalizable || gc_trace::has_suspects()) && gc_trace::take_maybe_dead() { - if any_finalizable { - self.drain_prompt_finalizers(); - } - // Re-probe cascade-skipped suspects (see `note_suspect`): a - // Rust-side transient that pinned one during its cascade has - // typically died by now, and CPython would have freed the - // object (and everything it anchors) at that instant. - if gc_trace::has_suspects() { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } - } else if hot & crate::hot_gates::PENDING_FINALIZERS != 0 - && crate::vm_singletons::has_pending_finalizers() - { - // An *untracked* finalizable instance freed by a plain `Rc` - // drop (its container temporary died between bytecodes — a - // `for k, v in d.items()` loop rebinding its tuple, dropping - // the overwritten value) parks its `__del__` via - // `PyInstance::drop`'s resurrection net. It never appears in - // the finalizable index, so the gate above stays cold; run - // the parked finalizers here to keep CPython's refcount - // timing (`test_dict.test_oob_indexing_dictiter_iternextitem` - // clears the dict from such a `__del__` mid-iteration). + // Prompt finalization. Between bytecodes — a safe point with no + // outstanding container borrows — run the `__del__`s and + // weakref callbacks queued by objects that died during the + // previous instruction (their `Drop` raised the gate), + // matching CPython's refcount-driven `tp_dealloc` timing. + if hot & crate::hot_gates::PENDING_FINALIZERS != 0 { + gc_trace::take_maybe_dead(); self.run_pending_finalizers(); } // A tracking site crossed the young threshold without polling @@ -6425,58 +4527,19 @@ impl Interpreter { self.run_pending_finalizers(); } } - // RFC 0047 (wave 5): reap objects whose last C-side reference - // was dropped *inside* an extension call since the previous - // instruction (a Cython setter decref'ing the old value of a - // `cdef` field). CPython would have run their dealloc chains in - // the call itself; the capi boundary parks them instead, and - // this between-bytecodes safe point — the first moment with no - // extension frame live — runs the refcount-guarded reap so - // weakrefs to anything that died clear before the next - // instruction can observe them. Gated on one thread-local read. - // - // Crucially, the drain waits for `cext_call_active()` to clear: - // bytecode a C extension re-enters (a Cython module-exec, a - // `PyObject_Call` back into Python) hits this safe point too, - // where every parked escaped subgraph would just be re-parked - // by `reap_dead_subgraph`'s C-frame guard after an O(subgraph) - // escape scan — re-scanning the whole queue on *every bytecode* - // of the re-entered code (pandas' import runs minutes long - // under that quadratic churn). Draining only at depth 0 runs - // each parked object through the cascade exactly once. - // Probe order matters: `has_pending_cext_drops` is one - // relaxed load when empty (the overwhelming case), while - // `cext_call_active` is a thread-local read — so ask - // "is there anything to do" before "may we do it". - // Native side exits share this queue: their obsolete pins - // can be reaped now that the continuation's frame is on - // the stack and its recursion depth is accounted for. + // Release the references C extension code dropped since the + // previous instruction (RFC 0047, wave 5). The capi boundary + // parks them rather than releasing them inside the call, + // where C may still borrow a body they back; this safe point + // with no extension frame live is the first moment they can + // go (`has_pending_cext_drops` is the cheap probe, so it + // comes first). if hot & crate::hot_gates::PENDING_CEXT != 0 && crate::vm_singletons::has_pending_cext_drops() && !crate::vm_singletons::cext_call_active() { - static REAP_TRACE: std::sync::OnceLock = std::sync::OnceLock::new(); - let reap_trace = *REAP_TRACE - .get_or_init(|| std::env::var_os("WEAVEPY_REAP_TRACE").is_some()); for dropped in crate::vm_singletons::drain_pending_cext_drops() { - if reap_trace { - let dead = Self::is_refcount_dead(&dropped, 1); - let sc = gc_trace::strong_count_for(&dropped); - eprintln!( - "[CEXT-REAP] {} id={:#x} sc={} dead={}", - dropped.type_name_owned(), - crate::weakref_registry::id_of(&dropped), - sc, - dead - ); - } - // Straight to the cascade: `prompt_reap_dropped`'s - // fast-path early return would plain-drop an untracked, - // finalizer-free container — but a C-dropped container - // (the old `blocks` tuple) is queued here precisely - // because its *tracked children* (Cython `Block`s) need - // the cascade to clear their weakrefs. - self.reap_dead_subgraph(dropped); + self.release(dropped); } } // RFC 0040 (WS7): surface any "unclosed file" `ResourceWarning` @@ -6659,21 +4722,6 @@ impl Interpreter { // always — stays free. Feeding any handler-raised error // through the same arm as `step`'s `Err` keeps it catchable // by a surrounding `try/except`, exactly as CPython. - // RFC 0040 (GC arc): note whether this instruction can drop the - // last reference to a finalizable object, so the next safe point's - // prompt-finalization sweep runs only when warranted. Gated on a - // live finalizable so non-`__del__` workloads pay nothing. A - // dropped reference shows up either as the operand stack shrinking - // (`POP_TOP`, every `STORE_*`/`*_SUBSCR`, `POP_EXCEPT`, `END_FOR`, - // a net-consuming `CALL`, …) or as one of the stack-neutral - // `DELETE_*`-name opcodes. - watch_drops = gc_trace::has_any_finalizable(); - stack_before = if watch_drops { frame.stack.len() } else { 0 }; - op_before = if watch_drops { - frame.code.instructions.get(frame.pc as usize).map(|i| i.op) - } else { - None - }; // `PyThreadState_SetAsyncExc`: an exception scheduled for this // thread from another thread (or itself) is raised here, at the // next instruction boundary — CPython's eval-breaker async-exc @@ -6779,39 +4827,6 @@ impl Interpreter { if instruction_ran { prev_pc = Some(cur_pc); } - // RFC 0059 WS1b: opcodes whose handlers account for every value - // they discard via `gc_trace::note_dropped` (audited: `POP_TOP`, - // `STORE_FAST`, and the `BINARY_OP`/`COMPARE_OP` arms — the - // specialized scalar fast paths of the latter two consume only - // pure-leaf variants by construction) are exempt from the coarse - // stack-shrink heuristic, so scalar-churning hot loops stop - // scheduling finalizer sweeps. Every other opcode keeps the - // conservative mark. - // RFC 0077 (WS2) widened the audited set to the opcodes the - // deltablue census showed driving ~95% of the coarse marks: - // `CALL` (operands graded by `reap_call_receiver`/`reap_call_ - // args`, moved arguments by the callee's frame-exit sweep, the - // raise path re-marks), `RETURN_VALUE` (the frame-exit sweep - // grades every slot), `STORE_ATTR` (receiver + displaced value - // on the instance hits; slot/generic paths re-mark), the - // `POP_JUMP_IF_*` family and `IS_OP` (operands noted in the - // handlers). - // One table byte per opcode (RFC 0077 WS3): bit 0 marks the - // audited (exempt) set above, bit 1 the stack-neutral - // `DELETE_*` opcodes that count as a drop regardless. - let drop_class = match op_before { - Some(op) => COARSE_DROP_CLASS[op as u8 as usize], - None => 0, - }; - if watch_drops - && drop_class & COARSE_EXEMPT == 0 - && (frame.stack.len() < stack_before || drop_class & COARSE_DELETE != 0) - { - if crate::specialize::stats_enabled() { - gc_trace::floor_stats::coarse_mark(op_before.map_or("?", OpCode::name)); - } - gc_trace::mark_maybe_dead(); - } match stepped { Ok(StepOutcome::Continue) => { // PEP 669 BRANCH/JUMP: fired after a control-flow @@ -7108,9 +5123,6 @@ impl Interpreter { break 'activation Err(e); } } - // The frame's locals and operands drop - // wholesale below; see `mark_bulk_drop`. - gc_trace::mark_bulk_drop(); break Err(e); } } @@ -7731,27 +5743,15 @@ impl Interpreter { } // Tear down the never-started generator: its frame // locals (bound arguments) die now, like CPython's - // refcount-driven dealloc. Drop the snapshot's own - // strong clones (mirror, materialised dict) *first* - // so the reap sees the true remaining refcount. - let caps = frame_reapables(&g); + // refcount-driven dealloc. *g.state.borrow_mut() = GeneratorState::Finished; Self::wipe_frame_locals(py); - for cap in caps { - self.prompt_reap_dropped(cap); - } return Ok(Object::None); } _ => {} } } Self::wipe_frame_locals(py); - // The wiped locals were pinned by a traceback until now - // (unittest's `assertRaises` clears them on exit); any of them - // may have been the last holder of a finalizable the hot/cold - // gate already demoted. Re-grade the whole index at the next - // safe point (see `gc_trace::mark_bulk_drop`). - gc_trace::mark_bulk_drop(); Ok(Object::None) } @@ -8881,8 +6881,6 @@ impl Interpreter { // consume the thread-local flag before the operands // actually drop — and before the `?`, so the error path's // unwind drop is covered too. - gc_trace::note_dropped(&a); - gc_trace::note_dropped(&b); let r = r_res?; // tracemalloc: a binary op that built a fresh container / // string / bytes (`b'x' * n`, `s1 + s2`, `list * 3`) is an @@ -8918,15 +6916,7 @@ impl Interpreter { /// keeps bound containers off the cascade. #[inline] fn retire_operand(&mut self, o: Object) { - match &o { - Object::Instance(_) => self.prompt_reap_dropped(o), - Object::List(_) | Object::Dict(_) | Object::Set(_) => { - if Self::looks_reapable_temporary(&o) { - self.prompt_reap_dropped(o); - } - } - _ => {} - } + self.release(o); } /// The `COMPARE_OP` handler body, split out of `step`. @@ -8953,8 +6943,6 @@ impl Interpreter { let r_res = self.rich_compare_obj(&a, &b, kind, &frame.globals); // RFC 0059 WS1b: same operand accounting (and ordering) // as BINARY_OP above. - gc_trace::note_dropped(&a); - gc_trace::note_dropped(&b); let r = r_res?; let r = if to_bool && !matches!(r, Object::Bool(_)) { let truthy = match &r { @@ -9065,7 +7053,6 @@ impl Interpreter { // coarse stack-shrink mark; note the discarded value here // (no-op for pure leaves — e.g. a discarded expression // statement's int/str result). - gc_trace::note_dropped(&v); // Discarding the last reference to a temporary mirrors // CPython's refcount-driven finalization (`f()` as a // statement finalizes the result immediately). This covers @@ -9089,9 +7076,7 @@ impl Interpreter { // and `frame.locals` always reflects live bindings — the // refcount guard therefore cannot false-positive on a value // still bound to a local. - if Self::local_needs_prompt_reap(&v) { - self.prompt_reap_dropped(v); - } + self.release(v); Ok(()) } @@ -9143,9 +7128,7 @@ impl Interpreter { } } }; - if let Some(stale) = stale { - gc_trace::note_dropped(&stale); - } + drop(stale); return Ok(()); } } @@ -9175,7 +7158,6 @@ impl Interpreter { // loop's coarse stack-shrink mark, so the displaced value // must be noted here (no-op for pure-leaf rebinds — loop // counters, accumulators, first bindings over `Unbound`). - gc_trace::note_dropped(&old); // CPython decrefs the value previously bound to the local; // when that was the last reference to a finalizable object // its `__del__` runs at the rebind, not at frame exit @@ -9192,10 +7174,7 @@ impl Interpreter { // held elsewhere" rebind (`for t in tasks:` displacing // the previous element) on one strong-count load before // the cascade's registry probes (RFC 0077 WS2). - if Self::local_needs_prompt_reap(&old) && Self::looks_reapable_temporary(&old) { - self.sync_py_locals(frame); - self.prompt_reap_dropped(old); - } + self.release(old); } Ok(()) } @@ -9216,15 +7195,13 @@ impl Interpreter { let truthy = match &v { Object::Instance(_) | Object::MappingProxyObj(_) => { let g = frame.globals.clone(); - self.obj_truthy(&v, &g) - .inspect_err(|_| gc_trace::mark_maybe_dead())? + self.obj_truthy(&v, &g)? } _ => v.is_truthy(), }; // RFC 0077 (WS2): the POP_JUMP family audits its one // discarded operand (exempt from the coarse stack-shrink // mark); the usual `bool` is a leaf and schedules nothing. - gc_trace::note_dropped(&v); if truthy == when { frame.pc += arg; } @@ -9295,8 +7272,6 @@ impl Interpreter { frame: &mut Frame, mut shell: QuietShell<'_>, snap_gen: u64, - fin: bool, - coarse: bool, prev_pc: &mut Option, ) -> QuietExit { // SAFETY (for the reads in `quiet_frame`): the flag is this @@ -9315,7 +7290,6 @@ impl Interpreter { root: frame, root_shell: (&raw mut shell).cast(), root_last: &raw mut last, - fin, maybe_dead, }; let mut entry = QuietEntry::Fresh; @@ -9330,7 +7304,6 @@ impl Interpreter { &mut *roots.root, &mut *roots.root_shell.cast::>(), snap_gen, - coarse, prev_pc, &mut *roots.root_last, entry, @@ -9354,7 +7327,6 @@ impl Interpreter { top_frame, &mut top_shell, snap_gen, - coarse, &mut top_prev, &mut top_last, entry, @@ -9407,7 +7379,7 @@ impl Interpreter { } else { let result = self.inline_finish(&mut done, exit); match inl.last_mut() { - None => self.inline_deliver(frame, shell, done, result), + None => self.inline_deliver(frame, done, result), Some(caller) => { let caller = &mut **caller; // SAFETY: as above. @@ -9416,8 +7388,7 @@ impl Interpreter { } else { &mut caller.frame }; - let mut caller_shell = QuietShell::Lazy(&mut caller.act); - self.inline_deliver(caller_frame, &mut caller_shell, done, result) + self.inline_deliver(caller_frame, done, result) } } } @@ -9436,13 +7407,12 @@ impl Interpreter { frame: &mut Frame, shell: &mut QuietShell<'_>, snap_gen: u64, - coarse: bool, prev_pc: &mut Option, last: &mut usize, entry: QuietEntry, roots: &SwitchRoots, ) -> FrameEv { - let (fin, maybe_dead) = (roots.fin, roots.maybe_dead); + let maybe_dead = roots.maybe_dead; // A burst's stop handed over from the activation it moved away // from (see `FrameEv::Switched`). let mut pending: Option = None; @@ -9462,7 +7432,7 @@ impl Interpreter { break 'run QuietExit::Yield; } // SAFETY: see `quiet_run`. - if fin && unsafe { (*maybe_dead).get() } { + if unsafe { (*maybe_dead).get() } { self.flush_lean(frame, shell); if self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen { break 'run QuietExit::Yield; @@ -9472,7 +7442,6 @@ impl Interpreter { QuietEntry::Raised { err, cur_pc } => { // The raise path drops the staged operands wholesale // (see the `CALL` arm of `step`). - gc_trace::mark_maybe_dead(); self.flush_lean(frame, shell); if *last != usize::MAX { *prev_pc = Some(*last); @@ -9499,7 +7468,6 @@ impl Interpreter { LeafStop::Breaker => break 'run QuietExit::Yield, LeafStop::Raised(err) => { // The `CALL` arm of `step` marks on the raise path. - gc_trace::mark_maybe_dead(); self.flush_lean(frame, shell); let cur_pc = frame.pc as usize - 1; if *last != usize::MAX { @@ -9511,12 +7479,10 @@ impl Interpreter { }); } LeafStop::Marked => { - if fin { - // Finalizers are arbitrary code: the spine must be - // complete before they run. - self.flush_lean(frame, shell); - self.drain_if_maybe_dead(); - } + // Finalizers are arbitrary code: the spine must be + // complete before they run. + self.flush_lean(frame, shell); + self.drain_if_maybe_dead(); // The drop may also have queued work of its own (an // unclosed file's ResourceWarning), which CPython // delivers at the instruction that dropped it. @@ -9584,7 +7550,6 @@ impl Interpreter { if let Err(err) = called { // The raise path drops the staged operands // wholesale (see the `CALL` arm of `step`). - gc_trace::mark_maybe_dead(); self.flush_lean(frame, shell); if *last != usize::MAX { *prev_pc = Some(*last); @@ -9603,7 +7568,7 @@ impl Interpreter { break 'run QuietExit::Yield; } // SAFETY: see `quiet_run`. - if fin && unsafe { (*maybe_dead).get() } { + if unsafe { (*maybe_dead).get() } { self.flush_lean(frame, shell); if self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen @@ -9641,7 +7606,6 @@ impl Interpreter { }; if let Some(loaded) = loaded { if let Err(err) = loaded { - gc_trace::mark_maybe_dead(); self.flush_lean(frame, shell); if *last != usize::MAX { *prev_pc = Some(*last); @@ -9660,7 +7624,7 @@ impl Interpreter { break 'run QuietExit::Yield; } // SAFETY: see `quiet_run`. - if fin && unsafe { (*maybe_dead).get() } { + if unsafe { (*maybe_dead).get() } { self.flush_lean(frame, shell); if self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen @@ -9689,22 +7653,10 @@ impl Interpreter { } } let live_shell = self.flush_lean(frame, shell); - let stack_before = frame.stack.len(); live_shell .lasti .store(frame.pc, std::sync::atomic::Ordering::Relaxed); let stepped = self.step_hot(frame); - if coarse { - let drop_class = match op_before { - Some(op) => COARSE_DROP_CLASS[op as u8 as usize], - None => 0, - }; - if drop_class & COARSE_EXEMPT == 0 - && (frame.stack.len() < stack_before || drop_class & COARSE_DELETE != 0) - { - gc_trace::mark_maybe_dead(); - } - } if !matches!(stepped, Ok(StepOutcome::Continue)) { return FrameEv::Exit(QuietExit::Outcome { stepped, cur_pc }); } @@ -9718,8 +7670,7 @@ impl Interpreter { break 'run QuietExit::Yield; } // SAFETY: see `quiet_run`. - if fin - && unsafe { (*maybe_dead).get() } + if unsafe { (*maybe_dead).get() } && self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen { @@ -10078,7 +8029,6 @@ impl Interpreter { Ok(FrameOutcome::Returned(v)) => { *gen.state.borrow_mut() = GeneratorState::Finished; let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(&gen); Ok(GenStep::Exhausted(v)) @@ -10093,7 +8043,6 @@ impl Interpreter { *gen.state.borrow_mut() = GeneratorState::Finished; let escaped = self.pep479_escape(&gen, err); let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(&gen); Err(escaped) @@ -10139,7 +8088,6 @@ impl Interpreter { // The exhausted generator leaves the stack (a full handler // takes the coarse mark for it). drop(it); - gc_trace::mark_maybe_dead(); QuietEntry::Returned { cur_pc: call_pc } } Err(err) => QuietEntry::Raised { @@ -10235,10 +8183,7 @@ impl Interpreter { } }; let result = match outcome { - Ok(FrameOutcome::Returned(v)) => { - self.reap_frame_locals_on_exit(frame, Some(&v)); - Ok(v) - } + Ok(FrameOutcome::Returned(v)) => Ok(v), Ok(FrameOutcome::Yielded(_)) => Err(RuntimeError::Internal( "lean frame yielded to a non-generator caller".to_owned(), )), @@ -10257,7 +8202,6 @@ impl Interpreter { fn inline_deliver( &mut self, frame: &mut Frame, - shell: &mut QuietShell<'_>, done: Box, result: Result, ) -> QuietEntry { @@ -10284,7 +8228,7 @@ impl Interpreter { match result { Ok(v) => { frame.stack.push(v); - self.drop_lean_callable(frame, shell, callable); + self.release(callable); QuietEntry::Returned { cur_pc: call_pc } } Err(err) => { @@ -10297,49 +8241,6 @@ impl Interpreter { } } - /// Release a lean call's callable after a normal return - /// (`reap_call_receiver`'s function arm): a function that was only a - /// call temporary goes through the cascade (which can run arbitrary - /// code); a bound one is just graded. The usual callee — a function - /// its namespace still holds (our clone, that binding, its collector - /// handle: three or more) with no weakref — needs neither. - #[inline(always)] - fn drop_lean_callable( - &mut self, - frame: &mut Frame, - shell: &mut QuietShell<'_>, - callable: Object, - ) { - let alive_elsewhere = match &callable { - Object::Function(f) => { - Rc::strong_count(f) > 2 - && !crate::weakref_registry::may_have_weakrefs(Rc::as_ptr(f) as usize as u64) - } - _ => false, - }; - if alive_elsewhere { - drop_hot(callable); - } else { - self.drop_lean_callable_slow(frame, shell, callable); - } - } - - /// [`Self::drop_lean_callable`] for a callable that may die here. - #[inline(never)] - fn drop_lean_callable_slow( - &mut self, - frame: &mut Frame, - shell: &mut QuietShell<'_>, - callable: Object, - ) { - if Self::looks_reapable_temporary(&callable) { - self.flush_lean(frame, shell); - self.prompt_reap_dropped(callable); - } else { - gc_trace::note_dropped(&callable); - } - } - /// Push the frame shells of every pending lean activation, outermost /// first, then this activation's own, and return it (see [`LeanAct`]). /// A no-op for an activation that already has its shell. @@ -10562,7 +8463,7 @@ impl Interpreter { Some(match result { Ok(v) => { frame.stack.push(v); - self.drop_lean_callable(frame, shell, callable); + self.release(callable); Ok(()) } Err(e) => Err(e), @@ -10817,11 +8718,6 @@ impl Interpreter { if let Some((f, receiver)) = bound { let bm = std::mem::replace(&mut frame.stack[callee_slot], Object::Function(f)); frame.stack[self_slot] = receiver; - // The bound method was a call temporary (or is still held - // elsewhere): grade its release like any dropped operand. - if gc_trace::note_dropped_marks(&bm) { - gc_trace::mark_maybe_dead(); - } drop(bm); } let has_self = !matches!(frame.stack[self_slot], Object::Unbound); @@ -11043,7 +8939,7 @@ impl Interpreter { Some(match result { Ok(v) => { frame.stack.push(v); - self.drop_lean_callable(frame, shell, callable); + self.release(callable); Ok(()) } Err(e) => Err(e), @@ -11262,7 +9158,7 @@ impl Interpreter { /// the callee's own references rather than at the next collection. fn release_call_operands(&mut self, stack: &mut Vec, callee_slot: usize) { let callee = std::mem::replace(&mut stack[callee_slot], Object::Unbound); - self.reap_call_receiver(callee); + self.release(callee); self.reap_call_args(&mut stack[callee_slot + 1..]); stack.truncate(callee_slot); } @@ -11329,8 +9225,7 @@ impl Interpreter { let result = self.run_lean_fn(frame, shell, snap_gen, pc, &getter, code, receiver); Some(result.map(|v| { if let Some(top) = frame.stack.last_mut() { - let old = std::mem::replace(top, v); - gc_trace::note_dropped(&old); + *top = v; } })) } @@ -11372,7 +9267,6 @@ impl Interpreter { // The exhausted generator leaves the stack (a full handler // takes the coarse mark for it). drop(it); - gc_trace::mark_maybe_dead(); Ok(()) } Err(e) => Err(e), @@ -11754,21 +9648,12 @@ impl Interpreter { crate::tier2::warm_compile(self, frame); } let exc_depth_on_entry = self.exc_info_len(); - let fin_live = gc_trace::has_any_finalizable(); - let fin = fin_live || gc_trace::active_suspects_present(); let mut act = LeanAct { frame: std::ptr::from_mut(frame), shell: None, }; let mut prev_pc: Option = None; - let exit = self.quiet_run( - frame, - QuietShell::Lazy(&mut act), - snap_gen, - fin, - fin_live, - &mut prev_pc, - ); + let exit = self.quiet_run(frame, QuietShell::Lazy(&mut act), snap_gen, &mut prev_pc); let outcome = match (exit, act.shell) { // The whole activation ran without ever needing its shell. ( @@ -11816,7 +9701,6 @@ impl Interpreter { match outcome { FrameOutcome::Returned(v) => { // The owner retires the frame (`retire_lean_frame`). - self.reap_frame_locals_on_exit(frame, Some(&v)); Ok(v) } FrameOutcome::Yielded(_) => Err(RuntimeError::Internal( @@ -11836,14 +9720,7 @@ impl Interpreter { if !gc_trace::take_maybe_dead() { return false; } - if gc_trace::has_any_finalizable() { - self.drain_prompt_finalizers(); - } - if gc_trace::has_suspects() { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } + self.run_pending_finalizers(); true } @@ -11891,8 +9768,13 @@ impl Interpreter { } } match self.leaf_core_helper(frame, last_pc) { + // A released receiver or argument may have queued + // a finalizer (see `after_release!` in the core). + // SAFETY: the running thread's own flag. + CoreAttr::Done if unsafe { (*sw.maybe_dead).get() } => { + return LeafStop::Marked; + } CoreAttr::Done => continue, - CoreAttr::Marked => return LeafStop::Marked, CoreAttr::Raised(e) => return LeafStop::Raised(e), CoreAttr::Full => return LeafStop::Step, CoreAttr::Decline => {} @@ -11904,6 +9786,8 @@ impl Interpreter { // SAFETY: as above. let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) }; match self.leaf_burst_slow(frame, snap_gen, last_pc) { + // SAFETY: as above. + LeafStop::Core if unsafe { (*sw.maybe_dead).get() } => return LeafStop::Marked, LeafStop::Core => {} stop => return stop, } @@ -11966,7 +9850,7 @@ impl Interpreter { _ => self.leaf_core_attr(frame, ins, pc), }; match done { - CoreAttr::Done | CoreAttr::Marked => { + CoreAttr::Done => { *last_pc = pc; frame.pc = pc as u32 + 1; } @@ -12089,6 +9973,23 @@ impl Interpreter { let mut len = stack.len(); let mut pc = frame.pc as usize; let mut last = *last_pc; + let pending_work = sw.maybe_dead; + // A dying object's `Drop` queued a finalizer or callback: the + // quiet loop runs it before the next instruction, as CPython + // does inside the releasing one's `Py_DECREF`. Checked after the + // arms that end a binding; any other release is noticed at the + // next back edge, call or return (queueing bumps the loop + // generation). + macro_rules! after_release { + () => { + // SAFETY: the running thread's own flag (see `quiet_run`). + #[allow(unused_unsafe)] + let queued = unsafe { (*pending_work).get() }; + if queued { + break Some(CoreExit::Stop(LeafStop::Marked)); + } + }; + } let stop = loop { if pc >= ninstrs { break Some(CoreExit::Stop(LeafStop::Step)); @@ -12553,76 +10454,20 @@ impl Interpreter { // Overwriting a scalar needs no drop. len -= 1; slot.write(base.add(len).read()); - } else if Self::core_shared_instance(&*slot) { - // A shared instance: the grading below would - // answer "escaped, no mark"; its release is a - // bare decrement. - len -= 1; - drop(std::mem::replace(&mut *slot, base.add(len).read())); } else { - // A displaced heap value the full arm would - // reap stays there — except a deferred-tracking - // instance, whose teardown is its plain drop - // (recycled when this was its last reference); - // any other is graded like every burst drop. - if Self::local_needs_prompt_reap(&*slot) - && Self::looks_reapable_temporary(&*slot) - { - match &*slot { - Object::Instance(i) if i.dies_by_plain_drop() => { - len -= 1; - if let Object::Instance(i) = - std::mem::replace(&mut *slot, base.add(len).read()) - { - PyInstance::try_recycle(i); - } - last = pc; - pc += 1; - continue; - } - // A tuple of scalars: the freelist is - // its teardown (see `prompt_reap_dropped`). - Object::Tuple(t) - if t.iter().all(Object::is_gc_atomic) - && !gc_trace::maybe_tracked( - crate::weakref_registry::id_of(&*slot), - ) => - { - len -= 1; - let old = - std::mem::replace(&mut *slot, base.add(len).read()); - self.maybe_donate_tuple(old); - last = pc; - pc += 1; - continue; - } - // An untracked container of scalars: - // a plain drop. - Object::List(_) | Object::Dict(_) - if Self::core_plain_last_container(&*slot) => - { - len -= 1; - drop(std::mem::replace( - &mut *slot, - base.add(len).read(), - )); - last = pc; - pc += 1; - continue; - } - _ => break None, - } + // The displaced value dies here if this was + // its last reference (the freelists take a + // dead plain instance's or tuple's + // allocation). + if !Self::core_droppable(&*slot) { + break None; } len -= 1; - let old = std::mem::replace(&mut *slot, base.add(len).read()); - let marked = gc_trace::note_dropped_marks(&old); - drop(old); - if marked { - gc_trace::mark_maybe_dead(); - last = pc; - pc += 1; - break Some(CoreExit::Stop(LeafStop::Marked)); - } + self.release(std::mem::replace(&mut *slot, base.add(len).read())); + last = pc; + pc += 1; + after_release!(); + continue; } } last = pc; @@ -12671,6 +10516,10 @@ impl Interpreter { ) { // SAFETY: as above. unsafe { drop_hot(base.add(len).read()) }; + last = pc; + pc += 1; + after_release!(); + continue; } last = pc; pc += 1; @@ -12974,6 +10823,7 @@ impl Interpreter { len = n; last = pc; pc += 1; + after_release!(); } // An instance whose class's `__getitem__` is a // native fast subscript the site cached. @@ -13007,6 +10857,7 @@ impl Interpreter { len += 1; last = pc; pc += 1; + after_release!(); } Err(e) => { pc += 1; @@ -13259,11 +11110,7 @@ impl Interpreter { *index += 1; v } - None if unique - && owner.is_none() - && Rc::strong_count(items) >= 2 - && !Self::iter_backing_list_dead(items) => - { + None if unique && owner.is_none() => { retire = true; Object::None } @@ -13454,11 +11301,7 @@ impl Interpreter { // `POP_ITER` pair (the full handler's shape). len -= 1; // SAFETY: the iterator slot leaves the stack. - let it = unsafe { base.add(len).read() }; - let marked = - gc_trace::maybe_tracked(crate::weakref_registry::id_of(&it)) - && gc_trace::note_dropped_marks(&it); - drop(it); + drop(unsafe { base.add(len).read() }); last = pc; pc += 1 + ins.arg as usize; let op_at = |pc: usize| { @@ -13471,10 +11314,6 @@ impl Interpreter { pc += 1; } } - if marked { - gc_trace::mark_maybe_dead(); - break Some(CoreExit::Stop(LeafStop::Marked)); - } continue; } // SAFETY: `len < cap`. @@ -14380,7 +12219,7 @@ impl Interpreter { // shared module leaves by a plain decrement). Object::Module(m) => { match Self::core_module_attr(code, m, pc, ins.arg) { - Some(v) if !gc_trace::note_dropped_marks(unsafe { &*top }) => { + Some(v) => { // SAFETY: the receiver is replaced in place. unsafe { drop_hot(std::mem::replace(&mut *top, v)) }; last = pc; @@ -14609,11 +12448,6 @@ impl Interpreter { }; let bm = std::mem::replace(&mut frame.stack[callee_slot], Object::Function(f)); frame.stack[self_slot] = receiver; - // The bound method was a call temporary (or is still held - // elsewhere): grade its release like any dropped operand. - if gc_trace::note_dropped_marks(&bm) { - gc_trace::mark_maybe_dead(); - } drop(bm); } @@ -14781,7 +12615,7 @@ impl Interpreter { // The spread tuple and a bound method were call temporaries (or // are still held elsewhere): grade their releases like any // dropped operand. - self.reap_call_receiver(callee); + self.release(callee); self.reap_call_args(&mut [spread, mapping]); let self_slot = callee_slot + 1; let act = self.inline_slot(); @@ -15183,7 +13017,6 @@ impl Interpreter { // A declined `__init__` may have stored into the instance // before it stopped; nothing else ever saw it, so it goes // (the framed call starts over on a fresh one). - gc_trace::note_dropped(&inst); drop(inst); false } @@ -15565,53 +13398,18 @@ impl Interpreter { DictStep::Item(item.0, item.1) } - /// Whether the core loop may release `v` with a plain drop: a scalar, - /// a string (freeing one runs no code), or a shared heap value whose - /// release is a bare decrement the collector need not hear about (the - /// leaf arms' `note_dropped_marks` grade answers no). + /// Whether the core loop may release `v` with a plain drop. Releasing + /// a value never runs Python (a dying object's `Drop` only queues work + /// for the next safe point), so only the fixtures that observe deaths + /// as they happen (the C-API dict/function watchers, the reftrace + /// printer) send the release to the full path. #[inline(always)] fn core_droppable(v: &Object) -> bool { - match v { - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None | Object::Str(_) => { - true - } - Object::Instance(i) => { - // Deferred tracking is an exact proof that the collector - // and weakref registry hold no owner. A stale Bloom-filter - // hit must not reject this nonfinal release. The last - // owner still takes ordinary teardown, and every store or - // weakref operation that requires tracking revokes the flag. - // Past two owners only a weakref-watched object can be at - // its dead line (see `gc_trace::note_dropped_marks`). - gc_trace::drop_survives_plainly(v) - || Rc::strong_count(i) > 1 - && (i.is_gc_deferred() || !gc_trace::note_dropped_marks(v)) - } - Object::List(l) if Rc::strong_count(l) > 1 => !gc_trace::note_dropped_marks(v), - Object::Dict(d) if Rc::strong_count(d) > 1 => !gc_trace::note_dropped_marks(v), - // The last owner of a container the collector never took, and - // no weakref watches, holding only scalars: freeing it runs no - // code (`prompt_reap_dropped` reaches the same plain drop the - // long way round). - Object::List(_) | Object::Dict(_) => Self::core_plain_last_container(v), - Object::Tuple(t) => ThinArc::strong_count(t) > 1 && !gc_trace::note_dropped_marks(v), - Object::Function(f) => Rc::strong_count(f) > 1 && !gc_trace::note_dropped_marks(v), - Object::Type(t) => Rc::strong_count(t) > 1 && !gc_trace::note_dropped_marks(v), - _ => false, - } - } - - /// [`Self::core_droppable`] for the last owner of an exact `list` or - /// `dict`: untracked (the miss filter proves it), unwatched, and - /// holding only scalars. - #[inline(never)] - fn core_plain_last_container(v: &Object) -> bool { - let id = crate::weakref_registry::id_of(v); - !gc_trace::maybe_tracked(id) - && !crate::weakref_registry::may_have_weakrefs(id) + // (A never-driven `asend` awaitable warns as it goes.) + !matches!(v, Object::AsyncGenAwait(_)) && !crate::capi_watchers::dicts_active() + && !crate::capi_watchers::funcs_active() && !crate::stdlib::testinternalcapi_mod::reftrace_print_active() - && Self::is_scalar_leaf_container(v) } /// Module scope for the core loop's `LOAD_NAME` / `STORE_NAME` arms: @@ -15715,21 +13513,6 @@ impl Interpreter { }) } - /// Whether `v` is an instance the frame-slot and drop graders would - /// both pass over (`looks_reapable_temporary` false and - /// `note_dropped_marks` false): held elsewhere beyond the collector's - /// handle and watched by no weakref — its release is a bare decrement. - #[inline(always)] - fn core_shared_instance(v: &Object) -> bool { - let Object::Instance(i) = v else { - return false; - }; - let sc = Rc::strong_count(i); - let id = Rc::as_ptr(i) as usize as u64; - (sc >= 3 || (sc == 2 && !gc_trace::maybe_tracked(id))) - && !crate::weakref_registry::may_have_weakrefs(id) - } - /// The `(function, has_self, effective argc)` of a plain-function /// `CALL` at `pc` (the shapes [`Self::core_call_cached`] serves). #[inline] @@ -16441,11 +14224,10 @@ impl Interpreter { if values[index] == 0 { if let Object::Instance(instance) = operand { let id = Rc::as_ptr(instance) as usize as u64; - eprintln!("First predicate instance drop miss: owners={}, deferred={}, maybe_tracked={}, tracked={}, weakrefs={}, weak_clones={}", + eprintln!("First predicate instance drop miss: owners={}, deferred={}, maybe_tracked={}, tracked={}, weakrefs={}", Rc::strong_count(instance), instance.is_gc_deferred(), gc_trace::maybe_tracked(id), gc_trace::is_tracked(id), - crate::weakref_registry::may_have_weakrefs(id), - crate::weakref_registry::strong_clone_count(id)); + crate::weakref_registry::may_have_weakrefs(id)); } } values[index] += 1; @@ -17045,7 +14827,7 @@ impl Interpreter { return true; } // SAFETY: see `quiet_run`. - if sw.fin && unsafe { (*sw.maybe_dead).get() } { + if unsafe { (*sw.maybe_dead).get() } { // SAFETY: as above. unsafe { self.flush_lean(&mut *cframe, &mut *cshell.cast::>()); @@ -17124,11 +14906,7 @@ impl Interpreter { // SAFETY: as above. unsafe { (*cframe).stack.push(v); - self.drop_lean_callable( - &mut *cframe, - &mut *cshell.cast::>(), - callable, - ); + self.release(callable); } entry = QuietEntry::Returned { cur_pc: call_pc }; } else { @@ -17152,14 +14930,7 @@ impl Interpreter { // SAFETY: the caller is the innermost remaining activation. (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; // SAFETY: as above. - entry = unsafe { - self.inline_deliver( - &mut *cframe, - &mut *cshell.cast::>(), - done, - result, - ) - }; + entry = unsafe { self.inline_deliver(&mut *cframe, done, result) }; } sw.cur = cframe; sw.last = if clast == &raw mut sw.scratch { @@ -17183,7 +14954,7 @@ impl Interpreter { return true; } // SAFETY: see `quiet_run`. - if sw.fin && unsafe { (*sw.maybe_dead).get() } { + if unsafe { (*sw.maybe_dead).get() } { // SAFETY: as above. unsafe { self.flush_lean(&mut *cframe, &mut *cshell.cast::>()); @@ -17244,11 +15015,6 @@ impl Interpreter { return CoreAttr::Decline; }; let v = std::mem::replace(top, it); - if gc_trace::note_dropped_marks(&v) { - gc_trace::mark_maybe_dead(); - drop(v); - return CoreAttr::Marked; - } drop(v); CoreAttr::Done } @@ -17289,11 +15055,6 @@ impl Interpreter { return CoreAttr::Decline; }; let receiver = std::mem::replace(top, v); - if gc_trace::note_dropped_marks(&receiver) { - gc_trace::mark_maybe_dead(); - drop(receiver); - return CoreAttr::Marked; - } drop(receiver); CoreAttr::Done } @@ -17309,15 +15070,8 @@ impl Interpreter { let Some(old) = Self::leaf_store_attr(code, stack, pc as u32, ins.arg) else { return CoreAttr::Decline; }; - let receiver = stack.pop().expect("receiver checked above"); - let marked = gc_trace::note_dropped_marks(&receiver) - | old.as_ref().is_some_and(gc_trace::note_dropped_marks); - drop(receiver); + drop(stack.pop().expect("receiver checked above")); drop(old); - if marked { - gc_trace::mark_maybe_dead(); - return CoreAttr::Marked; - } CoreAttr::Done } OpCode::LoadMethodAttr => { @@ -17342,11 +15096,6 @@ impl Interpreter { }; let cls = std::mem::replace(top, v); stack.push(Object::Unbound); - if gc_trace::note_dropped_marks(&cls) { - gc_trace::mark_maybe_dead(); - drop(cls); - return CoreAttr::Marked; - } drop(cls); return CoreAttr::Done; } @@ -17408,16 +15157,8 @@ impl Interpreter { let Some(v) = self.leaf_type_call(ty, &stack[first..]) else { return CoreAttr::Decline; }; - let mut marked = false; - for arg in stack.drain(self_slot..) { - marked |= gc_trace::note_dropped_marks(&arg); - } - stack.pop(); // the callee + stack.truncate(callee_slot); stack.push(v); - if marked { - gc_trace::mark_maybe_dead(); - return CoreAttr::Marked; - } return CoreAttr::Done; } let r = if argc == 0 @@ -17455,24 +15196,12 @@ impl Interpreter { let stack = &mut frame.stack; match r { Ok(v) => { - // The operands leave the stack: grade each one (the callee - // is a builtin: a leaf). A registered native body marks for - // any reference it releases itself (its registration - // contract). - let mut marked = false; + // The operands leave the stack. for arg in stack.drain(self_slot..) { - marked |= gc_trace::note_dropped_marks(&arg); drop_operand(arg); } - if matches!(stack[callee_slot], Object::BoundMethod(_)) { - marked |= gc_trace::note_dropped_marks(&stack[callee_slot]); - } stack.pop(); // the callee stack.push(v); - if marked { - gc_trace::mark_maybe_dead(); - return CoreAttr::Marked; - } CoreAttr::Done } Err(e) => { @@ -18219,79 +15948,20 @@ impl Interpreter { if matches!(new, Object::Cell(_)) { break; } - // A displaced value the full handler would run through - // the prompt-reap cascade stays there too — except a - // deferred-tracking instance, whose teardown is its - // plain drop (recycled when this was its last ref). - if Self::local_needs_prompt_reap(old) && Self::looks_reapable_temporary(old) { - // A container of nothing but scalars, untracked and - // unwatched, frees nothing that could finalize: the - // plain drop below is its whole teardown, so the - // burst need not end for a cascade that would find - // nothing. This is the shape a `x = [i, i]` loop - // displaces on every iteration. - if !gc_trace::dies_inert(old) { - match old { - Object::Instance(i) if i.dies_by_plain_drop() => { - let Some(v) = stack.pop() else { break }; - if let Object::Instance(i) = - std::mem::replace(&mut locals[slot], v) - { - PyInstance::try_recycle(i); - } - last = pc; - pc += 1; - continue; - } - _ => break, - } - } + if !Self::core_droppable(old) { + break; } let Some(v) = stack.pop() else { break }; let old = std::mem::replace(&mut locals[slot], v); last = pc; pc += 1; - // The usual displaced value is an unboxed scalar: no - // heap to release, nothing to grade (`Object` has no - // `Drop` of its own). - if matches!( - old, - Object::Int(_) - | Object::Float(_) - | Object::Bool(_) - | Object::None - | Object::Unbound - ) { - std::mem::forget(old); - } else if gc_trace::note_dropped_marks(&old) { - gc_trace::mark_maybe_dead(); - drop(old); - stop = LeafStop::Marked; - break; - } + self.release(old); } OpCode::PopTop => { - // Only pure leaves: every other discard runs the full - // handler's prompt-reap gate. match stack.last() { - Some( - Object::None - | Object::Bool(_) - | Object::Int(_) - | Object::Float(_) - | Object::Str(_), - ) => { - drop_operand(stack.pop().expect("checked above")); - } - Some(Object::Instance(_)) => { - if !Self::leaf_pop_instance(stack) { - break; - } - } - Some(Object::List(l)) if Rc::strong_count(l) == 2 => { - if !Self::leaf_pop_dead_list(stack) { - break; - } + Some(v) if Self::core_droppable(v) => { + let v = stack.pop().expect("checked above"); + self.release(v); } _ => break, } @@ -18545,12 +16215,6 @@ impl Interpreter { if is_none == (ins.op == OpCode::PopJumpIfNone) { pc += ins.arg as usize; } - if gc_trace::note_dropped_marks(&v) { - gc_trace::mark_maybe_dead(); - drop(v); - stop = LeafStop::Marked; - break; - } } OpCode::JumpForward => { last = pc; @@ -18617,7 +16281,6 @@ impl Interpreter { } // The iterator leaves the stack (a full handler // takes the coarse mark for it). - gc_trace::mark_maybe_dead(); drop(it); stop = LeafStop::Marked; break; @@ -18680,9 +16343,7 @@ impl Interpreter { // plain drop. range_done = it_unique && owner.is_none() - && items.try_borrow().is_ok_and(|xs| *index >= xs.len()) - && Rc::strong_count(items) >= 2 - && !Self::iter_backing_list_dead(items); + && items.try_borrow().is_ok_and(|xs| *index >= xs.len()); } v } @@ -18735,8 +16396,6 @@ impl Interpreter { // An untracked iterator holds nothing whose death it // could cause (its list survives, see above); any // other is graded like every burst drop. - let marked = gc_trace::maybe_tracked(crate::weakref_registry::id_of(&it)) - && gc_trace::note_dropped_marks(&it); drop(it); last = pc; pc += 1 + ins.arg as usize; @@ -18749,11 +16408,6 @@ impl Interpreter { pc += 1; } } - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } continue; }; stack.push(v); @@ -18897,15 +16551,9 @@ impl Interpreter { t.insert(k.clone(), v.clone()); } drop((t, s)); - let other = stack.pop().expect("length checked above"); + drop(stack.pop().expect("length checked above")); last = pc; pc += 1; - if gc_trace::note_dropped_marks(&other) { - gc_trace::mark_maybe_dead(); - drop(other); - stop = LeafStop::Marked; - break; - } } OpCode::Swap => { let depth = ins.arg as usize; @@ -19062,15 +16710,9 @@ impl Interpreter { _ => break, }; stack.truncate(n - 1); - let container = std::mem::replace(&mut stack[n - 2], r); + stack[n - 2] = r; last = pc; pc += 1; - if gc_trace::note_dropped_marks(&container) { - gc_trace::mark_maybe_dead(); - drop(container); - stop = LeafStop::Marked; - break; - } } OpCode::StoreSubscr => { // Stack: value, container, key (TOS). `list[int] = v` @@ -19096,11 +16738,6 @@ impl Interpreter { break; } let slot = &mut xs[i as usize]; - if Self::local_needs_prompt_reap(slot) - && Self::looks_reapable_temporary(slot) - { - break; - } let v = std::mem::replace(value_slot, Object::Unbound); std::mem::replace(slot, v) } @@ -19116,11 +16753,6 @@ impl Interpreter { }; let (old, changed) = match d.get_mut(&probe) { Some(slot) => { - if Self::local_needs_prompt_reap(slot) - && Self::looks_reapable_temporary(slot) - { - break; - } let v = std::mem::replace(value_slot, Object::Unbound); let old = std::mem::replace(slot, v); let changed = !old.is_same(slot); @@ -19149,15 +16781,8 @@ impl Interpreter { drop(key); last = pc; pc += 1; - let marked = gc_trace::note_dropped_marks(&container) - | gc_trace::note_dropped_marks(&old); drop(container); drop(old); - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } OpCode::LoadAttr => { // The instance-dict, slot, and module cache hits: a @@ -19196,15 +16821,9 @@ impl Interpreter { }, }; let Some(top) = stack.last_mut() else { break }; - let receiver = std::mem::replace(top, v); + *top = v; last = pc; pc += 1; - if gc_trace::note_dropped_marks(&receiver) { - gc_trace::mark_maybe_dead(); - drop(receiver); - stop = LeafStop::Marked; - break; - } } OpCode::LoadSuperAttr => { // Zero-argument `super().name` where the name resolves @@ -19343,19 +16962,11 @@ impl Interpreter { let Some(v) = self.leaf_type_call(ty, &stack[first..]) else { break; }; - let mut marked = false; - for arg in stack.drain(self_slot..) { - marked |= gc_trace::note_dropped_marks(&arg); - } + stack.truncate(self_slot); stack.pop(); // the callee stack.push(v); last = pc; pc += 1; - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } continue; } let r = if argc == 0 @@ -19396,23 +17007,13 @@ impl Interpreter { // (the callee is a builtin: a leaf). A registered // native body marks for any reference it // releases itself (its registration contract). - let mut marked = false; for arg in stack.drain(self_slot..) { - marked |= gc_trace::note_dropped_marks(&arg); drop_operand(arg); } - if matches!(stack[callee_slot], Object::BoundMethod(_)) { - marked |= gc_trace::note_dropped_marks(&stack[callee_slot]); - } stack.pop(); // the callee stack.push(v); last = pc; pc += 1; - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } Err(e) => { // As the full handler: the staged operands drop @@ -19443,15 +17044,8 @@ impl Interpreter { let receiver = stack.pop().expect("receiver checked above"); last = pc; pc += 1; - let marked = gc_trace::note_dropped_marks(&receiver) - | old.as_ref().is_some_and(gc_trace::note_dropped_marks); drop(receiver); drop(old); - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } OpCode::UnpackSequence => { let n = ins.arg as usize; @@ -19467,16 +17061,10 @@ impl Interpreter { _ => None, }; let Some(items) = pushed else { break }; - let seq = stack.pop().expect("sequence checked above"); + drop(stack.pop().expect("sequence checked above")); stack.extend(items); last = pc; pc += 1; - if gc_trace::note_dropped_marks(&seq) { - gc_trace::mark_maybe_dead(); - drop(seq); - stop = LeafStop::Marked; - break; - } } OpCode::BuildTuple => { // Tuples are never collector-tracked at build time and @@ -19524,15 +17112,9 @@ impl Interpreter { }; let it = Object::Iter(Rc::new(RefCell::new(it))); let Some(top) = stack.last_mut() else { break }; - let v = std::mem::replace(top, it); + *top = it; last = pc; pc += 1; - if gc_trace::note_dropped_marks(&v) { - gc_trace::mark_maybe_dead(); - drop(v); - stop = LeafStop::Marked; - break; - } } OpCode::UnaryOp => { // SAFETY: as in the full handler. @@ -19593,15 +17175,8 @@ impl Interpreter { let a = std::mem::replace(&mut stack[n - 2], Object::Bool(r)); last = pc; pc += 1; - let marked = - gc_trace::note_dropped_marks(&a) | gc_trace::note_dropped_marks(&b); drop(a); drop(b); - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } OpCode::ContainsOp => { let n = stack.len(); @@ -19616,15 +17191,8 @@ impl Interpreter { let item = std::mem::replace(&mut stack[n - 2], Object::Bool(r)); last = pc; pc += 1; - let marked = gc_trace::note_dropped_marks(&item) - | gc_trace::note_dropped_marks(&container); drop(item); drop(container); - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } OpCode::LoadDeref => { let v = match cells.get(ins.arg as usize) { @@ -19643,14 +17211,7 @@ impl Interpreter { let Some(cell) = cells.get(ins.arg as usize) else { break; }; - let reap = match cell.try_borrow() { - Ok(old) => { - Self::local_needs_prompt_reap(&old) - || matches!(*old, Object::Function(_)) - } - Err(_) => true, - }; - if reap { + if cell.try_borrow().is_err() { break; } let Some(v) = stack.pop() else { break }; @@ -19658,14 +17219,9 @@ impl Interpreter { Ok(mut slot) => std::mem::replace(&mut *slot, v), Err(_) => break, }; + drop(old); last = pc; pc += 1; - if gc_trace::note_dropped_marks(&old) { - gc_trace::mark_maybe_dead(); - drop(old); - stop = LeafStop::Marked; - break; - } } _ => break, } @@ -19996,20 +17552,12 @@ impl Interpreter { fast } LeafKind::Opaque => { - if Self::looks_reapable_temporary(&args[0]) { - return None; - } return Some((builtin.call)(args)); } _ => return None, } } }; - // The full handler promptly reaps a consumed temporary and any - // finalizable contents. Decline before the read can do work. - if Self::looks_reapable_temporary(&args[0]) { - return None; - } let completed = fast(args)?; #[cfg(test)] NATIVE_FAST_SUBSCRIPTS.with(|calls| calls.set(calls.get() + 1)); @@ -20420,11 +17968,6 @@ impl Interpreter { slot.set_non_leaf(bm); return None; } - // Collector handles and weakref-registry clones aren't program - // owners. Use the same conservative proof as prompt reaping. - if Self::looks_reapable_temporary(callable) { - return None; - } let Some(result) = self.leaf_builtin_call(kind, b, std::slice::from_ref(&bm.receiver)) else { // The same builtin can serve an exact receiver or an excluded @@ -21401,11 +18944,6 @@ impl Interpreter { let displaced = match slots.get_index_mut(key_idx as usize) { Some((key, slot)) if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => { - if Self::local_needs_prompt_reap(slot) - && Self::looks_reapable_temporary(slot) - { - return None; - } let val = std::mem::replace(value_slot, Object::Unbound); inst.note_slot_store(&val); Some(std::mem::replace(slot, val)) @@ -21471,15 +19009,10 @@ impl Interpreter { { // SAFETY: a leaf store runs no code while the // views below are live. - let (k, old) = unsafe { inst.attr_peek_index(key_idx as usize) }?; + let (k, _) = unsafe { inst.attr_peek_index(key_idx as usize) }?; if !slot_name_matches(code, name_idx, k) { return None; } - if Self::local_needs_prompt_reap(old) - && Self::looks_reapable_temporary(old) - { - return None; - } } // As the core loop's arm: an in-place value store. // SAFETY: as above. @@ -21494,18 +19027,8 @@ impl Interpreter { let Some(Object::Str(shared)) = code_name_obj(code, name_idx) else { return None; }; - { - // SAFETY: as above. - let split = unsafe { inst.dict.split_cell().peek() }?; - if let Some(i) = split.position(shared) { - let old = &split.values()[i]; - if Self::local_needs_prompt_reap(old) - && Self::looks_reapable_temporary(old) - { - return None; - } - } - } + // SAFETY: as above. + unsafe { inst.dict.split_cell().peek() }?; let val = std::mem::replace(value_slot, Object::Unbound); match inst.split_store(shared, val) { Ok(old) => old, @@ -21548,11 +19071,6 @@ impl Interpreter { &mut **d }; if let Some((_, slot)) = found.and_then(|i| d.get_index_mut(i)) { - if Self::local_needs_prompt_reap(slot) - && Self::looks_reapable_temporary(slot) - { - return None; - } let val = std::mem::replace(value_slot, Object::Unbound); let old = std::mem::replace(slot, val); // The name is in the dict: the indexed shape @@ -21594,57 +19112,6 @@ impl Interpreter { Some(displaced) } - /// `POP_TOP` of an instance whose tracking is still deferred: it - /// holds only atomic values and has no weakref or collector handle, - /// so its last reference frees (here: recycles) it. `false` leaves - /// the discard to the full handler's prompt-reap gate. Out of line - /// to keep the burst's `POP_TOP` arm as small as it was. - #[inline(never)] - fn leaf_pop_instance(stack: &mut Vec) -> bool { - match stack.last() { - Some(Object::Instance(i)) if i.dies_by_plain_drop() => { - if let Some(Object::Instance(i)) = stack.pop() { - PyInstance::try_recycle(i); - } - true - } - _ => false, - } - } - - /// `POP_TOP` of a list held only by the stack and its collector - /// handle, with no weakref and only atomic elements (a discarded - /// comprehension, a `list(range(...))` temporary): untracked and - /// freed here, the way CPython's decref frees it, instead of the - /// prompt-reap cascade. `false` leaves the discard to the full handler. - #[inline(never)] - fn leaf_pop_dead_list(stack: &mut Vec) -> bool { - let Some(Object::List(l)) = stack.last() else { - return false; - }; - let id = Rc::as_ptr(l) as usize as u64; - if crate::weakref_registry::may_have_weakrefs(id) - || crate::stdlib::testinternalcapi_mod::reftrace_print_active() - { - return false; - } - { - let Ok(items) = l.try_borrow() else { - return false; - }; - if items.len() > 64 || !items.iter().all(Object::is_gc_atomic) { - return false; - } - } - if !gc_trace::is_tracked(id) { - return false; - } - let obj = stack.pop().expect("checked above"); - gc_trace::untrack(&obj); - drop(obj); - true - } - /// Float arithmetic the leaf burst may run inline: the operators that /// cannot raise for the given operands. Same results as the generic /// `binary_op` float arms (`fresh_float` NaN identity included). @@ -21784,7 +19251,6 @@ impl Interpreter { specialize::record_dispatch(); frame.pc = pc + 1; let v = frame.pop()?; - gc_trace::note_dropped(&v); if matches!(v, Object::None) == (ins.op == OpCode::PopJumpIfNone) { frame.pc += ins.arg; } @@ -21970,7 +19436,7 @@ impl Interpreter { // Without this, the temp stayed pinned by its weakref // slot's strong clone until the next cyclic collection. if matches!(&v, Object::Instance(_)) { - self.prompt_reap_dropped(v); + self.release(v); } Ok(()) } @@ -22072,7 +19538,7 @@ impl Interpreter { // container as STORE_SUBSCR retires its stack operand, // and its weakrefs must clear on that refcount edge. if matches!(&target, Object::Instance(_)) { - self.prompt_reap_dropped(target); + self.release(target); } Ok(()) } @@ -22378,7 +19844,7 @@ impl Interpreter { // the binding's removal from `prompt_reap_dropped`'s // refcount test. self.sync_py_locals(frame); - self.prompt_reap_dropped(old); + self.release(old); } } OpCode::StoreName => { @@ -22452,9 +19918,7 @@ impl Interpreter { } }; if let Some(old) = old { - if Self::local_needs_prompt_reap(&old) { - self.prompt_reap_dropped(old); - } + self.release(old); } } } @@ -22484,9 +19948,7 @@ impl Interpreter { // `__del__` now, matching the `DeleteGlobal` path and CPython's // decref-on-store (module-scope `handle = None`). if let Some(old) = old { - if Self::local_needs_prompt_reap(&old) { - self.prompt_reap_dropped(old); - } + self.release(old); } } OpCode::DeleteName => { @@ -22517,7 +19979,7 @@ impl Interpreter { .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); if let Some(old) = old { - self.prompt_reap_dropped(old); + self.release(old); } } OpCode::DeleteGlobal => { @@ -22527,7 +19989,7 @@ impl Interpreter { .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); if let Some(old) = old { - self.prompt_reap_dropped(old); + self.release(old); } } OpCode::LoadDeref => { @@ -22565,10 +20027,7 @@ impl Interpreter { // `_formatArray` breaks its recurser cycle this way and // test_arrayprint's test_refcount asserts the array's // refcount recovers without a collect (RFC 0076 WS1). - if Self::local_needs_prompt_reap(&old) || matches!(old, Object::Function(_)) { - self.sync_py_locals(frame); - self.prompt_reap_dropped(old); - } + self.release(old); } OpCode::DeleteDeref => { // `del NAME` for a cell/free var empties the cell WITHOUT @@ -22583,7 +20042,7 @@ impl Interpreter { return Err(Self::unbound_deref(&frame.code, ins.arg as usize)); } let old = std::mem::replace(&mut *cell.borrow_mut(), Object::Unbound); - self.prompt_reap_dropped(old); + self.release(old); } OpCode::MakeCell => { let slot = ins.arg as usize; @@ -22658,13 +20117,13 @@ impl Interpreter { | Object::Coroutine(_) | Object::AsyncGenerator(_) ) { - self.prompt_reap_dropped(r); + self.release(r); } } } Err(e) => { if let Some(r) = receiver { - self.prompt_reap_dropped(r); + self.release(r); } return Err(e); } @@ -22819,7 +20278,7 @@ impl Interpreter { frame.push(r); // Operand retirement, as for BINARY_OP. if matches!(&v, Object::Instance(_)) { - self.prompt_reap_dropped(v); + self.release(v); } } OpCode::CompareOp => { @@ -22833,8 +20292,6 @@ impl Interpreter { // RFC 0077 (WS2): audited operand drops (exempt from the // coarse stack-shrink mark). `x is None` grades one leaf and // one live binding and schedules nothing. - gc_trace::note_dropped(&a); - gc_trace::note_dropped(&b); frame.push(Object::Bool(result)); } OpCode::ContainsOp => { @@ -22854,11 +20311,7 @@ impl Interpreter { // next cyclic collection (pdb's `'__return__' in // frame.f_locals` leaked one FrameLocalsProxy per stop — // test_pdb's test_pdb_frame_refleak counts the referrers). - if Self::local_needs_prompt_reap(&container) - && Self::looks_reapable_temporary(&container) - { - self.prompt_reap_dropped(container); - } + self.release(container); let result = if ins.arg == 1 { !found } else { found }; frame.push(Object::Bool(result)); } @@ -22910,9 +20363,6 @@ impl Interpreter { // or the callee's own frame-exit sweep for moved // arguments); the raise path drops the staged operands // wholesale on unwind, so it keeps the coarse mark. - if r.is_err() { - gc_trace::mark_maybe_dead(); - } r?; } OpCode::PushNull => { @@ -23026,13 +20476,13 @@ impl Interpreter { | Object::Coroutine(_) | Object::AsyncGenerator(_) ) { - self.prompt_reap_dropped(r); + self.release(r); } } } Err(e) => { if let Some(r) = receiver { - self.prompt_reap_dropped(r); + self.release(r); } return Err(e); } @@ -23155,7 +20605,7 @@ impl Interpreter { let callable = frame.pop()?; let r = self.run_py_exact_nofree(&f, locals)?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); return Ok(StepOutcome::Continue); } None => { @@ -23249,7 +20699,7 @@ impl Interpreter { let f = f.clone(); let r = self.call_python_owned(&f, std::mem::take(&mut pos_args), kw_pairs)?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); return Ok(StepOutcome::Continue); } let mut kw_pairs = kw_pairs; @@ -23277,7 +20727,7 @@ impl Interpreter { } let r = result?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut pos_args); for (_, v) in &mut kw_pairs { self.reap_call_args(std::slice::from_mut(v)); @@ -23320,7 +20770,7 @@ impl Interpreter { if let (Object::Instance(inst), Object::Tuple(_)) = (&callable, &args_obj) { if crate::stdlib::testcapi_call::instance_call_returns_args_tuple(inst) { frame.push(args_obj); - self.reap_call_receiver(callable); + self.release(callable); return Ok(StepOutcome::Continue); } } @@ -23411,7 +20861,7 @@ impl Interpreter { let r = self.call_c_profiled(&ctor, &call_args, &[], &frame.globals)?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut pos_args); return Ok(StepOutcome::Continue); } @@ -23436,7 +20886,7 @@ impl Interpreter { let r = self.call_c_profiled(&raw, &call_args, &[], &frame.globals)?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut pos_args); return Ok(StepOutcome::Continue); } @@ -23568,7 +21018,7 @@ impl Interpreter { } let r = result?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut pos_args); for (_, v) in &mut kw_pairs { self.reap_call_args(std::slice::from_mut(v)); @@ -23587,10 +21037,10 @@ impl Interpreter { // deadness guard inside leaves a still-referenced mapping // untouched, so a user-held dict passed as `**d` is safe. if let Some(kw) = kwargs_obj { - self.prompt_reap_dropped(kw); + self.release(kw); } // Same for the `*args` sequence temporary. - self.prompt_reap_dropped(args_obj); + self.release(args_obj); } OpCode::ReturnValue => { return Ok(StepOutcome::Return(frame.pop()?)); @@ -23603,14 +21053,12 @@ impl Interpreter { } OpCode::PopJumpIfNone => { let v = frame.pop()?; - gc_trace::note_dropped(&v); if matches!(v, Object::None) { frame.pc += ins.arg; } } OpCode::PopJumpIfNotNone => { let v = frame.pop()?; - gc_trace::note_dropped(&v); if !matches!(v, Object::None) { frame.pc += ins.arg; } @@ -23659,37 +21107,6 @@ impl Interpreter { // CPython frees a *temporary* iterable's elements by refcount // the instant the exhausted FOR_ITER pops the iterator (the // iterator held the last reference to the list). Our - // `next_value` detaches the backing list on exhaustion with a - // plain `Rc` drop, which skips finalizers — so a - // `for x in f():` loop over a list of `__del__`-bearing - // objects never ran them promptly (test_interpreters - // test_channels.test_list_all_closed: ChannelID handles from - // `_channels.list_all()` must decref their channels at loop - // end). Harvest a uniquely-held backing list's elements - // before the detach and run them through the reap cascade. - let harvested: Option> = match &it_obj { - Object::Iter(it) => { - let guard = it.borrow(); - if let crate::object::PyIterator::List { - items, - index, - owner, - } = &*guard - { - if owner.is_none() - && *index >= items.borrow().len() - && Self::iter_backing_list_dead(items) - { - Some(std::mem::take(&mut *items.borrow_mut())) - } else { - None - } - } else { - None - } - } - _ => None, - }; let next = match &it_obj { Object::Iter(it) => it.borrow_mut().next_value_checked()?, Object::LazyIter(l) => { @@ -23763,9 +21180,6 @@ impl Interpreter { frame.pop()?; frame.pc += ins.arg; frame.skip_end_for(); - if let Some(items) = harvested { - self.reap_iter_backing(items); - } } } } @@ -24899,104 +22313,14 @@ impl Interpreter { // discarded). let _prev = frame.pop()?; let popped = frame.exc_handlers.pop(); - // Drop the interpreter-wide `sys.exc_info()` entry first so - // its clone of the instance is gone before we test liveness. drop(self.exc_info_stack.borrow_mut().pop()); // CPython clears the just-handled exception at the end of an // `except` block (the implicit `del` of the bound name plus - // the per-frame exc-state pop). When that exception held the - // last reference to a finalizable object — `AttributeError.obj`, - // or an exception wrapping a file/process/lock in its args — - // its `__del__` must run *now*, matching CPython's refcount - // timing rather than waiting for the next cyclic collection - // (RFC 0040 deterministic-finalization arc: `test_io` / - // `test_subprocess` destructor-timing cases). + // the per-frame exc-state pop): it dies here unless something + // else holds it. self.recheck_frame_observed = true; if let Some((_, pe)) = popped { - // Deconstruct the handler's `PyException` *first*: its - // `context`/`cause` boxes and traceback entries hold - // their own strong clones of the chained exceptions and - // frames. Held across the reap below, they make the - // `__context__` chain look externally referenced and the - // cascade stops at the outer instance — leaving e.g. the - // `CancelledError` behind asyncio's `TimeoutError` (and - // every frame its traceback pins) to the next full - // collection (test_ssl.test_handshake_timeout_handler_leak). - let PyException { - instance, - traceback, - context, - cause, - .. - } = pe; - drop(traceback); - drop(context); - drop(cause); - let pe_instance = instance; - // A handled exception is itself an untracked, - // non-finalizable leaf, so it slips past the fast-path - // early-return in `prompt_reap_dropped`; when it anchors a - // finalizable object (`AttributeError.obj`, an exception - // wrapping a file/process) drive the cascade directly so - // that `__del__` runs at handler exit like CPython. - // - // A *weakref* to the exception (or, equivalently, to a - // value reachable only through it) must likewise observe - // the death now: CPython frees the just-handled exception - // by refcount the instant the `except` block ends, so - // `weakref.ref(e)()` is already `None` on the next line — - // even with the cycle GC disabled. `test_multiprocessing`'s - // `TestManagerExceptions` (bpo-106558, "Manager exceptions - // avoid creating cyclic references") asserts exactly this - // under `gc.disable()` with a *non-finalizable* `Empty()` - // /`RemoteError`, which the finalizable-only pre-check would - // leave pinned on its GC handle until the next collection. - // - // Both pre-checks keep the hot path cheap: the - // overwhelmingly common scalar-arg exception that nothing - // weakly references and has nothing to finalize skips the - // allocating cascade entirely (the O(1) `count_for` probe - // short-circuits before the depth-bounded - // `exc_has_finalizable` walk). Binding the exception to an - // outer name (`except E as e: saved = e`) or a pending - // re-raise keeps its refcount above the dead threshold, and - // every cascade node is independently refcount-guarded, so - // the reap is a no-op in those cases too. - // A GC-*tracked* exception must also reap: dead, it is - // otherwise pinned by its collector handle until the - // next cycle collection — and through its traceback it - // pins every frame (and their locals) it unwound. - // `pickle.load` ends with `except _Stop: return - // stopinst.value`, so a lingering `_Stop` kept the - // whole `_Unpickler` (memo included, i.e. every - // unpickled object) alive; `concurrent.futures`' - // `_process_worker` then never finalized call-item - // arguments after `del call_item` - // (test_concurrent_futures - // `test_ressources_gced_in_workers`). - let id = crate::weakref_registry::id_of(&pe_instance); - let weakly_observed = crate::weakref_registry::count_for(id) > 0; - // `anchors_tracked_child` too: an *untracked* exception - // can be the last holder of a GC-tracked one — via - // `__cause__`/`__context__`, or via its traceback's - // frames — asyncio's `TimeoutError` raised `from` the - // tracked `CancelledError` whose traceback pins the - // whole cancelled call chain (frames, transports, the - // SSLContext). A plain `Rc` drop frees the outer - // exception but leaves the tracked descendants on their - // collector handles until the next full collection - // (test_ssl.test_handshake_timeout_handler_leak). Only - // probed when the exception is actually about to die, - // so an `except E as e: saved = e` pays nothing. - if weakly_observed - || gc_trace::is_tracked(id) - || (!Self::exc_plainly_inert(&pe_instance) - && (Self::exc_has_finalizable(&pe_instance, 6) - || (Self::is_refcount_dead(&pe_instance, 1) - && Self::anchors_tracked_child(&pe_instance, 5)))) - { - self.reap_dead_subgraph(pe_instance); - } + self.release(pe.instance); } } OpCode::PrepReraiseStar => { @@ -25028,8 +22352,8 @@ impl Interpreter { // test_exception_refcycles_errors asserts // `gc.get_referrers(exc) == []` without a collect). drop(items); - self.prompt_reap_dropped(excs); - self.prompt_reap_dropped(orig); + self.release(excs); + self.release(orig); } OpCode::Reraise => { // CPython 3.13 `RERAISE n`: the exception is TOS; when @@ -26544,20 +23868,9 @@ impl Interpreter { let Some(Object::Dict(d)) = inst.native.get() else { return true; }; - // The instance's own slot (+ the collector's handle, if the - // dict is tracked) is the "unreferenced" baseline for the dict. - let dict_id = crate::weakref_registry::id_of(&Object::Dict(d.clone())); - let dict_allow = 1 + usize::from(gc_trace::is_tracked(dict_id)); - if Rc::strong_count(d) > dict_allow { - return true; - } - let obj = Object::Instance(inst.clone()); - let id = crate::weakref_registry::id_of(&obj); - // This table + `obj` + the collector's handle + weakref slots. - let allow = 2 - + usize::from(gc_trace::is_tracked(id)) - + crate::weakref_registry::strong_clone_count(id); - Rc::strong_count(inst) > allow + // The instance's own slot is the "unreferenced" baseline for + // the dict, and this table's for the instance. + Rc::strong_count(d) > 1 || Rc::strong_count(inst) > 1 }); } @@ -33039,7 +30352,7 @@ impl Interpreter { }; // The temporary result is released before hash() returns, including // an integer subclass with a finalizer. External references survive. - self.maybe_prompt_reap_replaced(result); + self.release(result); match value { Some(value) => Ok(Object::Int(if value == -1 { -2 } else { value })), None => Err(type_error("__hash__ method should return an integer")), @@ -33748,10 +31061,10 @@ impl Interpreter { if reverse { decorated.reverse(); } - // Undecorate, then promptly reap the dying key objects: CPython - // decrefs the keys inside `list.sort` while the list is still - // detached, so a key `__del__` that mutates the list is caught by - // the caller's mutation check (test_sort.test_key_with_mutating_del). + // Undecorate, then release the keys and run their finalizers now: + // CPython decrefs the keys inside `list.sort` while the list is + // still detached, so a key `__del__` that mutates the list is caught + // by the caller's mutation check (test_sort.test_key_with_mutating_del). let mut dead_keys: Vec = Vec::with_capacity(decorated.len()); *items = decorated .into_iter() @@ -33760,18 +31073,8 @@ impl Interpreter { v }) .collect(); - for k in dead_keys { - if matches!( - k, - Object::Instance(_) - | Object::Generator(_) - | Object::Coroutine(_) - | Object::AsyncGenerator(_) - ) && Self::is_refcount_dead(&k, 1) - { - self.reap_dead_subgraph(k); - } - } + drop(dead_keys); + self.run_pending_finalizers(); result } @@ -36924,21 +34227,7 @@ impl Interpreter { return Ok(Object::None); } } - // Snapshot reapable locals/stack before the frame is torn down: - // CPython's refcounting runs their `__del__` (and clears their - // weakrefs) the moment `close()` drops the frame (gh-142766), - // and tests assert on that promptness. - let frame_caps = match receiver { - Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { - frame_reapables(g) - } - _ => Vec::new(), - }; - let result = self.gen_method_close_inner(receiver); - for cap in frame_caps { - self.prompt_reap_dropped(cap); - } - result + self.gen_method_close_inner(receiver) } /// CPython `gen_close`'s fast paths: a never-started generator, or @@ -36991,7 +34280,6 @@ impl Interpreter { else { unreachable!("checked above"); }; - self.reap_dead_frame(&mut boxed); self.recycle_frame_allocs(&mut boxed); Self::release_finished_gen(g); drop(boxed); @@ -37455,11 +34743,6 @@ impl Interpreter { && !frame.sent_consumed { self.gil_countdown = crate::gil::GIL_CHECK_INTERVAL; - if !gc_trace::active_suspects_present() && gc_trace::has_suspects() { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } crate::gil::yield_checkpoint(); if crate::hot_gates::loop_gen() != snap_gen { break; @@ -37475,8 +34758,6 @@ impl Interpreter { fold.map(|sink| (std::ptr::from_mut(frame) as usize, sink)), ); let exc_depth_on_entry = self.exc_info_len(); - let fin_live = gc_trace::has_any_finalizable(); - let fin = fin_live || gc_trace::active_suspects_present(); let mut act = LeanAct { frame: std::ptr::from_mut(frame), shell: None, @@ -37484,24 +34765,10 @@ impl Interpreter { let mut prev_pc: Option = None; let run = if guard.depth() % 4 == 0 { stacker::maybe_grow(512 * 1024, 8 * 1024 * 1024, || { - self.quiet_run( - frame, - QuietShell::Lazy(&mut act), - snap_gen, - fin, - fin_live, - &mut prev_pc, - ) + self.quiet_run(frame, QuietShell::Lazy(&mut act), snap_gen, &mut prev_pc) }) } else { - self.quiet_run( - frame, - QuietShell::Lazy(&mut act), - snap_gen, - fin, - fin_live, - &mut prev_pc, - ) + self.quiet_run(frame, QuietShell::Lazy(&mut act), snap_gen, &mut prev_pc) }; self.sum_fold = prev_fold; match (run, act.shell) { @@ -37552,7 +34819,6 @@ impl Interpreter { Ok(FrameOutcome::Returned(v)) => { *gen.state.borrow_mut() = GeneratorState::Finished; let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(gen); Err(stop_iteration_with(v)) @@ -37567,7 +34833,6 @@ impl Interpreter { *gen.state.borrow_mut() = GeneratorState::Finished; let escaped = self.pep479_escape(gen, err); let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(gen); Err(escaped) @@ -37729,7 +34994,6 @@ impl Interpreter { // "")`. *gen.state.borrow_mut() = GeneratorState::Finished; let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); // RFC 0059 WS4: the exhausted frame's storage is dead — // donate it back to the frame pools. self.recycle_frame_allocs(frame); @@ -37746,7 +35010,6 @@ impl Interpreter { *gen.state.borrow_mut() = GeneratorState::Finished; let escaped = self.pep479_escape(gen, err); let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(gen); Err(escaped) @@ -39777,7 +37040,7 @@ impl Interpreter { normalize_index_msg(idx, xs.len(), "list assignment index out of range")?; std::mem::replace(&mut xs[i], value) }; - crate::vm_singletons::queue_container_removed(&evicted); + drop(evicted); specialize::record_hit(op_idx); Ok(true) } @@ -39790,10 +37053,7 @@ impl Interpreter { frame.stack.truncate(len - 2); let value = frame.pop()?; crate::builtins::ensure_dict_key(&key)?; - let old = crate::builtins::dict_insert(&d, key, value)?; - if let Some(old) = old { - crate::vm_singletons::queue_container_removed(&old); - } + drop(crate::builtins::dict_insert(&d, key, value)?); specialize::record_hit(op_idx); Ok(true) } @@ -40341,7 +37601,6 @@ impl Interpreter { // hits; the slot and generic paths (descriptors, `__setattr__`, // `member_set`) release values out of sight and keep the coarse // mark. - gc_trace::note_dropped(&receiver); match cache { IC::StoreAttrInstance { key_idx, ver } => { if let Object::Instance(inst) = &receiver { @@ -40383,7 +37642,7 @@ impl Interpreter { specialize::record_hit(op_idx); // CPython decrefs the overwritten value now; // reap it if this was its last live binding. - self.maybe_prompt_reap_replaced(old); + self.release(old); return Ok(()); } } @@ -40429,6 +37688,9 @@ impl Interpreter { // (the core loop's `STORE_ATTR` arm serves it). let mut upgrade_idx = false; let mut val = Some(val); + // A value whose store met a stored key only a + // user `__eq__` could equate to the name. + let mut exotic = None; let old = 'store: { // The split layout (see `core_store_new_attr`). if watch_value.is_none() && inst.dict.published().is_none() { @@ -40448,12 +37710,22 @@ impl Interpreter { } else { &mut **dict }; - let probe = code_name_key(code, name_idx); - let slot = match probe { - Some(k) => dict.get_mut(&k), - None => dict.get_mut(&crate::object::StrKey(name.as_str())), + // Never run Python under the borrow: the + // generic store compares an exotic key once, + // with nothing borrowed. + let probe = match code_name_key(code, name_idx) { + Some(k) => crate::object::LeafNameProbe::new(k.s, k.hash), + None => crate::object::LeafNameProbe::new( + name.as_str(), + crate::object::py_str_hash(name.as_str()), + ), }; - if let Some(slot) = slot { + let idx = dict.get_index_of(&probe); + if probe.saw_exotic() { + exotic = Some(val); + break 'store None; + } + if let Some((_, slot)) = idx.and_then(|i| dict.get_index_mut(i)) { let old = std::mem::replace(slot, val); upgrade_idx = true; Some(old) @@ -40470,6 +37742,11 @@ impl Interpreter { None } }; + if let Some(val) = exotic { + frame.push(val); + return self + .deopt_store_attr_taken(frame, cache_pc, name_idx, receiver); + } // Watched instance `__dict__` (test_watchers // test_object_dict): same events as the slow path. if let Some(v) = watch_value { @@ -40483,11 +37760,16 @@ impl Interpreter { } specialize::record_hit(op_idx); if upgrade_idx { - use crate::specialize::DictDataExt; + let probe = crate::object::LeafNameProbe::new( + name.as_str(), + crate::object::py_str_hash(name.as_str()), + ); let idx = inst .dict_cell() .borrow() - .index_of_key_str(name.as_str()) + .get_index_of(&probe) + .filter(|_| !probe.saw_exotic()) + .and_then(|i| u32::try_from(i).ok()) .map(|i| (i, inst.class.borrow().attr_version.get())); if let Some((key_idx, ver)) = idx { frame @@ -40497,7 +37779,7 @@ impl Interpreter { } } if let Some(old) = old { - self.maybe_prompt_reap_replaced(old); + self.release(old); } return Ok(()); } @@ -40539,7 +37821,6 @@ impl Interpreter { } }; drop(old); - gc_trace::mark_maybe_dead(); specialize::record_hit(op_idx); return Ok(()); } @@ -40558,7 +37839,6 @@ impl Interpreter { specialize::record_specialize_success(op_idx); } let val = frame.pop()?; - gc_trace::mark_maybe_dead(); self.store_attr(&receiver, &name, val) } IC::Cooldown(n) => { @@ -40570,13 +37850,11 @@ impl Interpreter { frame.code.caches.set(cache_pc, next); let val = frame.pop()?; let name = self.name_at(&frame.code, name_idx)?; - gc_trace::mark_maybe_dead(); self.store_attr(&receiver, &name, val) } _ => { let val = frame.pop()?; let name = self.name_at(&frame.code, name_idx)?; - gc_trace::mark_maybe_dead(); self.store_attr(&receiver, &name, val) } } @@ -40599,7 +37877,6 @@ impl Interpreter { .set(cache_pc, weavepy_compiler::InlineCache::Cooldown(COOLDOWN)); let val = frame.pop()?; let name = self.name_at(&frame.code, name_idx)?; - gc_trace::mark_maybe_dead(); self.store_attr(&receiver, &name, val) } @@ -40629,12 +37906,7 @@ impl Interpreter { match cache { IC::ForIterList => { let mut it = it_handle.borrow_mut(); - if let crate::object::PyIterator::List { - items, - index, - owner, - } = &mut *it - { + if let crate::object::PyIterator::List { items, index, .. } = &mut *it { let next = items.borrow().get(*index).cloned(); if let Some(v) = next { *index += 1; @@ -40648,24 +37920,11 @@ impl Interpreter { // generic path detaches but this fast path leaks a // live cursor (test_list.test_exhausted_iterator, // test_tier2_invalidates_iterator). - // - // A uniquely-held backing list (the loop iterated a - // temporary) has its elements harvested for the - // prompt-reap cascade — CPython frees them by - // refcount here (see the generic FOR_ITER arm). - let harvested = if owner.is_none() && Self::iter_backing_list_dead(items) { - Some(std::mem::take(&mut *items.borrow_mut())) - } else { - None - }; *items = Rc::new(RefCell::new(Vec::new())); drop(it); frame.pop()?; frame.pc += jump_arg; frame.skip_end_for(); - if let Some(items) = harvested { - self.reap_iter_backing(items); - } } specialize::record_hit(op_idx); return Ok(true); @@ -41660,7 +38919,7 @@ impl Interpreter { // close their files) the instant the log does // (test_tempfile.test_warnings_on_cleanup). if let Some(old) = old { - self.maybe_prompt_reap_replaced(old); + self.release(old); } Ok(()) } @@ -42483,7 +39742,7 @@ impl Interpreter { ); } if let Some(old) = old { - self.maybe_prompt_reap_replaced(old); + self.release(old); } Ok(()) } @@ -42583,7 +39842,7 @@ impl Interpreter { .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); if let Some(old) = removed { - self.maybe_prompt_reap_replaced(old); + self.release(old); Ok(()) } else { Err(attribute_error(format!( @@ -42619,7 +39878,7 @@ impl Interpreter { if let Some(old) = removed { // Same decref-at-removal semantics as the store arm // above / `DeleteGlobal`. - self.maybe_prompt_reap_replaced(old); + self.release(old); Ok(()) } else { Err(attribute_error(format!( @@ -42908,7 +40167,7 @@ impl Interpreter { // `del self._target, self._args, self._kwargs` releases the // thread's hold on its argument cycle without waiting for a GC // pass (`test_threading.test_no_refcycle_through_target`). - self.prompt_reap_dropped(removed); + self.release(removed); Ok(()) } @@ -43453,7 +40712,7 @@ impl Interpreter { normalize_index_msg(*i, items.len(), "list assignment index out of range")?; std::mem::replace(&mut items[idx], value) }; - crate::vm_singletons::queue_container_removed(&evicted); + drop(evicted); Ok(()) } (Object::List(items), Object::Slice(s)) => { @@ -43476,9 +40735,7 @@ impl Interpreter { let mut data = items.borrow_mut(); apply_slice_assignment(&mut data, s, replacement)? }; - for old in &evicted { - crate::vm_singletons::queue_container_removed(old); - } + drop(evicted); // The RHS was only read from — its *elements* were spliced in, // and the RHS container itself is discarded right here. A // literal RHS (`xs[-2:] = [obj]`) is GC-tracked the moment it @@ -43492,17 +40749,12 @@ impl Interpreter { // gh-106558). Elements stay alive through the target list; // an RHS still bound elsewhere fails the deadness test // untouched. - if Self::local_needs_prompt_reap(&value) && Self::looks_reapable_temporary(&value) { - self.prompt_reap_dropped(value); - } + self.release(value); Ok(()) } (Object::Dict(d), key) => { crate::builtins::ensure_dict_key(key)?; - let old = crate::builtins::dict_insert(d, key.clone(), value)?; - if let Some(old) = old { - crate::vm_singletons::queue_container_removed(&old); - } + drop(crate::builtins::dict_insert(d, key.clone(), value)?); Ok(()) } (Object::ByteArray(b), Object::Int(i)) => { @@ -43894,24 +41146,20 @@ impl Interpreter { normalize_index_msg(*i, items.len(), "list assignment index out of range")?; items.remove(idx) }; - crate::vm_singletons::queue_container_removed(&removed); + drop(removed); Ok(()) } (Object::List(items), Object::Slice(s)) => { let removed = apply_slice_deletion(&mut items.borrow_mut(), s)?; - for v in &removed { - crate::vm_singletons::queue_container_removed(v); - } + drop(removed); Ok(()) } (Object::Dict(d), key) => { crate::builtins::ensure_dict_key(key)?; let removed = crate::builtins::dict_remove(d, key)?; - let Some((k, v)) = removed else { + if removed.is_none() { return Err(key_error_object(key.clone())); - }; - crate::vm_singletons::queue_container_removed(&k); - crate::vm_singletons::queue_container_removed(&v); + } Ok(()) } (Object::ByteArray(b), Object::Int(i)) => { @@ -44608,27 +41856,6 @@ impl Interpreter { // drain the queue synchronously, matching CPython // semantics where `gc.collect()` returns *after* // every finaliser has fired. - // Heap introspection must observe CPython's *refcount* - // timing, not our deferred one: an object whose last - // program reference died a few instructions ago may still - // sit on the suspect list (its cascade was skipped while a - // Rust-side transient pinned it — see `note_suspect`). - // CPython would have freed it already, so - // `gc.get_objects()` / `gc.get_referrers()` must not - // enumerate it — worse, the returned snapshot list would - // *pin* the corpse through the caller's inspection loop, - // decaying its probe budget until it is evicted and leaks - // for good (test_taskgroups.test_exception_refcycles_* - // call `gc.get_referrers` right after an `except*` block - // and assert the handled group's ctor list is gone). - // Settle the pending reaps before taking the snapshot. - if matches!(b.name, ".gc.get_objects" | ".gc.get_referrers") - && gc_trace::has_suspects() - { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } if b.name == ".gc.collect" { // `generation` may be positional or the `generation=` // keyword (CPython accepts both). Validate it the way @@ -49864,7 +47091,7 @@ impl Interpreter { // collection). let mut positional = positional; for v in positional.drain(..) { - self.maybe_prompt_reap_replaced(v); + self.release(v); } self.recycle_scratch(positional); return result; @@ -50181,7 +47408,7 @@ impl Interpreter { )?; } frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut args); return Ok(()); } @@ -50532,7 +47759,7 @@ impl Interpreter { // Shared tail (skipped via early `?` only on error): reap the // bound-method receiver and any dead argument temporaries now that // the call has returned and the operands are about to be dropped. - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut args); self.recycle_scratch(args); Ok(()) @@ -57782,54 +55009,6 @@ fn normalize_bool_index(n: Object) -> Object { } } -/// Bit 0 of [`COARSE_DROP_CLASS`]: the opcode's handler grades every -/// reference it releases through `gc_trace::note_dropped` and is exempt -/// from the eval loop's coarse stack-shrink mark. -const COARSE_EXEMPT: u8 = 1; -/// Bit 1 of [`COARSE_DROP_CLASS`]: a stack-neutral `DELETE_*` opcode that -/// releases a binding and counts as a drop regardless of stack depth. -const COARSE_DELETE: u8 = 2; - -/// Per-opcode classification for the coarse maybe-dead mark in -/// `run_until_yield_or_return_impl` (RFC 0077 WS3): one byte load -/// replaces two `Option` match chains on every instruction. -static COARSE_DROP_CLASS: [u8; 256] = { - let mut t = [0u8; 256]; - let exempt = [ - OpCode::PopTop, - OpCode::StoreFast, - OpCode::BinaryOp, - OpCode::CompareOp, - OpCode::Call, - OpCode::ReturnValue, - OpCode::StoreAttr, - OpCode::PopJumpIfFalse, - OpCode::PopJumpIfTrue, - OpCode::PopJumpIfNone, - OpCode::PopJumpIfNotNone, - OpCode::IsOp, - // The yielded value moves to the consumer; nothing is released. - OpCode::YieldValue, - ]; - let mut i = 0; - while i < exempt.len() { - t[exempt[i] as u8 as usize] |= COARSE_EXEMPT; - i += 1; - } - let deletes = [ - OpCode::DeleteFast, - OpCode::DeleteName, - OpCode::DeleteGlobal, - OpCode::DeleteDeref, - ]; - let mut i = 0; - while i < deletes.len() { - t[deletes[i] as u8 as usize] |= COARSE_DELETE; - i += 1; - } - t -}; - /// What [`Interpreter::leaf_resolve_instance_attr`] found. enum LeafAttr { /// A plain value: the instance's own, or a non-descriptor class value. @@ -58237,7 +55416,7 @@ struct SwitchRoots { root: *mut Frame, root_shell: *mut QuietShell<'static>, root_last: *mut usize, - fin: bool, + /// This thread's pending-work flag (see `gc_trace::maybe_dead_flag`). maybe_dead: *const std::cell::Cell, } @@ -58274,7 +55453,6 @@ struct CoreSwitch { /// The last-executed-pc slot of inline activations other than the /// entry (never read back). scratch: usize, - fin: bool, maybe_dead: *const std::cell::Cell, /// This thread's recursion-depth cell (see `recursion::depth_cell`). depth_cell: *const std::cell::Cell, @@ -58308,7 +55486,6 @@ impl CoreSwitch { entry_dead: false, pending: None, scratch: usize::MAX, - fin: roots.fin, maybe_dead: roots.maybe_dead, depth_cell: crate::recursion::depth_cell(), } @@ -58983,7 +56160,8 @@ enum LeafStop { /// A back edge found the GIL countdown spent or the loop generation /// changed: the outer prologue must run. Breaker, - /// The last leaf instruction's discard scheduled a finalization sweep. + /// A dying object queued work (a finalizer, a weakref callback) for + /// the next safe point. Marked, /// A leaf builtin raised; `frame.pc` is past the call. Raised(RuntimeError), @@ -59017,8 +56195,6 @@ enum CoreExit { enum CoreAttr { /// Executed. Done, - /// Executed, and a discard scheduled a finalization sweep. - Marked, /// Not a cache hit: nothing touched. Decline, /// The site's first execution: the full handler must run (and @@ -67411,7 +64587,7 @@ mod tests { static HEAP_SNAPSHOT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); #[test] - fn finalizer_queue_claims_once_and_preserves_drop_fallback() { + fn finalizer_queue_claims_once_and_keeps_identity_on_last_release() { let code = compile_module( &parse_module( r" @@ -67471,15 +64647,17 @@ next(gen) ); inst }; - // An untracked last-Arc drop must still queue and run its copy once. + // The last release moves the reference itself into the queue, so + // the finalizer sees the same object, which dies after it runs. let dropped = drop_instance("drop"); let weak = Rc::downgrade(&dropped); drop(dropped); - assert!(weak.upgrade().is_none()); + assert!(weak.upgrade().is_some()); assert_eq!(interp.run_pending_finalizers(), 1); + assert!(weak.upgrade().is_none()); assert_eq!(events.borrow().len(), 3); - // A queue borrow that prevents Drop from scheduling must release the - // unscheduled copy without recursively resurrecting it again. + // A queue borrow that prevents Drop from scheduling releases the + // object without its finalizer. let rejected = drop_instance("rejected"); let weak = Rc::downgrade(&rejected); crate::vm_singletons::PENDING_FINALIZERS.with(|queue| { diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 77bac087..ffe285c8 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -2026,6 +2026,7 @@ pub fn c_contiguous_strides(shape: &[usize], itemsize: usize) -> Vec { impl Drop for PyMemoryView { fn drop(&mut self) { self.release(); + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); } } @@ -2880,6 +2881,13 @@ pub struct PyModule { pub dict: Rc>, } +/// A dying PyModule clears the weak references watching it. +impl Drop for PyModule { + fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); + } +} + impl fmt::Debug for PyModule { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "", self.name) @@ -4402,6 +4410,13 @@ pub struct PyFunction { pub defaults_override: OverrideFlag, } +/// A dying PyFunction clears the weak references watching it. +impl Drop for PyFunction { + fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); + } +} + /// A clonable set-once flag (see [`PyFunction::defaults_override`]). #[derive(Debug, Default)] pub struct OverrideFlag(std::sync::atomic::AtomicBool); @@ -4578,6 +4593,13 @@ pub struct BoundMethod { pub py_method: bool, } +/// A dying BoundMethod clears the weak references watching it. +impl Drop for BoundMethod { + fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); + } +} + impl BoundMethod { /// A fully-bound method (CPython `method`): calling it prepends /// `receiver` and calls `function` directly. @@ -4827,54 +4849,24 @@ impl fmt::Debug for PyGenerator { impl Drop for PyGenerator { fn drop(&mut self) { - // CPython finalizes a generator the moment its refcount dies: - // a *suspended* frame gets `GeneratorExit` thrown in so - // `finally:`/`with` cleanup runs. We can't run Python from - // `Drop`, so resurrect the live frame into the VM's - // pending-finalizer queue; `gc.collect()` and the module-exit - // path drain it. Created-but-never-started frames have run no - // user code, so (like CPython's `gen_close`) they are simply - // marked completed. - let Ok(mut state) = self.state.try_borrow_mut() else { - return; - }; - let prev = std::mem::replace(&mut *state, GeneratorState::Finished); - drop(state); - match prev { - GeneratorState::Suspended(frame) => { - // Finalized once already (CPython's `_PyGC_FINALIZED` bit): - // drop the frame for real instead of looping forever - // through resurrection → finalize → drop. - if self.finalize_ran.get() { - defer_generator_state_drop(GeneratorState::Suspended(frame)); - return; - } - let resurrected = Rc::new(PyGenerator { - name: RefCell::new(self.name.borrow().clone()), - qualname: RefCell::new(self.qualname.borrow().clone()), - kind: self.kind, - code: self.code.clone(), - state: RefCell::new(GeneratorState::Suspended(frame)), - origin: RefCell::new(self.origin.borrow().clone()), - hooks_inited: crate::sync::Cell::new(self.hooks_inited.get()), - finalizer: RefCell::new(self.finalizer.borrow().clone()), - finalize_ran: crate::sync::Cell::new(self.finalize_ran.get()), - }); - let obj = match self.kind { - CoroutineKind::Generator => Object::Generator(resurrected), - CoroutineKind::Coroutine => Object::Coroutine(resurrected), - CoroutineKind::AsyncGenerator => Object::AsyncGenerator(resurrected), - }; - crate::vm_singletons::try_push_pending_finalizer(obj); + // A suspended generator's `close()` (and a never-awaited + // coroutine's warning) already ran: `rc::Rc`'s `Drop` queued the + // generator for it at its last release. A created or suspended + // frame still owns locals that can hold the *next* generator of a + // pipeline (`chain(chain(chain(…)))`); dropping it inline recurses + // one native stack frame per link and overflows on long chains, so + // route it through the iterative trampoline below. + if let Ok(mut state) = self.state.try_borrow_mut() { + let prev = std::mem::replace(&mut *state, GeneratorState::Finished); + drop(state); + if matches!( + prev, + GeneratorState::Suspended(_) | GeneratorState::Created(_) + ) { + defer_generator_state_drop(prev); } - // A never-started frame still owns locals that can hold the - // *next* generator of a pipeline (`chain(chain(chain(…)))`). - // Dropping it inline recurses one native stack frame per - // link and overflows on long chains, so route it through - // the iterative trampoline below. - state @ GeneratorState::Created(_) => defer_generator_state_drop(state), - GeneratorState::Finished | GeneratorState::Running => {} } + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); } } @@ -7695,6 +7687,7 @@ impl fmt::Debug for PyFile { impl Drop for PyFile { fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); // CPython's fileio/buffered/textio deallocators emit a // `ResourceWarning("unclosed file %R")` when an *open*, fd-backed // stream is reclaimed without an explicit `close()`/`with` @@ -11995,9 +11988,7 @@ pub(crate) fn py_hash_value(obj: &Object) -> Option { // The common case first: a plain instance whose `__hash__` is // `object`'s hashes by identity, with no interpreter round trip. if inst.native.get().is_none() - && inst - .class_dunder(crate::types::Dunder::Hash) - .object_owner() + && inst.class_dunder(crate::types::Dunder::Hash).object_owner() { return Some(identity_hash(obj)); } diff --git a/crates/weavepy-vm/src/rc.rs b/crates/weavepy-vm/src/rc.rs index 34308d73..c2289ee4 100644 --- a/crates/weavepy-vm/src/rc.rs +++ b/crates/weavepy-vm/src/rc.rs @@ -132,13 +132,13 @@ macro_rules! rc_unsize { /// A reference-counted shared pointer with biased counting. See the module /// documentation. #[repr(transparent)] -pub struct Rc(ManuallyDrop>); +pub struct Rc(ManuallyDrop>); /// A weak reference to an [`Rc`] allocation. #[repr(transparent)] pub struct Weak(std::sync::Weak); -impl Rc { +impl Rc { #[inline] pub fn new(value: T) -> Self { Self(ManuallyDrop::new(Arc::new(value))) @@ -167,7 +167,7 @@ impl Rc { } } -impl Rc { +impl Rc { #[inline] pub fn from_arc(arc: Arc) -> Self { Self(ManuallyDrop::new(arc)) @@ -248,14 +248,14 @@ impl Rc { } } -impl Rc { +impl Rc { #[inline] pub fn make_mut(this: &mut Self) -> &mut T { Arc::make_mut(&mut this.0) } } -impl Clone for Rc { +impl Clone for Rc { #[inline(always)] fn clone(&self) -> Self { // SAFETY: `self` keeps the allocation alive, and the new owner @@ -267,20 +267,119 @@ impl Clone for Rc { } } -impl Drop for Rc { +impl Drop for Rc { #[inline(always)] fn drop(&mut self) { // SAFETY: this owner holds one strong reference, released exactly - // once: either here, or by Arc's own drop. + // once: either here, or by Arc's own drop, or by handing it on. unsafe { - if !try_release_shared(Arc::as_ptr(&self.0)) { - ManuallyDrop::drop(&mut self.0); + if try_release_shared(Arc::as_ptr(&self.0)) { + return; } + // The last reference to an object that owes a finalizer moves + // to the finalizer queue instead: the object stays alive, at + // its address, until its `__del__` (or generator close) has + // run, as CPython's `tp_finalize` before `tp_dealloc`. + if finalizable_kind::() && Arc::strong_count(&self.0) == 1 { + let arc = ManuallyDrop::take(&mut self.0); + crate::vm_singletons::finalize_on_last_release(arc); + return; + } + if nesting_kind::() { + release_nested(ManuallyDrop::take(&mut self.0)); + return; + } + ManuallyDrop::drop(&mut self.0); } } } -impl Deref for Rc { +/// Whether `T` is a container payload, whose release can release more +/// containers in turn. Constant per instantiation. +#[inline(always)] +fn nesting_kind() -> bool { + use crate::object::{ + BoundMethod, DictData, FrozenSetObj, PyFrame, PyFunction, PyGenerator, PyIterator, + PyTraceback, SetData, + }; + use crate::sync::RefCell; + use std::any::TypeId; + let t = TypeId::of::(); + t == TypeId::of::() + || t == TypeId::of::>>() + || t == TypeId::of::>() + || t == TypeId::of::>() + || t == TypeId::of::() + || t == TypeId::of::>() + || t == TypeId::of::>() + || t == TypeId::of::() + || t == TypeId::of::() + || t == TypeId::of::() + || t == TypeId::of::() + || t == TypeId::of::() +} + +/// How deep releases may nest before the trashcan takes over. +const TRASHCAN_DEPTH: u32 = 50; + +/// CPython's trashcan: releasing a deeply nested structure (a 500,000-deep +/// element tree, a long linked list of instances) recurses through every +/// level's drop glue, which would overflow the native stack. Past +/// [`TRASHCAN_DEPTH`] nested releases a container's last reference is +/// parked instead, and the outermost release frees the parked ones one at +/// a time. +struct Trashcan { + depth: std::cell::Cell, + parked: std::cell::RefCell>>, +} + +thread_local! { + static TRASHCAN: Trashcan = const { + Trashcan { + depth: std::cell::Cell::new(0), + parked: std::cell::RefCell::new(Vec::new()), + } + }; +} + +/// Release the last reference to a container (see [`Trashcan`]). +pub(crate) fn release_nested(last: R) { + let mut last = Some(last); + let _ = TRASHCAN.try_with(|t| { + let depth = t.depth.get(); + if depth >= TRASHCAN_DEPTH { + if let Ok(mut parked) = t.parked.try_borrow_mut() { + parked.push(Box::new(last.take())); + } + return; + } + t.depth.set(depth + 1); + drop(last.take()); + if depth == 0 { + loop { + let next = t.parked.try_borrow_mut().ok().and_then(|mut p| p.pop()); + match next { + Some(next) => drop(next), + None => break, + } + } + } + t.depth.set(depth); + }); + // Thread teardown, with the trashcan gone: release in place. + drop(last); +} + +/// Whether `T` is a payload whose last release may owe a finalizer (see +/// `vm_singletons::finalize_on_last_release`). Constant per instantiation. +#[inline(always)] +fn finalizable_kind() -> bool { + use std::any::TypeId; + let t = TypeId::of::(); + t == TypeId::of::() || t == TypeId::of::() +} + +impl Deref for Rc { type Target = T; #[inline(always)] fn deref(&self) -> &T { @@ -288,81 +387,81 @@ impl Deref for Rc { } } -impl AsRef for Rc { +impl AsRef for Rc { fn as_ref(&self) -> &T { self } } -impl Borrow for Rc { +impl Borrow for Rc { fn borrow(&self) -> &T { self } } -impl fmt::Debug for Rc { +impl fmt::Debug for Rc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&**self, f) } } -impl fmt::Display for Rc { +impl fmt::Display for Rc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&**self, f) } } -impl fmt::Pointer for Rc { +impl fmt::Pointer for Rc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Pointer::fmt(&Self::as_ptr(self), f) } } -impl PartialEq for Rc { +impl PartialEq for Rc { fn eq(&self, other: &Self) -> bool { **self == **other } } -impl Eq for Rc {} +impl Eq for Rc {} -impl PartialOrd for Rc { +impl PartialOrd for Rc { fn partial_cmp(&self, other: &Self) -> Option { (**self).partial_cmp(&**other) } } -impl Ord for Rc { +impl Ord for Rc { fn cmp(&self, other: &Self) -> std::cmp::Ordering { (**self).cmp(&**other) } } -impl Hash for Rc { +impl Hash for Rc { fn hash(&self, state: &mut H) { (**self).hash(state) } } -impl Default for Rc { +impl Default for Rc { fn default() -> Self { Self::new(T::default()) } } -impl From for Rc { +impl From for Rc { fn from(value: T) -> Self { Self::new(value) } } -impl From> for Rc { +impl From> for Rc { fn from(value: Box) -> Self { Self::from_arc(Arc::from(value)) } } -impl From> for Rc<[T]> { +impl From> for Rc<[T]> { fn from(value: Vec) -> Self { Self::from_arc(Arc::from(value)) } @@ -380,19 +479,19 @@ impl From for Rc { } } -impl From> for Rc { +impl From> for Rc { fn from(value: Arc) -> Self { Self::from_arc(value) } } -impl Weak { +impl Weak { pub const fn new() -> Self { Self(std::sync::Weak::new()) } } -impl Weak { +impl Weak { #[inline] pub fn upgrade(&self) -> Option> { self.0.upgrade().map(Rc::from_arc) @@ -421,7 +520,7 @@ impl Clone for Weak { } } -impl Default for Weak { +impl Default for Weak { fn default() -> Self { Self::new() } diff --git a/crates/weavepy-vm/src/shared_value.rs b/crates/weavepy-vm/src/shared_value.rs index 78b01224..e282dc88 100644 --- a/crates/weavepy-vm/src/shared_value.rs +++ b/crates/weavepy-vm/src/shared_value.rs @@ -30,6 +30,11 @@ mod sealed { /// Constructors must preserve this invariant for every live strong reference. pub unsafe trait ThinPayload: sealed::Sealed { type View: ?Sized; + /// Release the last reference to a payload. A payload whose release + /// can release more containers goes through `rc::release_nested`. + fn release_last(last: Arc) { + drop(last); + } fn pointer(data: *const usize, len: usize) -> *const Self; fn view(&self) -> &Self::View; } @@ -130,7 +135,7 @@ impl Drop for ThinArc { // Arc drops the payload and handles remaining weak references normally. unsafe { if !crate::rc::try_release_shared(self.raw()) { - drop(Arc::from_raw(self.raw())) + T::release_last(Arc::from_raw(self.raw())); } } } @@ -567,6 +572,9 @@ impl sealed::Sealed for crate::tuple_storage::TupleStorage {} // initialized [Object] tail. Only its constructors create these Arc payloads; // unique mutation changes elements/hash, never slice length or metadata. unsafe impl ThinPayload for crate::tuple_storage::TupleStorage { + fn release_last(last: Arc) { + crate::rc::release_nested(last); + } type View = Self; fn pointer(data: *const usize, len: usize) -> *const Self { ptr::slice_from_raw_parts(data.cast::(), len) as *const Self diff --git a/crates/weavepy-vm/src/stdlib/ast_build.rs b/crates/weavepy-vm/src/stdlib/ast_build.rs index 012b7797..9c973913 100644 --- a/crates/weavepy-vm/src/stdlib/ast_build.rs +++ b/crates/weavepy-vm/src/stdlib/ast_build.rs @@ -82,11 +82,7 @@ impl Builder<'_> { ) -> Result { let builder = self.recursive.resolve(function)?; let result = self.walk(vm, spec, &builder, &function.globals); - if result.is_ok() { - vm.reap_call_receiver(builder); - } else { - crate::gc_trace::mark_maybe_dead(); - } + vm.release(builder); result } @@ -178,11 +174,10 @@ impl Builder<'_> { let node = vm.call(&new, std::slice::from_ref(&cls), &[], globals); let node = match node { Ok(node) => { - vm.reap_call_receiver(new); + vm.release(new); node } Err(error) => { - crate::gc_trace::mark_maybe_dead(); return Err(error); } }; @@ -219,24 +214,20 @@ impl Builder<'_> { // setter's unused result. An ignored child can die here. let result = match result { Ok(result) => { - vm.reap_call_receiver(setter); + vm.release(setter); vm.reap_call_args(&mut args); result } Err(error) => { - crate::gc_trace::mark_maybe_dead(); return Err(error); } }; - crate::gc_trace::note_dropped(&result); - if Interpreter::local_needs_prompt_reap(&result) { - vm.prompt_reap_dropped(result); - } + vm.release(result); } Ok(()) })(); if let Err(error) = outcome { - vm.prompt_reap_dropped(node); + vm.release(node); return Err(error); } Ok(node) @@ -258,7 +249,7 @@ impl Builder<'_> { Ok(()) })(); if let Err(error) = outcome { - vm.prompt_reap_dropped(result); + vm.release(result); return Err(error); } Ok(result) diff --git a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs index 6206fe96..f917c5a0 100644 --- a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs +++ b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs @@ -1340,11 +1340,8 @@ fn future_iter_new(fut: &Object) -> Object { // Built outside the ordinary instantiation path, so enrol with the // collector by hand: a suspended `await` parks this iterator on the // coroutine's value stack, and its strong `fut` edge must be - // subtractable for task↔future cycles to collapse. Prompt-reclaim - // enrollment keeps its death by "refcount" observable — a lingering - // iterator pins the awaited Task, its coroutine frame, and every local - // in it (test_ssl's weakref leak tests watch exactly that chain). - crate::gc_trace::track_prompt_reclaim(out.clone()); + // subtractable for task↔future cycles to collapse. + crate::gc_trace::track(out.clone()); out } diff --git a/crates/weavepy-vm/src/stdlib/collections_native.rs b/crates/weavepy-vm/src/stdlib/collections_native.rs index e811d625..f207445f 100644 --- a/crates/weavepy-vm/src/stdlib/collections_native.rs +++ b/crates/weavepy-vm/src/stdlib/collections_native.rs @@ -285,9 +285,6 @@ fn deque_append(args: &[Object]) -> Result { Some(m) if f.d.len() - f.head() > m => Some(f.popleft()?), _ => None, }; - if trimmed.is_some() { - crate::gc_trace::mark_maybe_dead(); - } drop(trimmed); return Ok(Object::None); } @@ -319,10 +316,6 @@ fn deque_append(args: &[Object]) -> Result { }; } drop(st); - if trimmed.is_some() { - // A bounded deque released its oldest item (leaf contract). - crate::gc_trace::mark_maybe_dead(); - } drop(trimmed); Ok(Object::None) } @@ -343,9 +336,6 @@ fn deque_appendleft(args: &[Object]) -> Result { Some(m) if f.d.len() - h > m => f.d.pop(), _ => None, }; - if trimmed.is_some() { - crate::gc_trace::mark_maybe_dead(); - } drop(trimmed); return Ok(Object::None); } @@ -379,9 +369,6 @@ fn deque_appendleft(args: &[Object]) -> Result { }; } drop(st); - if trimmed.is_some() { - crate::gc_trace::mark_maybe_dead(); - } drop(trimmed); Ok(Object::None) } diff --git a/crates/weavepy-vm/src/stdlib/functools_mod.rs b/crates/weavepy-vm/src/stdlib/functools_mod.rs index e4fcc0f7..ae228f8e 100644 --- a/crates/weavepy-vm/src/stdlib/functools_mod.rs +++ b/crates/weavepy-vm/src/stdlib/functools_mod.rs @@ -632,11 +632,8 @@ fn scalar_lru_operation( data.insert(DictKey(key.clone()), value.clone()); order.push(limit).ok_or_else(invalid_lru_order)?; } - // Publish a consistent cache before releasing an evicted result. The - // prompt-reap queue preserves its finalizer's real calling frame. - if let Some((_, value)) = removed { - crate::vm_singletons::queue_container_removed(&value); - } + // Publish a consistent cache before releasing an evicted result. + drop(removed); Ok(ScalarLru::Complete(None)) } @@ -811,11 +808,7 @@ fn lru_cache_clear(args: &[Object], _kwargs: &[(String, Object)]) -> Result Result { .first() .ok_or_else(|| type_error("_strong_count() requires 1 argument"))?; let strong = gc_trace::strong_count_for(target) as i64; - let weak = crate::weakref_registry::strong_clone_count(id_of(target)) as i64; + // The registry holds its referents weakly: no strong clones. Ok(Object::new_tuple_array([ Object::Int(strong), - Object::Int(weak), + Object::Int(0), ])) } diff --git a/crates/weavepy-vm/src/stdlib/json_accel.rs b/crates/weavepy-vm/src/stdlib/json_accel.rs index 2609fb18..813ed6a5 100644 --- a/crates/weavepy-vm/src/stdlib/json_accel.rs +++ b/crates/weavepy-vm/src/stdlib/json_accel.rs @@ -1020,7 +1020,7 @@ fn parse_object( // CPython replaces duplicate values as it parses. Run // a displaced value's finalizer before the next field, // with no table borrow held across user code. - interp.prompt_reap_dropped(previous); + interp.release(previous); } } DecodedObject::Pairs(pairs) => pairs.push((key_obj, value)), diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index 0181d019..c71bfa92 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -2738,22 +2738,16 @@ fn stdlib_module_names_value() -> Object { } /// `sys.getrefcount(obj)` — best-effort, derived from the real -/// `Rc::strong_count` of the payload. Infrastructure references -/// (the cycle-GC registry's handle, weakref slots' strong clones) -/// are discounted so the number tracks *program-visible* bindings; -/// `+1` accounts for the argument reference, like CPython. The +/// `Rc::strong_count` of the payload. The collector and the weakref +/// registry hold no strong references; clones held only by the C-API +/// layer's caches are discounted so the number tracks *program-visible* +/// bindings. `+1` accounts for the argument reference, like CPython. The /// exact number is implementation-specific even in CPython. fn sys_getrefcount(args: &[Object]) -> Result { let Some(obj) = args.first() else { return Err(type_error("getrefcount() takes exactly 1 argument")); }; let strong = crate::gc_trace::strong_count_for(obj); - let id = crate::weakref_registry::id_of(obj); - let registry = usize::from(crate::gc_trace::is_tracked(id)); - let weak_clones = crate::weakref_registry::strong_clone_count(id); - // A dropped-but-registry-pinned memoryview (dead under CPython - // refcounting) must not count through its exporter edge. - let zombie_refs = crate::gc_trace::zombie_memoryview_refs_to(id); // Clones held only by the C-API layer's pin caches (a parked // argument-pinned identity box, a dead-except-pin scalar/tuple pin — // RFC 0076 WS1) are infrastructure too: on CPython the corresponding @@ -2768,9 +2762,6 @@ fn sys_getrefcount(args: &[Object]) -> Result { // unless CPython wouldn't have taken one at all (a borrowed load). let borrowed = borrowed_argument_discount(obj); let visible = strong - .saturating_sub(registry) - .saturating_sub(weak_clones) - .saturating_sub(zombie_refs) .saturating_sub(pinned) .saturating_sub(borrowed) .saturating_add(extra_c); diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs index 06eebcda..0e528ee0 100644 --- a/crates/weavepy-vm/src/stdlib/thread_real.rs +++ b/crates/weavepy-vm/src/stdlib/thread_real.rs @@ -1434,16 +1434,19 @@ fn spawn_python_worker( let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { worker_interp.run_thread_local_death_cleanup(synth_id as i64); })); - // Run any `__del__`/weakref-callback finalizers this worker - // deferred onto its *thread-local* pending queue (the prompt-reap - // cascade and the `PyInstance` `Drop` safety net both enqueue - // there). A blocked pool handler thread — `_handle_results` + // Release the target and its arguments, then run the + // `__del__`s and weakref callbacks this worker queued on its + // *thread-local* pending queue — the target function's death + // included (test_thread.test__count waits on its weakref + // callback). A blocked pool handler thread — `_handle_results` // parked in `outqueue.get()` — never reaches an eval-loop tick // to drain its own queue, so without this flush an object whose - // last reference died on that thread (e.g. a `test_release_task_refs` - // result copy) would be freed silently at thread teardown with its - // `__del__` skipped, leaking it. Drain while the GIL and the - // worker interpreter are both still live; isolate panics. + // last reference died on that thread would be freed with its + // `__del__` skipped. Drain while the GIL and the worker + // interpreter are both still live; isolate panics. + drop(worker_func); + drop(positional); + drop(kwargs_pairs); let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { worker_interp.run_pending_finalizers(); })); @@ -1455,17 +1458,13 @@ fn spawn_python_worker( let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe( super::greenlet_native::on_thread_teardown, )); - // Release this worker's Python references *while the GIL is - // still held*. The interpreter snapshot pins a large shared - // object graph; dropping it after the GIL is gone lets its - // `Arc` decrements (and the prompt-reap bookkeeping they - // trigger) race a peer thread's in-flight mark phase — which + // Release this worker's interpreter *while the GIL is still + // held*. Its snapshot pins a large shared object graph, and + // dropping it after the GIL is gone lets its `Arc` decrements + // race a peer thread's in-flight mark phase — which // intermittently classified the peer's *live* suspended // generators as garbage and closed them mid-`for` loop // (test_threading.test_foreign_thread's `wait_threads_exit`). - drop(worker_func); - drop(positional); - drop(kwargs_pairs); drop(worker_interp); crate::vm_singletons::clear_thread_python_tls(); // Drop the guard before marking finished so the parent's @@ -1730,13 +1729,6 @@ fn make_thread_handle_object(state: Arc, ident: Object) -> Ob return Err(runtime_error("thread not started")); } if join_state.done.load(Ordering::Acquire) { - // Already-finished fast path: still sweep — the worker's teardown - // (`del self._target, self._args, self._kwargs` in `Thread.run`) - // dropped containers that stay pinned by their cycle-GC handles - // until a sweep, and callers expect join() to have restored - // CPython's refcount timing regardless of who won the race - // (test_threading.test_no_refcycle_through_target). - crate::gc_trace::reap_dead_acyclic(); return Ok(Object::None); } // Joining a thread from itself would deadlock; CPython's handle @@ -1786,18 +1778,6 @@ fn make_thread_handle_object(state: Arc, ident: Object) -> Ob } }, } - // RFC 0039 (WS4): the just-joined worker ran its target's - // teardown — e.g. `threading.Thread.run`'s - // `del self._target, self._args, self._kwargs` — on its own - // thread, dropping the last *program* reference to any argument - // cycle. Objects this (joining) thread allocated are still pinned - // by its cycle-GC handle until a collection, so a `del` right - // after the join would still see the dead container as a live - // referrer. Refcounting would have freed it instantly in CPython; - // sweep this thread's dead acyclic garbage now so the caller's - // subsequent `del` finalizes the cycle without an explicit - // `gc.collect()` (test_threading.test_no_refcycle_through_target). - crate::gc_trace::reap_dead_acyclic(); Ok(Object::None) }; diff --git a/crates/weavepy-vm/src/stdlib/warnings_mod.rs b/crates/weavepy-vm/src/stdlib/warnings_mod.rs index 7f92cd7d..6e9739b3 100644 --- a/crates/weavepy-vm/src/stdlib/warnings_mod.rs +++ b/crates/weavepy-vm/src/stdlib/warnings_mod.rs @@ -614,16 +614,8 @@ fn call_show_warning( )?; let shown = ip.call_object_with_globals(&show_fn, &[msg.clone()], &[], &globals); // CPython decrefs the transient `WarningMessage` the moment - // `_showwarnmsg` returns; when the hook did not retain it (the stock - // stderr writer), the message — and, through `source=`, the very - // object whose finalizer emitted the warning — dies right here. A - // plain Rust drop would leave the tracked message pinned by its own - // GC handle until the next cyclic collection, keeping e.g. an - // unclosed `SpooledTemporaryFile`'s buffered fd alive across tests - // (test_tempfile.test_warnings_on_cleanup). The refcount guard - // inside leaves a *recorded* message (a `catch_warnings(record=True)` - // log holds it) untouched. - ip.maybe_prompt_reap_replaced(msg); + // `_showwarnmsg` returns: unless the hook kept it, it dies here. + drop(msg); shown?; Ok(()) } diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs index 3eeda5a7..7ec65ee8 100644 --- a/crates/weavepy-vm/src/stdlib/weakref_real.rs +++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs @@ -1104,9 +1104,9 @@ fn make_ref_object_with_class( class_override: Option>, ) -> Object { let target_id = id_of(&target); - // The registry's strong clone keeps the referent alive until the - // collector's prompt reap clears it, so a deferred instance is - // tracked from here on. + // A weakly referenced instance may die in a cycle, and only a + // collection can find that, so a deferred instance is tracked from + // here on. if let Object::Instance(inst) = &target { inst.ensure_gc_tracked(); } @@ -1124,27 +1124,6 @@ fn make_ref_object_with_class( let slot = Arc::new(slot); register(slot.clone()); - // RFC 0040 (GC arc): a weakref *with a callback* (`weakref.ref(obj, cb)`, - // `weakref.finalize`, `multiprocessing.util.Finalize`) must fire that - // callback the instant the referent's last strong reference drops, not at - // the next cyclic collection. Enroll the (tracked) referent in the cycle - // GC's prompt-finalization index so a refcount-death between bytecodes - // fires it — matching CPython's `tp_dealloc` weakref clear. - // - // Callback-*less* refs must clear promptly too (leak tests observe - // `wr()` going `None` right after the owning scope exits — test_ssl's - // SSLContext checks), but NOT via this index: enrolling every weakref - // target would leave `has_any_finalizable()` permanently true from the - // interpreter-boot weakrefs (`abc`/`typing` registries), turning the - // eval loop's drop safe point into a full index scan on every - // reference-dropping opcode (a 3-4x interpreter-wide slowdown, RFC 0054 - // WS5 re-measure). They are served by the opcode-level prompt-reap - // cascade (`reap_dead_subgraph` clears weakrefs) plus the suspect - // re-probe for deaths inside Rust transients. - if callback.is_some() { - crate::gc_trace::note_weakref_finalizable(target_id); - } - // The getter owns the slot. Registry entries and the wrapper's // back-pointer are weak; callback ownership stays in traced storage. let slot_for_get = slot.clone(); @@ -1536,7 +1515,7 @@ mod tests { TypeObject::new_user("WeakrefTarget", vec![], DictData::default()).unwrap(), ); let r = c_new_ref(target.clone(), None).unwrap(); - crate::gc_trace::sweep_weakref_only_targets(); + crate::weakref_registry::sweep_dead_targets(); let live = ref_type_call(std::slice::from_ref(&r)).unwrap(); assert!(live.is_same(&target)); let id = id_of(&target); diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index ae5f98b0..27fc9596 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -2198,62 +2198,20 @@ const RUNTIME_PIN_CAP: usize = 1 << 16; const RUNTIME_PIN_SOFT_LIMIT: usize = 1 << 12; /// Retire every pin after the activation's result or complete interpreter -/// state has been reconstructed. Entry pins can also become dead temporaries -/// when native code replaces a local or detaches an object from its owner. -/// Give them the same prompt-reap treatment as runtime pins and interpreter -/// locals. A suspended native activation must keep its entire table instead. -/// Returns `true` when a cascade could have run Python, requiring the caller -/// to revalidate its compiled resolutions. -fn drain_activation_pins(interp: &mut super::Interpreter, pins: &mut PinTable) -> bool { - let mut dirty = false; - for p in pins.drain(..) { - // Fresh lists are tracked just like interpreter-built lists. - // Dropping only the native pin leaves the collector's strong - // handle retaining a dead temporary until a later collection. - let o = match p { - Pin::List(list, _) => Object::List(list), - Pin::Obj(o) => o, - }; - // The common pin, a receiver or argument others still hold, - // releases plainly (the grades below would conclude the same). - if crate::gc_trace::drop_survives_plainly(&o) { - continue; - } - if super::Interpreter::local_needs_prompt_reap(&o) - && super::Interpreter::looks_reapable_temporary(&o) - { - dirty = true; - interp.maybe_prompt_reap_replaced(o); - } else { - crate::gc_trace::note_dropped(&o); - } - } - dirty -} - -/// Retire an activation's pins after its complete -/// interpreter frame has been rebuilt. Leaves can drop immediately, but -/// a dying container or finalizable object must wait for the interpreter -/// to install that frame and charge its recursion tick. The existing -/// pending-drop queue runs its cascade at the next bytecode safe point. -/// No Python runs here, and all live values already have rebuilt roots. +/// state has been reconstructed. A pin that held an object's last +/// reference frees it; a dying object only queues work for the next safe +/// point, so no Python runs here. Returns whether Python could have run +/// (it can't), for the callers that revalidate their compiled resolutions. +/// A suspended native activation must keep its entire table instead. +fn drain_activation_pins(_interp: &mut super::Interpreter, pins: &mut PinTable) -> bool { + pins.clear(); + false +} + +/// Retire an activation's pins after its complete interpreter frame has +/// been rebuilt (see [`drain_activation_pins`]). fn defer_activation_pins(pins: &mut PinTable) { - for pin in pins.drain(..) { - let obj = match pin { - Pin::List(list, _) => Object::List(list), - Pin::Obj(obj) => obj, - }; - if crate::gc_trace::drop_survives_plainly(&obj) { - continue; - } - if super::Interpreter::local_needs_prompt_reap(&obj) - && super::Interpreter::looks_reapable_temporary(&obj) - { - crate::vm_singletons::queue_parked_drop(&obj); - } else { - crate::gc_trace::note_dropped(&obj); - } - } + pins.clear(); } /// Reconstruct an [`Object`] from a `(bits, tag)` slot. `Boxed` never @@ -2903,13 +2861,7 @@ pub(crate) fn gc_sweep() { let dead: Vec<*const CodeObject> = st .cache .iter() - .filter(|(key, e)| { - // Python weakref slots temporarily hold strong clones; - // the collector discounts those from reachability too. - let weak_clones = - crate::weakref_registry::strong_clone_count(**key as usize as u64); - Rc::strong_count(&e.code) == 1 + weak_clones - }) + .filter(|(_, e)| Rc::strong_count(&e.code) == 1) .map(|(k, _)| *k) .collect(); for k in dead { @@ -4803,8 +4755,15 @@ unsafe fn try_native_ctor( // `__init__` is a procedure: its `None` rides the procedure lane. // A `try_native_call` rejection discards the fresh (empty, never // `__init__`-ed) instance and re-allocates on the interpreter path. - let status = - unsafe { try_native_call(jf, ctx, interp, nc, argc, SlotTag::None as u32, Some(&inst)) }?; + // Python never saw it, so it dies without its finalizer. + let Some(status) = + (unsafe { try_native_call(jf, ctx, interp, nc, argc, SlotTag::None as u32, Some(&inst)) }) + else { + if let Object::Instance(i) = &inst { + i.finalize_ran.set(true); + } + return None; + }; if status == CallStatus::Raised as i64 { // A raising `__init__` discards the instance (CPython's // `type_call` propagates before returning it). @@ -6643,25 +6602,11 @@ unsafe extern "C" fn wpjit_dict_set( let Some((d, key)) = dict_pin_and_key(ctx, pin, key_bits, key_tag) else { return 1; }; - // Python-free pre-probe: displaced-value discipline + deferral - // detection (a deferral means the insert below could run Python). - let old = match dict_probe_native(&d, &key) { - Ok(f) => f, - Err(()) => return 1, - }; - if let Some(old) = &old { - if !matches!( - old, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(old) - && super::Interpreter::looks_reapable_temporary(old) - { - return 1; - } + // Python-free pre-probe: a deferral means the insert below could run + // Python. + if dict_probe_native(&d, &key).is_err() { + return 1; } - // The displaced value (probed non-reapable above) drops by plain - // refcount, exactly like the no-op arm of the interpreter's - // prompt-reap check. match crate::builtins::dict_insert(&d, key, v) { Ok(_) => 0, Err(_) => 1, @@ -6700,21 +6645,9 @@ unsafe extern "C" fn wpjit_dict_del( Ok(Some(v)) => v, _ => return 1, }; - if !matches!( - old, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(&old) - && super::Interpreter::looks_reapable_temporary(&old) - { - return 1; - } drop(old); match crate::builtins::dict_remove(&d, &key) { - Ok(Some((k, v))) => { - crate::vm_singletons::queue_container_removed(&k); - crate::vm_singletons::queue_container_removed(&v); - 0 - } + Ok(Some(_)) => 0, _ => 1, } } @@ -8450,14 +8383,6 @@ unsafe extern "C" fn wpjit_attr_set(frame: *mut JitFrame, pin: i64, site: i64) - // and the reap; otherwise the overwrite drops it by plain // refcount, exactly like the no-op arm of // `maybe_prompt_reap_replaced`. - if !matches!( - dst, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(dst) - && super::Interpreter::looks_reapable_temporary(dst) - { - return 1; - } let old = std::mem::replace(dst, v); drop(old); 0 @@ -8476,18 +8401,8 @@ unsafe extern "C" fn wpjit_attr_set(frame: *mut JitFrame, pin: i64, site: i64) - // The split layout (see `Interpreter::core_store_new_attr`). let v = if inst.dict.published().is_none() { // SAFETY: a read between two native ops. - let Some(split) = (unsafe { inst.dict.split_cell().peek() }) else { + if unsafe { inst.dict.split_cell().peek() }.is_none() { return 1; - }; - if let Some(dst) = split.get(&g.name) { - if !matches!( - dst, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(dst) - && super::Interpreter::looks_reapable_temporary(dst) - { - return 1; - } } match inst.split_store(&g.name, v) { Ok(old) => { @@ -8521,14 +8436,6 @@ unsafe extern "C" fn wpjit_attr_set(frame: *mut JitFrame, pin: i64, site: i64) - RawEntryMut::Occupied(mut entry) => { let dst = entry.get_mut(); // The displaced-value discipline of the indexed arm. - if !matches!( - dst, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(dst) - && super::Interpreter::looks_reapable_temporary(dst) - { - return 1; - } Some(std::mem::replace(dst, v)) } RawEntryMut::Vacant(entry) => { @@ -11607,11 +11514,6 @@ impl NativeActivation { visit(&o); } } - - /// The pinned objects, cloned (for [`super::frame_reapables`]). - pub(crate) fn pinned_objects(&self) -> impl Iterator + '_ { - self.pins.iter().map(Pin::to_object) - } } /// Decide whether a `Yielded` exit can park (RFC 0073 WS4) and, if diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index c0543ad7..8561c9ea 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -741,6 +741,13 @@ pub struct TypeObject { pub exc_families: Cell, } +/// A dying TypeObject clears the weak references watching it. +impl Drop for TypeObject { + fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); + } +} + /// Per-class resolution of the `type.__call__` protocol, cached on the /// class ([`TypeObject::instance_plan`]) and invalidated by /// [`TypeObject::attr_version`]. Everything here is a pure function of @@ -2605,13 +2612,7 @@ pub struct PyInstance { /// `_PyGC_FINALIZED` bit. Set by `Vm::invoke_finalizer` the moment the /// finalizer is dispatched; read by [`PyInstance`]'s `Drop` to decide /// whether the dying instance still needs its `__del__` resurrected onto - /// the pending-finalizer queue. Without this, an acyclic finalizable - /// instance whose last `Arc` is dropped on a code path that *didn't* - /// route through the prompt-reap cascade (e.g. a cross-thread handoff - /// where a transient clone briefly inflated the refcount so the reap - /// bailed) is freed silently, skipping `__del__` (RFC 0040: - /// `test_multiprocessing_*` `test_release_task_refs` leaked one - /// `CountedObject` per race). + /// the pending-finalizer queue. pub finalize_ran: Cell, /// Cycle-collector tracking is still deferred: the collector has /// never seen this instance, and it holds only atomic values (see @@ -2939,19 +2940,8 @@ impl PyInstance { } impl Drop for PyInstance { - /// Last-resort finalizer safety net, mirroring - /// [`crate::object::PyGenerator`]'s `Drop`. WeavePy normally runs an - /// instance's `__del__` through the prompt-reap cascade the instant its - /// last program reference is dropped (matching CPython's refcount - /// timing). But that cascade is driven from specific eval-loop sites and - /// is gated on a refcount-dead test; when the final `Arc` is released - /// somewhere else — most often a cross-thread object handoff where a - /// transient clone on another thread briefly inflated the strong count so - /// the reap conservatively bailed — the instance would otherwise be freed - /// by a plain `Arc` drop with its `__del__` silently skipped (the cycle - /// collector never revisits acyclic objects). Catch that here: resurrect - /// a shallow copy that shares the dying instance's `__dict__`/slots/native - /// value onto the VM's pending-finalizer queue so `__del__` still runs. + /// An instance dies when its last reference goes, as in CPython, and + /// the weak references watching it clear. fn drop(&mut self) { // A deferred-tracking record names this instance; the dict may // outlive it (`d = obj.__dict__`), so retire the record first. @@ -2968,12 +2958,6 @@ impl Drop for PyInstance { } } } - if crate::hot_gates::env_flags::reap_trace() { - let name = self.cls().name.clone(); - if name.contains("Block") || name.contains("DataFrame") { - eprintln!("[INST-DROP] {name} body={:#x}", self.c_body.get()); - } - } // RFC 0045 (wave 3): release the faithful C inline body this // instance owns, if it ever crossed into a C extension that reads // its fields at fixed `tp_basicsize` offsets. Runs before the @@ -2988,44 +2972,9 @@ impl Drop for PyInstance { free(body); } } - // Already finalized (cascade/GC/this net's resurrected copy): the - // common case for finalizable instances and the *only* path for the - // overwhelmingly common finalizer-free instance — a single `Cell` - // read keeps the hot drop path cheap. - if self.finalize_ran.get() { - return; - } - // No `__del__` anywhere in the MRO ⇒ nothing to do. Cached on the - // type, so this is one more `Cell` read after the first instance. - if !self.cls().instances_need_finalize() { - return; - } - // CPython runs `tp_finalize` once: claim it so the resurrected copy - // (and any re-drop after the finalizer completes) can't loop. - self.finalize_ran.set(true); - // Can't run Python from `Drop`; resurrect onto the pending queue. The - // copy shares `dict`/`slots`/`native` (cloning the `Arc`s/contents), - // so `__del__` observes the same attributes; `try_*` tolerates TLS - // teardown and re-entrant borrows by dropping the request. - let resurrected = Object::Instance(Rc::new(PyInstance { - class: RefCell::new(self.cls()), - dict: self.dict.clone(), - native: match self.native.get() { - Some(v) => crate::sync::OnceBox::from(v.clone()), - None => crate::sync::OnceBox::new(), - }, - inline_values: Cell::new(self.inline_values.get()), - slots: RefCell::new(self.slots.borrow().clone()), - hash_cache: crate::sync::CachedHash::new(self.hash_cache.get()), - // This copy still owes its queued invocation. Dispatch claims - // its flag atomically; a failed enqueue suppresses its Drop. - finalize_ran: Cell::new(false), - // The resurrected copy is a distinct object that owns no C - // body (the dying `self` already freed its own above). - deferred: crate::sync::Cell::new(false), - c_body: CBody::default(), - })); - crate::vm_singletons::try_push_pending_finalizer(resurrected); + // A finalizer, if the class has one, already ran: `rc::Rc`'s `Drop` + // queued this instance for it at its last release. + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); } } diff --git a/crates/weavepy-vm/src/vm_singletons.rs b/crates/weavepy-vm/src/vm_singletons.rs index 4ef06da4..9c19ff57 100644 --- a/crates/weavepy-vm/src/vm_singletons.rs +++ b/crates/weavepy-vm/src/vm_singletons.rs @@ -68,6 +68,7 @@ pub fn push_pending_finalizer(obj: Object) { }); PENDING_FINALIZER_COUNT.fetch_add(1, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::PENDING_FINALIZERS); + crate::gc_trace::mark_maybe_dead(); } /// Like [`push_pending_finalizer`], but callable from `Drop` impls: @@ -88,6 +89,7 @@ pub fn try_push_pending_finalizer(obj: Object) { if pushed { PENDING_FINALIZER_COUNT.fetch_add(1, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::PENDING_FINALIZERS); + crate::gc_trace::mark_maybe_dead(); } else if let Some(obj) = pending { // No safe point can run this request during TLS teardown or a // reentrant queue borrow. Suppress its Drop fallback before releasing @@ -102,6 +104,55 @@ pub fn try_push_pending_finalizer(obj: Object) { } } +/// The last reference to `arc`'s object, a `PyInstance` or a `PyGenerator` +/// (see `rc::Rc`'s `Drop`), is going. One that owes a finalizer (an +/// instance whose class has `__del__`, a suspended generator, a coroutine +/// never awaited) is queued, still alive at its own address, for the next +/// safe point; anything else is released. +pub(crate) fn finalize_on_last_release(arc: std::sync::Arc) { + use crate::object::{CoroutineKind, GeneratorState, PyGenerator}; + use crate::types::PyInstance; + use std::any::TypeId; + use std::sync::Arc; + let t = TypeId::of::(); + if t == TypeId::of::() { + // SAFETY: `T` is `PyInstance` (the cast drops no metadata). + let inst = unsafe { Arc::from_raw(Arc::into_raw(arc).cast::()) }; + let owes = !inst.finalize_ran.get() + && inst + .class + .try_borrow() + .is_ok_and(|cls| cls.instances_need_finalize()); + if owes { + try_push_pending_finalizer(Object::Instance(Rc::from_arc(inst))); + } else { + drop(inst); + } + } else if t == TypeId::of::() { + // SAFETY: as above, for `PyGenerator`. + let g = unsafe { Arc::from_raw(Arc::into_raw(arc).cast::()) }; + let owes = !g.finalize_ran.get() + && g.state.try_borrow().is_ok_and(|state| match &*state { + GeneratorState::Suspended(_) => true, + GeneratorState::Created(_) => g.kind == CoroutineKind::Coroutine, + GeneratorState::Finished | GeneratorState::Running => false, + }); + if owes { + let kind = g.kind; + let g = Rc::from_arc(g); + try_push_pending_finalizer(match kind { + CoroutineKind::Generator => Object::Generator(g), + CoroutineKind::Coroutine => Object::Coroutine(g), + CoroutineKind::AsyncGenerator => Object::AsyncGenerator(g), + }); + } else { + drop(g); + } + } else { + drop(arc); + } +} + /// Drain the pending-finalizer queue. The eval loop calls this /// at every eval-breaker tick that has the GC flag set. pub fn drain_pending_finalizers() -> Vec { @@ -136,11 +187,17 @@ pub fn has_pending_finalizers() -> bool { /// Queue a weakref callback `(callback, weakref_obj)` for invocation at /// the next safe point. Teardown-safe (callable from sweep paths). pub fn push_pending_weakref_callback(callback: Object, weakref_obj: Object) { - let _ = PENDING_WEAKREF_CALLBACKS.try_with(|cell| { - if let Ok(mut queue) = cell.try_borrow_mut() { - queue.push((callback, weakref_obj)); - } - }); + let pushed = PENDING_WEAKREF_CALLBACKS + .try_with(|cell| { + cell.try_borrow_mut() + .map(|mut queue| queue.push((callback, weakref_obj))) + .is_ok() + }) + .unwrap_or(false); + if pushed { + crate::hot_gates::set(crate::hot_gates::PENDING_FINALIZERS); + crate::gc_trace::mark_maybe_dead(); + } } /// Is anything parked in this thread's weakref-callback queue? Cheap @@ -638,6 +695,7 @@ pub fn push_pending_resource_warning(message: String) { PENDING_RESOURCE_WARNINGS.with(|cell| cell.borrow_mut().push((message, None))); PENDING_RW_FLAG.store(true, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::RESOURCE_WARNINGS); + crate::gc_trace::mark_maybe_dead(); } /// As [`push_pending_resource_warning`], carrying the dying object's @@ -651,6 +709,7 @@ pub fn push_pending_resource_warning_with_source(message: String, source_key: us PENDING_RESOURCE_WARNINGS.with(|cell| cell.borrow_mut().push((message, Some(source_key)))); PENDING_RW_FLAG.store(true, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::RESOURCE_WARNINGS); + crate::gc_trace::mark_maybe_dead(); } /// Cheap probe for the eval-loop safe point: are any deferred resource @@ -805,65 +864,6 @@ thread_local! { static PENDING_CEXT_FLAG: std::cell::Cell = const { std::cell::Cell::new(false) }; } -thread_local! { - /// Object ids currently being torn down by the prompt reaper's cascade - /// on this thread (RFC 0047, wave 5). While an object is mid-cascade, - /// the teardown itself crosses it back into C — `traverse_object` runs - /// the GC bridge's `tp_traverse`, which mints a transient self box and - /// releases it — and that release would *re-queue* the object through - /// [`queue_cext_dropped`]. The queued clone then makes the cascade's - /// own deadness re-check see the object as externally alive, aborting - /// the teardown, and the drained clone starts the cascade over: a - /// livelock that pinned `repr(DataFrame)` at 100% CPU indefinitely. - /// A queue request for an id in this set is the cascade observing - /// itself and is dropped; requests for *other* objects (a child body - /// whose last C pin fell during the teardown) still queue normally. - static CASCADING_IDS: RefCell> = - RefCell::new(crate::fasthash::FxHashSet::default()); -} - -/// RAII marker for one object's trip through the prompt reaper's cascade; -/// see [`CASCADING_IDS`]. Created by [`enter_cascade`]. -#[derive(Debug)] -pub struct CascadeGuard { - id: u64, - owner: bool, -} - -impl Drop for CascadeGuard { - fn drop(&mut self) { - if self.owner { - let _ = CASCADING_IDS.try_with(|c| { - if let Ok(mut set) = c.try_borrow_mut() { - set.remove(&self.id); - } - }); - } - } -} - -/// Mark `id` as mid-cascade for the guard's lifetime. Nesting-safe: a -/// guard for an id already in the set is a no-op on drop (the outer -/// guard owns the entry). -pub fn enter_cascade(id: u64) -> CascadeGuard { - let owner = CASCADING_IDS - .try_with(|c| { - c.try_borrow_mut() - .map(|mut set| set.insert(id)) - .unwrap_or(false) - }) - .unwrap_or(false); - CascadeGuard { id, owner } -} - -/// Is `id` currently being torn down by the prompt reaper's cascade on -/// this thread? -fn in_cascade(id: u64) -> bool { - CASCADING_IDS - .try_with(|c| c.try_borrow().map(|set| set.contains(&id)).unwrap_or(true)) - .unwrap_or(false) -} - /// Park an object dropped by C extension code for a prompt-reap pass at /// the next eval-loop safe point. Only object kinds that can carry (or /// anchor) finalizers/weakrefs/tracked children are queued; scalars and @@ -887,23 +887,6 @@ pub fn queue_cext_dropped(obj: &Object) { queue_parked_drop(obj); } -/// Park a value evicted from a native container by a *mutating* method or -/// opcode — `dict.clear`/`__delitem__`/replacing `__setitem__`, `del d[k]`, -/// `list.remove`/`clear`/slice assignment, `set.discard`, … — for a -/// prompt-reap pass at the next eval-loop safe point. These mutators run as -/// plain builtin fns without interpreter access, so the reference they drop -/// can't go through `prompt_reap_dropped` inline; without the park, an -/// object whose *last* reference lived in the container stayed pinned by -/// its weakref registry entry / GC handle until the next cyclic collection. -/// CPython frees it on the spot (pandas' `_item_cache.clear()` relies on -/// the evicted Series — and its CoW `Block` — dying immediately: a stale -/// block kept `refs.has_reference()` true, misfiring the chained-assignment -/// `FutureWarning` in `Series.__setitem__`). Same kind filter as -/// [`queue_cext_dropped`]: scalars and other leaves drop inline as before. -pub fn queue_container_removed(obj: &Object) { - queue_cext_dropped(obj); -} - /// As [`queue_cext_dropped`] but with **no kind filter**. Native side exits /// also use this queue to retire temporary pins before the interpreter /// continuation runs, with callbacks deferred until its frame is installed. @@ -933,12 +916,6 @@ pub fn queue_parked_drop(obj: &Object) { if crate::gc_trace::collector_active() { return; } - // The prompt reaper's own teardown of this object (see - // [`CASCADING_IDS`]): the transient C crossings the cascade performs - // must not re-queue the object it is in the middle of freeing. - if in_cascade(crate::weakref_registry::id_of(obj)) { - return; - } let pushed = PENDING_CEXT_DROPS .try_with(|cell| { if let Ok(mut queue) = cell.try_borrow_mut() { @@ -950,33 +927,6 @@ pub fn queue_parked_drop(obj: &Object) { }) .unwrap_or(false); if pushed { - if crate::hot_gates::env_flags::reap_trace() { - eprintln!( - "[CEXT-DROP] queued {} id={:#x}", - obj.type_name_owned(), - crate::weakref_registry::id_of(obj) - ); - if std::env::var_os("WEAVEPY_REAP_BT").is_some() { - thread_local! { - static N: std::cell::Cell = const { std::cell::Cell::new(0) }; - } - let n = N.with(|c| { - let v = c.get() + 1; - c.set(v); - v - }); - let every: usize = std::env::var("WEAVEPY_REAP_BT_EVERY") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(5000); - if n.is_multiple_of(every) { - eprintln!( - "[CEXT-DROP-BT]\n{}", - std::backtrace::Backtrace::force_capture() - ); - } - } - } let _ = PENDING_CEXT_FLAG.try_with(|c| c.set(true)); PENDING_CEXT_COUNT.fetch_add(1, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::PENDING_CEXT); diff --git a/crates/weavepy-vm/src/weak_object.rs b/crates/weavepy-vm/src/weak_object.rs new file mode 100644 index 00000000..65cefc96 --- /dev/null +++ b/crates/weavepy-vm/src/weak_object.rs @@ -0,0 +1,136 @@ +//! A non-owning reference to any heap-backed [`Object`]. +//! +//! The cycle collector and the weak-reference registry hold their targets +//! through this type, so neither keeps an object alive: an object dies by +//! reference count the moment its last strong owner goes, as in CPython, +//! and the collector only has to find and break cycles. +//! +//! The weak handle keeps the allocation (not the payload) reserved, so an +//! object's address can't be reused while a handle to it exists. That's +//! what lets the collector and the registry key their tables by address. + +use crate::object::Object; +use crate::shared_value::ThinWeak; +use crate::sync::{Rc, Weak}; + +macro_rules! weak_object { + ($($variant:ident),* $(,)?) => { + /// A weak handle on a heap-backed [`Object`]; see the module docs. + #[allow(missing_debug_implementations)] + pub enum WeakObject { + Tuple(ThinWeak), + $($variant(weak_object!(@payload $variant)),)* + } + + impl WeakObject { + /// A weak handle on `obj`, or `None` for a value with no heap + /// allocation of its own (scalars, strings, bytes). + pub fn new(obj: &Object) -> Option { + Some(match obj { + Object::Tuple(t) => Self::Tuple(crate::shared_value::ThinArc::downgrade(t)), + $(Object::$variant(rc) => Self::$variant(Rc::downgrade(rc)),)* + _ => return None, + }) + } + + /// The object, while it is alive. + pub fn upgrade(&self) -> Option { + Some(match self { + Self::Tuple(w) => Object::Tuple(w.upgrade()?), + $(Self::$variant(w) => Object::$variant(w.upgrade()?),)* + }) + } + + /// The number of strong owners left (0 once the object died). + pub fn strong_count(&self) -> usize { + match self { + Self::Tuple(w) => w.strong_count(), + $(Self::$variant(w) => w.strong_count(),)* + } + } + + /// Whether the object has died. + #[inline] + pub fn is_dead(&self) -> bool { + self.strong_count() == 0 + } + } + }; + (@payload Long) => { Weak }; + (@payload Complex) => { Weak }; + (@payload List) => { Weak>> }; + (@payload Dict) => { Weak> }; + (@payload Range) => { Weak }; + (@payload Function) => { Weak }; + (@payload Builtin) => { Weak }; + (@payload BoundMethod) => { Weak }; + (@payload Code) => { Weak }; + (@payload Cell) => { Weak> }; + (@payload Iter) => { Weak> }; + (@payload Slice) => { Weak }; + (@payload Type) => { Weak }; + (@payload Instance) => { Weak }; + (@payload Module) => { Weak }; + (@payload Generator) => { Weak }; + (@payload Coroutine) => { Weak }; + (@payload AsyncGenerator) => { Weak }; + (@payload AsyncGenAwait) => { Weak }; + (@payload ByteArray) => { Weak>> }; + (@payload Set) => { Weak> }; + (@payload FrozenSet) => { Weak }; + (@payload File) => { Weak }; + (@payload Property) => { Weak }; + (@payload StaticMethod) => { Weak }; + (@payload ClassMethod) => { Weak }; + (@payload SlotDescriptor) => { Weak }; + (@payload Frame) => { Weak }; + (@payload Traceback) => { Weak }; + (@payload MemoryView) => { Weak }; + (@payload MappingProxy) => { Weak> }; + (@payload MappingProxyObj) => { Weak }; + (@payload DictView) => { Weak }; + (@payload SimpleNamespace) => { Weak> }; + (@payload LazyIter) => { Weak }; + (@payload Capsule) => { Weak }; + (@payload Foreign) => { Weak }; +} + +weak_object!( + Long, + Complex, + List, + Dict, + Range, + Function, + Builtin, + BoundMethod, + Code, + Cell, + Iter, + Slice, + Type, + Instance, + Module, + Generator, + Coroutine, + AsyncGenerator, + AsyncGenAwait, + ByteArray, + Set, + FrozenSet, + File, + Property, + StaticMethod, + ClassMethod, + SlotDescriptor, + Frame, + Traceback, + MemoryView, + MappingProxy, + MappingProxyObj, + DictView, + SimpleNamespace, + LazyIter, + Capsule, + Foreign, +); diff --git a/crates/weavepy-vm/src/weakref_registry.rs b/crates/weavepy-vm/src/weakref_registry.rs index 71ed558c..0dfc486c 100644 --- a/crates/weavepy-vm/src/weakref_registry.rs +++ b/crates/weavepy-vm/src/weakref_registry.rs @@ -93,9 +93,11 @@ pub struct WeakRefSlot { /// Identity of the referent. Used to look up the registry /// list when the referent dies. pub target_id: ObjectId, - /// `Some(strong_clone_of_target)` while the referent is - /// alive. Set to `None` by `notify_clear`. - pub target: RefCell>, + /// The referent, held weakly, while the slot is live. Set to `None` + /// by `notify_clear`. A weak reference never keeps its referent + /// alive: the referent dies by reference count, and its `Drop` (or the + /// collector's sweep, for kinds without one) clears the slot. + pub target: RefCell>, /// Whether a callback was supplied at creation. The callback /// *object* itself lives in the user-visible wrapper's traced slots /// or dictionary (under `__callback__`), never here: the wrapper is @@ -166,11 +168,45 @@ pub mod kind { pub const CALLABLE_PROXY: u8 = 2; } +/// A slot's referent. Every Python-visible referent has a heap allocation +/// and is held weakly; a value without one (only the registry's own unit +/// tests make such slots) is simply held. +#[allow(missing_debug_implementations)] +pub enum WeakTarget { + Weak(crate::weak_object::WeakObject), + Strong(Object), +} + +impl WeakTarget { + pub fn new(target: &Object) -> Self { + match crate::weak_object::WeakObject::new(target) { + Some(w) => Self::Weak(w), + None => Self::Strong(target.clone()), + } + } + + /// The referent, while it is alive. + pub fn upgrade(&self) -> Option { + match self { + Self::Weak(w) => w.upgrade(), + Self::Strong(o) => Some(o.clone()), + } + } + + /// Whether the referent has died. + pub fn is_dead(&self) -> bool { + match self { + Self::Weak(w) => w.is_dead(), + Self::Strong(_) => false, + } + } +} + impl WeakRefSlot { pub fn new(target_id: ObjectId, target: Object, has_callback: bool, kind: u8) -> Self { Self { target_id, - target: RefCell::new(Some(target)), + target: RefCell::new(Some(WeakTarget::new(&target))), has_callback, fixed_wrapper: false, identity_hash: target_id as i64, @@ -188,7 +224,17 @@ impl WeakRefSlot { if self.is_dead() { return None; } - self.target.borrow().clone() + self.target.borrow().as_ref()?.upgrade() + } + + /// Whether the referent died without this slot being cleared yet (a + /// kind with no death hook; see `sweep_dead_targets`). + pub fn target_died(&self) -> bool { + !self.is_dead() + && self + .target + .try_borrow() + .is_ok_and(|t| t.as_ref().is_some_and(WeakTarget::is_dead)) } /// Clear the slot. Returns the callback (if any) so the caller can @@ -354,17 +400,33 @@ impl WeakRefRegistry { /// list of `(weakref_object, callback_object)` pairs the /// caller should invoke. Removes the entry for `id`. pub fn notify_clear(&self, id: ObjectId) -> Vec<(Arc, Option)> { + let entries = Self::take_entries(&mut self.inner.borrow_mut(), id); + Self::clear_entries(entries) + } + + /// [`Self::notify_clear`] from a `Drop`, which may run while the + /// registry is borrowed: `None` then, leaving the slots for the + /// collector's sweep. + fn try_notify_clear(&self, id: ObjectId) -> Option, Option)>> { let entries = { - let mut g = self.inner.borrow_mut(); - // Hot path: this runs for every object the prompt reaper - // frees, and almost none of them was ever weakly referenced. - // Skip the map removal on the miss-filter's say-so, exactly - // as `count` does. - if g.slots.is_empty() || !g.filter.may_contain(id) { - return Vec::new(); - } - g.slots.remove(&id).unwrap_or_default() + let Ok(mut g) = self.inner.try_borrow_mut() else { + return None; + }; + Self::take_entries(&mut g, id) }; + Some(Self::clear_entries(entries)) + } + + fn take_entries(g: &mut RegistryInner, id: ObjectId) -> Vec> { + // Almost no object was ever weakly referenced: skip the map + // removal on the miss-filter's say-so, exactly as `count` does. + if g.slots.is_empty() || !g.filter.may_contain(id) { + return Vec::new(); + } + g.slots.remove(&id).unwrap_or_default() + } + + fn clear_entries(entries: Vec>) -> Vec<(Arc, Option)> { let mut out = Vec::with_capacity(entries.len()); // CPython fires a referent's weakref callbacks newest-first (its // weakref list inserts at the head), so a chain of @@ -394,28 +456,23 @@ impl WeakRefRegistry { .unwrap_or(0) } - /// How many *strong clones* of the referent the registry is - /// currently holding for `id`. Every live, un-cleared slot keeps - /// one `Object` clone of its target alive (see - /// [`WeakRefSlot::target`]). The cycle collector subtracts this - /// from an object's outer refcount so a weakref does **not** keep - /// its referent reachable — otherwise `weakref.ref(obj)()` would - /// stay live forever and `WeakKeyDictionary`/`WeakValueDictionary` - /// would never self-clean after `del obj; gc.collect()`. - pub fn strong_clone_count(&self, id: ObjectId) -> usize { - let g = self.inner.borrow(); - if g.slots.is_empty() || !g.filter.may_contain(id) { - return 0; - } + /// The ids whose referent died without clearing its slots: the kinds + /// with no death hook (`set`, `bytearray`, `SimpleNamespace`, code + /// objects), or a death that found the registry busy. Their slots are cleared by + /// [`sweep_dead_targets`]. + fn dead_target_ids(&self) -> Vec { + let Ok(g) = self.inner.try_borrow() else { + return Vec::new(); + }; g.slots - .get(&id) - .map(|v| { + .iter() + .filter(|(_, v)| { v.iter() .filter_map(Weak::upgrade) - .filter(|s| !s.is_dead() && s.target.borrow().is_some()) - .count() + .any(|slot| slot.target_died()) }) - .unwrap_or(0) + .map(|(id, _)| *id) + .collect() } /// Snapshot the live weakrefs targeting `id` as @@ -486,7 +543,7 @@ impl WeakRefRegistry { .filter_map(|(id, v)| { v.iter() .filter_map(Weak::upgrade) - .find_map(|s| s.target.borrow().clone()) + .find_map(|s| s.target.borrow().as_ref()?.upgrade()) .map(|t| (*id, t)) }) .collect() @@ -509,8 +566,8 @@ static REGISTRY: std::sync::LazyLock = std::sync::LazyLock::new(WeakRefRegistry::new); /// RFC 0065 (WS4): insert-only miss-filter over every id a weakref was -/// ever registered for. The prompt-reap drop paths ask "is anything -/// watching this object?" for *every* dropped container; the +/// ever registered for. Every death hook asks "is anything watching +/// this object?"; the /// overwhelming "no" answer is two relaxed loads here instead of the /// registry cell's borrow protocol. Stale bits (dead referents) are /// false positives that take the precise path — never a correctness @@ -575,14 +632,34 @@ pub fn count_for(id: ObjectId) -> usize { with_registry(|r| r.count(id)) } -/// Convenience: count of registry-held strong clones of `id` in the -/// current thread. Used by the cycle collector's refcount accounting. -pub fn strong_clone_count(id: ObjectId) -> usize { - // RFC 0065 (WS4): see `count_for`. - if !WEAKREF_FILTER.may_contain(id) { - return 0; +/// The death hook every weakly referenceable object's `Drop` calls with +/// its own id: clear the slots watching it and queue their callbacks for +/// the next safe point, as CPython's `tp_dealloc` does. One relaxed load +/// when no weakref was ever registered for `id`. +#[inline] +pub fn on_death(id: ObjectId) { + if WEAKREF_FILTER.may_contain(id) { + on_death_slow(id); + } +} + +#[cold] +fn on_death_slow(id: ObjectId) { + // A death during a registry borrow leaves the slots for + // `sweep_dead_targets`. + if let Some(cleared) = REGISTRY.try_notify_clear(id) { + queue_callbacks(cleared); + } +} + +/// Clear the slots of referents that died without their death hook +/// clearing them, and queue the callbacks. Run by every collection. +pub fn sweep_dead_targets() -> usize { + let ids = REGISTRY.dead_target_ids(); + for &id in &ids { + queue_callbacks(notify_clear(id)); } - with_registry(|r| r.strong_clone_count(id)) + ids.len() } /// Convenience: collect every live weakref for `id` in the @@ -705,11 +782,9 @@ mod tests { reg.register(last.clone()); drop(dead); assert_eq!(reg.count(64), 2); - assert_eq!(reg.strong_clone_count(64), 2); first.clear(); - // A still-owned, cleared slot is a live wrapper, not a target owner. + // A still-owned, cleared slot is a live wrapper. assert_eq!(reg.count(64), 2); - assert_eq!(reg.strong_clone_count(64), 1); reg.shrink(); let watchers = reg.collect_strong(64); assert_eq!(watchers.len(), 2); @@ -721,7 +796,6 @@ mod tests { assert!(Arc::ptr_eq(&cleared[1].0, &first)); assert!(last.upgrade().is_none()); assert_eq!(reg.count(64), 0); - assert_eq!(reg.strong_clone_count(64), 0); assert!(reg.notify_clear(64).is_empty()); } @@ -765,9 +839,7 @@ mod tests { let replacement = Arc::new(WeakRefSlot::new(16, Object::Int(7), false, kind::REF)); reg.register(replacement.clone()); assert_eq!(reg.count(16), 1); - assert_eq!(reg.strong_clone_count(16), 1); assert_eq!(reg.count(17), 0); - assert_eq!(reg.strong_clone_count(17), 0); assert!(reg.collect_strong(17).is_empty()); assert!(reg.notify_clear(17).is_empty()); } diff --git a/tests/regrtest/test_finalizer_constructor_fallback.py b/tests/regrtest/test_finalizer_constructor_fallback.py new file mode 100644 index 00000000..d8b5cc50 --- /dev/null +++ b/tests/regrtest/test_finalizer_constructor_fallback.py @@ -0,0 +1,38 @@ +# A constructor fast path that falls back to the generic call must not +# finalize the instance it allocated and then abandoned: Python never saw +# it, so its `__del__` would run on an object `__init__` never touched. + +log = [] +missing = [] + + +class Finalized: + def __init__(self, value): + self.value = value + + def __del__(self): + try: + log.append(self.value) + except AttributeError: + missing.append(len(log)) + + +class Holder: + pass + + +def rebind(n): + holder = Holder() + holder.item = 0 + for i in range(n): + holder.item = Finalized(i) + + +for unused in range(3): + log.clear() + rebind(3000) + assert missing == [], f"{len(missing)} unborn instances finalized" + # The last instance is still bound until `holder` dies with the frame. + assert log == list(range(3000)), (len(log), log[:5]) + +print("constructor fallback finalization: ok") From a6f63015c30f7969c65e336979188a1fca2c50a5 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 06:17:56 -0700 Subject: [PATCH 03/96] perf: compile leaf plans to native code and inline property getters Leaf plans (the frameless evaluator's register form of small methods) now compile with Cranelift once warm. Registers become SSA variables; moves, constants, branches, and integer compares and arithmetic run in line; other operations call helpers that share the interpreter's own implementation (factored out of the runner). A method call whose site last resolved a pure leaf compiles the callee's plan in line, guarded on the function's identity and code, so `self.output().value` runs without nested evaluations. A guard failure or any undecidable operation declines exactly as the runner does. Other changes: - Property getters run as inline activations in the core loop instead of a nested interpreter, with the getter cached per site and class version (3,844 to 2,166 instructions per access). - `getattr(obj, name[, default])` on a plain instance takes a leaf lane (3,614 to 1,543 instructions). - Comparisons of two instances of one class call the class's Python dunder directly, without a bound method. - Tier 2's compile-time probes no longer materialize split instance dictionaries, which had left probed instances on the slow dict path. - Weakref callbacks count toward the shared pending-work gate, so one thread's drain can't strand another thread's queued callbacks. - `_testinternalcapi.instance_layout` reports an instance's attribute layout without changing it. deltablue retires 4.6% fewer instructions. --- crates/weavepy-vm/Cargo.toml | 15 +- crates/weavepy-vm/src/leaf_plan.rs | 714 +++++---- crates/weavepy-vm/src/leaf_plan/jit.rs | 1297 +++++++++++++++++ crates/weavepy-vm/src/lib.rs | 361 ++++- .../src/stdlib/testinternalcapi_mod.rs | 24 + crates/weavepy-vm/src/tier2.rs | 17 +- crates/weavepy-vm/src/vm_singletons.rs | 28 +- tests/regrtest/test_leaf_builtin_calls.py | 11 + 8 files changed, 2094 insertions(+), 373 deletions(-) create mode 100644 crates/weavepy-vm/src/leaf_plan/jit.rs diff --git a/crates/weavepy-vm/Cargo.toml b/crates/weavepy-vm/Cargo.toml index a404cafa..c6aa37de 100644 --- a/crates/weavepy-vm/Cargo.toml +++ b/crates/weavepy-vm/Cargo.toml @@ -76,6 +76,12 @@ crossbeam-utils = { workspace = true } # RFC 0032 — tier-2 Cranelift JIT, behind the (default-off) `jit` feature. weavepy-jit = { workspace = true, optional = true } +# Native code for leaf plans (`leaf_plan::jit`), under the same feature. +cranelift-codegen = { workspace = true, optional = true } +cranelift-frontend = { workspace = true, optional = true } +cranelift-jit = { workspace = true, optional = true } +cranelift-module = { workspace = true, optional = true } +cranelift-native = { workspace = true, optional = true } stacker = "0.1.24" # RFC 0066 WS4: dedicated native stacks for greenlets (aarch64/x86_64 # switching with guard pages — the same contract as upstream greenlet's @@ -100,7 +106,14 @@ rustls-native-certs = { workspace = true } # `--no-default-features` for size-constrained embedders. default = ["jit"] # Compile the tier-2 JIT integration (pulls in Cranelift). -jit = ["dep:weavepy-jit"] +jit = [ + "dep:weavepy-jit", + "dep:cranelift-codegen", + "dep:cranelift-frontend", + "dep:cranelift-jit", + "dep:cranelift-module", + "dep:cranelift-native", +] [lints] workspace = true diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index 84115000..521f4c7f 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -12,6 +12,9 @@ use weavepy_compiler::{BinOpKind, CodeObject, CompareKind, OpCode, UnaryKind}; +#[cfg(feature = "jit")] +mod jit; + use crate::object::Object; use crate::sync::Rc; use crate::{CodeConstObjects, Interpreter}; @@ -107,6 +110,9 @@ pub(crate) fn compare(a: V, b: V, kind: CompareKind) -> Option { /// Registers a plan may address: the locals, then the operand stack. const REGS: usize = 32; +/// Nested leaf calls evaluated within one evaluation at most this deep. +const NEST: u8 = 3; + /// One plan operation. Registers are `u8` indices below [`REGS`]; `pc` /// is the source instruction (its inline caches and site slots), and /// `name` its `co_names` index. @@ -229,6 +235,13 @@ pub(crate) struct LeafPlan { /// reassigned), each to a different name: the buffered stores share /// one receiver and each is the latest to its attribute. unique_stores: bool, + /// Evaluations so far, toward compiling the plan (see [`jit`]). + #[cfg(feature = "jit")] + runs: std::sync::atomic::AtomicU32, + /// The native code for a pure and an effect evaluation, once compiled + /// (`None` inside: the plan stays interpreted). + #[cfg(feature = "jit")] + native: [std::sync::OnceLock>; 2], } // SAFETY: the constants' pointers name the code extension's immutable @@ -679,6 +692,10 @@ impl Builder<'_> { nargs: u8::try_from(crate::leaf_arity(self.code)).ok()?, nregs: u8::try_from(self.top.get().clamp(self.nl, REGS)).ok()?, unique_stores: self.stored.is_some() && self.code.arg_count > 0, + #[cfg(feature = "jit")] + runs: std::sync::atomic::AtomicU32::new(0), + #[cfg(feature = "jit")] + native: [std::sync::OnceLock::new(), std::sync::OnceLock::new()], }) } } @@ -741,6 +758,14 @@ impl Owned { } impl Owned { + #[inline(always)] + fn new() -> Self { + Owned { + buf: [const { std::mem::MaybeUninit::uninit() }; OWNED], + n: 0, + } + } + /// Whether `p` names one of the held values. #[inline(always)] fn holds(&self, p: *const Object) -> bool { @@ -792,6 +817,15 @@ struct Pending { } impl Pending { + #[inline(always)] + fn new() -> Self { + Pending { + buf: [const { std::mem::MaybeUninit::uninit() }; PENDING], + n: 0, + moved: 0, + } + } + fn get(&self, k: usize) -> &Store { debug_assert!(k < self.n); // SAFETY: the first `n` entries are initialized. @@ -915,9 +949,6 @@ impl Interpreter { args: &[*const Object], nest: u8, ) -> Option { - use weavepy_compiler::InlineCache as IC; - /// Nested leaf calls evaluated in place at most this deep. - const NEST: u8 = 3; /// A pure-leaf callee runs as a frame of this evaluation: its /// registers are the next `REGS` window, and its caller's state /// waits here until its return. @@ -936,6 +967,14 @@ impl Interpreter { if args.len() != nargs { return None; } + // A warm plan runs as native code (a getter's or constructor's + // evaluation stays here). + #[cfg(feature = "jit")] + if !GETTER && !FRESH { + if let Some(r) = jit::run::(self, code, ext, plan, f, args, nest) { + return r; + } + } let (mut code, mut ext, mut plan, mut f) = (code, ext, plan, f); let mut regs = [const { std::mem::MaybeUninit::::uninit() }; REGS]; for (k, &a) in args.iter().enumerate() { @@ -960,15 +999,8 @@ impl Interpreter { } let mut consts: &[V] = &plan.consts; let mut stamps: &[crate::StampSlot] = ext.stamp_slots.get().map_or(&[], |s| &s[..]); - let mut owned = Owned { - buf: [const { std::mem::MaybeUninit::uninit() }; OWNED], - n: 0, - }; - let mut pend = Pending { - buf: [const { std::mem::MaybeUninit::uninit() }; PENDING], - n: 0, - moved: 0, - }; + let mut owned = Owned::new(); + let mut pend = Pending::new(); let mut ops: &[Op] = &plan.ops; let mut ip = 0usize; loop { @@ -981,134 +1013,24 @@ impl Interpreter { Op::Move { dst, src } => set!(dst, get!(src)), // SAFETY: `k` names a plan constant (the translation). Op::Const { dst, k } => set!(dst, unsafe { *consts.get_unchecked(usize::from(k)) }), - Op::Global { dst, pc } => { - // The callee's own namespaces, stamp-validated as the - // core loop's `LOAD_GLOBAL` arm. - let pc = usize::from(pc); - let slot = stamps.get(pc)?; - let (gdict, bdict) = (f.globals.as_ptr(), f.builtins.as_ptr()); - let gid = crate::specialize::rc_id(&f.globals); - // SAFETY (raw dict reads): nothing runs code here. - let g_stamp = unsafe { (*gdict).mutation_stamp() }; - let hit = match code.caches.get(pc as u32) { - IC::LoadGlobalModule { - globals_id, - key_idx, - } if globals_id == gid && slot.get() == [gid, g_stamp, 0] => unsafe { - (*gdict).get_index(key_idx as usize) - }, - IC::LoadGlobalBuiltin { - builtins_id, - key_idx, - } if builtins_id == crate::specialize::rc_id(&f.builtins) - && !self.globals_missing_any.get() - && slot.get() - == [gid, g_stamp, unsafe { (*bdict).mutation_stamp() }] => - unsafe { (*bdict).get_index(key_idx as usize) }, - _ => return None, - }; - set!(dst, norm(hit?.1)); - } + Op::Global { dst, pc } => set!(dst, self.plan_global(code, stamps, f, pc)?), Op::Attr { dst, src, pc, name } => { - let V::R(p) = get!(src) else { - return None; - }; - let (pc, name) = (u32::from(pc), u32::from(name)); - // SAFETY: as `norm`. - let recv = unsafe { &*p }; - if EFFECT && pend.n > 0 { - // A buffered store to this attribute, the latest. - let hit = (0..pend.n) - .rev() - .map(|k| pend.get(k)) - // SAFETY: stable receivers (see `Store`). - .find(|(r, _, n, _)| *n == name && unsafe { (**r).is_same(recv) }); - if let Some((_, _, _, v)) = hit { - set!(dst, norm(v)); - continue; - } - } - let v = match recv { - Object::Instance(inst) => { - if GETTER && inst.cls_raw().native_kind.get() != 0 { - return None; - } - // SAFETY: the receiver remains rooted by an - // argument or owned scratch; no Python runs. - let hit = unsafe { - Self::leaf_cached_instance_field(ext, code, inst, pc, name) - } - .map(std::ptr::from_ref); - match hit { - Some(v) => norm(v), - // A stale or absent site cache resolves - // through the site's own entries. - None => owned.own(Self::leaf_attr_resolve_site( - code, inst, recv, pc, name, - )?)?, - } - } - Object::Type(cls) => { - if GETTER && !Self::plain_metaclass(cls) { - return None; - } - match stamps - .get(pc as usize) - .and_then(|s| crate::class_attr_hit(s, cls)) - { - Some(v) => owned.own(v)?, - None => { - owned.own(Self::leaf_load_type_attr(code, cls, pc, name)?)? - } - } - } - Object::Module(module) => { - if GETTER - && (crate::object::module_class(module).is_some() - || code.names.get(name as usize)?.starts_with("__")) - { - return None; - } - owned.own(Self::leaf_load_attr_recv(code, recv, pc, name)?)? - } - _ => return None, - }; + let v = self.plan_attr::( + code, + ext, + stamps, + &mut owned, + &pend, + get!(src), + pc, + name, + )?; set!(dst, v); } Op::Method { dst, src, pc, name } => { - // A method off the site's slot: the function under - // the receiver, or a class's function with an empty - // self slot (as the core loop's arm). - let V::R(p) = get!(src) else { - return None; - }; - let ms = crate::code_method_slot(code, u32::from(pc))?; - // SAFETY: as `norm`. - match unsafe { &*p } { - Object::Instance(inst) => { - let cls = inst.cls_raw(); - if !Self::default_getattribute(cls) { - return None; - } - let fp = ms.peek_fn(cls.attr_version.get())?; - // The instance's attributes must not shadow - // the method. - if crate::inst_may_shadow(inst, code, u32::from(name)) { - return None; - } - set!(dst, V::Fn(fp)); - set!(dst + 1, V::R(p)); - } - Object::Type(cls) => { - set!(dst, V::Fn(ms.peek_unbound(cls.attr_version.get())?)); - set!(dst + 1, V::Null); - } - recv => { - let b = self.leaf_builtin_method_ptr(ms, recv, code, name)?; - set!(dst, V::Bi(b)); - set!(dst + 1, V::R(p)); - } - } + let (func, recv) = self.plan_method(code, get!(src), pc, name)?; + set!(dst, func); + set!(dst + 1, recv); } Op::Compare { dst, a, b, kind } => { // SAFETY: the translation checked `kind` names a @@ -1117,77 +1039,11 @@ impl Interpreter { set!(dst, V::B(compare(get!(a), get!(b), kind)?)); } Op::Is { dst, a, b, invert } => { - let same = match (get!(a), get!(b)) { - (V::N, V::N) => true, - (V::B(x), V::B(y)) => x == y, - (V::I(x), V::I(y)) => Object::Int(x).is_same(&Object::Int(y)), - (V::F(x), V::F(y)) => Object::Float(x).is_same(&Object::Float(y)), - // SAFETY: as `norm`. - (V::R(p), V::R(q)) => unsafe { (*p).is_same(&*q) }, - _ => false, - }; - set!(dst, V::B(same != invert)); + set!(dst, V::B(is_same(get!(a), get!(b)) != invert)) } Op::Truth { dst, src } => set!(dst, V::B(truth(get!(src))?)), - Op::Unary { dst, src, kind } => { - let v = get!(src); - let r = match kind { - k if k == UnaryKind::Not as u8 => V::B(!truth(v)?), - k if k == UnaryKind::Neg as u8 => match v { - V::I(i) => V::I(i.checked_neg()?), - V::F(x) => V::F(-x), - _ => return None, - }, - k if k == UnaryKind::Pos as u8 => match v { - V::I(_) | V::F(_) => v, - _ => return None, - }, - k if k == UnaryKind::Invert as u8 => match v { - V::I(i) => V::I(!i), - _ => return None, - }, - _ => return None, - }; - set!(dst, r); - } - Op::Binary { dst, a, b, kind } => { - let r = match (get!(a), get!(b)) { - (V::I(a), V::I(b)) => V::I(match kind { - k if k == BinOpKind::Add as u8 => a.checked_add(b)?, - k if k == BinOpKind::Sub as u8 => a.checked_sub(b)?, - k if k == BinOpKind::Mult as u8 => a.checked_mul(b)?, - k if k == BinOpKind::BitAnd as u8 => a & b, - k if k == BinOpKind::BitOr as u8 => a | b, - k if k == BinOpKind::BitXor as u8 => a ^ b, - k if k == BinOpKind::RShift as u8 && (0..64).contains(&b) => a >> b, - k if k == BinOpKind::LShift as u8 - && (0..63).contains(&b) - && ((a << b) >> b) == a => - { - a << b - } - k if k == BinOpKind::FloorDiv as u8 && b > 0 && a >= 0 => a / b, - k if k == BinOpKind::Mod as u8 && b > 0 && a >= 0 => a % b, - _ => return None, - }), - (x, y) => { - let (x, y) = match (x, y) { - (V::F(x), V::F(y)) => (x, y), - (V::I(x), V::F(y)) => (x as f64, y), - (V::F(x), V::I(y)) => (x, y as f64), - _ => return None, - }; - // SAFETY: as the core loop's `BINARY_OP` arm - // (the compiler emits only valid kinds). - let kind: BinOpKind = unsafe { std::mem::transmute(kind) }; - match Self::leaf_float_op(x, y, kind)? { - Object::Float(x) => V::F(x), - _ => return None, - } - } - }; - set!(dst, r); - } + Op::Unary { dst, src, kind } => set!(dst, unary(get!(src), kind)?), + Op::Binary { dst, a, b, kind } => set!(dst, binary(get!(a), get!(b), kind)?), Op::Swap { a, b } => { let (x, y) = (get!(a), get!(b)); set!(a, y); @@ -1204,28 +1060,7 @@ impl Interpreter { } } Op::Jump { target } => ip = usize::from(target), - Op::New { dst, dict } => { - // Tracked like the core loop's; an allocation that - // would trigger a collection is left to it. - if crate::stdlib::tracemalloc_real::is_tracking() - || crate::stdlib::testinternalcapi_mod::reftrace_print_active() - || crate::gc_trace::auto_collect_due() - { - return None; - } - let obj = if dict { - Object::Dict(Rc::new(crate::sync::RefCell::new( - crate::object::DictData::with_capacity_and_hasher( - 0, - crate::fasthash::FxBuildHasher, - ), - ))) - } else { - Object::new_list(Vec::new()) - }; - crate::gc_trace::track(obj.clone()); - set!(dst, owned.own(obj)?); - } + Op::New { dst, dict } => set!(dst, new_container(&mut owned, dict)?), Op::Decline => return None, Op::Call { at, argc, pc } => { // A pure-leaf callee, evaluated in place: only while @@ -1401,73 +1236,7 @@ impl Interpreter { set!(c.at, v); continue; } - // A value this evaluation's scratch doesn't hold outlives - // it (a pure body stores nothing), so a nested caller - // borrows it rather than taking a reference to release. - if !EFFECT && !matches!(v, V::R(p) if owned.holds(p)) { - return Some(LeafRet::Borrowed(v)); - } - let r = to_object(v)?; - if EFFECT && pend.n > 0 { - // The latest store to each attribute is the one - // that lands; every one of them must go through - // before any does (a decline touched nothing, and - // the ordinary call runs the body instead). A - // lone store is its own check: it declines whole. - let unique = plan.unique_stores; - if pend.n > 1 { - // With one receiver, appends of new attributes - // land in order: `cursor` is its split length - // once the earlier ones have. - let mut cursor = None; - // A store the shortcut can't vouch for may - // move the layout: the rest check in full. - let mut split_ok = unique; - for k in 0..pend.n { - if !unique && !pend.latest(k) { - continue; - } - let (rp, spc, name, _) = pend.get(k); - // SAFETY: stable receivers (see `Store`). - let Object::Instance(inst) = (unsafe { &**rp }) else { - return None; - }; - let (spc, name) = (*spc as usize, *name); - let ready = if split_ok { - Self::core_store_attr_ready_split(ext, inst, spc, &mut cursor) - } else { - None - }; - split_ok &= ready.is_some(); - if !ready.unwrap_or_else(|| { - Self::core_store_attr_ready(code, inst, spc, name) - }) { - return None; - } - } - } - let n = pend.n; - for k in 0..n { - if !unique && !pend.latest(k) { - continue; - } - let (rp, spc, name, value) = pend.get(k); - // SAFETY: stable receivers (see `Store`). - let Object::Instance(inst) = (unsafe { &**rp }) else { - return None; - }; - // On `true` the value moved into the dict. - if Self::core_store_attr(code, inst, *spc as usize, *name, value) { - pend.moved |= 1 << k; - } else if n == 1 { - // The lone store declined, untouched. - return None; - } else { - debug_assert!(false, "a ready store declined"); - } - } - } - return Some(LeafRet::Owned(r)); + return self.plan_finish::(code, ext, plan, &owned, &mut pend, v); } Op::StoreAttr { recv, @@ -1501,24 +1270,355 @@ impl Interpreter { } continue; } - if !matches!(recv, Object::Instance(_)) || pend.n == PENDING { - return None; + plan_store(args, &mut owned, &mut pend, rp, get!(val), pc, name)?; + } + } + } + } +} + +/// `a is b` on two leaf values. +#[inline(always)] +pub(crate) fn is_same(a: V, b: V) -> bool { + match (a, b) { + (V::N, V::N) => true, + (V::B(x), V::B(y)) => x == y, + (V::I(x), V::I(y)) => Object::Int(x).is_same(&Object::Int(y)), + (V::F(x), V::F(y)) => Object::Float(x).is_same(&Object::Float(y)), + // SAFETY: as `norm`. + (V::R(p), V::R(q)) => unsafe { (*p).is_same(&*q) }, + _ => false, + } +} + +/// A unary operator on a scalar; `None` for any other operand. +#[inline(always)] +pub(crate) fn unary(v: V, kind: u8) -> Option { + Some(match kind { + k if k == UnaryKind::Not as u8 => V::B(!truth(v)?), + k if k == UnaryKind::Neg as u8 => match v { + V::I(i) => V::I(i.checked_neg()?), + V::F(x) => V::F(-x), + _ => return None, + }, + k if k == UnaryKind::Pos as u8 => match v { + V::I(_) | V::F(_) => v, + _ => return None, + }, + k if k == UnaryKind::Invert as u8 => match v { + V::I(i) => V::I(!i), + _ => return None, + }, + _ => return None, + }) +} + +/// A binary operator on two scalars; `None` for any other operands or a +/// result the interpreter must produce (an overflow, a negative operand +/// of a floor division). +#[inline(always)] +pub(crate) fn binary(a: V, b: V, kind: u8) -> Option { + Some(match (a, b) { + (V::I(a), V::I(b)) => V::I(match kind { + k if k == BinOpKind::Add as u8 => a.checked_add(b)?, + k if k == BinOpKind::Sub as u8 => a.checked_sub(b)?, + k if k == BinOpKind::Mult as u8 => a.checked_mul(b)?, + k if k == BinOpKind::BitAnd as u8 => a & b, + k if k == BinOpKind::BitOr as u8 => a | b, + k if k == BinOpKind::BitXor as u8 => a ^ b, + k if k == BinOpKind::RShift as u8 && (0..64).contains(&b) => a >> b, + k if k == BinOpKind::LShift as u8 && (0..63).contains(&b) && ((a << b) >> b) == a => { + a << b + } + k if k == BinOpKind::FloorDiv as u8 && b > 0 && a >= 0 => a / b, + k if k == BinOpKind::Mod as u8 && b > 0 && a >= 0 => a % b, + _ => return None, + }), + (x, y) => { + let (x, y) = match (x, y) { + (V::F(x), V::F(y)) => (x, y), + (V::I(x), V::F(y)) => (x as f64, y), + (V::F(x), V::I(y)) => (x, y as f64), + _ => return None, + }; + // SAFETY: as the core loop's `BINARY_OP` arm (the compiler + // emits only valid kinds). + let kind: BinOpKind = unsafe { std::mem::transmute(kind) }; + match Interpreter::leaf_float_op(x, y, kind)? { + Object::Float(x) => V::F(x), + _ => return None, + } + } + }) +} + +/// `BUILD_LIST 0` / `BUILD_MAP 0`: a new empty list or dict, tracked like +/// the core loop's and held by the owned scratch. An allocation that would +/// trigger a collection is left to the core loop. +#[inline(always)] +fn new_container(owned: &mut Owned, dict: bool) -> Option { + if crate::stdlib::tracemalloc_real::is_tracking() + || crate::stdlib::testinternalcapi_mod::reftrace_print_active() + || crate::gc_trace::auto_collect_due() + { + return None; + } + let obj = if dict { + Object::Dict(Rc::new(crate::sync::RefCell::new( + crate::object::DictData::with_capacity_and_hasher(0, crate::fasthash::FxBuildHasher), + ))) + } else { + Object::new_list(Vec::new()) + }; + crate::gc_trace::track(obj.clone()); + owned.own(obj) +} + +/// An effect leaf's `recv.name = val` (at store `pc`), buffered until the +/// return commits it. An argument receiver is used in place; any other (a +/// field's value) is held by the owned scratch. +#[inline(always)] +fn plan_store( + args: &[*const Object], + owned: &mut Owned, + pend: &mut Pending, + rp: *const Object, + val: V, + pc: u16, + name: u16, +) -> Option<()> { + // SAFETY: as `norm`. + let recv = unsafe { &*rp }; + if !matches!(recv, Object::Instance(_)) || pend.n == PENDING { + return None; + } + let value = to_object(val)?; + let rp = if args.iter().any(|&a| std::ptr::eq(a, rp)) { + rp + } else { + let V::R(held) = owned.own(crate::clone_hot(recv))? else { + return None; + }; + held + }; + pend.buf[pend.n].write((rp, u32::from(pc), u32::from(name), value)); + pend.n += 1; + Some(()) +} + +impl Interpreter { + /// `LOAD_GLOBAL` at `pc` of `code` in `f`'s namespaces, stamp-validated + /// as the core loop's arm. + #[inline(always)] + fn plan_global( + &self, + code: &CodeObject, + stamps: &[crate::StampSlot], + f: &crate::object::PyFunction, + pc: u16, + ) -> Option { + use weavepy_compiler::InlineCache as IC; + let pc = usize::from(pc); + let slot = stamps.get(pc)?; + let (gdict, bdict) = (f.globals.as_ptr(), f.builtins.as_ptr()); + let gid = crate::specialize::rc_id(&f.globals); + // SAFETY (raw dict reads): nothing runs code here. + let g_stamp = unsafe { (*gdict).mutation_stamp() }; + let hit = match code.caches.get(pc as u32) { + IC::LoadGlobalModule { + globals_id, + key_idx, + } if globals_id == gid && slot.get() == [gid, g_stamp, 0] => unsafe { + (*gdict).get_index(key_idx as usize) + }, + IC::LoadGlobalBuiltin { + builtins_id, + key_idx, + } if builtins_id == crate::specialize::rc_id(&f.builtins) + && !self.globals_missing_any.get() + && slot.get() == [gid, g_stamp, unsafe { (*bdict).mutation_stamp() }] => + unsafe { (*bdict).get_index(key_idx as usize) }, + _ => return None, + }; + Some(norm(hit?.1)) + } + + /// `src.name` (the `LOAD_ATTR` at `pc`): the latest buffered store to + /// it, a cached instance field, or what the site resolves; a value the + /// read hands back owned is held by the owned scratch. + #[allow(clippy::too_many_arguments)] + #[inline(always)] + fn plan_attr( + &self, + code: &CodeObject, + ext: &CodeConstObjects, + stamps: &[crate::StampSlot], + owned: &mut Owned, + pend: &Pending, + src: V, + pc: u16, + name: u16, + ) -> Option { + let V::R(p) = src else { + return None; + }; + let (pc, name) = (u32::from(pc), u32::from(name)); + // SAFETY: as `norm`. + let recv = unsafe { &*p }; + if EFFECT && pend.n > 0 { + // A buffered store to this attribute, the latest. + let hit = (0..pend.n) + .rev() + .map(|k| pend.get(k)) + // SAFETY: stable receivers (see `Store`). + .find(|(r, _, n, _)| *n == name && unsafe { (**r).is_same(recv) }); + if let Some((_, _, _, v)) = hit { + return Some(norm(v)); + } + } + match recv { + Object::Instance(inst) => { + if GETTER && inst.cls_raw().native_kind.get() != 0 { + return None; + } + // SAFETY: the receiver remains rooted by an argument or + // owned scratch; no Python runs. + match unsafe { Self::leaf_cached_instance_field(ext, code, inst, pc, name) } { + Some(v) => Some(norm(v)), + // A stale or absent site cache resolves through the + // site's own entries. + None => owned.own(Self::leaf_attr_resolve_site(code, inst, recv, pc, name)?), + } + } + Object::Type(cls) => { + if GETTER && !Self::plain_metaclass(cls) { + return None; + } + match stamps + .get(pc as usize) + .and_then(|s| crate::class_attr_hit(s, cls)) + { + Some(v) => owned.own(v), + None => owned.own(Self::leaf_load_type_attr(code, cls, pc, name)?), + } + } + Object::Module(module) => { + if GETTER + && (crate::object::module_class(module).is_some() + || code.names.get(name as usize)?.starts_with("__")) + { + return None; + } + owned.own(Self::leaf_load_attr_recv(code, recv, pc, name)?) + } + _ => None, + } + } + + /// The method-form load of `src.name` off the site's slot: the function + /// and the receiver, or a class's function and the empty self slot (as + /// the core loop's arm). + #[inline(always)] + fn plan_method(&self, code: &CodeObject, src: V, pc: u16, name: u16) -> Option<(V, V)> { + let V::R(p) = src else { + return None; + }; + let ms = crate::code_method_slot(code, u32::from(pc))?; + // SAFETY: as `norm`. + match unsafe { &*p } { + Object::Instance(inst) => { + let cls = inst.cls_raw(); + if !Self::default_getattribute(cls) { + return None; + } + let fp = ms.peek_fn(cls.attr_version.get())?; + // The instance's attributes must not shadow the method. + if crate::inst_may_shadow(inst, code, u32::from(name)) { + return None; + } + Some((V::Fn(fp), V::R(p))) + } + Object::Type(cls) => Some((V::Fn(ms.peek_unbound(cls.attr_version.get())?), V::Null)), + recv => { + let b = self.leaf_builtin_method_ptr(ms, recv, code, name)?; + Some((V::Bi(b), V::R(p))) + } + } + } + + /// The evaluation's return of `v`: borrowed when the evaluation's + /// scratch doesn't hold it (a pure body stores nothing, so it outlives + /// the evaluation), otherwise owned, with an effect leaf's buffered + /// stores committed first. The latest store to each attribute is the + /// one that lands; every one must go through before any does (a decline + /// touched nothing, and the ordinary call runs the body instead). A + /// lone store is its own check: it declines whole. + #[inline(always)] + fn plan_finish( + &self, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + owned: &Owned, + pend: &mut Pending, + v: V, + ) -> Option { + if !EFFECT && !matches!(v, V::R(p) if owned.holds(p)) { + return Some(LeafRet::Borrowed(v)); + } + let r = to_object(v)?; + if EFFECT && pend.n > 0 { + let unique = plan.unique_stores; + if pend.n > 1 { + // With one receiver, appends of new attributes land in + // order: `cursor` is its split length once the earlier ones + // have. A store the shortcut can't vouch for may move the + // layout: the rest check in full. + let mut cursor = None; + let mut split_ok = unique; + for k in 0..pend.n { + if !unique && !pend.latest(k) { + continue; } - let value = to_object(get!(val))?; - // An argument receiver is used in place; any other - // (a field's value) is held by the owned scratch. - let rp = if args.iter().any(|&a| std::ptr::eq(a, rp)) { - rp + let (rp, spc, name, _) = pend.get(k); + // SAFETY: stable receivers (see `Store`). + let Object::Instance(inst) = (unsafe { &**rp }) else { + return None; + }; + let (spc, name) = (*spc as usize, *name); + let ready = if split_ok { + Self::core_store_attr_ready_split(ext, inst, spc, &mut cursor) } else { - let V::R(held) = owned.own(crate::clone_hot(recv))? else { - return None; - }; - held + None }; - pend.buf[pend.n].write((rp, u32::from(pc), u32::from(name), value)); - pend.n += 1; + split_ok &= ready.is_some(); + if !ready.unwrap_or_else(|| Self::core_store_attr_ready(code, inst, spc, name)) + { + return None; + } + } + } + let n = pend.n; + for k in 0..n { + if !unique && !pend.latest(k) { + continue; + } + let (rp, spc, name, value) = pend.get(k); + // SAFETY: stable receivers (see `Store`). + let Object::Instance(inst) = (unsafe { &**rp }) else { + return None; + }; + // On `true` the value moved into the dict. + if Self::core_store_attr(code, inst, *spc as usize, *name, value) { + pend.moved |= 1 << k; + } else if n == 1 { + // The lone store declined, untouched. + return None; + } else { + debug_assert!(false, "a ready store declined"); } } } + Some(LeafRet::Owned(r)) } } diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs new file mode 100644 index 00000000..fd8b7731 --- /dev/null +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -0,0 +1,1297 @@ +//! Native code for leaf plans. +//! +//! A plan that keeps running is compiled, once per effect mode, into a +//! function over the same registers: each register is a pair of SSA +//! variables holding the [`V`] tag and payload words; moves, constants, +//! jumps, and scalar compares, truth tests and arithmetic run in line; and +//! every other operation calls a helper that runs the interpreter's own +//! implementation of it. Whatever the native code or a helper can't +//! settle declines the evaluation exactly where the interpreter's runner +//! would, so a native plan keeps the runner's contract: a result, or +//! nothing observable. +//! +//! A method call whose site last resolved a pure leaf is compiled with the +//! callee's plan in line, behind a guard on the function and its code; any +//! other call goes through [`Interpreter::leaf_eval_nested`] (and so runs +//! natively too once the callee's own plan is compiled). + +use super::{ + binary, compare, is_same, new_container, norm, plan_store, truth, unary, LeafPlan, LeafRet, Op, + Owned, Pending, NEST, V, +}; +use crate::object::{Object, PyFunction}; +use crate::sync::Rc; +use crate::{CodeConstObjects, Interpreter}; +use cranelift_codegen::ir::{ + condcodes::IntCC, types, AbiParam, Block, InstBuilder, MemFlags, SigRef, Signature, + StackSlotData, StackSlotKind, Value, +}; +use cranelift_codegen::settings::{self, Configurable}; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; +use cranelift_jit::{JITBuilder, JITModule}; +use cranelift_module::{Linkage, Module}; +use weavepy_compiler::{BinOpKind, CodeObject, CompareKind}; + +/// Evaluations of a plan (per effect mode) before it is compiled. +const WARM_RUNS: u32 = 64; + +/// How deep callee plans are compiled in line. +const INLINE_DEPTH: u8 = 2; + +/// The `repr(u64)` discriminants of [`V`]'s variants, in declaration order. +const T_FN: u64 = 0; +const T_BI: u64 = 1; +const T_NULL: u64 = 2; +const T_R: u64 = 3; +const T_I: u64 = 4; +const T_F: u64 = 5; +const T_B: u64 = 6; +const T_N: u64 = 7; + +/// A compiled plan: `(ctx, argument words) -> status`, where `0` returned +/// (the result's words in `ctx.out`) and anything else declined. +type NativeFn = unsafe extern "C" fn(*mut Ctx<'static>, *const u64) -> u32; + +/// A plan's native code and what it was compiled against. +pub(super) struct Native { + func: NativeFn, + /// The deepest in-line callee (the evaluation needs that much room + /// below the recursion limit). + depth: u8, + /// The in-line callees' frame descriptors, which the code addresses, + /// and what keeps their identity guards sound: a function's + /// allocation stays reserved (so no other function takes its + /// address), and its code stays alive. + // Boxed: the code embeds each descriptor's address. + #[allow(clippy::vec_box)] + _frames: Vec>, + _keep: Vec<(crate::sync::Weak, Rc)>, +} + +// SAFETY: the descriptors name immutable code objects and reserved +// function allocations; native code is only entered with the GIL held. +unsafe impl Send for Native {} +// SAFETY: as above. +unsafe impl Sync for Native {} + +/// `v` as its tag and payload words (the unused payload bytes zeroed). +#[inline(always)] +fn words(v: V) -> [u64; 2] { + match v { + V::Fn(p) => [T_FN, p as u64], + V::Bi(p) => [T_BI, p as u64], + V::Null => [T_NULL, 0], + V::R(p) => [T_R, p as u64], + V::I(i) => [T_I, i as u64], + V::F(x) => [T_F, x.to_bits()], + V::B(b) => [T_B, u64::from(b)], + V::N => [T_N, 0], + } +} + +/// The value with tag `t` and payload `p`. +#[inline(always)] +fn value(t: u64, p: u64) -> V { + match t { + T_FN => V::Fn(p as *const PyFunction), + T_BI => V::Bi(p as *const crate::object::BuiltinFn), + T_NULL => V::Null, + T_R => V::R(p as *const Object), + T_I => V::I(p as i64), + T_F => V::F(f64::from_bits(p)), + T_B => V::B((p & 0xff) != 0), + _ => V::N, + } +} + +/// Whether the discriminants above are `V`'s (checked once, before any +/// plan compiles). +fn tags_match() -> bool { + let tag = |v: V| { + // SAFETY: `V` is `repr(u64)`: its first word is the discriminant, + // always initialized. + unsafe { *std::ptr::from_ref(&v).cast::() } + }; + tag(V::Fn(std::ptr::null())) == T_FN + && tag(V::Bi(std::ptr::null())) == T_BI + && tag(V::Null) == T_NULL + && tag(V::R(std::ptr::null())) == T_R + && tag(V::I(0)) == T_I + && tag(V::F(0.0)) == T_F + && tag(V::B(false)) == T_B + && tag(V::N) == T_N + && std::mem::size_of::() == 16 +} + +/// What one frame of an evaluation runs: the code (its caches), its +/// constants and namespaces' function. +#[repr(C)] +pub(super) struct Frame { + code: *const CodeObject, + ext: *const CodeConstObjects, + f: *const PyFunction, +} + +impl Frame { + /// # Safety + /// + /// The frame's code and function are alive (the evaluation's own, or + /// an in-line callee whose guard passed). + #[inline(always)] + unsafe fn parts<'a>(&self) -> (&'a CodeObject, &'a CodeConstObjects, &'a PyFunction) { + // SAFETY: forwarded contract. + unsafe { (&*self.code, &*self.ext, &*self.f) } + } +} + +/// One native evaluation's state, shared with the helpers. +#[repr(C)] +pub(super) struct Ctx<'a> { + /// A helper's result words (a method load's two values; a return's + /// first). + out: [u64; 4], + /// The evaluation's own frame. + top: Frame, + interp: &'a Interpreter, + args: &'a [*const Object], + owned: Owned, + pend: Pending, + nest: u8, +} + +impl Ctx<'_> { + #[inline(always)] + fn put(&mut self, v: V) { + let [t, p] = words(v); + self.out[0] = t; + self.out[1] = p; + } +} + +/// `ctx.top`'s offset (native code passes the evaluation's own frame by +/// address). +const TOP_OFFSET: i32 = 32; +const _: () = assert!(std::mem::offset_of!(Ctx<'static>, top) == TOP_OFFSET as usize); + +/// Run `plan`'s native code for one evaluation, compiling it once it is +/// warm: `None` when it has none (the interpreter's runner evaluates), +/// otherwise the evaluation's outcome. +#[inline] +#[allow(clippy::too_many_arguments)] +pub(super) fn run( + interp: &Interpreter, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + f: &PyFunction, + args: &[*const Object], + nest: u8, +) -> Option> { + let slot = &plan.native[usize::from(EFFECT)]; + let native = match slot.get() { + Some(Some(native)) => native, + Some(None) => return None, + None => { + if plan.runs.fetch_add(1, std::sync::atomic::Ordering::Relaxed) < WARM_RUNS + || crate::tier2::jit_off_for_process() + || crate::gil::free_threading_enabled() + { + return None; + } + slot.get_or_init(|| compile(code, ext, plan, EFFECT)) + .as_ref()? + } + }; + // In-line callees run where the interpreter would nest frames: near + // the recursion limit, it decides. + if crate::recursion::current_depth() + usize::from(nest) + usize::from(native.depth) + 1 + >= crate::recursion::recursion_limit() + { + return None; + } + let mut aw = [0u64; 16]; + for (k, &a) in args.iter().enumerate() { + let [t, p] = words(norm(a)); + aw[2 * k] = t; + aw[2 * k + 1] = p; + } + let mut ctx = Ctx { + out: [0; 4], + top: Frame { code, ext, f }, + interp, + args, + owned: Owned::new(), + pend: Pending::new(), + nest, + }; + // SAFETY: the code was compiled from `plan` with this signature; the + // context and the argument words outlive the call, and the lifetime + // is only erased for its duration. + let status = unsafe { + (native.func)( + std::ptr::from_mut(&mut ctx).cast::>(), + aw.as_ptr(), + ) + }; + if status != 0 { + return Some(None); + } + let v = value(ctx.out[0], ctx.out[1]); + // In place: registers point into the scratch, which must not move. + Some(interp.plan_finish::(code, ext, plan, &ctx.owned, &mut ctx.pend, v)) +} + +// ---- helpers (the interpreter's operations, for native code) ---------- + +/// The context behind a native plan's first parameter. +/// +/// # Safety +/// +/// `ctx` is the live context [`run`] passed to the native code. +#[inline(always)] +unsafe fn cx<'a>(ctx: *mut Ctx<'static>) -> &'a mut Ctx<'a> { + // SAFETY: forwarded contract. + unsafe { &mut *ctx.cast::>() } +} + +/// A frame's stamp slots (its global and class attribute caches). +#[inline(always)] +fn stamps(ext: &CodeConstObjects) -> &[crate::StampSlot] { + ext.stamp_slots.get().map_or(&[], |s| &s[..]) +} + +unsafe extern "C" fn h_global(ctx: *mut Ctx<'static>, fr: *const Frame, pc: u32) -> u32 { + // SAFETY: called by native code with its context and a live frame. + let (c, (code, ext, f)) = unsafe { (cx(ctx), (*fr).parts()) }; + match c.interp.plan_global(code, stamps(ext), f, pc as u16) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +unsafe extern "C" fn h_attr( + ctx: *mut Ctx<'static>, + fr: *const Frame, + t: u64, + p: u64, + pc_name: u32, +) -> u32 { + // SAFETY: called by native code with its context and a live frame. + let (c, (code, ext, _)) = unsafe { (cx(ctx), (*fr).parts()) }; + let (pc, name) = (pc_name as u16, (pc_name >> 16) as u16); + match c.interp.plan_attr::( + code, + ext, + stamps(ext), + &mut c.owned, + &c.pend, + value(t, p), + pc, + name, + ) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +unsafe extern "C" fn h_method( + ctx: *mut Ctx<'static>, + fr: *const Frame, + t: u64, + p: u64, + pc_name: u32, +) -> u32 { + // SAFETY: called by native code with its context and a live frame. + let (c, (code, _, _)) = unsafe { (cx(ctx), (*fr).parts()) }; + let (pc, name) = (pc_name as u16, (pc_name >> 16) as u16); + match c.interp.plan_method(code, value(t, p), pc, name) { + Some((func, recv)) => { + let [a, b] = words(func); + let [d, e] = words(recv); + c.out = [a, b, d, e]; + 0 + } + None => 1, + } +} + +/// `0`/`1` for the comparison's result, `2` to decline. +extern "C" fn h_compare(ta: u64, pa: u64, tb: u64, pb: u64, kind: u32) -> u32 { + // SAFETY: the translation checked `kind` names a comparison. + let kind: CompareKind = unsafe { std::mem::transmute(kind as u8) }; + match compare(value(ta, pa), value(tb, pb), kind) { + Some(b) => u32::from(b), + None => 2, + } +} + +/// `0`/`1` for the operand's truth, `2` to decline. +extern "C" fn h_truth(t: u64, p: u64) -> u32 { + match truth(value(t, p)) { + Some(b) => u32::from(b), + None => 2, + } +} + +extern "C" fn h_is(ta: u64, pa: u64, tb: u64, pb: u64) -> u32 { + u32::from(is_same(value(ta, pa), value(tb, pb))) +} + +unsafe extern "C" fn h_unary(ctx: *mut Ctx<'static>, t: u64, p: u64, kind: u32) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + match unary(value(t, p), kind as u8) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +unsafe extern "C" fn h_binary( + ctx: *mut Ctx<'static>, + ta: u64, + pa: u64, + tb: u64, + pb: u64, + kind: u32, +) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + match binary(value(ta, pa), value(tb, pb), kind as u8) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +unsafe extern "C" fn h_new(ctx: *mut Ctx<'static>, dict: u32) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + match new_container(&mut c.owned, dict != 0) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +/// The evaluation's own `recv.name = val` (only the top frame stores). +unsafe extern "C" fn h_store( + ctx: *mut Ctx<'static>, + tr: u64, + pr: u64, + tv: u64, + pv: u64, + pc_name: u32, +) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + let V::R(rp) = value(tr, pr) else { + return 1; + }; + let (pc, name) = (pc_name as u16, (pc_name >> 16) as u16); + match plan_store( + c.args, + &mut c.owned, + &mut c.pend, + rp, + value(tv, pv), + pc, + name, + ) { + Some(()) => 0, + None => 1, + } +} + +/// A leaf call at `pc` of frame `fr` (at in-line `depth`) on the registers +/// whose words start at `w`: the callee, its self slot, then `argc` +/// arguments. A pure-leaf Python callee is evaluated one level down +/// (natively, once compiled); a read-only builtin runs on borrowed copies +/// of its arguments. +unsafe extern "C" fn h_call( + ctx: *mut Ctx<'static>, + fr: *const Frame, + w: *const u64, + argc_depth: u32, + pc: u32, +) -> u32 { + // SAFETY: called by native code with its context and a live frame. + let (c, (code, _, _)) = unsafe { (cx(ctx), (*fr).parts()) }; + let (argc, depth) = (argc_depth & 0xff, (argc_depth >> 8) as u8); + let nest = c.nest + depth; + // Only while no store is buffered (the callee would not see one). + if (EFFECT && c.pend.n > 0) || nest >= NEST { + return 1; + } + // SAFETY: native code spilled `argc + 2` registers' words at `w`. + let reg = |k: usize| unsafe { value(*w.add(2 * k), *w.add(2 * k + 1)) }; + let first = if matches!(reg(1), V::Null) { 2 } else { 1 }; + let n = argc as usize + 2 - first; + if n > 8 { + return 1; + } + let (fp, native) = match reg(0) { + V::Fn(fp) => (fp, None), + V::Bi(b) => (std::ptr::null(), Some((b, None))), + // SAFETY: as `norm`. + V::R(p) => match unsafe { &*p } { + Object::Function(func) => (Rc::as_ptr(func), None), + Object::Builtin(b) => (std::ptr::null(), Some((Rc::as_ptr(b), Some(b)))), + _ => return 1, + }, + _ => return 1, + }; + // Scalar arguments are staged as objects (no drop glue); heap ones are + // borrowed in place. + let mut staged = [const { std::mem::MaybeUninit::::uninit() }; 8]; + let mut ptrs = [std::ptr::null::(); 8]; + for k in 0..n { + let o = match reg(first + k) { + V::R(p) => { + ptrs[k] = p; + continue; + } + V::I(i) => Object::Int(i), + V::F(x) => Object::Float(x), + V::B(b) => Object::Bool(b), + V::N => Object::None, + V::Fn(_) | V::Bi(_) | V::Null => return 1, + }; + ptrs[k] = staged[k].write(o); + } + if let Some((b, rc)) = native { + // A read-only builtin, on borrowed copies of the arguments: never + // dropped, so no reference moves. + let mut copies = [const { std::mem::MaybeUninit::::uninit() }; 8]; + for k in 0..n { + // SAFETY: each pointer names a live object; the copy is never + // dropped. + copies[k].write(unsafe { std::ptr::read(ptrs[k]) }); + } + // SAFETY: the first `n` copies were written. + let args = unsafe { std::slice::from_raw_parts(copies.as_ptr().cast::(), n) }; + let Some(r) = c.interp.leaf_pure_builtin(code, pc as usize, b, rc, args) else { + return 1; + }; + return match c.owned.own(r) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + }; + } + // SAFETY: the class or the namespace holds the callee, and nothing + // here runs code that could release it. + let callee = unsafe { &*fp }; + // SAFETY: GIL-serialized raw read of the code cell. + let ccode: &Rc = unsafe { &*callee.code.as_ptr() }; + if !crate::code_is_pure_leaf(ccode) + || !Interpreter::leaf_code_ok(ccode) + || n != crate::leaf_arity(ccode) + || ccode.has_varkeywords + || crate::recursion::current_depth() + usize::from(nest) + 1 + >= crate::recursion::recursion_limit() + { + return 1; + } + let v = match c + .interp + .leaf_eval_nested(ccode, callee, &ptrs[..n], nest + 1) + { + Some(LeafRet::Borrowed(v)) => v, + Some(LeafRet::Owned(r)) => match c.owned.own(r) { + Some(v) => v, + None => return 1, + }, + None => return 1, + }; + c.put(v); + 0 +} + +// ---- the compiler ------------------------------------------------------- + +/// The process's plan compiler. Code it emits lives as long as the +/// process: plans are shared between threads, and their code is never +/// freed. +struct Engine { + module: JITModule, + ctx: cranelift_codegen::Context, + fbctx: FunctionBuilderContext, + ptr: types::Type, + next: u32, +} + +// SAFETY: the engine is only used under its mutex. +unsafe impl Send for Engine {} + +static ENGINE: std::sync::Mutex>> = std::sync::Mutex::new(None); + +impl Engine { + fn new() -> Option { + if !tags_match() { + return None; + } + let mut flags = settings::builder(); + flags.set("use_colocated_libcalls", "false").ok()?; + flags.set("is_pic", "false").ok()?; + flags.set("opt_level", "speed").ok()?; + if !cfg!(debug_assertions) && !verify() { + flags.set("enable_verifier", "false").ok()?; + } + let isa = cranelift_native::builder() + .ok()? + .finish(settings::Flags::new(flags)) + .ok()?; + let module = JITModule::new(JITBuilder::with_isa( + isa, + cranelift_module::default_libcall_names(), + )); + let ptr = module.target_config().pointer_type(); + let ctx = module.make_context(); + Some(Engine { + module, + ctx, + fbctx: FunctionBuilderContext::new(), + ptr, + next: 0, + }) + } +} + +/// Whether to verify the IR of every compiled plan and report failures +/// (`WEAVEPY_PLAN_VERIFY`, a debugging aid). +fn verify() -> bool { + static VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); + *VERIFY.get_or_init(|| std::env::var_os("WEAVEPY_PLAN_VERIFY").is_some()) +} + +/// Compile `plan` (the translation of `code`) for `effect` mode; `None` +/// leaves it to the runner. A compiler panic turns plan compilation off +/// for the process rather than taking the interpreter down. +fn compile( + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + effect: bool, +) -> Option { + let mut guard = ENGINE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let engine = guard.get_or_insert_with(Engine::new).as_mut()?; + let built = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + compile_with(engine, code, ext, plan, effect) + })); + match built { + Ok(native) => native, + Err(_) => { + *guard = Some(None); + None + } + } +} + +fn compile_with( + engine: &mut Engine, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + effect: bool, +) -> Option { + engine.module.clear_context(&mut engine.ctx); + let ptr = engine.ptr; + let sig = &mut engine.ctx.func.signature; + sig.params.push(AbiParam::new(ptr)); + sig.params.push(AbiParam::new(ptr)); + sig.returns.push(AbiParam::new(types::I32)); + let name = format!("wpplan_{}", engine.next); + engine.next += 1; + let id = engine + .module + .declare_function(&name, Linkage::Local, &engine.ctx.func.signature) + .ok()?; + let lowered = { + let b = FunctionBuilder::new(&mut engine.ctx.func, &mut engine.fbctx); + let mut lower = Lower { + b, + effect, + ptr, + ctx: Value::from_u32(0), + decline: Block::from_u32(0), + sigs: Sigs::default(), + depth: 0, + frames: Vec::new(), + keep: Vec::new(), + }; + let done = lower.lower(code, ext, plan); + let Lower { + b, + depth, + frames, + keep, + .. + } = lower; + done.map(|()| { + b.finalize(); + (depth, frames, keep) + }) + }; + let Some((depth, frames, keep)) = lowered else { + // An unfinished function leaves the builder's state behind. + engine.fbctx = FunctionBuilderContext::new(); + engine.module.clear_context(&mut engine.ctx); + return None; + }; + let defined = engine.module.define_function(id, &mut engine.ctx); + if let (Err(e), true) = (&defined, verify()) { + eprintln!( + "weavepy: plan for {:?} failed to compile: {e:?}", + code.qualname + ); + } + engine.module.clear_context(&mut engine.ctx); + defined.ok()?; + engine.module.finalize_definitions().ok()?; + let func = engine.module.get_finalized_function(id); + // SAFETY: `func` is a finalized function with exactly the signature + // declared above, and the module (never dropped) keeps it alive. + let func = unsafe { std::mem::transmute::<*const u8, NativeFn>(func) }; + Some(Native { + func, + depth, + _frames: frames, + _keep: keep, + }) +} + +/// The helper signatures one function imports, by shape. +#[derive(Default)] +struct Sigs { + map: Vec<(Vec, SigRef)>, +} + +/// One plan's registers and blocks while it is lowered (the evaluation's +/// own, or an in-line callee's). +struct FrameLower<'p> { + plan: &'p LeafPlan, + code: &'p CodeObject, + /// The frame descriptor the helpers get. + fr: Value, + /// In-line depth (0 for the evaluation's own frame). + depth: u8, + tags: Vec, + pays: Vec, + blocks: Vec, + /// Where a callee's return goes: the caller's result register (in the + /// caller's variables) and the block after the call. + ret: Option<(Variable, Variable, Block)>, + /// Whether an attribute store may have run before an op (only the + /// evaluation's own frame stores). + may_store: Vec, +} + +struct Lower<'b> { + b: FunctionBuilder<'b>, + effect: bool, + ptr: types::Type, + ctx: Value, + decline: Block, + sigs: Sigs, + /// The deepest in-line callee. + depth: u8, + // Boxed: the code embeds each descriptor's address. + #[allow(clippy::vec_box)] + frames: Vec>, + keep: Vec<(crate::sync::Weak, Rc)>, +} + +impl Lower<'_> { + fn sig(&mut self, params: &[types::Type]) -> SigRef { + if let Some((_, s)) = self.sigs.map.iter().find(|(p, _)| p == params) { + return *s; + } + let mut sig = Signature::new(self.b.func.signature.call_conv); + for &t in params { + sig.params.push(AbiParam::new(t)); + } + sig.returns.push(AbiParam::new(types::I32)); + let s = self.b.import_signature(sig); + self.sigs.map.push((params.to_vec(), s)); + s + } + + /// Call the helper at `addr`: its `u32` result. + fn call(&mut self, addr: usize, args: &[Value]) -> Value { + let params: Vec = args + .iter() + .map(|&a| self.b.func.dfg.value_type(a)) + .collect(); + let sig = self.sig(¶ms); + let callee = self.b.ins().iconst(self.ptr, addr as i64); + let inst = self.b.ins().call_indirect(sig, callee, args); + self.b.inst_results(inst)[0] + } + + /// Continue in a fresh block when `status` is `0`; decline otherwise. + fn check(&mut self, status: Value) { + let ok = self.b.create_block(); + self.b.ins().brif(status, self.decline, &[], ok, &[]); + self.b.switch_to_block(ok); + } + + fn get(&mut self, fl: &FrameLower<'_>, r: u8) -> (Value, Value) { + let r = usize::from(r); + (self.b.use_var(fl.tags[r]), self.b.use_var(fl.pays[r])) + } + + fn set(&mut self, fl: &FrameLower<'_>, r: u8, t: Value, p: Value) { + let r = usize::from(r); + self.b.def_var(fl.tags[r], t); + self.b.def_var(fl.pays[r], p); + } + + /// Load the result words a helper left at `ctx.out[k..k + 2]` into `r`. + fn set_out(&mut self, fl: &FrameLower<'_>, r: u8, k: i32) { + let flags = MemFlags::trusted(); + let t = self.b.ins().load(types::I64, flags, self.ctx, 8 * k); + let p = self.b.ins().load(types::I64, flags, self.ctx, 8 * k + 8); + self.set(fl, r, t, p); + } + + /// A boolean (`i8`) into register `r` as `V::B`. + fn set_bool(&mut self, fl: &FrameLower<'_>, r: u8, bit: Value) { + let t = self.b.ins().iconst(types::I64, T_B as i64); + let p = self.b.ins().uextend(types::I64, bit); + self.set(fl, r, t, p); + } + + fn u32c(&mut self, x: u32) -> Value { + self.b.ins().iconst(types::I32, i64::from(x)) + } + + /// Decline unless the `0`/`1` helper result `r` is not `2`; the `i8` + /// bit. + fn bit_or_decline(&mut self, r: Value) -> Value { + let declines = self.b.ins().icmp_imm(IntCC::Equal, r, 2); + let ok = self.b.create_block(); + self.b.ins().brif(declines, self.decline, &[], ok, &[]); + self.b.switch_to_block(ok); + self.b.ins().ireduce(types::I8, r) + } + + /// A register's truth as an `i8`, declining where the interpreter's + /// `truth` would: in line for the scalar tags, through the helper for + /// a heap value. + fn truth(&mut self, fl: &FrameLower<'_>, src: u8) -> Value { + let (t, p) = self.get(fl, src); + let merge = self.b.create_block(); + self.b.append_block_param(merge, types::I8); + let not_b = self.b.create_block(); + let not_i = self.b.create_block(); + let not_n = self.b.create_block(); + let is_b = self.b.ins().icmp_imm(IntCC::Equal, t, T_B as i64); + let low = self.b.ins().band_imm(p, 0xff); + let b_truth = self.b.ins().icmp_imm(IntCC::NotEqual, low, 0); + self.b + .ins() + .brif(is_b, merge, &[b_truth.into()], not_b, &[]); + self.b.switch_to_block(not_b); + let is_i = self.b.ins().icmp_imm(IntCC::Equal, t, T_I as i64); + let i_truth = self.b.ins().icmp_imm(IntCC::NotEqual, p, 0); + self.b + .ins() + .brif(is_i, merge, &[i_truth.into()], not_i, &[]); + self.b.switch_to_block(not_i); + let is_n = self.b.ins().icmp_imm(IntCC::Equal, t, T_N as i64); + let falsy = self.b.ins().iconst(types::I8, 0); + self.b.ins().brif(is_n, merge, &[falsy.into()], not_n, &[]); + self.b.switch_to_block(not_n); + let r = self.call(h_truth as *const () as usize, &[t, p]); + let bit = self.bit_or_decline(r); + self.b.ins().jump(merge, &[bit.into()]); + self.b.switch_to_block(merge); + self.b.block_params(merge)[0] + } + + fn lower(&mut self, code: &CodeObject, ext: &CodeConstObjects, plan: &LeafPlan) -> Option<()> { + let entry = self.b.create_block(); + self.b.append_block_params_for_function_params(entry); + self.b.switch_to_block(entry); + self.ctx = self.b.block_params(entry)[0]; + let argw = self.b.block_params(entry)[1]; + self.decline = self.b.create_block(); + let fr = self.b.ins().iadd_imm(self.ctx, i64::from(TOP_OFFSET)); + let fl = self.frame(plan, code, ext, fr, 0, None); + let flags = MemFlags::trusted(); + for k in 0..usize::from(plan.nargs) { + let t = self.b.ins().load(types::I64, flags, argw, (16 * k) as i32); + let p = self + .b + .ins() + .load(types::I64, flags, argw, (16 * k + 8) as i32); + self.set(&fl, k as u8, t, p); + } + self.b.ins().jump(fl.blocks[0], &[]); + self.body(&fl)?; + self.b.switch_to_block(self.decline); + let one = self.b.ins().iconst(types::I32, 1); + self.b.ins().return_(&[one]); + self.b.seal_all_blocks(); + Some(()) + } + + /// Declare a frame's registers (zeroed in the current block) and its + /// op blocks. + fn frame<'p>( + &mut self, + plan: &'p LeafPlan, + code: &'p CodeObject, + ext: &'p CodeConstObjects, + fr: Value, + depth: u8, + ret: Option<(Variable, Variable, Block)>, + ) -> FrameLower<'p> { + let nregs = usize::from(plan.nregs); + let mut tags = Vec::with_capacity(nregs); + let mut pays = Vec::with_capacity(nregs); + let zero = self.b.ins().iconst(types::I64, 0); + for _ in 0..nregs { + let t = self.b.declare_var(types::I64); + let p = self.b.declare_var(types::I64); + self.b.def_var(t, zero); + self.b.def_var(p, zero); + tags.push(t); + pays.push(p); + } + let blocks = (0..plan.ops.len()).map(|_| self.b.create_block()).collect(); + // Conservatively: a store anywhere earlier in the op order. + let mut seen = false; + let may_store = plan + .ops + .iter() + .map(|op| { + let before = seen; + seen |= matches!(op, Op::StoreAttr { .. }); + before + }) + .collect(); + let _ = ext; + FrameLower { + plan, + code, + fr, + depth, + tags, + pays, + blocks, + ret, + may_store, + } + } + + /// Lower every op of a frame into its blocks. + fn body(&mut self, fl: &FrameLower<'_>) -> Option<()> { + for (i, op) in fl.plan.ops.iter().enumerate() { + self.b.switch_to_block(fl.blocks[i]); + if self.op(fl, i, op)? { + // Falls through (the translation ends every path in a + // return or a jump, so a next op exists). + let next = *fl.blocks.get(i + 1)?; + self.b.ins().jump(next, &[]); + } + } + Some(()) + } + + /// The pure-leaf callee to compile in line for the `CALL` at op `i` + /// whose callee register `at` a method load filled: what that load's + /// site last resolved, with its plan and arity (self included). + fn inline_target( + &self, + fl: &FrameLower<'_>, + i: usize, + at: u8, + argc: u8, + ) -> Option<( + Rc, + Rc, + &'static CodeConstObjects, + &'static LeafPlan, + )> { + if fl.depth >= INLINE_DEPTH || (self.effect && fl.depth == 0 && fl.may_store[i]) { + return None; + } + let (pc, _) = fl.plan.ops[..i].iter().rev().find_map(|op| match *op { + Op::Method { dst, pc, name, .. } if dst == at => Some((pc, name)), + _ => None, + })?; + let f = crate::code_method_slot(fl.code, u32::from(pc))?.last_fn()?; + let ccode = f.code(); + if !crate::code_is_pure_leaf(&ccode) + || !Interpreter::leaf_code_ok(&ccode) + || crate::leaf_arity(&ccode) != usize::from(argc) + 1 + || ccode.has_varkeywords + { + return None; + } + let cext = crate::code_vm_ext(&ccode)?; + let cplan = cext + .leaf_plan + .get_or_init(|| super::build(&ccode, cext).map(Box::new)) + .as_deref()?; + // A pure callee stores nothing. + if cplan + .ops + .iter() + .any(|op| matches!(op, Op::StoreAttr { .. })) + { + return None; + } + // SAFETY: `ccode` (kept alive with the compiled code) owns its + // extension and plan, which never move. + let (cext, cplan) = unsafe { (&*std::ptr::from_ref(cext), &*std::ptr::from_ref(cplan)) }; + Some((f, ccode, cext, cplan)) + } + + /// Lower one op; whether it falls through to the next. + #[allow(clippy::too_many_lines)] + fn op(&mut self, fl: &FrameLower<'_>, i: usize, op: &Op) -> Option { + let effect = self.effect && fl.depth == 0; + match *op { + Op::Move { dst, src } => { + let (t, p) = self.get(fl, src); + self.set(fl, dst, t, p); + } + Op::Const { dst, k } => { + let [t, p] = words(*fl.plan.consts.get(usize::from(k))?); + let t = self.b.ins().iconst(types::I64, t as i64); + let p = self.b.ins().iconst(types::I64, p as i64); + self.set(fl, dst, t, p); + } + Op::Global { dst, pc } => { + let pc = self.u32c(u32::from(pc)); + let st = self.call(h_global as *const () as usize, &[self.ctx, fl.fr, pc]); + self.check(st); + self.set_out(fl, dst, 0); + } + Op::Attr { dst, src, pc, name } => { + let (t, p) = self.get(fl, src); + let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); + let addr = if effect { + h_attr:: as *const () as usize + } else { + h_attr:: as *const () as usize + }; + let st = self.call(addr, &[self.ctx, fl.fr, t, p, pn]); + self.check(st); + self.set_out(fl, dst, 0); + } + Op::Method { dst, src, pc, name } => { + let (t, p) = self.get(fl, src); + let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); + let st = self.call(h_method as *const () as usize, &[self.ctx, fl.fr, t, p, pn]); + self.check(st); + self.set_out(fl, dst, 0); + self.set_out(fl, dst + 1, 2); + } + Op::Compare { dst, a, b, kind } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + let cc = match kind { + k if k == CompareKind::Lt as u8 => IntCC::SignedLessThan, + k if k == CompareKind::LtE as u8 => IntCC::SignedLessThanOrEqual, + k if k == CompareKind::Eq as u8 => IntCC::Equal, + k if k == CompareKind::NotEq as u8 => IntCC::NotEqual, + k if k == CompareKind::Gt as u8 => IntCC::SignedGreaterThan, + k if k == CompareKind::GtE as u8 => IntCC::SignedGreaterThanOrEqual, + _ => return None, + }; + // Two ints compare in line. + let ia = self.b.ins().icmp_imm(IntCC::Equal, ta, T_I as i64); + let ib = self.b.ins().icmp_imm(IntCC::Equal, tb, T_I as i64); + let both = self.b.ins().band(ia, ib); + let fast = self.b.create_block(); + let slow = self.b.create_block(); + let merge = self.b.create_block(); + self.b.append_block_param(merge, types::I8); + self.b.ins().brif(both, fast, &[], slow, &[]); + self.b.switch_to_block(fast); + let bit = self.b.ins().icmp(cc, pa, pb); + self.b.ins().jump(merge, &[bit.into()]); + self.b.switch_to_block(slow); + let k = self.u32c(u32::from(kind)); + let r = self.call(h_compare as *const () as usize, &[ta, pa, tb, pb, k]); + let bit = self.bit_or_decline(r); + self.b.ins().jump(merge, &[bit.into()]); + self.b.switch_to_block(merge); + let bit = self.b.block_params(merge)[0]; + self.set_bool(fl, dst, bit); + } + Op::Is { dst, a, b, invert } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + let r = self.call(h_is as *const () as usize, &[ta, pa, tb, pb]); + let r = if invert { + self.b.ins().bxor_imm(r, 1) + } else { + r + }; + let bit = self.b.ins().ireduce(types::I8, r); + self.set_bool(fl, dst, bit); + } + Op::Truth { dst, src } => { + let bit = self.truth(fl, src); + self.set_bool(fl, dst, bit); + } + Op::Unary { dst, src, kind } => { + let (t, p) = self.get(fl, src); + let k = self.u32c(u32::from(kind)); + let st = self.call(h_unary as *const () as usize, &[self.ctx, t, p, k]); + self.check(st); + self.set_out(fl, dst, 0); + } + Op::Binary { dst, a, b, kind } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + let int_op = [BinOpKind::Add, BinOpKind::Sub, BinOpKind::Mult] + .into_iter() + .find(|k| *k as u8 == kind); + let merge = self.b.create_block(); + self.b.append_block_param(merge, types::I64); + self.b.append_block_param(merge, types::I64); + let slow = self.b.create_block(); + if let Some(k) = int_op { + // Two ints add, subtract and multiply in line; an + // overflow declines (the interpreter's `binary` does). + let ia = self.b.ins().icmp_imm(IntCC::Equal, ta, T_I as i64); + let ib = self.b.ins().icmp_imm(IntCC::Equal, tb, T_I as i64); + let both = self.b.ins().band(ia, ib); + let fast = self.b.create_block(); + self.b.ins().brif(both, fast, &[], slow, &[]); + self.b.switch_to_block(fast); + let (r, ovf) = match k { + BinOpKind::Add => { + let r = self.b.ins().iadd(pa, pb); + let axr = self.b.ins().bxor(pa, r); + let bxr = self.b.ins().bxor(pb, r); + let and = self.b.ins().band(axr, bxr); + (r, self.b.ins().icmp_imm(IntCC::SignedLessThan, and, 0)) + } + BinOpKind::Sub => { + let r = self.b.ins().isub(pa, pb); + let axb = self.b.ins().bxor(pa, pb); + let axr = self.b.ins().bxor(pa, r); + let and = self.b.ins().band(axb, axr); + (r, self.b.ins().icmp_imm(IntCC::SignedLessThan, and, 0)) + } + _ => { + let lo = self.b.ins().imul(pa, pb); + let hi = self.b.ins().smulhi(pa, pb); + let sign = self.b.ins().sshr_imm(lo, 63); + (lo, self.b.ins().icmp(IntCC::NotEqual, hi, sign)) + } + }; + let ok = self.b.create_block(); + self.b.ins().brif(ovf, self.decline, &[], ok, &[]); + self.b.switch_to_block(ok); + let ti = self.b.ins().iconst(types::I64, T_I as i64); + self.b.ins().jump(merge, &[ti.into(), r.into()]); + } else { + self.b.ins().jump(slow, &[]); + } + self.b.switch_to_block(slow); + let kv = self.u32c(u32::from(kind)); + let st = self.call( + h_binary as *const () as usize, + &[self.ctx, ta, pa, tb, pb, kv], + ); + self.check(st); + let flags = MemFlags::trusted(); + let t = self.b.ins().load(types::I64, flags, self.ctx, 0); + let p = self.b.ins().load(types::I64, flags, self.ctx, 8); + self.b.ins().jump(merge, &[t.into(), p.into()]); + self.b.switch_to_block(merge); + let (t, p) = (self.b.block_params(merge)[0], self.b.block_params(merge)[1]); + self.set(fl, dst, t, p); + } + Op::Swap { a, b } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + self.set(fl, a, tb, pb); + self.set(fl, b, ta, pa); + } + Op::BranchIf { src, when, target } => { + let bit = self.truth(fl, src); + let to = *fl.blocks.get(usize::from(target))?; + let next = self.b.create_block(); + if when { + self.b.ins().brif(bit, to, &[], next, &[]); + } else { + self.b.ins().brif(bit, next, &[], to, &[]); + } + self.b.switch_to_block(next); + } + Op::BranchNone { src, when, target } => { + let (t, _) = self.get(fl, src); + let is_none = self.b.ins().icmp_imm(IntCC::Equal, t, T_N as i64); + let to = *fl.blocks.get(usize::from(target))?; + let next = self.b.create_block(); + if when { + self.b.ins().brif(is_none, to, &[], next, &[]); + } else { + self.b.ins().brif(is_none, next, &[], to, &[]); + } + self.b.switch_to_block(next); + } + Op::Jump { target } => { + let to = *fl.blocks.get(usize::from(target))?; + self.b.ins().jump(to, &[]); + return Some(false); + } + Op::Call { at, argc, pc } => self.call_op(fl, i, at, argc, pc)?, + Op::Return { src } => { + let (t, p) = self.get(fl, src); + if let Some((rt, rp, cont)) = fl.ret { + // An in-line callee's return: the caller's result + // register, and on past its call. + self.b.def_var(rt, t); + self.b.def_var(rp, p); + self.b.ins().jump(cont, &[]); + } else { + let flags = MemFlags::trusted(); + self.b.ins().store(flags, t, self.ctx, 0); + self.b.ins().store(flags, p, self.ctx, 8); + let zero = self.b.ins().iconst(types::I32, 0); + self.b.ins().return_(&[zero]); + } + return Some(false); + } + Op::New { dst, dict } => { + let d = self.u32c(u32::from(dict)); + let st = self.call(h_new as *const () as usize, &[self.ctx, d]); + self.check(st); + self.set_out(fl, dst, 0); + } + Op::Decline => { + self.b.ins().jump(self.decline, &[]); + return Some(false); + } + Op::StoreAttr { + recv, + val, + pc, + name, + } => { + if !effect { + self.b.ins().jump(self.decline, &[]); + return Some(false); + } + let (tr, pr) = self.get(fl, recv); + let (tv, pv) = self.get(fl, val); + let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); + let st = self.call( + h_store as *const () as usize, + &[self.ctx, tr, pr, tv, pv, pn], + ); + self.check(st); + } + } + Some(true) + } + + /// A `CALL` (op `i`): the callee its site last resolved in line behind + /// a guard, falling back to the helper. + fn call_op(&mut self, fl: &FrameLower<'_>, i: usize, at: u8, argc: u8, pc: u16) -> Option<()> { + let cont = self.b.create_block(); + if let Some((f, ccode, cext, cplan)) = self.inline_target(fl, i, at, argc) { + // The guard: the register holds this function, called with an + // instance receiver, and the function still runs this code. + let (tf, pf) = self.get(fl, at); + let (ts, _) = self.get(fl, at + 1); + let is_fn = self.b.ins().icmp_imm(IntCC::Equal, tf, T_FN as i64); + let same = self + .b + .ins() + .icmp_imm(IntCC::Equal, pf, Rc::as_ptr(&f) as i64); + let has_self = self.b.ins().icmp_imm(IntCC::Equal, ts, T_R as i64); + let ok = self.b.ins().band(is_fn, same); + let ok = self.b.ins().band(ok, has_self); + let code_check = self.b.create_block(); + let inline = self.b.create_block(); + let generic = self.b.create_block(); + self.b.ins().brif(ok, code_check, &[], generic, &[]); + self.b.switch_to_block(code_check); + let cell = f.code.as_ptr(); + // SAFETY: the cell holds an `Rc` (one pointer word); the guard + // above proved the function alive wherever this load runs. + let bits = unsafe { *cell.cast::() }; + let addr = self.b.ins().iconst(self.ptr, cell as i64); + let now = self.b.ins().load(types::I64, MemFlags::trusted(), addr, 0); + let same_code = self.b.ins().icmp_imm(IntCC::Equal, now, bits as i64); + self.b.ins().brif(same_code, inline, &[], generic, &[]); + self.b.switch_to_block(inline); + let frame = Box::new(Frame { + code: Rc::as_ptr(&ccode), + ext: cext, + f: Rc::as_ptr(&f), + }); + let fr = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(&*frame) as i64); + let depth = fl.depth + 1; + self.depth = self.depth.max(depth); + let ret = Some((fl.tags[usize::from(at)], fl.pays[usize::from(at)], cont)); + let cfl = self.frame(cplan, &ccode, cext, fr, depth, ret); + // The receiver and the arguments become the callee's first + // registers. + for k in 0..=usize::from(argc) { + let (t, p) = self.get(fl, at + 1 + k as u8); + self.set(&cfl, k as u8, t, p); + } + self.b.ins().jump(cfl.blocks[0], &[]); + self.frames.push(frame); + self.keep.push((Rc::downgrade(&f), ccode.clone())); + self.body(&cfl)?; + self.b.switch_to_block(generic); + } + let n = usize::from(argc) + 2; + let slot = self.b.create_sized_stack_slot(StackSlotData::new( + StackSlotKind::ExplicitSlot, + (16 * n) as u32, + 3, + )); + for k in 0..n { + let (t, p) = self.get(fl, at + k as u8); + self.b.ins().stack_store(t, slot, (16 * k) as i32); + self.b.ins().stack_store(p, slot, (16 * k + 8) as i32); + } + let w = self.b.ins().stack_addr(self.ptr, slot, 0); + let ad = self.u32c(u32::from(argc) | u32::from(fl.depth) << 8); + let pc = self.u32c(u32::from(pc)); + let addr = if self.effect && fl.depth == 0 { + h_call:: as *const () as usize + } else { + h_call:: as *const () as usize + }; + let st = self.call(addr, &[self.ctx, fl.fr, w, ad, pc]); + self.check(st); + self.set_out(fl, at, 0); + self.b.ins().jump(cont, &[]); + self.b.switch_to_block(cont); + Some(()) + } +} diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 8ad1530b..99138a4d 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -7595,8 +7595,14 @@ impl Interpreter { return FrameEv::Call(act); } } - // `inst.prop` for a plain `property`: its getter runs as a - // lean activation, like a lean call; `for x in gen` resumes + // `inst.prop` for a plain `property`: its getter runs as an + // inline activation, like an inline call. + if op_before == Some(OpCode::LoadAttr) && self.inline_calls_ok() { + if let Some(act) = self.try_inline_property(frame, shell, cur_pc) { + return FrameEv::Call(act); + } + } + // Otherwise as a lean activation; `for x in gen` resumes // the generator as one. if matches!(op_before, Some(OpCode::LoadAttr | OpCode::ForIter)) { let loaded = if op_before == Some(OpCode::LoadAttr) { @@ -9171,9 +9177,65 @@ impl Interpreter { } } + /// The getter of the `property` that the `LOAD_ATTR` at `pc` of + /// `frame` loads off the plain instance at TOS, when it's a plain + /// one-argument Python function a lean activation can run. + fn lean_property_getter( + frame: &Frame, + pc: usize, + ) -> Option<( + Rc, + Rc, + Rc, + )> { + use weavepy_compiler::InlineCache as IC; + let name_idx = frame.code.instructions.get(pc)?.arg; + // A first execution still specializes the site. + if matches!(frame.code.caches.get(pc as u32), IC::Empty) { + return None; + } + let Some(Object::Instance(inst)) = frame.stack.last() else { + return None; + }; + // A getter's `AttributeError` falls back to `__getattr__`, which + // the inline return can't do. + if inst + .cls_raw() + .dunder(crate::types::Dunder::GetAttr) + .present() + { + return None; + } + let LeafAttr::Property(prop) = + Self::leaf_resolve_instance_attr(&frame.code, inst, name_idx)? + else { + return None; + }; + // SAFETY: GIL-serialized raw read of the getter cell (the burst is + // off in free-threaded mode); the clone is taken before anything + // else runs. + let Object::Function(f) = (unsafe { &*prop.fget.as_ptr() }) else { + return None; + }; + let code = f.code(); + if code.arg_count != 1 + || code.is_generator + || code.is_coroutine + || code.is_async_generator + || code.has_varargs + || code.has_varkeywords + || code.kwonly_count != 0 + || !Self::lean_code_ok(&code) + { + return None; + } + f.lean_cells_ref(&code)?; + let f = f.clone(); + Some((prop, f, code)) + } + /// `LOAD_ATTR` of a `property` on a plain instance, from the quiet - /// loop: the getter — a plain one-argument Python function the lean - /// path can run — is called with the receiver, and its result + /// loop: the getter is called with the receiver, and its result /// replaces the receiver at TOS. `None` leaves the load to the full /// handler with nothing touched. fn try_lean_property( @@ -9183,44 +9245,7 @@ impl Interpreter { snap_gen: u64, pc: usize, ) -> Option> { - use weavepy_compiler::InlineCache as IC; - let name_idx = frame.code.instructions.get(pc)?.arg; - // A first execution still specializes the site. - if matches!(frame.code.caches.get(pc as u32), IC::Empty) { - return None; - } - let getter = { - let Some(Object::Instance(inst)) = frame.stack.last() else { - return None; - }; - let LeafAttr::Property(prop) = - Self::leaf_resolve_instance_attr(&frame.code, inst, name_idx)? - else { - return None; - }; - // SAFETY: GIL-serialized raw read of the getter cell (the - // burst is off in free-threaded mode); the clone is taken - // before anything else runs. - let Object::Function(f) = (unsafe { &*prop.fget.as_ptr() }) else { - return None; - }; - let f = f.clone(); - let code = f.code(); - if code.arg_count != 1 - || code.is_generator - || code.is_coroutine - || code.is_async_generator - || code.has_varargs - || code.has_varkeywords - || code.kwonly_count != 0 - || !Self::lean_code_ok(&code) - { - return None; - } - f.lean_cells_ref(&code)?; - (f, code) - }; - let (getter, code) = (getter.0, getter.1); + let (_, getter, code) = Self::lean_property_getter(frame, pc)?; let receiver = frame.stack.last()?.clone(); let result = self.run_lean_fn(frame, shell, snap_gen, pc, &getter, code, receiver); Some(result.map(|v| { @@ -9230,6 +9255,56 @@ impl Interpreter { })) } + /// The getter the `LOAD_ATTR` site at `pc` of `frame` cached for the + /// class of the instance at TOS (see [`MethodSlot::getter`]). + #[inline] + fn cached_property_getter( + frame: &Frame, + pc: usize, + ) -> Option<(Rc, Rc)> { + let Some(Object::Instance(inst)) = frame.stack.last() else { + return None; + }; + let slot = code_vm_ext(&frame.code)?.method_slots.get()?.get(pc)?; + slot.getter(inst.cls_raw().attr_version.get()) + } + + /// [`Self::try_lean_property`] as an inline activation (CPython's + /// `LOAD_ATTR_PROPERTY`): the receiver leaves the stack as the + /// getter's argument, and the getter's return pushes its result. + fn try_inline_property( + &mut self, + frame: &mut Frame, + shell: &mut QuietShell<'_>, + pc: usize, + ) -> Option> { + let (getter, code) = match Self::cached_property_getter(frame, pc) { + Some(hit) => hit, + None => { + let (prop, getter, code) = Self::lean_property_getter(frame, pc)?; + if let (Some(Object::Instance(inst)), Some(slot)) = + (frame.stack.last(), code_method_slot(&frame.code, pc as u32)) + { + slot.set_getter(inst.cls_raw().attr_version.get(), &prop, &getter, &code); + } + (getter, code) + } + }; + // Past the recursion limit the nested lean path raises. + let crate::recursion::Enter::Ok(guard) = crate::recursion::enter() else { + return None; + }; + // Committed. + let receiver = frame.stack.pop()?; + let act = self.inline_slot(); + // SAFETY: a parked slot's locals storage is its own, and empty. + let locals = unsafe { &mut *act.frame.locals.as_ptr() }; + locals.push(receiver); + fill_unbound(locals, code.varnames.len()); + frame.pc = pc as u32 + 1; + Some(self.inline_bind(frame, shell, pc, act, code, Object::Function(getter), guard)) + } + /// `FOR_ITER` over a generator, from the quiet loop: the generator /// resumes as a lean activation (see `generator_send_lean`); its /// value is pushed, or on exhaustion the loop exits exactly as the @@ -9759,7 +9834,7 @@ impl Interpreter { CoreExit::Helper => { // SAFETY: see `CoreSwitch` (the running activation's // handles; nothing else borrows them here). - let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) }; + let frame = unsafe { &mut *sw.cur }; if burst_stats::enabled() { if let Some(ins) = frame.code.instructions.get(frame.pc as usize) { burst_stats::note_helper(ins.op); @@ -9767,6 +9842,20 @@ impl Interpreter { burst_stats::note_site(&frame.code.qualname, frame.pc, ins.op, recv); } } + // A cached property site skips the helper's resolution. + let pc = frame.pc as usize; + if frame + .code + .instructions + .get(pc) + .is_some_and(|i| i.op == OpCode::LoadAttr) + && Self::cached_property_getter(frame, pc).is_some() + && self.core_property(sw) + { + continue; + } + // SAFETY: as above (`core_property` declined untouched). + let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) }; match self.leaf_core_helper(frame, last_pc) { // A released receiver or argument may have queued // a finalizer (see `after_release!` in the core). @@ -9777,7 +9866,16 @@ impl Interpreter { CoreAttr::Done => continue, CoreAttr::Raised(e) => return LeafStop::Raised(e), CoreAttr::Full => return LeafStop::Step, - CoreAttr::Decline => {} + CoreAttr::Decline => { + let load_attr = frame + .code + .instructions + .get(frame.pc as usize) + .is_some_and(|i| i.op == OpCode::LoadAttr); + if load_attr && self.core_property(sw) { + continue; + } + } } } CoreExit::Slow => {} @@ -12451,6 +12549,35 @@ impl Interpreter { drop(bm); } + /// The core loop's `LOAD_ATTR` of a `property` at `sw`'s (synced) + /// running activation's pc: the getter's activation + /// ([`Self::try_inline_property`]) is pushed and made the running one, + /// as [`Self::core_call`] does for a call. `false` touches nothing. + #[inline(never)] + fn core_property(&mut self, sw: &mut CoreSwitch) -> bool { + if !self.inline_calls_ok() { + return false; + } + let mut tmp = None; + // SAFETY: see `CoreSwitch` (as in `core_call`). + let act = unsafe { + let depth = (*sw.inl).len(); + let (frame, _, shell) = sw.activation(depth, &mut tmp); + let pc = (*frame).pc as usize; + self.try_inline_property(&mut *frame, &mut *shell.cast::>(), pc) + }; + let Some(mut act) = act else { + return false; + }; + let callee: *mut Frame = &raw mut *act.frame; + // SAFETY: as above. + unsafe { (*sw.inl).push(act) }; + sw.cur = callee; + sw.scratch = usize::MAX; + sw.last = &raw mut sw.scratch; + true + } + /// [`Self::core_call`] for the `CALL_KW` at `pc`: the quiet loop's /// inline keyword call ([`Self::try_inline_call_kw`]), pushed and made /// the running activation. `false` touches nothing. @@ -17639,7 +17766,11 @@ impl Interpreter { if let Some(Object::Builtin(f)) = b.get(&crate::object::StrKey("next")) { next_ptr = Rc::as_ptr(f) as usize; } - for (name, kind) in [("len", LeafKind::Len), ("isinstance", LeafKind::Isinstance)] { + for (name, kind) in [ + ("len", LeafKind::Len), + ("isinstance", LeafKind::Isinstance), + ("getattr", LeafKind::GetAttr), + ] { if let Some(Object::Builtin(f)) = b.get(&crate::object::StrKey(name)) { calls.insert(Rc::as_ptr(f) as usize, kind); if name == "len" { @@ -18068,6 +18199,59 @@ impl Interpreter { Some(kind) } + /// `getattr(obj, name[, default])` on a plain instance whose lookup + /// runs no code: an instance attribute, a plain class value, or a + /// method to bind, and for a missing attribute with no `__getattr__`, + /// the default. `None` for anything else (descriptors, dunders, a + /// missing attribute's `AttributeError`), which the full call handles. + fn leaf_getattr(args: &[Object]) -> Option> { + let (Object::Instance(inst), Object::Str(name)) = (args.first()?, args.get(1)?) else { + return None; + }; + let name: &str = name; + if args.len() > 3 || name.starts_with("__") || crate::object::exotic_str_keys_possible() { + return None; + } + let cls = inst.cls_raw(); + if cls.native_kind.get() != 0 + || inst.native.get().is_some() + || !Self::default_getattribute(cls) + { + return None; + } + // Only class values without `__get__` (or a function, which binds) + // leave the instance dictionary in charge. + let on_class = cls.lookup(name); + if !matches!( + on_class, + None | Some( + Object::Function(_) + | Object::Int(_) + | Object::Float(_) + | Object::Bool(_) + | Object::None + | Object::Str(_) + | Object::Bytes(_) + | Object::Tuple(_) + ) + ) { + return None; + } + if let Some(v) = inst.attr_get_str(name) { + return Some(Ok(v)); + } + Some(Ok(match on_class { + Some(f @ Object::Function(_)) => { + Object::BoundMethod(Rc::new(BoundMethod::new(args[0].clone(), f))) + } + Some(v) => v, + None if args.len() == 3 && !cls.dunder(crate::types::Dunder::GetAttr).present() => { + args[2].clone() + } + None => return None, + })) + } + /// Call a leaf builtin if `args` (receiver first for methods) has an /// admitted shape; `None` sends the call down the full path. fn leaf_builtin_call( @@ -18102,6 +18286,7 @@ impl Interpreter { ) } K::Isinstance => return Self::core_isinstance(args), + K::GetAttr => return Self::leaf_getattr(args), // `list.append` and `list.pop` in line (as `list_append` and // `list_pop` do them). K::ListAppend => { @@ -36115,6 +36300,22 @@ impl Interpreter { if let Some(r) = native_scalar_compare(a, b, op) { return Ok(Object::Bool(r)); } + // Two instances of one class whose comparison dunder is a Python + // function: the forward call runs directly, with no bound method + // and none of the special operand shapes below. A `NotImplemented` + // answer continues the full protocol, past the forward call. + let mut forward_ran = false; + if let (Object::Instance(ia), Object::Instance(ib)) = (a, b) { + if std::ptr::eq(ia.cls_raw(), ib.cls_raw()) { + if let Some(f @ Object::Function(_)) = ia.cls_raw().lookup(cmp_dunder(op).0) { + let r = self.call(&f, &[a.clone(), b.clone()], &[], globals)?; + if !r.is_same(&crate::vm_singletons::not_implemented()) { + return Ok(r); + } + forward_ran = true; + } + } + } // An object-backed `mappingproxy` compares as the wrapped mapping // (CPython `mappingproxy_richcompare` delegates unconditionally). if matches!(a, Object::MappingProxyObj(_)) || matches!(b, Object::MappingProxyObj(_)) { @@ -36264,7 +36465,9 @@ impl Interpreter { Err(e) => return Err(e), } } - if let Some(method) = self.cmp_method(a, dunder, globals) { + if forward_ran { + a_slot_ran = true; + } else if let Some(method) = self.cmp_method(a, dunder, globals) { a_slot_ran = true; let r = self.call(&method, std::slice::from_ref(b), &[], globals)?; if !r.is_same(¬_impl) { @@ -55041,6 +55244,8 @@ enum LeafKind { ObjectNew, Len, Isinstance, + /// `getattr` (see `Interpreter::leaf_getattr`). + GetAttr, ListAppend, ListPop, ListInsert, @@ -61171,6 +61376,14 @@ enum MethodSlotFn { /// A rejected native bound-call body, weakly held so a call site /// doesn't extend its lifetime. The key is the registry generation. NonLeaf(crate::sync::Weak), + /// A `LOAD_ATTR` site's `property` getter for instances of the class + /// at the key's version, and the code it was checked against (see + /// `Interpreter::try_inline_property`). + Getter { + prop: crate::sync::Weak, + func: crate::sync::Weak, + code: crate::sync::Weak, + }, } // SAFETY: read and written only from the dispatch loop with the GIL @@ -61336,6 +61549,16 @@ impl MethodSlot { } } + /// The plain function the slot resolved last, if it's still alive: a + /// compiler's guess at what the site will call next. + pub(crate) fn last_fn(&self) -> Option> { + // SAFETY: GIL-serialized; no `&mut` escapes `set`. + match unsafe { &*self.0.get() } { + (_, MethodSlotFn::Py(w)) => w.upgrade(), + _ => None, + } + } + /// [`Self::peek_fn`] for a class receiver: a plain function or a /// static method's function, either called with an empty self slot. #[inline] @@ -61481,6 +61704,52 @@ impl MethodSlot { } } + /// The property getter the slot names for instances of the class at + /// `ver`, while the property still holds that function and the + /// function that code. The class holds the property for as long as it + /// reads `ver` (see `get_held`), so no upgrade is needed. + #[inline] + fn getter(&self, ver: u64) -> Option<(Rc, Rc)> { + // SAFETY: GIL-serialized; no `&mut` escapes `set_getter`. + let (v, MethodSlotFn::Getter { prop, func, code }) = (unsafe { &*self.0.get() }) else { + return None; + }; + if *v != ver || prop.strong_count() == 0 { + return None; + } + // SAFETY: the class holds the property (see above); a raw read of + // its getter cell, compared before anything runs. + let Object::Function(f) = (unsafe { &*(*prop.as_ptr()).fget.as_ptr() }) else { + return None; + }; + if Rc::as_ptr(f) != func.as_ptr() { + return None; + } + let c = f.code(); + (Rc::as_ptr(&c) == code.as_ptr()).then(|| (f.clone(), c)) + } + + #[inline] + fn set_getter( + &self, + ver: u64, + prop: &Rc, + func: &Rc, + code: &Rc, + ) { + // SAFETY: as `set`. + unsafe { + *self.0.get() = ( + ver, + MethodSlotFn::Getter { + prop: Rc::downgrade(prop), + func: Rc::downgrade(func), + code: Rc::downgrade(code), + }, + ); + }; + } + #[inline] fn set_native_subscript(&self, ver: u64, generation: u64, fast: leaf_builtins::Fast) { // SAFETY: as `set`; neither value owns Python objects. diff --git a/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs index bcef325e..c7f2e974 100644 --- a/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs +++ b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs @@ -1259,6 +1259,21 @@ fn has_inline_values(args: &[Object]) -> Result { Ok(Object::Bool(inline)) } +/// `_testinternalcapi.instance_layout(obj)` (WeavePy-only): how an +/// instance stores its attributes, without changing it — `"split"` (the +/// class's shared names), `"dict"` (a published dictionary), or `"none"`. +fn instance_layout(args: &[Object]) -> Result { + let layout = match args.first() { + Some(Object::Instance(inst)) if inst.dict.published().is_some() => "dict", + Some(Object::Instance(inst)) => match inst.dict.split_cell().try_borrow() { + Ok(split) if !split.is_empty() => "split", + _ => "none", + }, + _ => "none", + }; + Ok(Object::from_static(layout)) +} + /// `_testinternalcapi.run_in_subinterp_with_config(code, config)` — /// CPython's helper spins up a `Py_NewInterpreterFromConfig` interpreter, /// runs `code` with `PyRun_SimpleString`, and tears it down, returning @@ -2139,6 +2154,15 @@ pub fn build(_cache: &ModuleCache) -> Rc { call_kw: None, })), ); + d.insert( + DictKey(Object::from_static("instance_layout")), + Object::Builtin(Rc::new(BuiltinFn { + name: "instance_layout", + binds_instance: false, + call: Box::new(instance_layout), + call_kw: None, + })), + ); d.insert( DictKey(Object::from_static("has_inline_values")), Object::Builtin(Rc::new(BuiltinFn { diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 27fc9596..55ba0a3a 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -2499,13 +2499,8 @@ fn attr_chain_step(obj: &Object, name: &str) -> Option { }; let v = match storage { AttrStorage::Slot(_) => inst.slot_get(name)?, - AttrStorage::Indexed(key_idx) => inst - .dict - .get()? - .borrow() - .get_index(key_idx as usize)? - .1 - .clone(), + // Either layout, without materializing a split one. + AttrStorage::Indexed(key_idx) => inst.attr_index_map(key_idx as usize, |_, v| v.clone())?, AttrStorage::NewKey => return None, }; matches!(v, Object::Instance(_)).then_some(v) @@ -2707,11 +2702,9 @@ fn probe_method_entry( if crate::specialize::type_has_attr_override(&cls) { return None; } - if inst - .dict - .get() - .is_some_and(|dict| dict.borrow().get(&StrKey(name)).is_some()) - { + // Read without materializing a split layout: a probe must not change + // how the instance stores its attributes. + if inst.attr_get_str(name).is_some() { return None; } let Some(Object::Function(f)) = cls.lookup(name) else { diff --git a/crates/weavepy-vm/src/vm_singletons.rs b/crates/weavepy-vm/src/vm_singletons.rs index 9c19ff57..01873b63 100644 --- a/crates/weavepy-vm/src/vm_singletons.rs +++ b/crates/weavepy-vm/src/vm_singletons.rs @@ -51,12 +51,15 @@ thread_local! { const { RefCell::new(Vec::new()) }; } -/// Process-wide count of parked `__del__` requests across all threads' -/// [`PENDING_FINALIZERS`] queues (RFC 0058 WS2). The eval loop probes -/// for pending finalizers *every instruction*; a macOS thread-local +/// Process-wide count of parked `__del__` requests and weakref callbacks +/// across all threads' [`PENDING_FINALIZERS`] and +/// [`PENDING_WEAKREF_CALLBACKS`] queues (RFC 0058 WS2). The eval loop +/// probes for pending work *every instruction*; a macOS thread-local /// access plus a `RefCell` borrow there is measurably expensive, so -/// this relaxed atomic is the fast gate and the thread-local queue -/// stays the precise, per-thread source of truth. +/// this relaxed atomic is the fast gate and the thread-local queues +/// stay the precise, per-thread source of truth. A drain re-raises the +/// shared hot gate while it's nonzero, so one thread's drain never +/// strands work another thread queued. static PENDING_FINALIZER_COUNT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); @@ -195,6 +198,7 @@ pub fn push_pending_weakref_callback(callback: Object, weakref_obj: Object) { }) .unwrap_or(false); if pushed { + PENDING_FINALIZER_COUNT.fetch_add(1, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::PENDING_FINALIZERS); crate::gc_trace::mark_maybe_dead(); } @@ -212,7 +216,11 @@ pub fn has_pending_weakref_callbacks() -> bool { /// Drain the pending weakref-callback queue. pub fn drain_pending_weakref_callbacks() -> Vec<(Object, Object)> { - PENDING_WEAKREF_CALLBACKS.with(|cell| std::mem::take(&mut *cell.borrow_mut())) + let taken = PENDING_WEAKREF_CALLBACKS.with(|cell| std::mem::take(&mut *cell.borrow_mut())); + if !taken.is_empty() { + PENDING_FINALIZER_COUNT.fetch_sub(taken.len(), std::sync::atomic::Ordering::Release); + } + taken } /// Build a singleton instance of the given built-in registry type. @@ -456,7 +464,13 @@ pub fn clear_thread_python_tls() { PENDING_FINALIZER_COUNT.fetch_sub(n, std::sync::atomic::Ordering::Release); } } - let _ = PENDING_WEAKREF_CALLBACKS.try_with(|cell| cell.borrow_mut().clear()); + let dropped_callbacks = + PENDING_WEAKREF_CALLBACKS.try_with(|cell| std::mem::take(&mut *cell.borrow_mut()).len()); + if let Ok(n) = dropped_callbacks { + if n > 0 { + PENDING_FINALIZER_COUNT.fetch_sub(n, std::sync::atomic::Ordering::Release); + } + } let _ = CURRENT_THREAD_HANDLES.try_with(|cell| cell.borrow_mut().clear()); let dropped_cext = PENDING_CEXT_DROPS.try_with(|cell| std::mem::take(&mut *cell.borrow_mut()).len()); diff --git a/tests/regrtest/test_leaf_builtin_calls.py b/tests/regrtest/test_leaf_builtin_calls.py index f0d9da2e..b8802b8b 100644 --- a/tests/regrtest/test_leaf_builtin_calls.py +++ b/tests/regrtest/test_leaf_builtin_calls.py @@ -121,3 +121,14 @@ def f_floor(x): assert f_get({Key(1): 2}, Key(1)) == 2 assert Key.calls == before + 1 print("ok") + + +# A native leaf that returns a value it owns (a builtin call's fresh +# result) hands back its own reference, not a pointer into its scratch. +def f_fresh(s, a): + return s.format(a) + + +for i in range(3000): + assert f_fresh("{}!", i) == "%d!" % i +print("fresh ok") From 16b876a1fb1ca895be2bcd7a9993ba6e59c0fa6e Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 06:18:05 -0700 Subject: [PATCH 04/96] build: record the VM crate's direct Cranelift dependencies in the lockfile --- Cargo.lock | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 1036cdee..a3fef29c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2935,6 +2935,11 @@ dependencies = [ "bzip2", "chrono", "corosensei", + "cranelift-codegen", + "cranelift-frontend", + "cranelift-jit", + "cranelift-module", + "cranelift-native", "crc32fast", "crossbeam-channel", "crossbeam-utils", From 8f2bec87b608a60159ebb919718cbf8dda9592fe Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 06:56:07 -0700 Subject: [PATCH 05/96] perf: release returning frames' locals directly and widen native plan fast paths A returning inline activation released its locals in line only when every heap local had escaped (a check the old prompt-reap discipline needed); now any unshared locals vector is released in line, and an object that dies with it queues its finalizer for the caller's next check. The exit-reap check is gone, and `inline_finish` takes the same shortcut. Native leaf plans gain in-line float compares and float add, subtract, multiply and divide (NaN results and zero divisors take the helper), and attribute, method and global helpers that receive their site's cache slot directly. deltablue retires 10% fewer instructions than before plan compilation; a non-leaf method call costs 832 instructions instead of 1,069. --- crates/weavepy-vm/src/leaf_plan.rs | 27 +++- crates/weavepy-vm/src/leaf_plan/jit.rs | 190 ++++++++++++++++++++++--- crates/weavepy-vm/src/lib.rs | 146 ++++--------------- 3 files changed, 218 insertions(+), 145 deletions(-) diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index 521f4c7f..e39f1495 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -1416,10 +1416,21 @@ impl Interpreter { stamps: &[crate::StampSlot], f: &crate::object::PyFunction, pc: u16, + ) -> Option { + self.plan_global_at(code, stamps.get(usize::from(pc))?, f, pc) + } + + /// [`Self::plan_global`] with the site's stamp slot in hand. + #[inline(always)] + fn plan_global_at( + &self, + code: &CodeObject, + slot: &crate::StampSlot, + f: &crate::object::PyFunction, + pc: u16, ) -> Option { use weavepy_compiler::InlineCache as IC; let pc = usize::from(pc); - let slot = stamps.get(pc)?; let (gdict, bdict) = (f.globals.as_ptr(), f.builtins.as_ptr()); let gid = crate::specialize::rc_id(&f.globals); // SAFETY (raw dict reads): nothing runs code here. @@ -1520,10 +1531,22 @@ impl Interpreter { /// the core loop's arm). #[inline(always)] fn plan_method(&self, code: &CodeObject, src: V, pc: u16, name: u16) -> Option<(V, V)> { + let ms = crate::code_method_slot(code, u32::from(pc))?; + self.plan_method_at(code, ms, src, name) + } + + /// [`Self::plan_method`] with the site's method slot in hand. + #[inline(always)] + fn plan_method_at( + &self, + code: &CodeObject, + ms: &crate::MethodSlot, + src: V, + name: u16, + ) -> Option<(V, V)> { let V::R(p) = src else { return None; }; - let ms = crate::code_method_slot(code, u32::from(pc))?; // SAFETY: as `norm`. match unsafe { &*p } { Object::Instance(inst) => { diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index fd8b7731..e9ac77bb 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -23,8 +23,9 @@ use crate::object::{Object, PyFunction}; use crate::sync::Rc; use crate::{CodeConstObjects, Interpreter}; use cranelift_codegen::ir::{ - condcodes::IntCC, types, AbiParam, Block, InstBuilder, MemFlags, SigRef, Signature, - StackSlotData, StackSlotKind, Value, + condcodes::{FloatCC, IntCC}, + types, AbiParam, Block, InstBuilder, MemFlags, SigRef, Signature, StackSlotData, StackSlotKind, + Value, }; use cranelift_codegen::settings::{self, Configurable}; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; @@ -260,10 +261,16 @@ fn stamps(ext: &CodeConstObjects) -> &[crate::StampSlot] { ext.stamp_slots.get().map_or(&[], |s| &s[..]) } -unsafe extern "C" fn h_global(ctx: *mut Ctx<'static>, fr: *const Frame, pc: u32) -> u32 { - // SAFETY: called by native code with its context and a live frame. - let (c, (code, ext, f)) = unsafe { (cx(ctx), (*fr).parts()) }; - match c.interp.plan_global(code, stamps(ext), f, pc as u16) { +unsafe extern "C" fn h_global( + ctx: *mut Ctx<'static>, + fr: *const Frame, + slot: *const crate::StampSlot, + pc: u32, +) -> u32 { + // SAFETY: called by native code with its context, a live frame, and + // the site's stamp slot (its code keeps the table). + let (c, (code, _, f), slot) = unsafe { (cx(ctx), (*fr).parts(), &*slot) }; + match c.interp.plan_global_at(code, slot, f, pc as u16) { Some(v) => { c.put(v); 0 @@ -300,17 +307,48 @@ unsafe extern "C" fn h_attr( } } -unsafe extern "C" fn h_method( +/// [`h_attr`] with the site's field slot in hand: a cached split field of +/// a plain instance answers without the general read. +unsafe extern "C" fn h_field( ctx: *mut Ctx<'static>, fr: *const Frame, + slot: *const crate::FieldSlot, t: u64, p: u64, pc_name: u32, ) -> u32 { - // SAFETY: called by native code with its context and a live frame. - let (c, (code, _, _)) = unsafe { (cx(ctx), (*fr).parts()) }; - let (pc, name) = (pc_name as u16, (pc_name >> 16) as u16); - match c.interp.plan_method(code, value(t, p), pc, name) { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + if t == T_R && (!EFFECT || c.pend.n == 0) { + // SAFETY: as `norm`; the slot's code keeps the table. + if let (Object::Instance(inst), slot) = unsafe { (&*(p as *const Object), &*slot) } { + let (ver, idx) = slot.get(); + if inst.cls_raw().attr_version.get() == ver { + // SAFETY: the receiver is rooted and nothing runs code + // while the view is read. + if let Some(v) = unsafe { inst.split_field(idx as usize) } { + c.put(norm(v)); + return 0; + } + } + } + } + // SAFETY: forwarded. + unsafe { h_attr::(ctx, fr, t, p, pc_name) } +} + +unsafe extern "C" fn h_method( + ctx: *mut Ctx<'static>, + fr: *const Frame, + ms: *const crate::MethodSlot, + t: u64, + p: u64, + name: u32, +) -> u32 { + // SAFETY: called by native code with its context, a live frame, and + // the site's method slot (its code keeps the table). + let (c, (code, _, _), ms) = unsafe { (cx(ctx), (*fr).parts(), &*ms) }; + match c.interp.plan_method_at(code, ms, value(t, p), name as u16) { Some((func, recv)) => { let [a, b] = words(func); let [d, e] = words(recv); @@ -778,6 +816,11 @@ impl Lower<'_> { self.set(fl, r, t, p); } + /// A payload word as the float it holds. + fn as_f64(&mut self, p: Value) -> Value { + self.b.ins().bitcast(types::F64, MemFlags::new(), p) + } + fn u32c(&mut self, x: u32) -> Value { self.b.ins().iconst(types::I32, i64::from(x)) } @@ -982,27 +1025,59 @@ impl Lower<'_> { self.set(fl, dst, t, p); } Op::Global { dst, pc } => { + let Some(slot) = crate::code_stamp_slot(fl.code, u32::from(pc)) else { + self.b.ins().jump(self.decline, &[]); + return Some(false); + }; + let slot = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(slot) as i64); let pc = self.u32c(u32::from(pc)); - let st = self.call(h_global as *const () as usize, &[self.ctx, fl.fr, pc]); + let st = self.call(h_global as *const () as usize, &[self.ctx, fl.fr, slot, pc]); self.check(st); self.set_out(fl, dst, 0); } Op::Attr { dst, src, pc, name } => { let (t, p) = self.get(fl, src); let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); - let addr = if effect { - h_attr:: as *const () as usize - } else { - h_attr:: as *const () as usize + let st = match crate::code_field_slot(fl.code, u32::from(pc)) { + Some(slot) => { + let slot = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(slot) as i64); + let addr = if effect { + h_field:: as *const () as usize + } else { + h_field:: as *const () as usize + }; + self.call(addr, &[self.ctx, fl.fr, slot, t, p, pn]) + } + None => { + let addr = if effect { + h_attr:: as *const () as usize + } else { + h_attr:: as *const () as usize + }; + self.call(addr, &[self.ctx, fl.fr, t, p, pn]) + } }; - let st = self.call(addr, &[self.ctx, fl.fr, t, p, pn]); self.check(st); self.set_out(fl, dst, 0); } Op::Method { dst, src, pc, name } => { + let Some(ms) = crate::code_method_slot(fl.code, u32::from(pc)) else { + self.b.ins().jump(self.decline, &[]); + return Some(false); + }; let (t, p) = self.get(fl, src); - let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); - let st = self.call(h_method as *const () as usize, &[self.ctx, fl.fr, t, p, pn]); + let ms = self.b.ins().iconst(self.ptr, std::ptr::from_ref(ms) as i64); + let nm = self.u32c(u32::from(name)); + let st = self.call( + h_method as *const () as usize, + &[self.ctx, fl.fr, ms, t, p, nm], + ); self.check(st); self.set_out(fl, dst, 0); self.set_out(fl, dst + 1, 2); @@ -1019,18 +1094,42 @@ impl Lower<'_> { k if k == CompareKind::GtE as u8 => IntCC::SignedGreaterThanOrEqual, _ => return None, }; - // Two ints compare in line. + let fcc = match kind { + k if k == CompareKind::Lt as u8 => FloatCC::LessThan, + k if k == CompareKind::LtE as u8 => FloatCC::LessThanOrEqual, + k if k == CompareKind::Eq as u8 => FloatCC::Equal, + k if k == CompareKind::NotEq as u8 => FloatCC::NotEqual, + k if k == CompareKind::Gt as u8 => FloatCC::GreaterThan, + _ => FloatCC::GreaterThanOrEqual, + }; + // Two ints, or two floats, compare in line. let ia = self.b.ins().icmp_imm(IntCC::Equal, ta, T_I as i64); let ib = self.b.ins().icmp_imm(IntCC::Equal, tb, T_I as i64); let both = self.b.ins().band(ia, ib); let fast = self.b.create_block(); + let not_int = self.b.create_block(); + let float = self.b.create_block(); let slow = self.b.create_block(); let merge = self.b.create_block(); self.b.append_block_param(merge, types::I8); - self.b.ins().brif(both, fast, &[], slow, &[]); + self.b.ins().brif(both, fast, &[], not_int, &[]); self.b.switch_to_block(fast); let bit = self.b.ins().icmp(cc, pa, pb); self.b.ins().jump(merge, &[bit.into()]); + self.b.switch_to_block(not_int); + let fa = self.b.ins().icmp_imm(IntCC::Equal, ta, T_F as i64); + let fb = self.b.ins().icmp_imm(IntCC::Equal, tb, T_F as i64); + let both = self.b.ins().band(fa, fb); + self.b.ins().brif(both, float, &[], slow, &[]); + self.b.switch_to_block(float); + let (xa, xb) = (self.as_f64(pa), self.as_f64(pb)); + // A NaN declines, as the interpreter's `compare` does. + let nan = self.b.ins().fcmp(FloatCC::Unordered, xa, xb); + let ordered = self.b.create_block(); + self.b.ins().brif(nan, self.decline, &[], ordered, &[]); + self.b.switch_to_block(ordered); + let bit = self.b.ins().fcmp(fcc, xa, xb); + self.b.ins().jump(merge, &[bit.into()]); self.b.switch_to_block(slow); let k = self.u32c(u32::from(kind)); let r = self.call(h_compare as *const () as usize, &[ta, pa, tb, pb, k]); @@ -1069,6 +1168,7 @@ impl Lower<'_> { let int_op = [BinOpKind::Add, BinOpKind::Sub, BinOpKind::Mult] .into_iter() .find(|k| *k as u8 == kind); + let float_div = kind == BinOpKind::Div as u8; let merge = self.b.create_block(); self.b.append_block_param(merge, types::I64); self.b.append_block_param(merge, types::I64); @@ -1080,7 +1180,8 @@ impl Lower<'_> { let ib = self.b.ins().icmp_imm(IntCC::Equal, tb, T_I as i64); let both = self.b.ins().band(ia, ib); let fast = self.b.create_block(); - self.b.ins().brif(both, fast, &[], slow, &[]); + let not_int = self.b.create_block(); + self.b.ins().brif(both, fast, &[], not_int, &[]); self.b.switch_to_block(fast); let (r, ovf) = match k { BinOpKind::Add => { @@ -1109,6 +1210,51 @@ impl Lower<'_> { self.b.switch_to_block(ok); let ti = self.b.ins().iconst(types::I64, T_I as i64); self.b.ins().jump(merge, &[ti.into(), r.into()]); + // Two floats in line too; a NaN result takes the helper + // (which gives it its identity). + self.b.switch_to_block(not_int); + let fa = self.b.ins().icmp_imm(IntCC::Equal, ta, T_F as i64); + let fb = self.b.ins().icmp_imm(IntCC::Equal, tb, T_F as i64); + let both = self.b.ins().band(fa, fb); + let float = self.b.create_block(); + self.b.ins().brif(both, float, &[], slow, &[]); + self.b.switch_to_block(float); + let (xa, xb) = (self.as_f64(pa), self.as_f64(pb)); + let r = match k { + BinOpKind::Add => self.b.ins().fadd(xa, xb), + BinOpKind::Sub => self.b.ins().fsub(xa, xb), + _ => self.b.ins().fmul(xa, xb), + }; + let nan = self.b.ins().fcmp(FloatCC::Unordered, r, r); + let fine = self.b.create_block(); + self.b.ins().brif(nan, slow, &[], fine, &[]); + self.b.switch_to_block(fine); + let bits = self.b.ins().bitcast(types::I64, MemFlags::new(), r); + let tf = self.b.ins().iconst(types::I64, T_F as i64); + self.b.ins().jump(merge, &[tf.into(), bits.into()]); + } else if float_div { + // Two floats divide in line by a nonzero divisor; a NaN + // result takes the helper. + let fa = self.b.ins().icmp_imm(IntCC::Equal, ta, T_F as i64); + let fb = self.b.ins().icmp_imm(IntCC::Equal, tb, T_F as i64); + let both = self.b.ins().band(fa, fb); + let float = self.b.create_block(); + self.b.ins().brif(both, float, &[], slow, &[]); + self.b.switch_to_block(float); + let (xa, xb) = (self.as_f64(pa), self.as_f64(pb)); + let zero = self.b.ins().f64const(0.0); + let by_zero = self.b.ins().fcmp(FloatCC::Equal, xb, zero); + let nonzero = self.b.create_block(); + self.b.ins().brif(by_zero, slow, &[], nonzero, &[]); + self.b.switch_to_block(nonzero); + let r = self.b.ins().fdiv(xa, xb); + let nan = self.b.ins().fcmp(FloatCC::Unordered, r, r); + let fine = self.b.create_block(); + self.b.ins().brif(nan, slow, &[], fine, &[]); + self.b.switch_to_block(fine); + let bits = self.b.ins().bitcast(types::I64, MemFlags::new(), r); + let tf = self.b.ins().iconst(types::I64, T_F as i64); + self.b.ins().jump(merge, &[tf.into(), bits.into()]); } else { self.b.ins().jump(slow, &[]); } diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 99138a4d..7fff1c39 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -7049,33 +7049,8 @@ impl Interpreter { #[inline] fn pop_top_step(&mut self, frame: &mut Frame) -> Result<(), RuntimeError> { let v = frame.pop()?; - // RFC 0059 WS1b: POP_TOP is exempt from the eval loop's - // coarse stack-shrink mark; note the discarded value here - // (no-op for pure leaves — e.g. a discarded expression - // statement's int/str result). - // Discarding the last reference to a temporary mirrors - // CPython's refcount-driven finalization (`f()` as a - // statement finalizes the result immediately). This covers - // not just directly-finalizable values (instances with - // `__del__`, live generators/coroutines) but *containers* - // that anchor a dead finalizable acyclic subgraph: a - // discarded `pool.map(...)` returns a `list` of unpickled, - // GC-tracked result copies, and if it isn't reaped here the - // copies linger as dead-but-tracked garbage until *some* - // thread's next collection finalizes them — which, across the - // statement's trailing `del`/`sleep`, is frequently a - // background pool thread, so the copies' `__del__` then races - // the main thread's (CPython runs every one on the thread that - // dropped the ref). The `local_needs_prompt_reap` kinds match - // `reap_frame_locals_on_exit`; `prompt_reap_dropped`'s refcount - // guard leaves anything still referenced elsewhere alone - // (RFC 0040: `test_multiprocessing_*` `test_release_task_refs`). - // No `sync_py_locals` is needed: a discarded statement result - // is an operand-stack temporary that was never a frame local, - // so the live-locals mirror cannot hold a stale clone of it, - // and `frame.locals` always reflects live bindings — the - // refcount guard therefore cannot false-positive on a value - // still bound to a local. + // A discarded temporary dies here, finalizers and all (`f()` as a + // statement finalizes its result at once, as in CPython). self.release(v); Ok(()) } @@ -8123,31 +8098,17 @@ impl Interpreter { None, ) => { // Only leaf instructions and inline calls ran here, so the - // frame's other fields are untouched. Scalar-only locals - // (and no leftover operands or cells) owe the exit reap - // nothing: they die here with no drop glue. + // frame's other fields are untouched: with no leftover + // operands or cells, the locals are released here. done.clean = true; if frame.stack.is_empty() && frame.code.cellvars.is_empty() && Rc::strong_count(&frame.locals) == 1 { // SAFETY: sole owner; nothing else reaches the vector. - let locals = unsafe { &mut *frame.locals.as_ptr() }; - if locals.iter().all(|o| { - matches!( - o, - Object::Int(_) - | Object::Float(_) - | Object::Bool(_) - | Object::None - | Object::Unbound - ) - }) { - // SAFETY: scalars own nothing (no drop skipped). - unsafe { locals.set_len(0) }; - drop(done.guard.take()); - return Ok(v); - } + unsafe { (*frame.locals.as_ptr()).clear() }; + drop(done.guard.take()); + return Ok(v); } Ok(FrameOutcome::Returned(v)) } @@ -13573,73 +13534,6 @@ impl Interpreter { Some(Ok(Object::Bool(r))) } - /// Whether a returning frame's `locals` owe the exit reap - /// (`reap_frame_locals_on_exit`) nothing: scalars and strings, and - /// instances or containers held beyond every slot of the frame that - /// could name them plus the collector's handle, with no weakref - /// watching — the reap's own "escaped" verdict, and a grade of no mark - /// for each release. - #[inline] - fn core_escaped_locals(locals: &[Object]) -> bool { - // Each heap local's identity and strong count: up to eight are - // compared exactly (how many slots here name each object); past - // that, every heap slot is assumed to name every object. - const EXACT: usize = 8; - let mut heap = [const { std::mem::MaybeUninit::<(u64, usize)>::uninit() }; EXACT]; - let mut n = 0usize; - for o in locals { - let (id, sc) = match o { - Object::Int(_) - | Object::Float(_) - | Object::Bool(_) - | Object::None - | Object::Unbound - | Object::Str(_) => continue, - Object::Instance(i) => (Rc::as_ptr(i) as usize as u64, Rc::strong_count(i)), - Object::List(l) => (Rc::as_ptr(l) as usize as u64, Rc::strong_count(l)), - Object::Dict(d) => (Rc::as_ptr(d) as usize as u64, Rc::strong_count(d)), - Object::Tuple(t) => ( - ThinArc::as_ptr(t).cast::<()>() as usize as u64, - ThinArc::strong_count(t), - ), - _ => return false, - }; - if n < EXACT { - heap[n].write((id, sc)); - } - n += 1; - } - if n == 0 { - return true; - } - if n > EXACT { - return locals.iter().all(|o| { - let (sc, id) = match o { - Object::Instance(i) => (Rc::strong_count(i), Rc::as_ptr(i) as usize as u64), - Object::List(l) => (Rc::strong_count(l), Rc::as_ptr(l) as usize as u64), - Object::Dict(d) => (Rc::strong_count(d), Rc::as_ptr(d) as usize as u64), - Object::Tuple(t) => ( - ThinArc::strong_count(t), - ThinArc::as_ptr(t).cast::<()>() as usize as u64, - ), - _ => return true, - }; - sc > n + usize::from(gc_trace::maybe_tracked(id)) - && !crate::weakref_registry::may_have_weakrefs(id) - }); - } - // SAFETY: the first `n` entries were written. - let heap: &[(u64, usize)] = unsafe { std::slice::from_raw_parts(heap.as_ptr().cast(), n) }; - heap.iter().all(|&(id, sc)| { - // Held past every slot here and a collector handle needs no - // count of the slots that name it. - (sc > n + 1 || { - let held = heap.iter().filter(|&&(other, _)| other == id).count(); - sc > held + usize::from(gc_trace::maybe_tracked(id)) - }) && !crate::weakref_registry::may_have_weakrefs(id) - }) - } - /// The `(function, has_self, effective argc)` of a plain-function /// `CALL` at `pc` (the shapes [`Self::core_call_cached`] serves). #[inline] @@ -15003,20 +14897,17 @@ impl Interpreter { sw.entry_dead = true; } // `inline_finish`'s shell-less return, its common case in line: no - // leftover operands or cells, and locals that die here with no drop - // glue (scalars) or whose release the exit reap would pass over - // (see `core_escaped_locals`). + // leftover operands or cells, and locals only this activation + // holds. They are released here; an object that dies with them + // queues its finalizer, which runs below before the caller goes on. let clean = frame.stack.is_empty() && frame.code.cellvars.is_empty() - && Rc::strong_count(&frame.locals) == 1 - // SAFETY: sole owner; nothing else reaches the vector. - && Self::core_escaped_locals(unsafe { &*frame.locals.as_ptr() }); + && Rc::strong_count(&frame.locals) == 1; let mut tmp = None; let (cframe, clast, cshell, entry); if clean && done.init_inst.is_none() { done.clean = true; - // SAFETY: as above; the values drop by plain decrements (or - // own nothing). + // SAFETY: as above. for v in unsafe { (*frame.locals.as_ptr()).drain(..) } { drop_hot(v); } @@ -61243,6 +61134,19 @@ fn drop_hot(v: Object) { } } +/// The field slot for `cache_pc`, allocating the table on first use. +#[inline] +fn code_field_slot(code: &CodeObject, cache_pc: u32) -> Option<&FieldSlot> { + let ext = code_vm_ext(code)?; + ext.field_slots + .get_or_init(|| { + (0..code.instructions.len()) + .map(|_| FieldSlot::empty()) + .collect() + }) + .get(cache_pc as usize) +} + /// The stamp slot for `cache_pc`, allocating the table on first use. #[inline] fn code_stamp_slot(code: &CodeObject, cache_pc: u32) -> Option<&StampSlot> { From 603677ce0124b6e9828074e03b048ea6d31d80a7 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 07:13:31 -0700 Subject: [PATCH 06/96] perf: fast-step generators whose prologue runs other instructions A generator qualified for fast steps only if every instruction in its body was one a step runs, so a body that builds its iterator first (`for j in range(k): yield j`) never did. Eligibility now covers what a resume reaches from a yield; the first resume's step stops at the prologue and the general loop runs it once. Resuming such a generator costs 647 instructions instead of 1,187. --- crates/weavepy-vm/src/gen_fast.rs | 44 ++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs index d2902740..b9ed6706 100644 --- a/crates/weavepy-vm/src/gen_fast.rs +++ b/crates/weavepy-vm/src/gen_fast.rs @@ -96,8 +96,11 @@ fn op_supported(op: OpCode) -> bool { ) } -/// Whether `code` is a plain generator whose every instruction is one a -/// fast step runs (cached in the code's extension table). +/// Whether `code` is a plain generator whose steady state, everything a +/// resume can reach from a `yield`, is instructions a fast step runs +/// (cached in the code's extension table). The prologue may hold others +/// (`range(n)` built before the loop): the first resume's step stops at +/// them, and the general loop runs them once. fn code_ok(code: &CodeObject) -> bool { use std::sync::atomic::Ordering; let Some(ext) = crate::code_vm_ext(code) else { @@ -113,13 +116,46 @@ fn code_ok(code: &CodeObject) -> bool { && !code.is_async_generator && code.cellvars.is_empty() && code.freevars.is_empty() - && code.instructions.iter().all(|i| op_supported(i.op)) - && in_bounds(code, ext.objects.len()); + && in_bounds(code, ext.objects.len()) + && steady_state_supported(code); ext.gen_fast .store(if ok { 2 } else { 1 }, Ordering::Relaxed); ok } +/// Whether every instruction a resume reaches from one of `code`'s yields +/// is one a fast step runs (jump targets are in range: see [`in_bounds`]). +fn steady_state_supported(code: &CodeObject) -> bool { + let instrs = &code.instructions; + let n = instrs.len(); + let mut seen = vec![false; n]; + let mut work: Vec = (0..n) + .filter(|&p| instrs[p].op == OpCode::YieldValue) + .map(|p| p + 1) + .collect(); + while let Some(p) = work.pop() { + if p >= n || std::mem::replace(&mut seen[p], true) { + continue; + } + let ins = instrs[p]; + if !op_supported(ins.op) { + return false; + } + let arg = ins.arg as usize; + match ins.op { + OpCode::ReturnValue | OpCode::Reraise => {} + OpCode::JumpForward => work.push(p + 1 + arg), + OpCode::JumpBackward => work.push((p + 1).saturating_sub(arg)), + OpCode::PopJumpIfFalse | OpCode::PopJumpIfTrue | OpCode::ForIter => { + work.push(p + 1); + work.push(p + 1 + arg); + } + _ => work.push(p + 1), + } + } + true +} + /// Whether every jump, local and constant `code`'s instructions name is /// in range, and the last instruction can't fall through: the step then /// reads instructions, locals and constants without per-use checks. From 7e9d50b04a2f2716c93fd3a998818057c5e9360f Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:06:05 -0700 Subject: [PATCH 07/96] perf: run exception handlers off the full prologue and match own-class handlers directly While a traceback holds a frame's object, the frame can't take the quiet loop, so its handler ran every instruction through the full prologue. When nothing that prologue serves is pending, each instruction now only brings the frame's `lasti` current and steps. An `except` clause naming the raised exception's own class matches by identity, without walking either class's MRO. A raise caught in the same frame costs about 7,000 instructions instead of 8,375. --- crates/weavepy-vm/src/lib.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 7fff1c39..46000d60 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -4501,6 +4501,25 @@ impl Interpreter { self.step_hot(frame) } } + } else if frame_blocks_quiet { + // Quiet but for a frame object something outside holds (an + // exception's traceback, while its handler runs): nothing + // the full prologue serves is pending, so each instruction + // only brings the frame's `lasti` current before it runs. + cur_pc = frame.pc as usize; + shell + .lasti + .store(frame.pc, std::sync::atomic::Ordering::Relaxed); + if shell + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) + { + self.ensure_top_py_frame(&mut py_frame_slot) + .lasti + .set(frame.pc); + } + instruction_ran = true; + self.step_hot(frame) } else { // RFC 0059 (WS2): the unified eval-breaker word. One relaxed // load answers "is *any* deferred work pending?" for the six @@ -23630,6 +23649,14 @@ impl Interpreter { } fn exception_matches(&self, exc: &Object, ty: &Object) -> Result { + // The commonest handler names the raised exception's own class, + // which (being raised) derives from BaseException. + if let (Object::Instance(inst), Object::Type(t)) = (exc, ty) { + if !crate::gil::free_threading_enabled() && std::ptr::eq(inst.cls_raw(), Rc::as_ptr(t)) + { + return Ok(true); + } + } // Every entry must be a BaseException subclass *before* any // matching happens — `except (ValueError, 42):` is a TypeError // even when the raised exception would match ValueError From cdee10da703e2da59ddfa53faa87123c4e548531 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 08:37:30 -0700 Subject: [PATCH 08/96] perf: cache native plans' field positions per class and global locations per namespace state Each attribute site in a native plan keeps the split-field positions of up to four receiver classes, so methods a base class shares (the constraint classes in deltablue all read `self.direction`) stop missing the single-class field slot. Each global site remembers which namespace holds its value and at what index, under the globals' and builtins' process-unique stamps, and reads the current value there on a hit. deltablue retires about 3% fewer instructions. --- crates/weavepy-vm/src/leaf_plan/jit.rs | 173 +++++++++++++++++++------ crates/weavepy-vm/src/lib.rs | 13 -- 2 files changed, 130 insertions(+), 56 deletions(-) diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index e9ac77bb..61dcbab1 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -67,6 +67,30 @@ pub(super) struct Native { #[allow(clippy::vec_box)] _frames: Vec>, _keep: Vec<(crate::sync::Weak, Rc)>, + /// The attribute and global sites' caches, which the code addresses. + #[allow(clippy::vec_box)] + _fields: Vec>, + #[allow(clippy::vec_box)] + _globals: Vec>, +} + +/// An attribute site's split-field positions by receiver class version: +/// a site several classes reach (methods a base class shares) keeps one +/// for each. A version names one class in one state, and a class's shared +/// names never move, so a hit needs only the instance to hold the field. +#[derive(Default)] +pub(super) struct FieldCache { + entries: [std::cell::Cell<(u64, u32)>; 4], + next: std::cell::Cell, +} + +/// A global site's resolution: the globals and builtins stamps it was +/// found under (process-unique, so they name the dicts and their key +/// layouts), which of the two holds it, and at what index. +#[derive(Default)] +pub(super) struct GlobalCache { + stamps: std::cell::Cell<(u64, u64)>, + at: std::cell::Cell<(bool, u32)>, } // SAFETY: the descriptors name immutable code objects and reserved @@ -265,18 +289,47 @@ unsafe extern "C" fn h_global( ctx: *mut Ctx<'static>, fr: *const Frame, slot: *const crate::StampSlot, + cache: *const GlobalCache, pc: u32, ) -> u32 { - // SAFETY: called by native code with its context, a live frame, and - // the site's stamp slot (its code keeps the table). - let (c, (code, _, f), slot) = unsafe { (cx(ctx), (*fr).parts(), &*slot) }; - match c.interp.plan_global_at(code, slot, f, pc as u16) { - Some(v) => { - c.put(v); - 0 + // SAFETY: called by native code with its context, a live frame, the + // site's stamp slot (its code keeps the table), and its cache. + let (c, (code, _, f), slot, cache) = unsafe { (cx(ctx), (*fr).parts(), &*slot, &*cache) }; + // SAFETY (raw dict reads): nothing runs code here. + let (gs, bs) = unsafe { + ( + (*f.globals.as_ptr()).mutation_stamp(), + (*f.builtins.as_ptr()).mutation_stamp(), + ) + }; + if cache.stamps.get() == (gs, bs) { + let (builtin, idx) = cache.at.get(); + let dict = if builtin { &f.builtins } else { &f.globals }; + // SAFETY: as above. + if let Some((_, v)) = unsafe { (*dict.as_ptr()).get_index(idx as usize) } { + c.put(norm(v)); + return 0; } - None => 1, } + let Some(v) = c.interp.plan_global_at(code, slot, f, pc as u16) else { + return 1; + }; + // Remember where the value lives (the site's inline cache says which + // namespace), under the stamps it was read with. + use weavepy_compiler::InlineCache as IC; + match code.caches.get(pc) { + IC::LoadGlobalModule { key_idx, .. } => { + cache.stamps.set((gs, bs)); + cache.at.set((false, key_idx)); + } + IC::LoadGlobalBuiltin { key_idx, .. } => { + cache.stamps.set((gs, bs)); + cache.at.set((true, key_idx)); + } + _ => {} + } + c.put(v); + 0 } unsafe extern "C" fn h_attr( @@ -307,34 +360,59 @@ unsafe extern "C" fn h_attr( } } -/// [`h_attr`] with the site's field slot in hand: a cached split field of -/// a plain instance answers without the general read. +/// [`h_attr`] through the site's [`FieldCache`]: a split field of a plain +/// instance whose class the site has seen answers without the general +/// read; a general read that found the instance's own field remembers it. unsafe extern "C" fn h_field( ctx: *mut Ctx<'static>, fr: *const Frame, - slot: *const crate::FieldSlot, + cache: *const FieldCache, t: u64, p: u64, pc_name: u32, ) -> u32 { - // SAFETY: called by native code with its context. - let c = unsafe { cx(ctx) }; - if t == T_R && (!EFFECT || c.pend.n == 0) { - // SAFETY: as `norm`; the slot's code keeps the table. - if let (Object::Instance(inst), slot) = unsafe { (&*(p as *const Object), &*slot) } { - let (ver, idx) = slot.get(); - if inst.cls_raw().attr_version.get() == ver { + // SAFETY: called by native code with its context and the site's + // cache. + let (c, cache) = unsafe { (cx(ctx), &*cache) }; + // SAFETY: as `norm`. + let inst = match (t == T_R).then(|| unsafe { &*(p as *const Object) }) { + Some(Object::Instance(inst)) => Some(inst), + _ => None, + }; + if let Some(inst) = inst.filter(|_| !EFFECT || c.pend.n == 0) { + let ver = inst.cls_raw().attr_version.get(); + for e in &cache.entries { + let (v, idx) = e.get(); + if v == ver && v != 0 { // SAFETY: the receiver is rooted and nothing runs code // while the view is read. if let Some(v) = unsafe { inst.split_field(idx as usize) } { c.put(norm(v)); return 0; } + break; } } } // SAFETY: forwarded. - unsafe { h_attr::(ctx, fr, t, p, pc_name) } + let status = unsafe { h_attr::(ctx, fr, t, p, pc_name) }; + if let (0, Some(inst)) = (status, inst) { + // SAFETY: a live frame. + let (code, _, _) = unsafe { (*fr).parts() }; + let ver = inst.cls_raw().attr_version.get(); + // The instance's own field (not a class value or descriptor), + // from its split layout. + if let Some((_, Some(idx))) = + Interpreter::leaf_resolve_instance_attr_ix(code, inst, pc_name >> 16) + { + if ver != 0 && inst.dict.published().is_none() { + let k = usize::from(cache.next.get()) % cache.entries.len(); + cache.entries[k].set((ver, idx)); + cache.next.set(cache.next.get().wrapping_add(1)); + } + } + } + status } unsafe extern "C" fn h_method( @@ -673,6 +751,8 @@ fn compile_with( depth: 0, frames: Vec::new(), keep: Vec::new(), + fields: Vec::new(), + globals: Vec::new(), }; let done = lower.lower(code, ext, plan); let Lower { @@ -680,14 +760,16 @@ fn compile_with( depth, frames, keep, + fields, + globals, .. } = lower; done.map(|()| { b.finalize(); - (depth, frames, keep) + (depth, frames, keep, fields, globals) }) }; - let Some((depth, frames, keep)) = lowered else { + let Some((depth, frames, keep, fields, globals)) = lowered else { // An unfinished function leaves the builder's state behind. engine.fbctx = FunctionBuilderContext::new(); engine.module.clear_context(&mut engine.ctx); @@ -712,6 +794,8 @@ fn compile_with( depth, _frames: frames, _keep: keep, + _fields: fields, + _globals: globals, }) } @@ -754,6 +838,10 @@ struct Lower<'b> { #[allow(clippy::vec_box)] frames: Vec>, keep: Vec<(crate::sync::Weak, Rc)>, + #[allow(clippy::vec_box)] + fields: Vec>, + #[allow(clippy::vec_box)] + globals: Vec>, } impl Lower<'_> { @@ -1033,36 +1121,35 @@ impl Lower<'_> { .b .ins() .iconst(self.ptr, std::ptr::from_ref(slot) as i64); + let cache = Box::::default(); + let cache_v = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(&*cache) as i64); + self.globals.push(cache); let pc = self.u32c(u32::from(pc)); - let st = self.call(h_global as *const () as usize, &[self.ctx, fl.fr, slot, pc]); + let st = self.call( + h_global as *const () as usize, + &[self.ctx, fl.fr, slot, cache_v, pc], + ); self.check(st); self.set_out(fl, dst, 0); } Op::Attr { dst, src, pc, name } => { let (t, p) = self.get(fl, src); let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); - let st = match crate::code_field_slot(fl.code, u32::from(pc)) { - Some(slot) => { - let slot = self - .b - .ins() - .iconst(self.ptr, std::ptr::from_ref(slot) as i64); - let addr = if effect { - h_field:: as *const () as usize - } else { - h_field:: as *const () as usize - }; - self.call(addr, &[self.ctx, fl.fr, slot, t, p, pn]) - } - None => { - let addr = if effect { - h_attr:: as *const () as usize - } else { - h_attr:: as *const () as usize - }; - self.call(addr, &[self.ctx, fl.fr, t, p, pn]) - } + let cache = Box::::default(); + let cache_v = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(&*cache) as i64); + self.fields.push(cache); + let addr = if effect { + h_field:: as *const () as usize + } else { + h_field:: as *const () as usize }; + let st = self.call(addr, &[self.ctx, fl.fr, cache_v, t, p, pn]); self.check(st); self.set_out(fl, dst, 0); } diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 46000d60..d48292f4 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -61161,19 +61161,6 @@ fn drop_hot(v: Object) { } } -/// The field slot for `cache_pc`, allocating the table on first use. -#[inline] -fn code_field_slot(code: &CodeObject, cache_pc: u32) -> Option<&FieldSlot> { - let ext = code_vm_ext(code)?; - ext.field_slots - .get_or_init(|| { - (0..code.instructions.len()) - .map(|_| FieldSlot::empty()) - .collect() - }) - .get(cache_pc as usize) -} - /// The stamp slot for `cache_pc`, allocating the table on first use. #[inline] fn code_stamp_slot(code: &CodeObject, cache_pc: u32) -> Option<&StampSlot> { From f3e50e8d8a419715271803260c32113a4585756a Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:51:54 -0700 Subject: [PATCH 09/96] perf: read __slots__ scalar fields in tier 2's attribute fast path A tier-2 attribute read of a `__slots__` field took the general path (closures, a guard re-check, a named lookup) every time. Scalar slot fields now read at their usual position alongside indexed instance fields. attr_access retires about 5% fewer instructions. --- crates/weavepy-vm/src/tier2.rs | 39 ++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 55ba0a3a..7dfaddd8 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -7673,21 +7673,38 @@ unsafe extern "C" fn wpjit_attr_get(frame: *mut JitFrame, pin: i64, site: i64) - ctx.pins.get(pin as usize), ctx.attr_guards.get(site as usize), ) { - if let (AttrStorage::Indexed(key_idx), JitType::Int | JitType::Float | JitType::Bool) = - (g.storage, g.lane) - { - if attr_class_ok(inst, g.ver) { - // SAFETY: a read between two native ops; nothing here runs - // code (see `GilCell::peek`). - if let Some((k, v)) = unsafe { inst.attr_peek_index(key_idx as usize) } { - if key_is(k, &g.name) { - if let Some(bits) = pack(v, g.lane) { - jf.ret_bits = bits; - return 0; + match (g.storage, g.lane) { + (AttrStorage::Indexed(key_idx), JitType::Int | JitType::Float | JitType::Bool) => { + if attr_class_ok(inst, g.ver) { + // SAFETY: a read between two native ops; nothing here + // runs code (see `GilCell::peek`). + if let Some((k, v)) = unsafe { inst.attr_peek_index(key_idx as usize) } { + if key_is(k, &g.name) { + if let Some(bits) = pack(v, g.lane) { + jf.ret_bits = bits; + return 0; + } + } + } + } + } + // A `__slots__` field at its usual position. + (AttrStorage::Slot(key_idx), JitType::Int | JitType::Float | JitType::Bool) => { + if attr_class_ok(inst, g.ver) { + // SAFETY: as above. + if let Some((k, v)) = + unsafe { inst.slots.peek() }.and_then(|s| s.get_index(key_idx as usize)) + { + if key_is(k, &g.name) { + if let Some(bits) = pack(v, g.lane) { + jf.ret_bits = bits; + return 0; + } } } } } + _ => {} } } // Scoped so the receiver borrow of `ctx.pins` ends before an From eedc36a9eaefa23bab90b8f2dc2942845e6d8648 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:55:53 -0700 Subject: [PATCH 10/96] perf: give Object a defined layout for native code repr(u8) fixes the tag byte at offset 0 and each payload at its natural alignment, so code compiled at run time can read and write values in place. The size stays 16 bytes (asserted). --- crates/weavepy-vm/src/object.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index ffe285c8..77a0a640 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -85,7 +85,12 @@ pub(crate) fn str_char_len(s: &SharedStr) -> usize { } /// A Python value as seen by the interpreter. +/// +/// `repr(u8)` fixes the layout native code relies on (see `frame_jit`): +/// the variant's index in a tag byte at offset 0, and each payload at its +/// natural alignment after it (every payload is one word or smaller). #[derive(Clone)] +#[repr(u8)] pub enum Object { None, /// The "no value" marker for local-variable slots — CPython's NULL @@ -12884,3 +12889,7 @@ mod tests { assert!(d.is_none()); } } + +// Every payload is one word or smaller, so a value is a tag and a word. +const _: () = assert!(std::mem::size_of::() == 16); +const _: () = assert!(std::mem::size_of::>() == 16); From ae879b6d0d228e0051b53e0948162e7d383c47f9 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:37:06 -0700 Subject: [PATCH 11/96] perf: evaluate leaf property getters in place and nurse young instances - A cached property whose getter is a pure leaf is evaluated on the receiver with no activation (t_property: 2026 -> 1276 instructions per read). - Newborn instances wait in a young set of weak handles instead of being entered in the collector's index: most die before the next collection, which first hands the survivors over, as do the reflective APIs and the finalization pass. Their births still pace gen-0 collections. - gc_trace::track takes a reference (callers no longer clone to register). - The cached __del__ verdict no longer pays for the MRO walk's prologue. --- crates/weavepy-capi/src/gc_bridge.rs | 2 +- crates/weavepy-vm/src/builtin_types.rs | 22 +- crates/weavepy-vm/src/builtins.rs | 20 +- crates/weavepy-vm/src/gc_trace.rs | 195 +++++++++++++++--- crates/weavepy-vm/src/import.rs | 2 +- crates/weavepy-vm/src/leaf_plan.rs | 2 +- crates/weavepy-vm/src/lib.rs | 101 ++++++--- crates/weavepy-vm/src/stdlib/asyncio_mod.rs | 2 +- crates/weavepy-vm/src/stdlib/functools_mod.rs | 2 +- crates/weavepy-vm/src/stdlib/gc_real.rs | 6 +- crates/weavepy-vm/src/stdlib/io.rs | 4 +- crates/weavepy-vm/src/stdlib/os.rs | 2 +- crates/weavepy-vm/src/stdlib/pickle_accel.rs | 2 +- .../src/stdlib/pickle_accel/encode.rs | 2 +- crates/weavepy-vm/src/stdlib/sys.rs | 2 +- crates/weavepy-vm/src/stdlib/weakref_real.rs | 2 +- crates/weavepy-vm/src/stdlib/win_console.rs | 2 +- crates/weavepy-vm/src/tier2.rs | 10 +- crates/weavepy-vm/src/types.rs | 16 +- 19 files changed, 290 insertions(+), 106 deletions(-) diff --git a/crates/weavepy-capi/src/gc_bridge.rs b/crates/weavepy-capi/src/gc_bridge.rs index e1027162..6aa96bed 100644 --- a/crates/weavepy-capi/src/gc_bridge.rs +++ b/crates/weavepy-capi/src/gc_bridge.rs @@ -212,7 +212,7 @@ pub unsafe extern "C" fn PyObject_GC_Track(op: *mut c_void) { } let obj = unsafe { crate::object::clone_object(op as *mut PyObject) }; if matches!(obj, Object::Instance(_)) { - weavepy_vm::gc_trace::track(obj); + weavepy_vm::gc_trace::track(&obj); } } diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index 46ae4e32..c51543c1 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -428,7 +428,7 @@ impl BuiltinTypes { ); pi.dict = dict.into(); let inst = Object::Instance(Rc::new(pi)); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } } @@ -491,7 +491,7 @@ impl BuiltinTypes { let mut pi = crate::types::PyInstance::with_native(cls.clone(), ns); pi.dict = dict.into(); let inst = Object::Instance(Rc::new(pi)); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } } @@ -500,7 +500,7 @@ impl BuiltinTypes { // through a closure cell is a cycle the collector must // see (test_interpreters' captured_thread_exception // left its Thread in `threading._dangling`). - crate::gc_trace::track(ns.clone()); + crate::gc_trace::track(&ns); Ok(ns) }; let mut ns_dict = simple_namespace_.dict.borrow_mut(); @@ -1834,7 +1834,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { Object::new_tuple(new_args.to_vec()), ); let obj = Object::Instance(inst); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); return Ok(obj); } // `tuple.__new__(tuple, it)` / `int.__new__(int, x)` … on the *built-in @@ -1846,7 +1846,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { // name/doc seeding lives in `module.__init__` only). if cls.is_subclass_of(&builtin_types().module_) { let inst = Object::Instance(Rc::new(PyInstance::new(cls))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } // A `type` subclass reaching the generic allocator is a wrong-arity @@ -1929,7 +1929,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { let interp = unsafe { &mut *ptr }; let s = interp.type_call_default(&bt.str_, &args[1..], &[])?; let inst = Object::Instance(Rc::new(PyInstance::with_native(cls.clone(), s))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } } @@ -1955,7 +1955,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { let interp = unsafe { &mut *ptr }; let v = interp.type_call_default(&bt.int_, &args[1..], &[])?; let inst = Object::Instance(Rc::new(PyInstance::with_native(cls.clone(), v))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } } @@ -1970,7 +1970,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { // `__init__` / `__setstate__` / the `_reconstruct` append-and-update loop. if let Some(native) = native_seed_for_new(&cls, args.get(1)) { let inst = Object::Instance(Rc::new(PyInstance::with_native(cls, native))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } // RFC 0024: explicit `object.__new__(cls)` / `super().__new__(cls)` @@ -1982,7 +1982,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { // A natively served class's instances are never tracked (see // `stdlib::datetime_native`). if !native { - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); } Ok(inst) } @@ -5237,7 +5237,7 @@ pub(crate) fn exception_group_new( // An exception group always anchors other exception instances, so // it can participate in reference cycles — GC-track it like // `build_exception_instance` does for enriched exceptions. - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); Ok(inst) } @@ -5817,7 +5817,7 @@ fn int_new_kw( return Some(Ok(v)); } let inst = Object::Instance(Rc::new(PyInstance::with_native(cls.clone(), v))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); Some(Ok(inst)) } diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 716756d9..71b3ef99 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -7013,7 +7013,7 @@ fn b_list(args: &[Object]) -> Result { let obj = Object::new_list(out); // CPython tracks every list; keep `list(...)` consistent with the // `[]` literal path so `gc.is_tracked` and cycle collection agree. - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); // tracemalloc parity with the `[]` literal path // (`test_tracemalloc.test_reset_peak` builds `list(range(100000))` // and expects the peak to reflect it). @@ -7047,7 +7047,7 @@ fn b_tuple(args: &[Object]) -> Result { fn b_dict(args: &[Object]) -> Result { if args.is_empty() { let obj = Object::new_dict(); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); return Ok(obj); } if args.len() > 1 { @@ -7063,7 +7063,7 @@ fn b_dict(args: &[Object]) -> Result { // Clone the stored table so copying never rehashes user keys. let d = src.borrow().clone(); let obj = Object::Dict(Rc::new(RefCell::new(d))); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); return Ok(obj); } // Mapping path for user-defined classes (`__keys__` style) is @@ -7117,7 +7117,7 @@ fn b_dict(args: &[Object]) -> Result { i += 1; } let obj = Object::Dict(Rc::new(RefCell::new(d))); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } @@ -7154,7 +7154,7 @@ fn b_set(args: &[Object]) -> Result { }; let obj = Object::Set(Rc::new(RefCell::new(out))); // CPython tracks every set (`gc.is_tracked(set())` is True). - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } @@ -8243,7 +8243,7 @@ fn b_sorted(args: &[Object]) -> Result { crate::compare_op(a, b, weavepy_compiler::CompareKind::Lt) })?; let obj = Object::new_list(buf); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } @@ -10904,7 +10904,7 @@ fn str_split_whitespace(s: &str, maxsplit: i64) -> Vec { /// through either the interpreter or a native string helper. fn new_str_split_list(items: Vec) -> Object { let result = Object::new_list(items); - crate::gc_trace::track(result.clone()); + crate::gc_trace::track(&result); result } @@ -13447,7 +13447,7 @@ fn dict_copy(args: &[Object]) -> Result { // CPython's `PyDict_Copy` preserves GC tracking: the copy is tracked // iff the source is (test_dict `test_copy_maintains_tracking`). if crate::gc_trace::is_tracked(crate::weakref_registry::id_of(&Object::Dict(d))) { - crate::gc_trace::track(out.clone()); + crate::gc_trace::track(&out); } Ok(out) } @@ -13539,7 +13539,7 @@ fn dict_or(args: &[Object]) -> Result { out.insert(k.clone(), v.clone()); } let obj = Object::Dict(Rc::new(RefCell::new(out))); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } @@ -13554,7 +13554,7 @@ fn dict_ror(args: &[Object]) -> Result { out.insert(k.clone(), v.clone()); } let obj = Object::Dict(Rc::new(RefCell::new(out))); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 641f70dc..671c4600 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -381,6 +381,14 @@ pub struct GcState { deferred: RefCell>, /// Length at which [`GcState::sweep_deferred`] compacts `deferred`. deferred_limit: AtomicUsize, + /// Instances born since the last collection, held weakly: most die + /// young, and registering one with the collector (an index entry and + /// a handle) costs far more than its life. Their births count toward + /// the gen-0 threshold as eager registration's would, and every + /// collection, reflective API and finalization pass first hands the + /// collector the ones still alive ([`GcState::flush_young`]), so it + /// sees exactly the population eager tracking would have shown it. + young: RefCell>>, /// Frozen handles. `gc.freeze()` moves all tracked objects /// here; they are skipped by future collections until /// `gc.unfreeze()` runs. @@ -428,6 +436,7 @@ impl GcState { gen0_gauge: AtomicU64::new(DEFAULT_THRESHOLDS[0] as u64), deferred: RefCell::new(Vec::new()), deferred_limit: AtomicUsize::new(DEFERRED_FLOOR), + young: RefCell::new(Vec::new()), frozen: RefCell::new(Vec::new()), garbage: RefCell::new(Vec::new()), callbacks: RefCell::new(Vec::new()), @@ -464,6 +473,7 @@ impl GcState { RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).thresholds)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).counts)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).deferred)); + RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).young)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).frozen)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).garbage)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).callbacks)); @@ -502,17 +512,89 @@ impl GcState { /// Track `obj` for cycle detection. Idempotent — if `obj` /// is already tracked, this is a no-op. - pub fn track(&self, obj: Object) { + pub fn track(&self, obj: &Object) { // A container holding only atomic values cannot anchor a cycle // yet, so it stays off the GC's books until it holds something // that could close one. `defer_container` remembers it so the // next collection can promote it first. - if !container_can_cycle(&obj) && self.defer_container(&obj) { + if !container_can_cycle(obj) && self.defer_container(obj) { return; } + if let Object::Instance(inst) = obj { + if self.nurse(inst) { + return; + } + } self.track_now(obj); } + /// Register a newborn instance in the young set (see [`Self::young`]). + /// `false` leaves it to eager registration. + fn nurse(&self, inst: &crate::Rc) -> bool { + let over = { + // SAFETY: nothing below runs code while the set is borrowed + // (a dead predecessor's release only frees memory). + let Some(young) = (unsafe { self.young.peek_mut() }) else { + // A flush is walking the set, or cells are shared. + return false; + }; + // Churn (a loop that builds an object and drops the previous + // one) leaves its dead predecessors at the tail: reclaim them + // here, so the set neither grows nor parks dead allocations. + let n = young.len(); + for i in (n.saturating_sub(2)..n).rev() { + if young[i].strong_count() == 0 { + young.swap_remove(i); + } + } + young.push(crate::Rc::downgrade(inst)); + young.len() >= YOUNG_CAP + }; + // As in `track_now`: an id recycled from a finalized object starts + // un-finalized (a finalizer that runs from here on is recorded). + // SAFETY: a read that runs no code. + if unsafe { self.finalized_ids.peek() }.is_none_or(|f| !f.is_empty()) { + let id = crate::Rc::as_ptr(inst) as usize as ObjectId; + self.finalized_ids.borrow_mut().remove(&id); + } + self.note_gen0_alloc(); + if over { + self.flush_young(); + } + true + } + + /// Register the young instances still alive with the collector. + fn flush_young(&self) { + let young = match self.young.try_borrow_mut() { + Ok(mut young) if !young.is_empty() => std::mem::take(&mut *young), + _ => return, + }; + for w in young { + if let Some(inst) = w.upgrade() { + self.register(&Object::Instance(inst), false); + } + } + } + + /// The young instances still alive (the dead ones are dropped). + fn young_live(&self) -> usize { + let Ok(mut young) = self.young.try_borrow_mut() else { + return 0; + }; + young.retain(|w| w.strong_count() > 0); + young.len() + } + + /// Whether the instance `id` is in the young set. + fn is_young(&self, id: ObjectId) -> bool { + self.young.try_borrow().is_ok_and(|young| { + young + .iter() + .any(|w| w.as_ptr() as usize as ObjectId == id && w.strong_count() > 0) + }) + } + /// Remember `obj` weakly instead of tracking it. Returns `false` when /// the kind has no deferred form (the caller tracks it as before). /// @@ -563,6 +645,7 @@ impl GcState { /// enough for the collector to see every cycle it would have seen /// with eager tracking. fn sweep_deferred(&self, promote_all: bool) { + self.flush_young(); let mut promote: Vec = Vec::new(); { let Ok(mut deferred) = self.deferred.try_borrow_mut() else { @@ -590,7 +673,7 @@ impl GcState { ); } for obj in promote { - self.track_now(obj); + self.track_now(&obj); } } @@ -603,44 +686,71 @@ impl GcState { /// [`Self::track`] without the deferral filter: register `obj` with the /// collector unconditionally. For objects the caller has already decided /// must be tracked (a promoted deferral, `gc.is_tracked`, `gc.get_objects`). - pub fn track_now(&self, obj: Object) { - let new_id = id_of(&obj); + pub fn track_now(&self, obj: &Object) { + if self.register(obj, true) { + self.note_gen0_alloc(); + } + } + + /// Enter `obj` in the index and generation 0 (`false`: it was already + /// there, or has no heap allocation of its own). A `fresh` object's id + /// may be a finalized one's, recycled; a young instance's own + /// finalizer may already have run, and its entry says so. + fn register(&self, obj: &Object, fresh: bool) -> bool { + let new_id = id_of(obj); { let mut index = self.index.borrow_mut(); // One probe decides the dedupe and the insert. An entry whose // object died can't sit at this address: its weak handle keeps // the allocation reserved until the entry is pruned. let entry = match index.entry(new_id) { - std::collections::hash_map::Entry::Occupied(_) => return, + std::collections::hash_map::Entry::Occupied(_) => return false, std::collections::hash_map::Entry::Vacant(e) => e, }; - let Some(handle) = TrackedHandle::new(&obj, 0) else { - return; + let Some(handle) = TrackedHandle::new(obj, 0) else { + return false; }; let handle = HandleRc::new(handle); // Publish to the miss-filter *before* the insert becomes // observable (we hold the index borrow). self.tracked_filter.insert(new_id); entry.insert(handle.clone()); + // `finalized_ids` is keyed by object id (a pointer), which the + // allocator recycles. A freshly tracked object at a recycled + // address must start *un*-finalized (`test_is_finalized`). + // Almost always empty (only `__del__`-bearing objects ever land + // there). + if !self.finalized_ids.borrow().is_empty() { + if fresh { + self.finalized_ids.borrow_mut().remove(&new_id); + } else if self.finalized_ids.borrow().contains(&new_id) { + handle.finalized.store(true, Ordering::Release); + } + } let mut gens = self.generations.borrow_mut(); handle.slot.store(gens[0].handles.len(), Ordering::Release); gens[0].handles.push(handle); } - // `finalized_ids` is keyed by object id (a pointer), which the - // allocator recycles. A freshly tracked object at a recycled address - // must start *un*-finalized (`test_is_finalized`). Almost always - // empty (only `__del__`-bearing objects ever land there). - if !self.finalized_ids.borrow().is_empty() { - self.finalized_ids.borrow_mut().remove(&new_id); - } serial_add(&self.tracked_count, 1); serial_add(&self.tracked_version, 1); - self.note_gen0_alloc(); + true } /// Stop tracking `obj`. Backs the explicit `gc._untrack(obj)` extension /// and the C-API `PyObject_GC_UnTrack`. pub fn untrack_id(&self, id: ObjectId) { + if let Ok(mut young) = self.young.try_borrow_mut() { + if let Some(i) = young + .iter() + .position(|w| w.as_ptr() as usize as ObjectId == id) + { + young.swap_remove(i); + let mut counts = self.counts.borrow_mut(); + counts[0] = counts[0].saturating_sub(1); + self.sync_gen0_gauge(counts[0], None); + return; + } + } let Some(handle) = self.index.borrow_mut().remove(&id) else { return; }; @@ -772,6 +882,11 @@ impl GcState { self.index.borrow().contains_key(&id) } + /// [`Self::is_tracked`] for an instance, which may be young. + pub fn is_tracked_instance(&self, id: ObjectId) -> bool { + self.is_tracked(id) || self.is_young(id) + } + /// O(1) handle lookup by object id (any generation or frozen). pub fn handle_for(&self, id: ObjectId) -> Option> { if !self.tracked_filter.may_contain(id) { @@ -785,6 +900,7 @@ impl GcState { /// finalize objects that are still alive at exit. The per-handle /// `finalized` flag guarantees each `__del__` runs at most once. pub fn finalization_candidates(&self) -> Vec<(HandleRc, Object)> { + self.flush_young(); let mut out = Vec::new(); let mut consider = |h: &HandleRc| { // A finalizer already queued by a collection (but not yet @@ -852,7 +968,14 @@ impl GcState { /// every allocation site polls `maybe_auto_collect` itself /// (`MAKE_FUNCTION`, `list(it)`, …). fn note_gen0_alloc(&self) { - self.bump_count(0); + // SAFETY: nothing runs while the counts are updated. + match unsafe { self.counts.peek_mut() } { + Some(counts) => { + counts[0] = counts[0].saturating_add(1); + self.sync_gen0_gauge(counts[0], None); + } + None => self.bump_count(0), + } let gauge = self.gen0_gauge.load(Ordering::Relaxed); let threshold = gauge & 0xffff_ffff; if threshold != 0 && (gauge >> 32) == threshold && self.is_enabled() { @@ -919,7 +1042,7 @@ impl GcState { if !due { return false; } - let live = self.prune(); + let live = self.prune() + self.young_live(); if live < threshold0 / 2 { let mut counts = self.counts.borrow_mut(); counts[0] = live; @@ -934,6 +1057,7 @@ impl GcState { /// Total population (across all generations + frozen). pub fn population(&self) -> usize { + self.flush_young(); let gens = self.generations.borrow(); let mut n = 0; for g in gens.iter() { @@ -966,6 +1090,7 @@ impl GcState { /// `gc.freeze()` — mark every currently-tracked object as /// frozen so it is ignored by future collections. pub fn freeze_all(&self) { + self.flush_young(); let mut gens = self.generations.borrow_mut(); let mut frozen = self.frozen.borrow_mut(); for g in gens.iter_mut() { @@ -2161,8 +2286,8 @@ pub fn with_state(f: impl FnOnce(&GcState) -> R) -> R { } /// Convenience: track `obj` in the shared, process-global GC. -pub fn track(obj: Object) { - if let Object::Instance(inst) = &obj { +pub fn track(obj: &Object) { + if let Object::Instance(inst) = obj { // The instance may have been born with deferred tracking; its // `__dict__` no longer needs to guard it. inst.clear_deferred_tracking(); @@ -2181,12 +2306,12 @@ pub fn track(obj: Object) { // that never went through BuildMap) would otherwise never be a // candidate. CPython tracks every dict; we pair the tracking with // the objects that make the dict cycle-capable. - if let Object::Module(m) = &obj { + if let Object::Module(m) = obj { let dict = Object::Dict(m.dict.clone()); - with_state(|s| s.track(dict)); - } else if let Object::Function(f) = &obj { + with_state(|s| s.track(&dict)); + } else if let Object::Function(f) = obj { let dict = Object::Dict(f.globals.clone()); - with_state(|s| s.track(dict)); + with_state(|s| s.track(&dict)); } with_state(|s| s.track(obj)); } @@ -2241,7 +2366,7 @@ pub fn track_deferred_owner(owner: usize) { // instance's so `is_gc_deferred` stops claiming the collector has // never seen it. inst.deferred.set(false); - track(Object::Instance(inst)); + track(&Object::Instance(inst)); } /// True while a collection (mark/sweep or weakref-only pass) is in @@ -2332,6 +2457,10 @@ pub fn is_atomic(obj: &Object) -> bool { /// sweep's fixed cost outweighs what it reclaims. const DEFERRED_FLOOR: usize = 4096; +/// How many young instances wait before the set is handed to the +/// collector regardless (collections normally flush it well before). +const YOUNG_CAP: usize = 8192; + /// How many containers may stay deferred at once. Past this the sweep /// hands the whole set to the collector, which both bounds the weak /// references parked here and restores the allocation pacing a program @@ -2438,13 +2567,13 @@ fn container_can_cycle(obj: &Object) -> bool { pub fn track_memoryview_exporter(mv: &Object, exporter: &Object) { debug_assert!(matches!(mv, Object::MemoryView(_))); if !is_atomic(exporter) { - track(mv.clone()); + track(&mv); } } pub fn track_if_cyclic(obj: &Object) -> bool { if container_can_cycle(obj) { - track(obj.clone()); + track(&obj); true } else { false @@ -2611,7 +2740,7 @@ mod tests { .map(|_| Object::Dict(Rc::new(RefCell::new(DictData::default())))) .collect(); for root in &roots { - state.track_now(root.clone()); + state.track_now(&root); } if frozen { state.freeze_all(); @@ -2661,7 +2790,7 @@ mod tests { .collect(); for root in &roots { // See `track_and_untrack` on `track_now`. - state.track_now(root.clone()); + state.track_now(&root); } for (collection, expected) in [(0, 1), (1, 2), (2, 2), (2, 2)] { assert_eq!(state.collect(collection), 0); @@ -2713,7 +2842,7 @@ mod tests { // `track_now`, not `track`: an empty dict holds nothing that could // close a cycle, so `track` would defer it (see `defer_container`). // This test is about the index's bookkeeping, not that policy. - s.track_now(d.clone()); + s.track_now(&d); assert!(s.is_tracked(id_of(&d))); s.untrack_id(id_of(&d)); assert!(!s.is_tracked(id_of(&d))); @@ -2724,7 +2853,7 @@ mod tests { let s = GcState::new(); let dict = Rc::new(RefCell::new(DictData::default())); let outer = Object::Dict(dict.clone()); - s.track_now(outer.clone()); + s.track_now(&outer); // The dict references itself: a 1-cycle. dict.borrow_mut().insert( crate::object::DictKey(Object::from_static("self")), @@ -2745,7 +2874,7 @@ mod tests { let s = GcState::new(); let dict = Rc::new(RefCell::new(DictData::default())); let obj = Object::Dict(dict.clone()); - s.track_now(obj.clone()); + s.track_now(&obj); let weak = Rc::downgrade(&dict); drop(obj); drop(dict); @@ -2761,7 +2890,7 @@ mod tests { let s = GcState::new(); let d = Object::Dict(Rc::new(RefCell::new(DictData::default()))); // See `track_and_untrack` on `track_now`. - s.track_now(d.clone()); + s.track_now(&d); s.freeze_all(); assert_eq!(s.freeze_count(), 1); s.unfreeze_all(); diff --git a/crates/weavepy-vm/src/import.rs b/crates/weavepy-vm/src/import.rs index 98e05586..e53d928a 100644 --- a/crates/weavepy-vm/src/import.rs +++ b/crates/weavepy-vm/src/import.rs @@ -158,7 +158,7 @@ impl ModuleCache { // is True); a module's globals routinely close cycles (a function // defined in the module captures the module's dict). Track on install. if matches!(module, Object::Module(_)) { - crate::gc_trace::track(module.clone()); + crate::gc_trace::track(&module); } self.modules .borrow_mut() diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index e39f1495..fecf6478 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -1370,7 +1370,7 @@ fn new_container(owned: &mut Owned, dict: bool) -> Option { } else { Object::new_list(Vec::new()) }; - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); owned.own(obj) } diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index d48292f4..544d188b 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -9822,17 +9822,30 @@ impl Interpreter { burst_stats::note_site(&frame.code.qualname, frame.pc, ins.op, recv); } } - // A cached property site skips the helper's resolution. + // A cached property site skips the helper's resolution: a + // pure leaf getter is evaluated in place, any other runs + // as an inline activation. let pc = frame.pc as usize; if frame .code .instructions .get(pc) .is_some_and(|i| i.op == OpCode::LoadAttr) - && Self::cached_property_getter(frame, pc).is_some() - && self.core_property(sw) { - continue; + if let Some((getter, code)) = Self::cached_property_getter(frame, pc) { + // SAFETY: as above. + let last_pc = unsafe { &mut *sw.last }; + if self.core_leaf_property(frame, last_pc, pc, &getter, &code) { + // SAFETY: the running thread's own flag. + if unsafe { (*sw.maybe_dead).get() } { + return LeafStop::Marked; + } + continue; + } + if self.core_property(sw) { + continue; + } + } } // SAFETY: as above (`core_property` declined untouched). let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) }; @@ -10570,7 +10583,7 @@ impl Interpreter { .collect(); len -= n; let obj = Object::new_list(items); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); // SAFETY: `len < cap` (the operands' slots were freed). unsafe { base.add(len).write(obj) }; len += 1; @@ -12529,6 +12542,40 @@ impl Interpreter { drop(bm); } + /// The `LOAD_ATTR` at `pc` of `frame` of a property whose cached + /// getter is a pure leaf: the getter evaluated on the receiver at TOS + /// (no activation), its result replacing the receiver. `false` + /// touches nothing. + #[inline(never)] + fn core_leaf_property( + &self, + frame: &mut Frame, + last_pc: &mut usize, + pc: usize, + getter: &PyFunction, + code: &Rc, + ) -> bool { + if !code_is_pure_leaf(code) + || !pure_leaf_warm(code) + || crate::recursion::current_depth() >= crate::recursion::recursion_limit() + { + return false; + } + let Some(recv) = frame.stack.last() else { + return false; + }; + let Some(v) = self.pure_leaf_eval::(code, getter, &[std::ptr::from_ref(recv)]) + else { + return false; + }; + if let Some(top) = frame.stack.last_mut() { + drop_hot(std::mem::replace(top, v)); + } + *last_pc = pc; + frame.pc = pc as u32 + 1; + true + } + /// The core loop's `LOAD_ATTR` of a `property` at `sw`'s (synced) /// running activation's pc: the getter's activation /// ([`Self::try_inline_property`]) is pushed and made the running one, @@ -16506,7 +16553,7 @@ impl Interpreter { } let items = stack.split_off(stack.len() - n); let obj = Object::new_list(items); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); stack.push(obj); last = pc; pc += 1; @@ -16543,7 +16590,7 @@ impl Interpreter { } stack.truncate(split); let obj = Object::Dict(Rc::new(RefCell::new(d))); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); stack.push(obj); last = pc; pc += 1; @@ -17885,7 +17932,7 @@ impl Interpreter { _ => return None, }; let obj = Object::new_list(items); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); return Some(obj); } None @@ -21294,7 +21341,7 @@ impl Interpreter { // missed entirely. Track unconditionally so the collector both // reports it via `gc.get_objects()`/`is_tracked` and can break // the cycle when it becomes unreachable. - gc_trace::track(obj.clone()); + gc_trace::track(&obj); frame.push(obj); if gc_trace::maybe_auto_collect() { self.run_pending_finalizers(); @@ -21351,7 +21398,7 @@ impl Interpreter { self.record_alloc(&obj); // CPython tracks every set (`gc.is_tracked(set())` is True); // track unconditionally like lists. - gc_trace::track(obj.clone()); + gc_trace::track(&obj); frame.push(obj); if gc_trace::maybe_auto_collect() { self.run_pending_finalizers(); @@ -21383,7 +21430,7 @@ impl Interpreter { self.record_alloc(&obj); // CPython tracks dicts at creation; track unconditionally so a // dict that becomes cyclic (`d = {}; d[0] = d`) is collectable. - gc_trace::track(obj.clone()); + gc_trace::track(&obj); frame.push(obj); if gc_trace::maybe_auto_collect() { self.run_pending_finalizers(); @@ -21962,7 +22009,7 @@ impl Interpreter { // `gc.collect()` can reclaim those cycles and `gc.is_tracked(f)` // reports True, matching CPython (functions are GC objects). let obj = Object::Function(Rc::new(f)); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); // PEP 590-era function watchers: PyFunction_EVENT_CREATE // fires from `PyFunction_New*` (test_capi.test_watchers). if crate::capi_watchers::funcs_active() { @@ -22046,7 +22093,7 @@ impl Interpreter { } } let obj = Object::Function(pf); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); frame.push(obj); } // RFC 0068 — CPython prologue unit; WeavePy's frame setup @@ -29456,7 +29503,7 @@ impl Interpreter { // literal and `b_list` paths, or a cycle closed through the // result (`x = list(...); x[0].back = x`) is never collected. let obj = Object::new_list(collected); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); Ok(obj) } else { Ok(Object::new_tuple(collected)) @@ -29477,7 +29524,7 @@ impl Interpreter { // match the `{}` literal and `b_dict` paths so a cycle through // the copy is collectable. if let Some(d) = &out { - gc_trace::track(d.clone()); + gc_trace::track(&d); } Ok(out) } @@ -30839,7 +30886,7 @@ impl Interpreter { .find_map(|(k, v)| (k == "key").then(|| v.clone())); self.sort_with_key(&mut items, key_fn.as_ref(), reverse, globals)?; let obj = Object::new_list(items); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); Ok(obj) } @@ -37744,7 +37791,7 @@ impl Interpreter { // on the instance must join the cycle collector // (see `generic_setattr_instance`). if matches!(val, Object::BoundMethod(_)) { - gc_trace::track(val.clone()); + gc_trace::track(&val); } // The slot still exists; reach in by index and // overwrite, recovering the displaced value. We @@ -37797,7 +37844,7 @@ impl Interpreter { // method escaping into an instance attribute // joins the cycle collector. if matches!(val, Object::BoundMethod(_)) { - gc_trace::track(val.clone()); + gc_trace::track(&val); } let watch_value = if crate::capi_watchers::dicts_active() { Some(val.clone()) @@ -39335,7 +39382,7 @@ impl Interpreter { // is reclaimable and the drop-time "unclosed file" // `ResourceWarning` still fires // (test_io `test_garbage_collection`). - gc_trace::track(obj.clone()); + gc_trace::track(&obj); Ok(()) } }, @@ -39812,7 +39859,7 @@ impl Interpreter { // instance attribute so the cycle collector can reclaim them // (CPython's `method` type is always GC-tracked — `test_method`). if matches!(value, Object::BoundMethod(_)) { - gc_trace::track(value.clone()); + gc_trace::track(&value); } // `insert` returns the value this store displaces; reap it if the // slot held its last live binding, mirroring CPython's decref of the @@ -45436,7 +45483,7 @@ impl Interpreter { ) } else { let inst = Object::Instance(Rc::new(PyInstance::new(cls.clone()))); - gc_trace::track(inst.clone()); + gc_trace::track(&inst); (inst, true) } } @@ -46064,7 +46111,7 @@ impl Interpreter { })?; } let out = Object::Dict(Rc::new(RefCell::new(d))); - crate::gc_trace::track(out.clone()); + crate::gc_trace::track(&out); return Ok(out); } } @@ -46111,7 +46158,7 @@ impl Interpreter { })?; } let out = Object::Dict(Rc::new(RefCell::new(d))); - crate::gc_trace::track(out.clone()); + crate::gc_trace::track(&out); return Ok(out); } // `int(x)` / `float(x)` honour the user's `__int__` / @@ -46356,7 +46403,7 @@ impl Interpreter { // type whose `tp_traverse` is non-NULL — for us that's // every Python-defined class (they all carry a dict). if !deferred { - gc_trace::track(inst.clone()); + gc_trace::track(&inst); } // CPython's allocator runs a threshold-driven young // collection right here; without it the tracked set @@ -46714,7 +46761,7 @@ impl Interpreter { // `KeyError('k')`, every internal `raise`) stays off the GC books, // matching the container-untracking fast path used for list/dict/set. if exception_holds_nonatomic(&obj) { - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); } obj } @@ -47327,7 +47374,7 @@ impl Interpreter { // RFC 0024: generator frames can participate in // reference cycles (a local that holds the generator // itself), so track them like instances. - gc_trace::track(obj.clone()); + gc_trace::track(&obj); // Threshold-driven young collection at the // allocation site, as in the instance path. if gc_trace::maybe_auto_collect() { @@ -47391,7 +47438,7 @@ impl Interpreter { } else { Object::Generator(gen) }; - gc_trace::track(obj.clone()); + gc_trace::track(&obj); if gc_trace::maybe_auto_collect() { self.run_pending_finalizers(); } diff --git a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs index f917c5a0..6524b66f 100644 --- a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs +++ b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs @@ -1341,7 +1341,7 @@ fn future_iter_new(fut: &Object) -> Object { // collector by hand: a suspended `await` parks this iterator on the // coroutine's value stack, and its strong `fut` edge must be // subtractable for task↔future cycles to collapse. - crate::gc_trace::track(out.clone()); + crate::gc_trace::track(&out); out } diff --git a/crates/weavepy-vm/src/stdlib/functools_mod.rs b/crates/weavepy-vm/src/stdlib/functools_mod.rs index ae228f8e..948bd333 100644 --- a/crates/weavepy-vm/src/stdlib/functools_mod.rs +++ b/crates/weavepy-vm/src/stdlib/functools_mod.rs @@ -311,7 +311,7 @@ fn lru_cache_wrapper_new(args: &[Object]) -> Result { // wrapper counted as an external root that made that namespace // immortal. let wrapper = Object::Instance(inst); - crate::gc_trace::track(wrapper.clone()); + crate::gc_trace::track(&wrapper); Ok(wrapper) } diff --git a/crates/weavepy-vm/src/stdlib/gc_real.rs b/crates/weavepy-vm/src/stdlib/gc_real.rs index 57f4c4a6..8108c34d 100644 --- a/crates/weavepy-vm/src/stdlib/gc_real.rs +++ b/crates/weavepy-vm/src/stdlib/gc_real.rs @@ -504,7 +504,7 @@ fn object_is_tracked(target: &Object) -> bool { // answers as CPython's (always tracked) and is tracked from here on. Object::Instance(inst) => { inst.ensure_gc_tracked(); - gc_trace::with_state(|s| s.is_tracked(id_of(target))) + gc_trace::with_state(|s| s.is_tracked_instance(id_of(target))) } // A list/dict/set born holding only scalars has its tracking // deferred; CPython tracks it from birth, so answer as CPython @@ -513,7 +513,7 @@ fn object_is_tracked(target: &Object) -> bool { let id = id_of(target); gc_trace::with_state(|s| { if !s.is_tracked(id) { - s.track_now(target.clone()); + s.track_now(&target); } s.is_tracked(id) }) @@ -585,7 +585,7 @@ fn get_stats(_args: &[Object]) -> Result { fn track_obj(args: &[Object]) -> Result { if let Some(o) = args.first() { - gc_trace::track(o.clone()); + gc_trace::track(&o); } Ok(Object::None) } diff --git a/crates/weavepy-vm/src/stdlib/io.rs b/crates/weavepy-vm/src/stdlib/io.rs index 01e39199..a3df33c3 100644 --- a/crates/weavepy-vm/src/stdlib/io.rs +++ b/crates/weavepy-vm/src/stdlib/io.rs @@ -1349,7 +1349,7 @@ fn ind_new(args: &[Object]) -> Result { } }; let inst = Object::Instance(Rc::new(crate::types::PyInstance::new(cls))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); Ok(inst) } @@ -4754,7 +4754,7 @@ fn bw_new(args: &[Object]) -> Result { _ => return Err(type_error("Buffered.__new__(X): X is not a type object")), }; let inst = Object::Instance(Rc::new(crate::types::PyInstance::new(cls))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); Ok(inst) } diff --git a/crates/weavepy-vm/src/stdlib/os.rs b/crates/weavepy-vm/src/stdlib/os.rs index 962e7cdf..427a3f2d 100644 --- a/crates/weavepy-vm/src/stdlib/os.rs +++ b/crates/weavepy-vm/src/stdlib/os.rs @@ -7497,7 +7497,7 @@ fn struct_seq_replace( let obj = Object::Instance(Rc::new(new_inst)); // CPython allocates the copy through the GC heap, so it is tracked from // birth (test_structseq.test_replace_gc_tracked builds a cycle out of it). - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs index 3f9d8c96..e116bcd3 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs @@ -1323,7 +1323,7 @@ impl Objects { Object::List(_) | Object::Dict(_) | Object::Instance(_) ) { - crate::gc_trace::track(value.clone()); + crate::gc_trace::track(&value); } } diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs index 92847caa..2d11af98 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs @@ -520,7 +520,7 @@ impl Encoder<'_> { continue; } let names = Object::new_list(names); - crate::gc_trace::track(names.clone()); + crate::gc_trace::track(&names); dict.insert(DictKey(Object::from_static("__slotnames__")), names); drop(dict); class.bump_attr_version(); diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index c71bfa92..3aa35a92 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -304,7 +304,7 @@ pub fn build_with_state( // pinned its whole importlib graph (and everything reachable from // it) for the life of the process. let tracked = |o: Object| { - crate::gc_trace::track(o.clone()); + crate::gc_trace::track(&o); o }; d.insert( diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs index 7ec65ee8..f7f5ef48 100644 --- a/crates/weavepy-vm/src/stdlib/weakref_real.rs +++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs @@ -1215,7 +1215,7 @@ fn make_ref_object_with_class( // is its own callback) and bound methods are tracked as well. let callback_can_cycle = !matches!(&callback, None | Some(Object::Builtin(_))); if callback_can_cycle { - crate::gc_trace::track(wrapper.clone()); + crate::gc_trace::track(&wrapper); } wrapper } diff --git a/crates/weavepy-vm/src/stdlib/win_console.rs b/crates/weavepy-vm/src/stdlib/win_console.rs index 33426e7b..eb1d0d9a 100644 --- a/crates/weavepy-vm/src/stdlib/win_console.rs +++ b/crates/weavepy-vm/src/stdlib/win_console.rs @@ -767,6 +767,6 @@ fn wcio_new(args: &[Object]) -> Result { } }; let inst = Object::Instance(Rc::new(PyInstance::new(cls))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); Ok(inst) } diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 7dfaddd8..0138f8d7 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -7140,7 +7140,7 @@ unsafe extern "C" fn wpjit_build_list( // (`gc.is_tracked([])` is True and any list can close a cycle by // later mutation); a natively built list is no different, and a // comprehension accumulator in particular always escapes. - crate::gc_trace::track(Object::List(list.clone())); + crate::gc_trace::track(&Object::List(list.clone())); let idx = ctx.pins.len() as i64; ctx.pins.push(Pin::List(list, elem)); idx @@ -7333,7 +7333,7 @@ unsafe extern "C" fn wpjit_build_map(frame: *mut JitFrame, n: i64) -> i64 { let obj = Object::Dict(Rc::new(GilRefCell::new(d))); // The interpreter tracks every dict it builds; a natively built // dict is no different. - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); let idx = ctx.pins.len() as i64; ctx.pins.push(Pin::Obj(obj)); idx @@ -7377,7 +7377,7 @@ unsafe extern "C" fn wpjit_list_repeat(frame: *mut JitFrame, pin: i64, count: i6 drop(items); let fresh = Rc::new(crate::sync::RefCell::new(out)); // See `wpjit_build_list`: every built list is GC-tracked. - crate::gc_trace::track(Object::List(fresh.clone())); + crate::gc_trace::track(&Object::List(fresh.clone())); let idx = ctx.pins.len() as i64; ctx.pins.push(Pin::List(fresh, elem)); idx @@ -7409,7 +7409,7 @@ unsafe extern "C" fn wpjit_list_from_range(frame: *mut JitFrame, start: i64, sto let items: Vec = (start..start + len).map(Object::Int).collect(); let list = Rc::new(crate::sync::RefCell::new(items)); let obj = Object::List(list.clone()); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); if crate::stdlib::tracemalloc_real::is_tracking() { crate::stdlib::tracemalloc_real::track_new_object(&obj); } @@ -7465,7 +7465,7 @@ unsafe extern "C" fn wpjit_list_slice( drop(items); let fresh = Rc::new(crate::sync::RefCell::new(out)); // See `wpjit_build_list`: every built list is GC-tracked. - crate::gc_trace::track(Object::List(fresh.clone())); + crate::gc_trace::track(&Object::List(fresh.clone())); let idx = ctx.pins.len() as i64; ctx.pins.push(Pin::List(fresh, elem)); idx diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index 8561c9ea..a1b7af1b 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -1107,7 +1107,7 @@ impl TypeObject { // free it — and weakrefs to it (or to methods in its dict) // would never clear. Built-ins are immortal; skip them. if !ty.flags.is_builtin { - crate::gc_trace::track(Object::Type(ty.clone())); + crate::gc_trace::track(&Object::Type(ty.clone())); } Ok(ty) } @@ -1544,12 +1544,20 @@ impl TypeObject { /// conservative `true` on any borrow conflict: a spurious resurrection /// is harmless because `Vm::invoke_finalizer` simply no-ops when the /// instance turns out to have no `__del__`. + #[inline] pub fn instances_need_finalize(&self) -> bool { match self.has_del.get() { - 1 => return false, - 2 => return true, - _ => {} + 1 => false, + 2 => true, + _ => self.instances_need_finalize_slow(), } + } + + /// [`Self::instances_need_finalize`] with no cached verdict: the MRO + /// walk, which caches one. + #[cold] + #[inline(never)] + fn instances_need_finalize_slow(&self) -> bool { let Ok(mro) = self.mro.try_borrow() else { return true; }; From 21eb465517119282e7c9f19bc30229079ce6cd94 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:09:30 -0700 Subject: [PATCH 12/96] perf: read and write scalar instance fields in line in tier 2 Compiled code now reads a pinned instance's field, and stores a scalar over a scalar field, without calling a helper when the instance's values are split over its class's shared names: the pin, the guard's class version, the split block's names and the field's index are checked in line, and anything else takes the helper as before. The VM measures the offsets with offset_of! on the owning types and checks them against the first live instance a probe sees before publishing them (WEAVEPY_JIT_NO_INLINE_ATTRS=1 turns this off). A loop of two reads and a write drops from 432 to 171 instructions per iteration; float_math retires 14% fewer instructions. --- crates/weavepy-jit/src/lib.rs | 1 + crates/weavepy-jit/src/lower.rs | 158 ++++++++++++++++++++- crates/weavepy-jit/src/runtime.rs | 67 +++++++++ crates/weavepy-vm/src/inst_dict.rs | 14 ++ crates/weavepy-vm/src/lazy_arc.rs | 3 + crates/weavepy-vm/src/sync.rs | 11 ++ crates/weavepy-vm/src/tier2.rs | 168 ++++++++++++++++++++++- tests/regrtest/test_jit_inline_fields.py | 129 +++++++++++++++++ 8 files changed, 549 insertions(+), 2 deletions(-) create mode 100644 tests/regrtest/test_jit_inline_fields.py diff --git a/crates/weavepy-jit/src/lib.rs b/crates/weavepy-jit/src/lib.rs index 5e04cb84..09ae43d8 100644 --- a/crates/weavepy-jit/src/lib.rs +++ b/crates/weavepy-jit/src/lib.rs @@ -42,6 +42,7 @@ pub use ir::{ TOp, TStmt, TTerm, }; pub use runtime::{ + set_obj_layout, ObjLayout, register_attr_get_chain_helper, register_attr_helpers, register_build_map_helper, register_build_set_helper, register_cached_attr_chain_helper, register_call_dyn_helper, register_call_dyn_int_helper, register_call_method_helper, register_call_py_helper, diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 9f327aac..ab519273 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -2719,6 +2719,36 @@ impl<'a, 'b> Lowerer<'a, 'b> { let trusted = MemFlags::trusted(); let snapshot = self.vstack.clone(); let (pin, _) = self.pop(); + // A scalar field of a split-layout instance, read in line; any + // other shape takes the helper below. + let inline = match (runtime::obj_layout(), out) { + (Some(l), JitType::Int | JitType::Float | JitType::Bool) => Some(*l), + _ => None, + }; + let mut join = None; + if let Some(l) = inline { + let miss = self.b.create_block(); + let done = self.b.create_block(); + self.b.append_block_param(done, Self::cl_ty(out)); + let addr = self.split_field_addr(&l, pin, site, miss, true); + let (want, off, ty) = match out { + JitType::Int => (l.tag_int, 8, types::I64), + JitType::Float => (l.tag_float, 8, types::F64), + _ => (l.tag_bool, 1, types::I8), + }; + let tag = self.b.ins().uload8(types::I32, trusted, addr, 0); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, tag, i64::from(want)); + self.miss_if(other, miss); + let v = self.b.ins().load(ty, trusted, addr, off); + let v = if ty == types::I8 { + self.b.ins().uextend(types::I64, v) + } else { + v + }; + self.b.ins().jump(done, &[BlockArg::from(v)]); + self.b.switch_to_block(miss); + join = Some(done); + } let sig = self.list_helper_sig(); let helper = self .b @@ -2737,9 +2767,97 @@ impl<'a, 'b> Lowerer<'a, 'b> { .b .ins() .load(Self::cl_ty(out), trusted, self.frame_ptr, OFF_RET_BITS); + let res = match join { + Some(done) => { + self.b.ins().jump(done, &[BlockArg::from(res)]); + self.b.switch_to_block(done); + self.b.block_params(done)[0] + } + None => res, + }; self.vstack.push((res, out)); } + /// Branch to `miss` when `cond`; continue in a fresh block otherwise. + fn miss_if(&mut self, cond: Value, miss: Block) { + let ok = self.b.create_block(); + self.b.ins().brif(cond, miss, &[], ok, &[]); + self.b.switch_to_block(ok); + } + + /// The address of the field the attribute guard of `site` names in the + /// pinned instance `pin`'s split values (see [`runtime::ObjLayout`]): + /// an instance pin whose class still has the guard's version, whose + /// values are split over its class's names, unborrowed (`read`) or + /// unborrowed and exclusive (a write), and hold the guard's index. + /// Anything else branches to `miss`. + fn split_field_addr( + &mut self, + l: &runtime::ObjLayout, + pin: Value, + site: u32, + miss: Block, + read: bool, + ) -> Value { + let t = MemFlags::trusted(); + let ptr = self.ptr_ty; + let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); + // The pin. + let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); + self.miss_if(out, miss); + let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); + let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); + let p = self.b.ins().iadd(buf, off); + let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); + let not_obj = self.b.ins().icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); + self.miss_if(not_obj, miss); + let otag = self.b.ins().uload8(types::I32, t, p, l.pin_obj); + let not_inst = self.b.ins().icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + self.miss_if(not_inst, miss); + let inst = self.b.ins().load(ptr, t, p, l.pin_obj + 8); + // The guard. + let guards = self.b.ins().load(ptr, t, ctx, l.ctx_guards); + let gbuf = self.b.ins().load(ptr, t, guards, l.guards_buf); + let g = self + .b + .ins() + .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); + let ver = self.b.ins().load(types::I64, t, g, l.guard_ver); + let idx = self.b.ins().uload32(t, g, l.guard_split_idx); + // The class. + let cls = self.b.ins().load(ptr, t, inst, l.inst_class); + let cver = self.b.ins().load(types::I64, t, cls, l.type_attr_version); + let stale = self.b.ins().icmp(IntCC::NotEqual, cver, ver); + self.miss_if(stale, miss); + // The split values. + let lazy = self.b.ins().load(ptr, t, inst, l.inst_dict_lazy); + self.miss_if(lazy, miss); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I32, t, flag, 0); + self.miss_if(shared, miss); + let borrow = self.b.ins().sload32(t, inst, l.inst_split_borrow); + let busy = if read { + self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0) + } else { + self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0) + }; + self.miss_if(busy, miss); + let block = self.b.ins().load(ptr, t, inst, l.inst_split_block); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + self.miss_if(empty, miss); + let keys = self.b.ins().load(ptr, t, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, t, cls, l.type_shared_keys); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + self.miss_if(foreign, miss); + let len = self.b.ins().uload32(t, block, l.split_len); + let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + self.miss_if(absent, miss); + let off = self.b.ins().ishl_imm(idx, 4); + let at = self.b.ins().iadd(block, off); + self.b.ins().iadd_imm(at, i64::from(l.split_values)) + } + /// Consecutive callback-free reads can replay from the first read on a /// later miss. Preserve its original receiver in the deopt snapshot. fn emit_attr_get_chain( @@ -2892,7 +3010,41 @@ impl<'a, 'b> Lowerer<'a, 'b> { let trusted = MemFlags::trusted(); let snapshot = self.vstack.clone(); let (pin, _) = self.pop(); - let (val, _) = self.pop(); + let (val, lane) = self.pop(); + // A scalar over a scalar field of a split-layout instance, stored + // in line (no release, no collector bookkeeping); any other shape + // takes the helper below. + let inline = match (runtime::obj_layout(), lane) { + (Some(l), JitType::Int | JitType::Float | JitType::Bool) => Some(*l), + _ => None, + }; + let mut done = None; + if let Some(l) = inline { + let miss = self.b.create_block(); + let stored = self.b.create_block(); + let addr = self.split_field_addr(&l, pin, site, miss, false); + let old = self.b.ins().uload8(types::I64, trusted, addr, 0); + let one = self.b.ins().iconst(types::I64, 1); + let bit = self.b.ins().ishl(one, old); + let scalars = (1i64 << l.tag_int) + | (1i64 << l.tag_float) + | (1i64 << l.tag_bool) + | (1i64 << l.tag_none); + let m = self.b.ins().band_imm(bit, scalars); + let heap = self.b.ins().icmp_imm(IntCC::Equal, m, 0); + self.miss_if(heap, miss); + let (tag, off, v) = match lane { + JitType::Int => (l.tag_int, 8, val), + JitType::Float => (l.tag_float, 8, val), + _ => (l.tag_bool, 1, self.b.ins().ireduce(types::I8, val)), + }; + let tagv = self.b.ins().iconst(types::I8, i64::from(tag)); + self.b.ins().store(trusted, tagv, addr, 0); + self.b.ins().store(trusted, v, addr, off); + self.b.ins().jump(stored, &[]); + self.b.switch_to_block(miss); + done = Some(stored); + } self.b .ins() .store(trusted, val, self.frame_ptr, OFF_RET_BITS); @@ -2910,6 +3062,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let bad = self.b.ins().icmp_imm(IntCC::NotEqual, status, 0); let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); + if let Some(stored) = done { + self.b.ins().jump(stored, &[]); + self.b.switch_to_block(stored); + } } /// RFC 0076 WS6 — closure-cell read via `wpjit_cell_get`. The cell diff --git a/crates/weavepy-jit/src/runtime.rs b/crates/weavepy-jit/src/runtime.rs index bd9e6b57..e924f99e 100644 --- a/crates/weavepy-jit/src/runtime.rs +++ b/crates/weavepy-jit/src/runtime.rs @@ -150,6 +150,73 @@ impl SlotTag { } } +/// Where the embedder keeps what a pinned instance's field read or write +/// touches, so compiled code can do the common one in line: byte offsets +/// (from the pointer named in each field's doc), the object tags, and the +/// address of the flag that says borrow-free reads are off. The embedder +/// measures and checks them; compiled code falls back to its helpers on +/// anything else. +#[derive(Clone, Copy, Debug, Default)] +pub struct ObjLayout { + /// [`JitFrame::ctx`] → the pin table's buffer pointer and length. + pub ctx_pins_ptr: i32, + pub ctx_pins_len: i32, + /// A pin: its size, its discriminant byte and the object pin's value, + /// and where that object sits in it. + pub pin_size: i32, + pub pin_tag: i32, + pub pin_obj_tag: u8, + pub pin_obj: i32, + /// [`JitFrame::ctx`] → the attribute guards' shared pointer; that + /// pointer → the guard array. + pub ctx_guards: i32, + pub guards_buf: i32, + /// A guard: its size, its class version (`u64`), and the split-values + /// index its name sits at (`u32`, `u32::MAX` for none). + pub guard_size: i32, + pub guard_ver: i32, + pub guard_split_idx: i32, + /// Object tags (the first byte of a value); a word payload is at 8, a + /// `bool`'s at 1. + pub tag_instance: u8, + pub tag_int: u8, + pub tag_float: u8, + pub tag_bool: u8, + pub tag_none: u8, + /// An instance value's payload pointer → the instance's class pointer, + /// its published dict pointer (null while split), its split values' + /// borrow counter (`i32`) and block pointer (null while empty). + pub inst_class: i32, + pub inst_dict_lazy: i32, + pub inst_split_borrow: i32, + pub inst_split_block: i32, + /// A class pointer → its attribute version (`u64`) and its shared + /// names' pointer. + pub type_attr_version: i32, + pub type_shared_keys: i32, + /// A split block → its names' pointer, its length (`u32`) and its + /// first value. + pub split_keys: i32, + pub split_len: i32, + pub split_values: i32, + /// The byte that is nonzero once cells are shared between threads. + pub cells_unguarded: usize, +} + +static OBJ_LAYOUT: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Publish the embedder's object layout (once; later calls are ignored). +/// Frames compiled from then on read pinned instances' fields in line. +pub fn set_obj_layout(layout: ObjLayout) { + let _ = OBJ_LAYOUT.set(layout); +} + +/// The published object layout. +#[must_use] +pub(crate) fn obj_layout() -> Option<&'static ObjLayout> { + OBJ_LAYOUT.get() +} + /// The exchange buffer the VM passes to a compiled frame. /// /// The VM owns the backing storage (`Vec` / `Vec`); this diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs index 57451d59..6a521f4f 100644 --- a/crates/weavepy-vm/src/inst_dict.rs +++ b/crates/weavepy-vm/src/inst_dict.rs @@ -225,6 +225,14 @@ impl Drop for SplitValues { } impl SplitValues { + /// Where native code finds a split block's parts: the block pointer + /// (null when empty) within the values, and the shared names' pointer, + /// the `u32` length and the first value within a block. + pub(crate) const BLOCK_OFFSET: usize = std::mem::offset_of!(Self, block); + pub(crate) const KEYS_OFFSET: usize = std::mem::offset_of!(SplitHeader, keys); + pub(crate) const LEN_OFFSET: usize = std::mem::offset_of!(SplitHeader, len); + pub(crate) const VALUES_OFFSET: usize = std::mem::size_of::(); + #[inline] fn layout(cap: usize) -> std::alloc::Layout { std::alloc::Layout::new::() @@ -675,6 +683,12 @@ impl std::fmt::Debug for InstDict { } impl InstDict { + /// Where native code finds the published dictionary's pointer (null + /// while the values are split) and the split values' cell. + pub(crate) const LAZY_OFFSET: usize = + std::mem::offset_of!(Self, lazy) + LazyArc::>::POINTER_OFFSET; + pub(crate) const SPLIT_OFFSET: usize = std::mem::offset_of!(Self, split); + pub fn new() -> Self { Self { lazy: LazyArc::new(), diff --git a/crates/weavepy-vm/src/lazy_arc.rs b/crates/weavepy-vm/src/lazy_arc.rs index afa5ea57..655a6251 100644 --- a/crates/weavepy-vm/src/lazy_arc.rs +++ b/crates/weavepy-vm/src/lazy_arc.rs @@ -18,6 +18,9 @@ pub struct LazyArc { } impl LazyArc { + /// Where the payload pointer (null until set) sits, for native code. + pub(crate) const POINTER_OFFSET: usize = std::mem::offset_of!(Self, pointer); + pub const fn new() -> Self { Self { pointer: AtomicPtr::new(ptr::null_mut()), diff --git a/crates/weavepy-vm/src/sync.rs b/crates/weavepy-vm/src/sync.rs index 1d8b2ef0..165d0fbc 100644 --- a/crates/weavepy-vm/src/sync.rs +++ b/crates/weavepy-vm/src/sync.rs @@ -346,6 +346,12 @@ fn cells_unguarded() -> bool { CELLS_UNGUARDED.load(Ordering::Relaxed) } +/// The flag [`GilCell::peek`] tests, for native code that peeks (a byte: +/// nonzero once cells are shared between threads). +pub(crate) fn cells_unguarded_flag() -> *const bool { + CELLS_UNGUARDED.as_ptr() +} + /// Whether reference counts must be updated atomically: set whenever the /// cell bias is revoked, and also before spawning a thread that may touch /// objects before it registers (see [`crate::rc`]). Never cleared. @@ -658,6 +664,11 @@ impl fmt::Display for BorrowMutError { impl std::error::Error for BorrowMutError {} impl GilCell { + /// Where the borrow counter (an `i32`: negative while mutably + /// borrowed) and the payload sit, for native code that peeks. + pub(crate) const BORROW_OFFSET: usize = std::mem::offset_of!(Self, borrow); + pub(crate) const DATA_OFFSET: usize = std::mem::offset_of!(Self, data); + /// Build a cell holding `value`. `const`-callable so the /// codebase's existing `thread_local!{ static FOO: RefCell<…> = /// const { RefCell::new(None) }; }` patterns keep working. diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 0138f8d7..ca45bd61 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -88,6 +88,11 @@ struct AttrGuard { /// The class value can't acquire descriptor hooks independently of /// the receiver's class version. Only native scalar updates use this. stable_descriptor: bool, + /// The index the name holds in its class's shared names, when the + /// site's index is that one (`u32::MAX` otherwise): compiled code + /// reads and writes a split-layout instance's field there in line + /// (see [`obj_layout`]). + split_idx: u32, /// The value lane the site was compiled with. lane: JitType, /// The class's `attr_version` at compile time. Tokens are globally unique @@ -2165,6 +2170,9 @@ fn method_ret_info( /// slot bits tagged [`SlotTag::ListPin`] / [`SlotTag::ObjPin`] index /// this table, which keeps the object alive and reachable for the /// access helpers and the deopt rebuild. +// `repr(C, u8)`: compiled code reads an object pin's discriminant and +// value in place (see `obj_layout`). +#[repr(C, u8)] enum Pin { /// A pinned list plus the element lane the compile assumed. List(Rc>>, JitType), @@ -2614,6 +2622,7 @@ fn attr_site_guard( lane: site.lane, ver: cls.attr_version.get(), storage: AttrStorage::Indexed(*field_idx), + split_idx: split_index(&cls, AttrStorage::Indexed(*field_idx), &site.name), last_result_pin: Cell::new(usize::MAX), }); } @@ -2632,7 +2641,7 @@ fn attr_site_guard( else { return None; }; - let Object::Instance(_) = &recv else { + let Object::Instance(inst) = &recv else { return None; }; return Some(AttrGuard { @@ -2642,6 +2651,7 @@ fn attr_site_guard( lane: site.lane, ver, storage: AttrStorage::Indexed(field_idx), + split_idx: split_index(&inst.cls(), AttrStorage::Indexed(field_idx), &site.name), last_result_pin: Cell::new(usize::MAX), }); } @@ -2660,6 +2670,10 @@ fn attr_site_guard( && matches!(storage, AttrStorage::Indexed(_)) && scalar_field_update_shape(&frame.code) && scalar_update_class_value_stable(&recv, &site.name, ver); + let split_idx = match &recv { + Object::Instance(inst) => split_index(&inst.cls(), storage, &site.name), + _ => u32::MAX, + }; Some(AttrGuard { name: shared_name(), name_hash: crate::object::py_str_hash(&site.name), @@ -2667,10 +2681,161 @@ fn attr_site_guard( lane: site.lane, ver, storage, + split_idx, last_result_pin: Cell::new(usize::MAX), }) } +/// [`AttrGuard::split_idx`] for an indexed site of `cls`: the index, when +/// the class's shared names hold `name` there (they never move). +fn split_index(cls: &TypeObject, storage: AttrStorage, name: &str) -> u32 { + let AttrStorage::Indexed(i) = storage else { + return u32::MAX; + }; + match cls.shared_keys.get().and_then(|keys| keys.get(i as usize)) { + Some(DictKey(Object::Str(s))) if &**s == name => i, + _ => u32::MAX, + } +} + +/// Measure where pinned instances keep what compiled code's in-line field +/// reads and writes touch (see [`weavepy_jit::ObjLayout`]), check every +/// offset against `obj` (a live instance), and publish the layout. Once: +/// later calls return straight away. +fn ensure_obj_layout(obj: &Object) { + static DONE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + if DONE.load(std::sync::atomic::Ordering::Relaxed) { + return; + } + let Object::Instance(inst) = obj else { + return; + }; + DONE.store(true, std::sync::atomic::Ordering::Relaxed); + if let Some(layout) = obj_layout(obj, inst) { + weavepy_jit::set_obj_layout(layout); + } +} + +/// The layout [`ensure_obj_layout`] publishes, `None` when any offset +/// doesn't read back what the safe accessors say about `obj`. +fn obj_layout(obj: &Object, inst: &Rc) -> Option { + use crate::inst_dict::{InstDict, SplitValues}; + use crate::sync::GilCell; + use crate::types::PyInstance; + if crate::gil::free_threading_enabled() || std::env::var_os("WEAVEPY_JIT_NO_INLINE_ATTRS").is_some() { + return None; + } + let word = |p: *const u8, off: usize| -> usize { + // SAFETY: every read below is inside a live object the caller + // holds, at an offset measured from its own type. + unsafe { p.add(off).cast::().read_unaligned() } + }; + let tag = |v: &Object| -> u8 { + // SAFETY: `Object` is `repr(u8)`: its first byte is the tag. + unsafe { *std::ptr::from_ref(v).cast::() } + }; + let obj_p = std::ptr::from_ref(obj).cast::(); + // The payload word is the `Arc` allocation; the instance follows its + // two counts. + let arc = word(obj_p, 8); + let data = Rc::as_ptr(inst) as usize; + let arc_data = data.checked_sub(arc)?; + let i32_of = |v: usize| i32::try_from(v).ok(); + let inst_class = arc_data + std::mem::offset_of!(PyInstance, class) + GilCell::>::DATA_OFFSET; + let dict = arc_data + std::mem::offset_of!(PyInstance, dict); + let inst_dict_lazy = dict + InstDict::LAZY_OFFSET; + let inst_split_borrow = dict + InstDict::SPLIT_OFFSET + GilCell::::BORROW_OFFSET; + let inst_split_block = + dict + InstDict::SPLIT_OFFSET + GilCell::::DATA_OFFSET + SplitValues::BLOCK_OFFSET; + let arc_p = arc as *const u8; + // The class pointer is its `Arc` allocation too. + let cls = inst.cls(); + let cls_arc = word(arc_p, inst_class); + if cls_arc + arc_data != Rc::as_ptr(&cls) as usize { + return None; + } + let type_attr_version = arc_data + std::mem::offset_of!(TypeObject, attr_version); + let type_shared_keys = arc_data + + std::mem::offset_of!(TypeObject, shared_keys) + + crate::sync::LazyArc::::POINTER_OFFSET; + let cls_p = cls_arc as *const u8; + // SAFETY: as `word`, a `u64` field. + let ver = unsafe { cls_p.add(type_attr_version).cast::().read_unaligned() }; + let keys = cls.shared_keys.get().map_or(0, |k| std::ptr::from_ref(k) as usize); + let published = inst.dict.published().is_some(); + if ver != cls.attr_version.get() + || word(cls_p, type_shared_keys) != keys + || (word(arc_p, inst_dict_lazy) != 0) != published + { + return None; + } + // SAFETY: as `word`, the `i32` borrow counter. + let borrow = unsafe { arc_p.add(inst_split_borrow).cast::().read_unaligned() }; + if borrow != 0 { + return None; + } + // A pin: `repr(C, u8)`. + let sample = Pin::Obj(obj.clone()); + let sample_p = std::ptr::from_ref(&sample).cast::(); + let pin_obj = match &sample { + Pin::Obj(o) => std::ptr::from_ref(o) as usize - sample_p as usize, + Pin::List(..) => return None, + }; + // SAFETY: the discriminant byte of a `repr(C, u8)` enum. + let pin_obj_tag = unsafe { *sample_p }; + if word(sample_p, pin_obj + 8) != arc { + return None; + } + drop(sample); + // A vector's buffer pointer and length, measured on one whose three + // words all differ. + let mut v: Vec = Vec::with_capacity(4); + v.push(Pin::Obj(Object::None)); + let vp = std::ptr::from_ref(&v).cast::(); + let find = |want: usize| (0..3).map(|k| k * 8).find(|&off| word(vp, off) == want); + let (vec_ptr, vec_len) = (find(v.as_ptr() as usize)?, find(1)?); + drop(v); + let guards: StdRc> = StdRc::new(Vec::with_capacity(2)); + let rc_box = { + // SAFETY: `std::rc::Rc` is one pointer to its allocation. + unsafe { std::mem::transmute_copy::>, usize>(&guards) } + }; + let guards_vec = std::ptr::from_ref::>(&guards) as usize - rc_box; + let guards_buf = guards_vec + vec_ptr; + if word(rc_box as *const u8, guards_buf) != guards.as_ptr() as usize { + return None; + } + let ctx_pins = std::mem::offset_of!(CallCtx, pins); + Some(weavepy_jit::ObjLayout { + ctx_pins_ptr: i32_of(ctx_pins + vec_ptr)?, + ctx_pins_len: i32_of(ctx_pins + vec_len)?, + pin_size: i32_of(std::mem::size_of::())?, + pin_tag: 0, + pin_obj_tag, + pin_obj: i32_of(pin_obj)?, + ctx_guards: i32_of(std::mem::offset_of!(CallCtx, attr_guards))?, + guards_buf: i32_of(guards_buf)?, + guard_size: i32_of(std::mem::size_of::())?, + guard_ver: i32_of(std::mem::offset_of!(AttrGuard, ver))?, + guard_split_idx: i32_of(std::mem::offset_of!(AttrGuard, split_idx))?, + tag_instance: tag(obj), + tag_int: tag(&Object::Int(0)), + tag_float: tag(&Object::Float(0.0)), + tag_bool: tag(&Object::Bool(false)), + tag_none: tag(&Object::None), + inst_class: i32_of(inst_class)?, + inst_dict_lazy: i32_of(inst_dict_lazy)?, + inst_split_borrow: i32_of(inst_split_borrow)?, + inst_split_block: i32_of(inst_split_block)?, + type_attr_version: i32_of(type_attr_version)?, + type_shared_keys: i32_of(type_shared_keys)?, + split_keys: i32_of(SplitValues::KEYS_OFFSET)?, + split_len: i32_of(SplitValues::LEN_OFFSET)?, + split_values: i32_of(SplitValues::VALUES_OFFSET)?, + cells_unguarded: crate::sync::cells_unguarded_flag() as usize, + }) +} + /// RFC 0069 WS1 — the compile-time method probe: resolve `name` on the /// class of the instance currently in local `slot`, when the shape is /// eligible for a burned-in method call: @@ -2790,6 +2955,7 @@ fn attr_fingerprint_obj( let Object::Instance(inst) = obj else { return None; }; + ensure_obj_layout(obj); // RFC 0070 WS3 / RFC 0071 WS2 — the tier-1 predicate classifies // the storage: an indexed instance-dict hit, a `__slots__` member // (read and written through the slot side table by name), or the diff --git a/tests/regrtest/test_jit_inline_fields.py b/tests/regrtest/test_jit_inline_fields.py new file mode 100644 index 00000000..903293b1 --- /dev/null +++ b/tests/regrtest/test_jit_inline_fields.py @@ -0,0 +1,129 @@ +"""Compiled code reads and writes instance fields in line. + +Each loop runs long enough for the tier-2 compiler to take it, then the +receiver or its class changes shape under the compiled code: every change +must reach the generic path with the interpreter's result. +""" + +import unittest + + +class Point: + def __init__(self, x, y): + self.x = x + self.y = y + + +class Other: + def __init__(self): + self.y = 100 + self.x = 200 + + +def read_sum(p, n): + t = 0 + for _ in range(n): + t += p.x + return t + + +def bump(p, n): + for _ in range(n): + p.y = p.y + 1.5 + return p.y + + +class InlineFieldTests(unittest.TestCase): + N = 3000 + + def test_plain_reads_and_writes(self): + p = Point(2, 0.5) + self.assertEqual(read_sum(p, self.N), 2 * self.N) + self.assertEqual(bump(p, self.N), 0.5 + 1.5 * self.N) + + def test_class_gains_a_property(self): + class C: + def __init__(self): + self.x = 1 + + def loop(o, n): + t = 0 + for _ in range(n): + t += o.x + return t + + c = C() + self.assertEqual(loop(c, self.N), self.N) + C.x = property(lambda self: 7) + self.assertEqual(loop(c, self.N), 7 * self.N) + + def test_class_swap(self): + class A: + def __init__(self): + self.x = 1 + self.y = 2 + + class B: + def __init__(self): + self.y = 3 + self.x = 4 + + def loop(o, n): + t = 0 + for _ in range(n): + t += o.x + return t + + a = A() + self.assertEqual(loop(a, self.N), self.N) + b = B() + self.assertEqual(loop(b, self.N), 4 * self.N) + a.__class__ = B + self.assertEqual(loop(a, self.N), self.N) + + def test_materialized_dict(self): + p = Point(3, 1.0) + self.assertEqual(read_sum(p, self.N), 3 * self.N) + d = vars(p) + d["x"] = 5 + self.assertEqual(read_sum(p, self.N), 5 * self.N) + del d["y"] + d["y"] = 1.0 + self.assertEqual(bump(p, 10), 16.0) + + def test_deleted_and_readded_field(self): + p = Point(1, 0.0) + self.assertEqual(read_sum(p, self.N), self.N) + del p.x + with self.assertRaises(AttributeError): + read_sum(p, 1) + p.x = 9 + self.assertEqual(read_sum(p, self.N), 9 * self.N) + + def test_non_scalar_store_and_read(self): + p = Point(1, 0.0) + self.assertEqual(bump(p, self.N), 1.5 * self.N) + p.y = [1] + with self.assertRaises(TypeError): + bump(p, 1) + p.y = 2.0 + self.assertEqual(bump(p, 2), 5.0) + + def test_field_order_differs_between_instances(self): + p = Point(6, 0.0) + o = Other() + self.assertEqual(read_sum(p, self.N), 6 * self.N) + self.assertEqual(read_sum(o, self.N), 200 * self.N) + self.assertEqual(read_sum(p, self.N), 6 * self.N) + + def test_type_change_of_field(self): + p = Point(1, 0.0) + self.assertEqual(read_sum(p, self.N), self.N) + p.x = 2.5 + self.assertEqual(read_sum(p, 4), 10.0) + p.x = True + self.assertEqual(read_sum(p, 4), 4) + + +if __name__ == "__main__": + unittest.main() From 48cedc1b7122a33912ec9048b26626db1e842a15 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:30:01 -0700 Subject: [PATCH 13/96] perf: read leaf plans' cached instance fields in line A native plan's attribute read now checks its site's first field-cache entry in line (the receiver's class version, its split values over the class's names, the cached position) and produces the field's register words without calling h_field; anything else calls it as before. The object layout tier 2 measures is shared with the plans, and the plan runner publishes it at its first instance read. The in-line checks are grouped so each site adds few blocks. deltablue: -2.8% instructions per iteration. --- crates/weavepy-jit/src/lower.rs | 33 +++---- crates/weavepy-vm/src/leaf_plan.rs | 4 + crates/weavepy-vm/src/leaf_plan/jit.rs | 122 ++++++++++++++++++++++++- crates/weavepy-vm/src/tier2.rs | 12 ++- 4 files changed, 149 insertions(+), 22 deletions(-) diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index ab519273..8eacb1d5 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -2810,11 +2810,11 @@ impl<'a, 'b> Lowerer<'a, 'b> { let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); let p = self.b.ins().iadd(buf, off); let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); - let not_obj = self.b.ins().icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); - self.miss_if(not_obj, miss); let otag = self.b.ins().uload8(types::I32, t, p, l.pin_obj); + let not_obj = self.b.ins().icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); let not_inst = self.b.ins().icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); - self.miss_if(not_inst, miss); + let bad = self.b.ins().bor(not_obj, not_inst); + self.miss_if(bad, miss); let inst = self.b.ins().load(ptr, t, p, l.pin_obj + 8); // The guard. let guards = self.b.ins().load(ptr, t, ctx, l.ctx_guards); @@ -2825,34 +2825,35 @@ impl<'a, 'b> Lowerer<'a, 'b> { .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); let ver = self.b.ins().load(types::I64, t, g, l.guard_ver); let idx = self.b.ins().uload32(t, g, l.guard_split_idx); - // The class. + // The class and the split values (grouped by what each group's + // loads need proven: fewer blocks compile faster). let cls = self.b.ins().load(ptr, t, inst, l.inst_class); let cver = self.b.ins().load(types::I64, t, cls, l.type_attr_version); - let stale = self.b.ins().icmp(IntCC::NotEqual, cver, ver); - self.miss_if(stale, miss); - // The split values. let lazy = self.b.ins().load(ptr, t, inst, l.inst_dict_lazy); - self.miss_if(lazy, miss); let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); - let shared = self.b.ins().uload8(types::I32, t, flag, 0); - self.miss_if(shared, miss); + let shared = self.b.ins().uload8(types::I64, t, flag, 0); let borrow = self.b.ins().sload32(t, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, t, inst, l.inst_split_block); + let stale = self.b.ins().icmp(IntCC::NotEqual, cver, ver); let busy = if read { self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0) } else { self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0) }; - self.miss_if(busy, miss); - let block = self.b.ins().load(ptr, t, inst, l.inst_split_block); let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); - self.miss_if(empty, miss); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(stale, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, miss); let keys = self.b.ins().load(ptr, t, block, l.split_keys); let ckeys = self.b.ins().load(ptr, t, cls, l.type_shared_keys); - let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); - self.miss_if(foreign, miss); let len = self.b.ins().uload32(t, block, l.split_len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); - self.miss_if(absent, miss); + let bad = self.b.ins().bor(foreign, absent); + self.miss_if(bad, miss); let off = self.b.ins().ishl_imm(idx, 4); let at = self.b.ins().iadd(block, off); self.b.ins().iadd_imm(at, i64::from(l.split_values)) diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index fecf6478..0cc0053e 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -1489,6 +1489,10 @@ impl Interpreter { } match recv { Object::Instance(inst) => { + // (Native plans read fields in line once the layout is + // known.) + #[cfg(feature = "jit")] + crate::tier2::ensure_obj_layout(recv); if GETTER && inst.cls_raw().native_kind.get() != 0 { return None; } diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index 61dcbab1..836d5d07 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -196,6 +196,14 @@ impl Ctx<'_> { /// `ctx.top`'s offset (native code passes the evaluation's own frame by /// address). const TOP_OFFSET: i32 = 32; +/// Where native code finds the count of buffered stores, and a field +/// cache's first entry's version and position. +const PEND_N_OFFSET: i32 = + (std::mem::offset_of!(Ctx<'static>, pend) + std::mem::offset_of!(Pending, n)) as i32; +const FIELD_VER_OFFSET: i32 = (std::mem::offset_of!(FieldCache, entries) + + std::mem::offset_of!((u64, u32), 0)) as i32; +const FIELD_IDX_OFFSET: i32 = (std::mem::offset_of!(FieldCache, entries) + + std::mem::offset_of!((u64, u32), 1)) as i32; const _: () = assert!(std::mem::offset_of!(Ctx<'static>, top) == TOP_OFFSET as usize); /// Run `plan`'s native code for one evaluation, compiling it once it is @@ -889,6 +897,96 @@ impl Lower<'_> { self.b.def_var(fl.pays[r], p); } + /// Branch to `miss` when `cond`; continue in a fresh block otherwise. + fn miss_if(&mut self, cond: Value, miss: Block) { + let ok = self.b.create_block(); + self.b.ins().brif(cond, miss, &[], ok, &[]); + self.b.switch_to_block(ok); + } + + /// [`h_field`]'s first cache entry in line: the receiver register + /// `(t, p)` is a plain instance whose class has the entry's version + /// and whose values are split over its class's names, holding the + /// entry's position; the field's value as register words. Anything + /// else (and, with buffered stores, everything) branches to `miss`. + fn inline_field( + &mut self, + l: &weavepy_jit::ObjLayout, + t: Value, + p: Value, + cache: i64, + effect: bool, + miss: Block, + ) -> (Value, Value) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + // (The checks are grouped by what each group's loads need proven: + // fewer blocks compile faster.) + let not_ref = self.b.ins().icmp_imm(IntCC::NotEqual, t, T_R as i64); + self.miss_if(not_ref, miss); + let otag = self.b.ins().uload8(types::I32, f, p, 0); + let mut bad = self.b.ins().icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + if effect { + let n = self.b.ins().load(types::I64, f, self.ctx, PEND_N_OFFSET); + let pending = self.b.ins().icmp_imm(IntCC::NotEqual, n, 0); + bad = self.b.ins().bor(bad, pending); + } + self.miss_if(bad, miss); + // An instance: its class, its split values, and the site's entry. + let inst = self.b.ins().load(ptr, f, p, 8); + let cls = self.b.ins().load(ptr, f, inst, l.inst_class); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let at = self.b.ins().iconst(ptr, cache); + let want = self.b.ins().load(types::I64, f, at, FIELD_VER_OFFSET); + let idx = self.b.ins().uload32(f, at, FIELD_IDX_OFFSET); + let lazy = self.b.ins().load(ptr, f, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, f, flag, 0); + let borrow = self.b.ins().sload32(f, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, f, inst, l.inst_split_block); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, miss); + // A block over the class's names that holds the position. + let keys = self.b.ins().load(ptr, f, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, f, cls, l.type_shared_keys); + let len = self.b.ins().uload32(f, block, l.split_len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let bad = self.b.ins().bor(foreign, absent); + self.miss_if(bad, miss); + let off = self.b.ins().ishl_imm(idx, 4); + let v = self.b.ins().iadd(block, off); + let v = self.b.ins().iadd_imm(v, i64::from(l.split_values)); + // `norm`: the scalars by value, anything else by reference. + let vt = self.b.ins().uload8(types::I64, f, v, 0); + let word = self.b.ins().load(types::I64, f, v, 8); + let byte = self.b.ins().uload8(types::I64, f, v, 1); + let zero = self.b.ins().iconst(types::I64, 0); + let mut rt = self.b.ins().iconst(types::I64, T_R as i64); + let mut rp = v; + for (tag, vtag, pay) in [ + (l.tag_int, T_I, word), + (l.tag_float, T_F, word), + (l.tag_bool, T_B, byte), + (l.tag_none, T_N, zero), + ] { + let hit = self.b.ins().icmp_imm(IntCC::Equal, vt, i64::from(tag)); + let tv = self.b.ins().iconst(types::I64, vtag as i64); + rt = self.b.ins().select(hit, tv, rt); + rp = self.b.ins().select(hit, pay, rp); + } + (rt, rp) + } + /// Load the result words a helper left at `ctx.out[k..k + 2]` into `r`. fn set_out(&mut self, fl: &FrameLower<'_>, r: u8, k: i32) { let flags = MemFlags::trusted(); @@ -1139,11 +1237,19 @@ impl Lower<'_> { let (t, p) = self.get(fl, src); let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); let cache = Box::::default(); - let cache_v = self - .b - .ins() - .iconst(self.ptr, std::ptr::from_ref(&*cache) as i64); + let cache_at = std::ptr::from_ref(&*cache) as i64; + let cache_v = self.b.ins().iconst(self.ptr, cache_at); self.fields.push(cache); + // The site's most recent field, read in line. + let done = self.b.create_block(); + self.b.append_block_param(done, types::I64); + self.b.append_block_param(done, types::I64); + if let Some(l) = crate::tier2::published_obj_layout() { + let slow = self.b.create_block(); + let (vt, vp) = self.inline_field(l, t, p, cache_at, effect, slow); + self.b.ins().jump(done, &[vt.into(), vp.into()]); + self.b.switch_to_block(slow); + } let addr = if effect { h_field:: as *const () as usize } else { @@ -1151,7 +1257,13 @@ impl Lower<'_> { }; let st = self.call(addr, &[self.ctx, fl.fr, cache_v, t, p, pn]); self.check(st); - self.set_out(fl, dst, 0); + let flags = MemFlags::trusted(); + let rt = self.b.ins().load(types::I64, flags, self.ctx, 0); + let rp = self.b.ins().load(types::I64, flags, self.ctx, 8); + self.b.ins().jump(done, &[rt.into(), rp.into()]); + self.b.switch_to_block(done); + let (rt, rp) = (self.b.block_params(done)[0], self.b.block_params(done)[1]); + self.set(fl, dst, rt, rp); } Op::Method { dst, src, pc, name } => { let Some(ms) = crate::code_method_slot(fl.code, u32::from(pc)) else { diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index ca45bd61..31bfbe5e 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -2702,7 +2702,8 @@ fn split_index(cls: &TypeObject, storage: AttrStorage, name: &str) -> u32 { /// reads and writes touch (see [`weavepy_jit::ObjLayout`]), check every /// offset against `obj` (a live instance), and publish the layout. Once: /// later calls return straight away. -fn ensure_obj_layout(obj: &Object) { +#[inline] +pub(crate) fn ensure_obj_layout(obj: &Object) { static DONE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); if DONE.load(std::sync::atomic::Ordering::Relaxed) { return; @@ -2712,10 +2713,19 @@ fn ensure_obj_layout(obj: &Object) { }; DONE.store(true, std::sync::atomic::Ordering::Relaxed); if let Some(layout) = obj_layout(obj, inst) { + let _ = PUBLISHED_LAYOUT.set(layout); weavepy_jit::set_obj_layout(layout); } } +static PUBLISHED_LAYOUT: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// The object layout [`ensure_obj_layout`] published, for the VM's other +/// native code (the leaf plans'). +pub(crate) fn published_obj_layout() -> Option<&'static weavepy_jit::ObjLayout> { + PUBLISHED_LAYOUT.get() +} + /// The layout [`ensure_obj_layout`] publishes, `None` when any offset /// doesn't read back what the safe accessors say about `obj`. fn obj_layout(obj: &Object, inst: &Rc) -> Option { From aaaf3a6735dc54df0f7af36a97e110935e54fa05 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:59:58 -0700 Subject: [PATCH 14/96] perf: read, write, append to and iterate scalar lists in line in tier 2 Compiled code now reads and writes an int or float element of a pinned list of that lane, appends a scalar when the list has room, and steps a list loop over int, float or bool elements without calling a helper: the pin's lane, the list cell's borrow counter, the index and the element's tag are checked in line, and anything else takes the helper. The object layout is now measured on a private instance of object at the first compile, so code that never reads an instance attribute gets the in-line paths too. An attribute site with no split position (a __slots__ member) leaves for its helper before touching the pin. nbody -9%, float_math -10%, list_ops -4% instructions. WEAVEPY_JIT_QUICK=1 compiles with Cranelift's quick settings (a tuning knob: 13-89M fewer instructions of compilation, 6-32% slower code). --- crates/weavepy-jit/src/engine.rs | 4 + crates/weavepy-jit/src/lower.rs | 260 ++++++++++++++++++++++-- crates/weavepy-jit/src/runtime.rs | 11 + crates/weavepy-vm/src/leaf_plan.rs | 4 - crates/weavepy-vm/src/leaf_plan/jit.rs | 5 + crates/weavepy-vm/src/tier2.rs | 80 +++++++- tests/regrtest/test_jit_inline_lists.py | 126 ++++++++++++ 7 files changed, 465 insertions(+), 25 deletions(-) create mode 100644 tests/regrtest/test_jit_inline_lists.py diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index e5408456..66ded5a4 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -250,6 +250,10 @@ impl JitEngine { flag_builder.set("is_pic", "false").ok()?; // Favour fast compiles over the last few percent of codegen. flag_builder.set("opt_level", "speed").ok()?; + if std::env::var_os("WEAVEPY_JIT_QUICK").is_some() { + flag_builder.set("opt_level", "none").ok()?; + flag_builder.set("regalloc_algorithm", "single_pass").ok()?; + } // The IR verifier re-checks every function it compiles: a debug // aid whose cost lands on every warm-up in release builds. if !cfg!(debug_assertions) { diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 8eacb1d5..99c4d400 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -595,6 +595,40 @@ impl<'a, 'b> Lowerer<'a, 'b> { // args, and a header deopt spills it. let snapshot = self.vstack.clone(); let args = self.block_args(); + // A scalar element of a scalar-lane list, or the end of + // the list, in line. + let got_b = self.b.create_block(); + self.b.append_block_param(got_b, Self::cl_ty(elem)); + let mut native_done = None; + if let (Some(l), JitType::Int | JitType::Float | JitType::Bool) = + (runtime::obj_layout(), elem) + { + let l = *l; + let miss = self.b.create_block(); + let (items, len, _) = self.pinned_list(&l, seq, &[elem], miss, true); + let negative = self.b.ins().icmp_imm(IntCC::SignedLessThan, idx, 0); + self.miss_if(negative, miss); + let at_end = self.b.ins().icmp(IntCC::SignedGreaterThanOrEqual, idx, len); + let more = self.b.create_block(); + let end = self.b.create_block(); + self.b.ins().brif(at_end, end, &[], more, &[]); + self.b.switch_to_block(more); + let off = self.b.ins().ishl_imm(idx, 4); + let e = self.b.ins().iadd(items, off); + let (want, voff, ty) = Self::lane_tag(&l, elem).expect("scalar lane"); + let tag = self.b.ins().uload8(types::I32, trusted, e, 0); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, tag, i64::from(want)); + self.miss_if(other, miss); + let v = self.b.ins().load(ty, trusted, e, voff); + let v = if ty == types::I8 { + self.b.ins().uextend(types::I64, v) + } else { + v + }; + self.b.ins().jump(got_b, &[BlockArg::from(v)]); + self.b.switch_to_block(miss); + native_done = Some(end); + } let sig = self.list_helper_sig(); let helper = self .b @@ -605,15 +639,23 @@ impl<'a, 'b> Lowerer<'a, 'b> { .ins() .call_indirect(sig, helper, &[self.frame_ptr, seq, idx]); let status = self.b.inst_results(call)[0]; - let got_b = self.b.create_block(); + let fetched = self.b.create_block(); let rest_b = self.b.create_block(); let is_got = self.b.ins().icmp_imm(IntCC::Equal, status, 0); - self.b.ins().brif(is_got, got_b, &[], rest_b, &[]); - self.b.switch_to_block(got_b); + self.b.ins().brif(is_got, fetched, &[], rest_b, &[]); + self.b.switch_to_block(fetched); let res = self.b .ins() .load(Self::cl_ty(elem), trusted, self.frame_ptr, OFF_RET_BITS); + self.b.ins().jump(got_b, &[BlockArg::from(res)]); + if let Some(end) = native_done { + self.b.switch_to_block(end); + let eb = self.cl_blocks[exit]; + self.b.ins().jump(eb, &args); + } + self.b.switch_to_block(got_b); + let res = self.b.block_params(got_b)[0]; self.b.def_var(loop_var, res); let next = self.b.ins().iadd_imm(idx, 1); self.b.def_var(idx_var, next); @@ -2656,6 +2698,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { let snapshot = self.vstack.clone(); let (val, ty) = self.pop(); let (pin, _) = self.pop(); + let appended = self.inline_list_append(pin, val, ty); self.b .ins() .store(trusted, val, self.frame_ptr, OFF_RET_BITS); @@ -2676,10 +2719,51 @@ impl<'a, 'b> Lowerer<'a, 'b> { let bad = self.b.ins().icmp_imm(IntCC::NotEqual, status, 0); let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); + if let Some(done) = appended { + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } // `append` returns `None`, which the following `POP_TOP` // consumes — neither ever exists on the native stack. } + /// `append` of a scalar to a pinned list with room for it, in line; + /// the block to continue at, with the current block the helper's + /// path (`None` when not in line). + fn inline_list_append(&mut self, pin: Value, val: Value, ty: JitType) -> Option { + let l = *runtime::obj_layout()?; + // The lanes the helper boxes this scalar into. + let lanes: &[JitType] = match ty { + JitType::Int => &[JitType::Int, JitType::Obj], + JitType::Float => &[JitType::Float, JitType::Obj], + JitType::Bool => &[JitType::Obj], + _ => return None, + }; + let t = MemFlags::trusted(); + let miss = self.b.create_block(); + let done = self.b.create_block(); + let (items, len, list) = self.pinned_list(&l, pin, lanes, miss, false); + let cap = self.b.ins().load(types::I64, t, list, l.list_cap); + let full = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, len, cap); + self.miss_if(full, miss); + let off = self.b.ins().ishl_imm(len, 4); + let at = self.b.ins().iadd(items, off); + let (tag, voff, _) = Self::lane_tag(&l, ty).expect("scalar lane"); + let v = if ty == JitType::Bool { + self.b.ins().ireduce(types::I8, val) + } else { + val + }; + let tagv = self.b.ins().iconst(types::I8, i64::from(tag)); + self.b.ins().store(t, tagv, at, 0); + self.b.ins().store(t, v, at, voff); + let len1 = self.b.ins().iadd_imm(len, 1); + self.b.ins().store(t, len1, list, l.list_len); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(miss); + Some(done) + } + /// RFC 0073 WS1 — `LIST_APPEND` inside an inlined comprehension: /// identical to [`Self::emit_list_append`] except the accumulator /// pin stays on the native stack (it is live across the whole @@ -2802,6 +2886,17 @@ impl<'a, 'b> Lowerer<'a, 'b> { let t = MemFlags::trusted(); let ptr = self.ptr_ty; let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); + // The guard first: a site whose field has no split position (a + // `__slots__` member) leaves straight away. + let guards = self.b.ins().load(ptr, t, ctx, l.ctx_guards); + let gbuf = self.b.ins().load(ptr, t, guards, l.guards_buf); + let g = self + .b + .ins() + .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); + let idx = self.b.ins().uload32(t, g, l.guard_split_idx); + let none = self.b.ins().icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); + self.miss_if(none, miss); // The pin. let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); @@ -2816,15 +2911,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { let bad = self.b.ins().bor(not_obj, not_inst); self.miss_if(bad, miss); let inst = self.b.ins().load(ptr, t, p, l.pin_obj + 8); - // The guard. - let guards = self.b.ins().load(ptr, t, ctx, l.ctx_guards); - let gbuf = self.b.ins().load(ptr, t, guards, l.guards_buf); - let g = self - .b - .ins() - .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); let ver = self.b.ins().load(types::I64, t, g, l.guard_ver); - let idx = self.b.ins().uload32(t, g, l.guard_split_idx); // The class and the split values (grouped by what each group's // loads need proven: fewer blocks compile faster). let cls = self.b.ins().load(ptr, t, inst, l.inst_class); @@ -2859,6 +2946,78 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.b.ins().iadd_imm(at, i64::from(l.split_values)) } + /// The pinned list `pin`'s items (buffer pointer and length; and its + /// capacity, and where the length lives, for an append): a list pin + /// whose element lane is one of `lanes`, unborrowed (`read`) or + /// unborrowed and exclusive (a write). Anything else branches to + /// `miss`. + fn pinned_list( + &mut self, + l: &runtime::ObjLayout, + pin: Value, + lanes: &[JitType], + miss: Block, + read: bool, + ) -> (Value, Value, Value) { + let t = MemFlags::trusted(); + let ptr = self.ptr_ty; + let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); + let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); + self.miss_if(out, miss); + let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); + let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); + let p = self.b.ins().iadd(buf, off); + let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); + let lane = self.b.ins().uload8(types::I32, t, p, l.pin_list_elem); + let mut bad = self.b.ins().icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_list_tag)); + let mut ok_lane = self.b.ins().iconst(types::I8, 0); + for &want in lanes { + let hit = self.b.ins().icmp_imm(IntCC::Equal, lane, want as i64); + ok_lane = self.b.ins().bor(ok_lane, hit); + } + let wrong_lane = self.b.ins().bxor_imm(ok_lane, 1); + bad = self.b.ins().bor(bad, wrong_lane); + self.miss_if(bad, miss); + let list = self.b.ins().load(ptr, t, p, l.pin_list); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I32, t, flag, 0); + let borrow = self.b.ins().sload32(t, list, l.list_borrow); + let busy = if read { + self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0) + } else { + self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0) + }; + let shared = self.b.ins().icmp_imm(IntCC::NotEqual, shared, 0); + let bad = self.b.ins().bor(busy, shared); + self.miss_if(bad, miss); + let items = self.b.ins().load(ptr, t, list, l.list_ptr); + let len = self.b.ins().load(types::I64, t, list, l.list_len); + (items, len, list) + } + + /// The element address for a (possibly negative) in-range `idx` of + /// `len` items at `items`; out of range branches to `miss`. + fn list_elem(&mut self, items: Value, len: Value, idx: Value, miss: Block) -> Value { + let neg = self.b.ins().icmp_imm(IntCC::SignedLessThan, idx, 0); + let wrapped = self.b.ins().iadd(idx, len); + let i = self.b.ins().select(neg, wrapped, idx); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, i, len); + self.miss_if(out, miss); + let off = self.b.ins().ishl_imm(i, 4); + self.b.ins().iadd(items, off) + } + + /// The object tag of a scalar lane. + fn lane_tag(l: &runtime::ObjLayout, lane: JitType) -> Option<(u8, i32, Type)> { + match lane { + JitType::Int => Some((l.tag_int, 8, types::I64)), + JitType::Float => Some((l.tag_float, 8, types::F64)), + JitType::Bool => Some((l.tag_bool, 1, types::I8)), + _ => None, + } + } + /// Consecutive callback-free reads can replay from the first read on a /// later miss. Preserve its original receiver in the deopt snapshot. fn emit_attr_get_chain( @@ -3212,6 +3371,24 @@ impl<'a, 'b> Lowerer<'a, 'b> { let snapshot = self.vstack.clone(); let (idx, _) = self.pop(); let (pin, _) = self.pop(); + // A scalar element of a scalar-lane list, read in line. + let mut join = None; + if let (Some(l), JitType::Int | JitType::Float) = (runtime::obj_layout(), elem) { + let l = *l; + let miss = self.b.create_block(); + let done = self.b.create_block(); + self.b.append_block_param(done, Self::cl_ty(elem)); + let (items, len, _) = self.pinned_list(&l, pin, &[elem], miss, true); + let e = self.list_elem(items, len, idx, miss); + let (want, off, ty) = Self::lane_tag(&l, elem).expect("scalar lane"); + let tag = self.b.ins().uload8(types::I32, trusted, e, 0); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, tag, i64::from(want)); + self.miss_if(other, miss); + let v = self.b.ins().load(ty, trusted, e, off); + self.b.ins().jump(done, &[BlockArg::from(v)]); + self.b.switch_to_block(miss); + join = Some(done); + } let sig = self.list_helper_sig(); let helper = self .b @@ -3229,6 +3406,14 @@ impl<'a, 'b> Lowerer<'a, 'b> { .b .ins() .load(Self::cl_ty(elem), trusted, self.frame_ptr, OFF_RET_BITS); + let res = match join { + Some(done) => { + self.b.ins().jump(done, &[BlockArg::from(res)]); + self.b.switch_to_block(done); + self.b.block_params(done)[0] + } + None => res, + }; self.vstack.push((res, elem)); } @@ -3240,7 +3425,21 @@ impl<'a, 'b> Lowerer<'a, 'b> { let snapshot = self.vstack.clone(); let (idx, _) = self.pop(); let (pin, _) = self.pop(); - let (val, _) = self.pop(); + let (val, vty) = self.pop(); + // A scalar over a scalar element of a list of that lane, stored in + // line (no release). + let mut stored = None; + if let (Some(l), JitType::Int | JitType::Float) = (runtime::obj_layout(), vty) { + let l = *l; + let miss = self.b.create_block(); + let done = self.b.create_block(); + let (items, len, _) = self.pinned_list(&l, pin, &[vty], miss, false); + let e = self.list_elem(items, len, idx, miss); + self.store_scalar_over_scalar(&l, e, val, vty, miss); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(miss); + stored = Some(done); + } // Typed store: an F64 value lands as its bit pattern. self.b .ins() @@ -3258,6 +3457,43 @@ impl<'a, 'b> Lowerer<'a, 'b> { let bad = self.b.ins().icmp_imm(IntCC::NotEqual, status, 0); let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); + if let Some(done) = stored { + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + } + + /// Store the scalar `val` (of `lane`) over the value at `at` when that + /// one is a scalar too (nothing to release); otherwise branch to + /// `miss`, untouched. + fn store_scalar_over_scalar( + &mut self, + l: &runtime::ObjLayout, + at: Value, + val: Value, + lane: JitType, + miss: Block, + ) { + let t = MemFlags::trusted(); + let old = self.b.ins().uload8(types::I64, t, at, 0); + let one = self.b.ins().iconst(types::I64, 1); + let bit = self.b.ins().ishl(one, old); + let scalars = (1i64 << l.tag_int) + | (1i64 << l.tag_float) + | (1i64 << l.tag_bool) + | (1i64 << l.tag_none); + let m = self.b.ins().band_imm(bit, scalars); + let heap = self.b.ins().icmp_imm(IntCC::Equal, m, 0); + self.miss_if(heap, miss); + let (tag, off, _) = Self::lane_tag(l, lane).expect("scalar lane"); + let v = if lane == JitType::Bool { + self.b.ins().ireduce(types::I8, val) + } else { + val + }; + let tagv = self.b.ins().iconst(types::I8, i64::from(tag)); + self.b.ins().store(t, tagv, at, 0); + self.b.ins().store(t, v, at, off); } /// The shared `(frame, pin, idx) -> status` signature of the diff --git a/crates/weavepy-jit/src/runtime.rs b/crates/weavepy-jit/src/runtime.rs index e924f99e..5b122b28 100644 --- a/crates/weavepy-jit/src/runtime.rs +++ b/crates/weavepy-jit/src/runtime.rs @@ -201,6 +201,17 @@ pub struct ObjLayout { pub split_values: i32, /// The byte that is nonzero once cells are shared between threads. pub cells_unguarded: usize, + /// A list pin: its discriminant byte's value, where the list pointer + /// and the element lane (a [`crate::JitType`] byte) sit in it. + pub pin_list_tag: u8, + pub pin_list: i32, + pub pin_list_elem: i32, + /// A list pointer → its cell's borrow counter (`i32`), and its items' + /// buffer pointer, length and capacity. + pub list_borrow: i32, + pub list_ptr: i32, + pub list_len: i32, + pub list_cap: i32, } static OBJ_LAYOUT: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index 0cc0053e..fecf6478 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -1489,10 +1489,6 @@ impl Interpreter { } match recv { Object::Instance(inst) => { - // (Native plans read fields in line once the layout is - // known.) - #[cfg(feature = "jit")] - crate::tier2::ensure_obj_layout(recv); if GETTER && inst.cls_raw().native_kind.get() != 0 { return None; } diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index 836d5d07..a9b54053 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -673,6 +673,10 @@ impl Engine { flags.set("use_colocated_libcalls", "false").ok()?; flags.set("is_pic", "false").ok()?; flags.set("opt_level", "speed").ok()?; + if std::env::var_os("WEAVEPY_JIT_QUICK").is_some() { + flags.set("opt_level", "none").ok()?; + flags.set("regalloc_algorithm", "single_pass").ok()?; + } if !cfg!(debug_assertions) && !verify() { flags.set("enable_verifier", "false").ok()?; } @@ -712,6 +716,7 @@ fn compile( plan: &LeafPlan, effect: bool, ) -> Option { + crate::tier2::ensure_obj_layout(); let mut guard = ENGINE .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 31bfbe5e..9b8e0bca 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -1237,6 +1237,7 @@ impl JitState { _ => None, } }; + ensure_obj_layout(); let r = engine.compile_frame_direct(code, &mut classify, &mut jit_probes, &mut direct); if let Some(t0) = t0 { eprintln!("jit compile-time {:?} {:?}", code.name, t0.elapsed()); @@ -2703,16 +2704,24 @@ fn split_index(cls: &TypeObject, storage: AttrStorage, name: &str) -> u32 { /// offset against `obj` (a live instance), and publish the layout. Once: /// later calls return straight away. #[inline] -pub(crate) fn ensure_obj_layout(obj: &Object) { +pub(crate) fn ensure_obj_layout() { static DONE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); if DONE.load(std::sync::atomic::Ordering::Relaxed) { return; } - let Object::Instance(inst) = obj else { - return; - }; DONE.store(true, std::sync::atomic::Ordering::Relaxed); - if let Some(layout) = obj_layout(obj, inst) { + publish_obj_layout(); +} + +#[cold] +#[inline(never)] +fn publish_obj_layout() { + // A private instance of `object` to measure against (never seen by + // Python code; nothing tracks it). + let cls = crate::builtin_types::builtin_types().object_.clone(); + let inst = Rc::new(crate::types::PyInstance::new(cls)); + let obj = Object::Instance(inst.clone()); + if let Some(layout) = obj_layout(&obj, &inst) { let _ = PUBLISHED_LAYOUT.set(layout); weavepy_jit::set_obj_layout(layout); } @@ -2797,14 +2806,56 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option = Vec::with_capacity(4); v.push(Pin::Obj(Object::None)); let vp = std::ptr::from_ref(&v).cast::(); let find = |want: usize| (0..3).map(|k| k * 8).find(|&off| word(vp, off) == want); - let (vec_ptr, vec_len) = (find(v.as_ptr() as usize)?, find(1)?); + let (vec_ptr, vec_len, vec_cap) = (find(v.as_ptr() as usize)?, find(1)?, find(v.capacity())?); drop(v); + // A list pin: the list's `Arc` allocation and its element lane. + let list = Rc::new(GilRefCell::new(vec![Object::Int(1)])); + let list_arc = { + // SAFETY: `Rc` is one pointer to its allocation. + unsafe { std::mem::transmute_copy::>>, usize>(&list) } + }; + let sample = Pin::List(list.clone(), JitType::Float); + let sample_p = std::ptr::from_ref(&sample).cast::(); + // SAFETY: the discriminant byte of a `repr(C, u8)` enum. + let pin_list_tag = unsafe { *sample_p }; + let (pin_list, pin_list_elem) = match &sample { + Pin::List(l, e) => ( + std::ptr::from_ref(l) as usize - sample_p as usize, + std::ptr::from_ref(e) as usize - sample_p as usize, + ), + Pin::Obj(_) => return None, + }; + // SAFETY: the element lane's byte. + let lane_byte = unsafe { *sample_p.add(pin_list_elem) }; + if word(sample_p, pin_list) != list_arc + || lane_byte != JitType::Float as u8 + || pin_list_tag == pin_obj_tag + || std::mem::size_of::() != 1 + { + return None; + } + drop(sample); + let list_data = list_arc_data(&list, list_arc)?; + let list_borrow = list_data + GilCell::>::BORROW_OFFSET; + let list_vec = list_data + GilCell::>::DATA_OFFSET; + let list_p = list_arc as *const u8; + // SAFETY: as `word`, the `i32` borrow counter. + let list_free = unsafe { list_p.add(list_borrow).cast::().read_unaligned() } == 0; + let items = list.borrow(); + if !list_free + || word(list_p, list_vec + vec_ptr) != items.as_ptr() as usize + || word(list_p, list_vec + vec_len) != items.len() + || word(list_p, list_vec + vec_cap) != items.capacity() + { + return None; + } + drop(items); let guards: StdRc> = StdRc::new(Vec::with_capacity(2)); let rc_box = { // SAFETY: `std::rc::Rc` is one pointer to its allocation. @@ -2843,9 +2894,21 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option>>, arc: usize) -> Option { + (Rc::as_ptr(list) as usize).checked_sub(arc) +} + /// RFC 0069 WS1 — the compile-time method probe: resolve `name` on the /// class of the instance currently in local `slot`, when the shape is /// eligible for a burned-in method call: @@ -2965,7 +3028,6 @@ fn attr_fingerprint_obj( let Object::Instance(inst) = obj else { return None; }; - ensure_obj_layout(obj); // RFC 0070 WS3 / RFC 0071 WS2 — the tier-1 predicate classifies // the storage: an indexed instance-dict hit, a `__slots__` member // (read and written through the slot side table by name), or the diff --git a/tests/regrtest/test_jit_inline_lists.py b/tests/regrtest/test_jit_inline_lists.py new file mode 100644 index 00000000..c46f72e9 --- /dev/null +++ b/tests/regrtest/test_jit_inline_lists.py @@ -0,0 +1,126 @@ +"""Compiled code reads, writes, appends to and iterates scalar lists in line. + +Each loop runs long enough for the tier-2 compiler to take it; the lists +then change shape under the compiled code, and every change must reach +the generic path with the interpreter's result. +""" + +import unittest + + +def total(xs, n): + t = 0 + for i in range(n): + t += xs[i % len(xs)] + xs[-1] + return t + + +def scale(xs, n, k): + for i in range(n): + j = i % len(xs) + xs[j] = xs[j] * k + return xs + + +def fill(n): + out = [] + for i in range(n): + out.append(i * 2) + return out + + +def fill_floats(n): + out = [] + for i in range(n): + out.append(i * 0.5) + return out + + +def walk(xs): + t = 0 + for x in xs: + t += x + return t + + +class InlineListTests(unittest.TestCase): + N = 3000 + + def test_reads(self): + xs = list(range(10)) + self.assertEqual(total(xs, self.N), sum(xs[i % 10] + 9 for i in range(self.N))) + + def test_out_of_range(self): + xs = [1, 2, 3] + + def get(xs, n, idx): + t = 0 + for _ in range(n): + t += xs[idx] + return t + + self.assertEqual(get(xs, self.N, -3), self.N) + with self.assertRaises(IndexError): + get(xs, 1, 3) + with self.assertRaises(IndexError): + get(xs, 1, -4) + + def test_element_type_changes(self): + xs = list(range(10)) + self.assertEqual(total(xs, self.N), total(list(range(10)), self.N)) + xs[3] = 2.5 + self.assertEqual(total(xs, 10), sum(xs) + 9 * 10) + xs[3] = "a" + with self.assertRaises(TypeError): + total(xs, 10) + + def test_writes(self): + xs = [1.0] * 8 + scale(xs, self.N, 1.0001) + ref = [1.0] * 8 + for i in range(self.N): + ref[i % 8] = ref[i % 8] * 1.0001 + self.assertEqual(xs, ref) + + def test_write_over_heap_element(self): + xs = [1.0] * 4 + scale(xs, self.N, 1.0) + marker = [1.0] + xs[1] = marker + with self.assertRaises(TypeError): + scale(xs, 4, 2.0) + + def test_appends_grow(self): + self.assertEqual(fill(self.N), [i * 2 for i in range(self.N)]) + self.assertEqual(fill_floats(self.N), [i * 0.5 for i in range(self.N)]) + self.assertEqual(fill(5), [0, 2, 4, 6, 8]) + + def test_iteration(self): + xs = list(range(self.N)) + self.assertEqual(walk(xs), sum(xs)) + self.assertEqual(walk([]), 0) + ys = [1, 2, 3.5] + self.assertEqual(walk(ys), 6.5) + with self.assertRaises(TypeError): + walk([1, "x"]) + + def test_iteration_sees_growth(self): + def grow(xs, limit): + n = 0 + for x in xs: + n += 1 + if len(xs) < limit: + xs.append(x + 1) + return n + + xs = [0] + self.assertEqual(grow(xs, self.N), self.N) + self.assertEqual(xs[-1], self.N - 1) + + def test_bools(self): + flags = [True, False] * 100 + self.assertEqual(walk(flags * 10), 1000) + + +if __name__ == "__main__": + unittest.main() From 01ade8dbb7633d85b6cf73b566ffe5b8b775bf94 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:17:55 -0700 Subject: [PATCH 15/96] perf: memoise str lengths in the string and build string results in place `str` payloads get their own header word: the code-point count, computed on first use or known at birth. It replaces the thread-local length and ASCII caches (whose one-slot ASCII cache missed on every new string, and whose length cache held strong references to up to 1,024 strings), so `len(s)`, index and slice bounds, and every ASCII check are O(1) reads. Strings derived from ASCII text (split fields, case mappings, slices, joins of counted parts, replacements) are born knowing their counts. The string builders write results straight into the final allocation through a sequential writer, with no intermediate `String`, zeroing, or second copy, and copy short pieces inline rather than calling `memcpy`: - `upper`/`lower`/`casefold`/`title`/`capitalize`/`swapcase` of ASCII text map whole buffers with branch-free, vectorized loops. - `join` of exact strings sizes the result once and stores a one-byte separator directly. - `replace` records its matches in one `memchr` scan and builds an exact-size result, returning the receiver itself when nothing matched (as CPython does); `split` with a one-byte separator scans with `memchr`, and an unsplit receiver is returned as the list's item. - `split()` presizes for prose, settles most bytes with one comparison, and defers its all-`str` result list from GC tracking at any length. - `%`-formatting borrows an exact `str` template, copies literal runs at once, and counts the error index only when raising (which now names a non-ASCII conversion character by its code point, as CPython does). A text or bytes payload's last owner frees it with plain loads while the reference-count bias holds, instead of `Arc`'s two locked decrements. str_methods (WORK=15000 vs 3000, retired instructions per unit): 41,839 -> 27,002. Wall time on a loaded machine (min of 9, interleaved): python3.14 97.2 ms, before 114.0 ms (1.17x), after 67.6 ms (0.70x). --- crates/weavepy-vm/src/builtins.rs | 335 +++++++++++++---- crates/weavepy-vm/src/gc_trace.rs | 14 + crates/weavepy-vm/src/lib.rs | 51 ++- crates/weavepy-vm/src/object.rs | 45 +-- crates/weavepy-vm/src/rc.rs | 49 +++ crates/weavepy-vm/src/shared_value.rs | 506 ++++++++++++++++++++------ crates/weavepy-vm/src/stdlib/sys.rs | 2 +- crates/weavepy-vm/src/unicode_case.rs | 117 ++++-- 8 files changed, 867 insertions(+), 252 deletions(-) diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 71b3ef99..5cd9bd8f 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -2662,7 +2662,7 @@ fn slot_sizeof(args: &[Object]) -> Result { // is 56 bytes + (len+1) units of the kind width (1 for latin-1, // 2 for BMP, 4 beyond). Object::Str(s) => { - let len = str_char_len(s) as i64; + let len = crate::object::str_char_len(s) as i64; let max_cp = s.chars().map(u32::from).max().unwrap_or(0); match max_cp { 0..=0x7f => 40 + len + 1, @@ -10777,14 +10777,41 @@ fn str_arity(name: &str, args: &[Object], min: usize, max: usize) -> Result<(), Ok(()) } +/// An ASCII `str` receiver's case mapping, written straight into the +/// result, which is ASCII too and born knowing its count; `None` for any +/// other receiver. +fn ascii_case_result(args: &[Object], case: crate::unicode_case::AsciiCase) -> Option { + let Some(Object::Str(s)) = args.first() else { + return None; + }; + if !SharedStr::is_ascii(s) { + return None; + } + // SAFETY: an ASCII case mapping of ASCII text is ASCII of the same + // length, all of it written (see `ascii_case_into`). + Some(Object::Str(unsafe { + SharedStr::build_unchecked(s.len(), Some(s.len()), |out| { + out.push_with(s.len(), |buf| { + crate::unicode_case::ascii_case_into(case, s.as_bytes(), buf); + }); + }) + })) +} + fn str_upper(args: &[Object]) -> Result { str_arity("upper", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Upper) { + return Ok(out); + } let up = crate::unicode_case::upper(&str_self(args)?); Ok(str_result(args, up)) } fn str_lower(args: &[Object]) -> Result { str_arity("lower", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Lower) { + return Ok(out); + } let lo = crate::unicode_case::lower(&str_self(args)?); Ok(str_result(args, lo)) } @@ -10794,6 +10821,9 @@ fn str_lower(args: &[Object]) -> Result { /// and folding is context-free (no Greek final-sigma special-casing). fn str_casefold(args: &[Object]) -> Result { str_arity("casefold", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Lower) { + return Ok(out); + } let out = crate::unicode_case::casefold(&str_self(args)?); Ok(str_result(args, out)) } @@ -10850,33 +10880,38 @@ fn ascii_space_table() -> &'static [bool; 128] { TABLE.get_or_init(|| std::array::from_fn(|i| crate::unicode_case::is_space(i as u8 as char))) } -fn str_split_whitespace(s: &str, maxsplit: i64) -> Vec { +fn str_split_whitespace(s: &str, ascii: bool, maxsplit: i64) -> Vec { use crate::unicode_case::is_space; - // Presized: `collect()` over a filtered split carries no size hint, - // so the vector grew several times for a sentence-length string. - let mut out = Vec::with_capacity((s.len() / 8).min(64) + 1); + // Presized: `collect()` over a filtered split carries no size hint. + // English prose averages under five letters a word, so a quarter of + // the length covers a sentence without growing (each growth copies). + let mut out = Vec::with_capacity((s.len() / 4).min(256) + 1); let limit = if maxsplit < 0 { i64::MAX } else { maxsplit }; let mut splits = 0i64; - if str_is_ascii_cached(s) { + if ascii { + // Fields of ASCII text are ASCII: each is born knowing its count. let table = ascii_space_table(); + // Every ASCII space is at most `' '`, so most bytes are settled + // by one comparison. + let is_space = |c: u8| c <= b' ' && table[usize::from(c)]; let b = s.as_bytes(); let mut i = 0; loop { - while i < b.len() && table[b[i] as usize] { + while i < b.len() && is_space(b[i]) { i += 1; } if i == b.len() { break; } if splits >= limit { - out.push(Object::Str(SharedStr::from(&s[i..]))); + out.push(Object::Str(SharedStr::from_ascii(&s[i..]))); break; } let start = i; - while i < b.len() && !table[b[i] as usize] { + while i < b.len() && !is_space(b[i]) { i += 1; } - out.push(Object::Str(SharedStr::from(&s[start..i]))); + out.push(Object::Str(SharedStr::from_ascii(&s[start..i]))); splits += 1; } return out; @@ -10903,8 +10938,14 @@ fn str_split_whitespace(s: &str, maxsplit: i64) -> Vec { /// initial elements are all strings. Register them before returning /// through either the interpreter or a native string helper. fn new_str_split_list(items: Vec) -> Object { + // Plain `str` fields can't close a cycle yet, however many there are. + let inert = items.iter().all(|o| matches!(o, Object::Str(_))); let result = Object::new_list(items); - crate::gc_trace::track(&result); + if inert { + crate::gc_trace::with_state(|s| s.track_inert(&result)); + } else { + crate::gc_trace::track(&result); + } result } @@ -10928,28 +10969,57 @@ fn str_split(args: &[Object], kwargs: &[(String, Object)]) -> Result = match sep { - None | Some(Object::None) => str_split_whitespace(&s, maxsplit), + let (_, ascii) = recv_char_len(args, &s); + let mut out: Vec = match sep { + None | Some(Object::None) => str_split_whitespace(&s, ascii, maxsplit), Some(sep_obj) => { let sep = str_arg_bridged(sep_obj) .ok_or_else(|| type_error("must be str or None, not other"))?; if sep.is_empty() { return Err(value_error("empty separator")); } - if maxsplit < 0 { - s.split(&*sep) - .map(|s| Object::Str(SharedStr::from(s))) - .collect() - } else { - s.splitn((maxsplit as usize).saturating_add(1), &*sep) - .map(|s| Object::Str(SharedStr::from(s))) - .collect() - } + str_split_sep(&s, ascii, &sep, maxsplit) } }; + // CPython returns an exact `str` that needed no splitting as the + // list's one item itself, not a copy. + if let ([Object::Str(only)], Some(recv @ Object::Str(_))) = (&out[..], args.first()) { + if only.len() == s.len() { + out[0] = recv.clone(); + } + } Ok(new_str_split_list(wrap(out))) } +/// `s.split(sep, maxsplit)` for a non-empty separator. A one-byte +/// separator scans with `memchr` rather than `str::split`'s searcher, +/// whose setup costs more than splitting a short line; fields of ASCII +/// text are born knowing their counts. +fn str_split_sep(s: &str, ascii: bool, sep: &str, maxsplit: i64) -> Vec { + let piece = |text: &str| { + Object::Str(if ascii { + SharedStr::from_ascii(text) + } else { + SharedStr::from(text) + }) + }; + let limit = usize::try_from(maxsplit).unwrap_or(usize::MAX); + if let [byte] = *sep.as_bytes() { + let mut out = Vec::new(); + let mut start = 0; + for at in memchr::memchr_iter(byte, s.as_bytes()) { + if out.len() == limit { + break; + } + out.push(piece(&s[start..at])); + start = at + 1; + } + out.push(piece(&s[start..])); + return out; + } + s.splitn(limit.saturating_add(1), sep).map(piece).collect() +} + fn str_join(args: &[Object]) -> Result { use std::borrow::Cow; @@ -10960,10 +11030,10 @@ fn str_join(args: &[Object]) -> Result { // Exact built-in sequences of exact strings need no iteration hooks, // reference clones, or intermediate copies. Hold a list's read guard // across both passes so concurrent mutation cannot change its size. - if matches!(args.first(), Some(Object::Str(_))) { + if let Some(Object::Str(sep)) = args.first() { let joined = match &args[1] { - Object::List(items) => join_exact_strings(&items.borrow(), &sep), - Object::Tuple(items) => join_exact_strings(items, &sep), + Object::List(items) => join_exact_strings(&items.borrow(), sep), + Object::Tuple(items) => join_exact_strings(items, sep), _ => None, }; if let Some(joined) = joined { @@ -11017,27 +11087,50 @@ fn str_join(args: &[Object]) -> Result { } /// Return `None` for shapes that need the general Unicode/iterator path. -fn join_exact_strings(items: &[Object], sep: &str) -> Option { - let mut size = sep.len().checked_mul(items.len().saturating_sub(1))?; +/// The result is written straight into its final allocation, and knows its +/// code-point count when every part (and the separator) already does. +fn join_exact_strings(items: &[Object], sep: &SharedStr) -> Option { + let gaps = items.len().saturating_sub(1); + let mut size = sep.len().checked_mul(gaps)?; + // The separator is usually a constant: counting it once memoises it. + let mut chars = SharedStr::char_count(sep).checked_mul(gaps); for item in items { let Object::Str(s) = item else { return None; }; size = size.checked_add(s.len())?; + chars = chars.and_then(|n| Some(n + SharedStr::known_char_count(s)?)); } if items.len() == 1 { return Some(items[0].clone()); } - let mut joined = String::with_capacity(size); - for (i, item) in items.iter().enumerate() { - if i != 0 { - joined.push_str(sep); + let fill = |out: &mut crate::shared_value::TextWriter<'_>| { + let mut items = items.iter().filter_map(|item| match item { + Object::Str(s) => Some(s.as_bytes()), + _ => None, + }); + if let Some(first) = items.next() { + out.push(first); } - if let Object::Str(s) = item { - joined.push_str(s); + // The usual one-byte separator (`" "`, `","`) is stored directly. + if let [byte] = *sep.as_bytes() { + for item in items { + out.push_byte(byte); + out.push(item); + } + } else { + for item in items { + out.push(sep.as_bytes()); + out.push(item); + } } - } - Some(Object::from_str(joined)) + }; + // SAFETY: a concatenation of `str`s is UTF-8, `size` bytes long (the + // caller holds the list's guard, so its items cannot change), and + // `chars` sums their counts. + Some(Object::Str(unsafe { + SharedStr::build_unchecked(size, chars, fill) + })) } fn str_startswith(args: &[Object]) -> Result { @@ -11182,6 +11275,94 @@ fn str_match_prefix_suffix( } } +/// `recv.replace(from, to, limit)` for an exact `str` receiver and a +/// non-empty `from`: one scan records the matches, so the result is +/// written straight into an allocation of its exact size, and a receiver +/// with no match is returned itself (CPython's `unicode_result_unchanged`). +/// `None` when the result's size overflows, for the general path to +/// report. +fn str_replace_exact(recv: &SharedStr, from: &str, to: &str, limit: usize) -> Option { + /// Match offsets, on the stack for the usual handful. + struct Hits { + inline: [usize; 32], + len: usize, + spill: Vec, + } + impl Hits { + fn push(&mut self, at: usize) { + if self.len < self.inline.len() { + self.inline[self.len] = at; + } else { + if self.spill.is_empty() { + self.spill.extend_from_slice(&self.inline); + } + self.spill.push(at); + } + self.len += 1; + } + fn as_slice(&self) -> &[usize] { + if self.len <= self.inline.len() { + &self.inline[..self.len] + } else { + &self.spill + } + } + } + let (h, n) = (recv.as_bytes(), from.as_bytes()); + let mut hits = Hits { + inline: [0; 32], + len: 0, + spill: Vec::new(), + }; + if n.len() <= 16 && n[0].is_ascii() { + // A short needle that starts with an ASCII byte (never inside a + // multibyte character, so every hit is a boundary) is found by a + // `memchr` scan for that byte; comparing at most 16 bytes per + // candidate keeps it linear. + if let Some(last_start) = h.len().checked_sub(n.len()) { + let mut i = 0; + while i <= last_start && hits.len < limit { + let Some(off) = memchr::memchr(n[0], &h[i..=last_start]) else { + break; + }; + let at = i + off; + if short_bytes_eq(&h[at + 1..at + n.len()], &n[1..]) { + hits.push(at); + i = at + n.len(); + } else { + i = at + 1; + } + } + } + } else { + for (at, _) in recv.match_indices(from).take(limit) { + hits.push(at); + } + } + let hits = hits.as_slice(); + if hits.is_empty() { + return Some(Object::Str(recv.clone())); + } + let size = (h.len() - hits.len() * n.len()).checked_add(hits.len().checked_mul(to.len())?)?; + // ASCII text with ASCII replacements stays ASCII. + let chars = (SharedStr::is_ascii(recv) && to.is_ascii()).then_some(size); + let fill = |out: &mut crate::shared_value::TextWriter<'_>| { + let mut copied = 0; + for &at in hits { + out.push(&h[copied..at]); + out.push(to.as_bytes()); + copied = at + n.len(); + } + out.push(&h[copied..]); + }; + // SAFETY: the result splices whole `str`s at character boundaries + // (every match is one), `size` bytes in all; `chars` is given only + // when every byte is ASCII. + Some(Object::Str(unsafe { + SharedStr::build_unchecked(size, chars, fill) + })) +} + fn str_replace_kw(args: &[Object], kwargs: &[(String, Object)]) -> Result { let s = str_self(args)?; let positional = args.len().saturating_sub(1); @@ -11255,10 +11436,16 @@ fn str_replace_kw(args: &[Object], kwargs: &[(String, Object)]) -> Result 3631). s.replace(from, to) } else { s.replacen(from, to, count as usize) @@ -11454,15 +11641,10 @@ fn str_find(args: &[Object]) -> Result { Some(p) => p, None => return Err(type_error("find() expected str")), }; - // One ASCII probe for the whole call: each of the char/byte - // conversions below consults the same thread-local cache, and the - // TLS access costs more than the work it guards. - let ascii = str_is_ascii_cached(s); - let total_chars = if ascii { - s.len() as i64 - } else { - str_char_len(s) as i64 - }; + // One ASCII probe for the whole call, O(1) from the receiver's + // memoised count. + let (total_chars, ascii) = recv_char_len(args, s); + let total_chars = total_chars as i64; let Some((start, end)) = str_search_window(args, total_chars) else { return Ok(Object::Int(-1)); }; @@ -11488,6 +11670,17 @@ fn str_find(args: &[Object]) -> Result { } } +/// The method receiver's length in code points, and whether it is pure +/// ASCII (one byte per code point): O(1) from a `str`'s memoised count, a +/// scan of the bridged text for a `WStr`. +fn recv_char_len(args: &[Object], s: &str) -> (usize, bool) { + let n = match args.first() { + Some(Object::Str(recv)) => SharedStr::char_count(recv), + _ => s.chars().count(), + }; + (n, n == s.len()) +} + /// Whether `s` is pure ASCII, memoized by buffer identity. CPython's PEP 393 /// layout makes char↔byte offset mapping O(1); our UTF-8 `SharedStr` needs a /// scan. Callers like `str.find(sub, start)`-in-a-loop (email's @@ -11511,15 +11704,6 @@ pub(crate) fn str_is_ascii_cached(s: &str) -> bool { }) } -/// Total `len()` in code points, O(1) for (cached-)ASCII strings. -pub(crate) fn str_char_len(s: &str) -> usize { - if str_is_ascii_cached(s) { - s.len() - } else { - s.chars().count() - } -} - fn char_offset_to_byte(s: &str, n: usize) -> usize { if n == 0 { return 0; @@ -11539,18 +11723,27 @@ fn byte_offset_to_char(s: &str, byte: usize) -> usize { fn str_title(args: &[Object]) -> Result { str_arity("title", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Title) { + return Ok(out); + } let out = crate::unicode_case::title(&str_self(args)?); Ok(str_result(args, out)) } fn str_capitalize(args: &[Object]) -> Result { str_arity("capitalize", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Capitalize) { + return Ok(out); + } let out = crate::unicode_case::capitalize(&str_self(args)?); Ok(str_result(args, out)) } fn str_swapcase(args: &[Object]) -> Result { str_arity("swapcase", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Swap) { + return Ok(out); + } let out = crate::unicode_case::swapcase(&str_self(args)?); Ok(str_result(args, out)) } @@ -11733,17 +11926,27 @@ fn str_rfind(args: &[Object]) -> Result { Some(p) => p, None => return Err(type_error("rfind() expected str")), }; - let total_chars = str_char_len(s) as i64; - let Some((start, end)) = str_search_window(args, total_chars) else { + let (total_chars, ascii) = recv_char_len(args, s); + let Some((start, end)) = str_search_window(args, total_chars as i64) else { return Ok(Object::Int(-1)); }; - let start_byte = char_offset_to_byte(s, start as usize); - let end_byte = char_offset_to_byte(s, end as usize); + let (start_byte, end_byte) = if ascii { + ((start as usize).min(s.len()), (end as usize).min(s.len())) + } else { + ( + char_offset_to_byte(s, start as usize), + char_offset_to_byte(s, end as usize), + ) + }; let hay = &s[start_byte..end_byte]; match substr_rfind(hay, &sub) { Some(byte_idx) => { let abs_byte = byte_idx + start_byte; - Ok(Object::Int(byte_offset_to_char(s, abs_byte) as i64)) + Ok(Object::Int(if ascii { + abs_byte as i64 + } else { + byte_offset_to_char(s, abs_byte) as i64 + })) } None => Ok(Object::Int(-1)), } @@ -11779,12 +11982,8 @@ fn str_count(args: &[Object]) -> Result { None => return Err(type_error("count() expected str")), }; // One ASCII probe for the whole call, as `str.find` does. - let ascii = str_is_ascii_cached(s); - let total_chars = if ascii { - s.len() as i64 - } else { - str_char_len(s) as i64 - }; + let (total_chars, ascii) = recv_char_len(args, s); + let total_chars = total_chars as i64; let Some((start, end)) = str_search_window(args, total_chars) else { return Ok(Object::Int(0)); }; @@ -16290,7 +16489,7 @@ pub(crate) fn file_readlines(args: &[Object]) -> Result { loop { let line = file_readline(&[Object::File(f.clone())])?; let len = match &line { - Object::Str(s) => str_char_len(s), + Object::Str(s) => crate::object::str_char_len(s), Object::WStr(cps) => cps.len(), Object::Bytes(b) => b.len(), _ => 0, diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 671c4600..a44cb4c5 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -595,6 +595,20 @@ impl GcState { }) } + /// [`Self::track`] for a container its builder knows holds only + /// atomic values (a `str.split` result): deferred whatever its size, + /// without the element scan, which `track` caps at `SCAN_CAP` + /// elements and so registers every longer list eagerly. + pub fn track_inert(&self, obj: &Object) { + debug_assert!(match obj { + Object::List(l) => l.borrow().iter().all(element_is_inert), + _ => false, + }); + if !self.defer_container(obj) { + self.track_now(obj); + } + } + /// Remember `obj` weakly instead of tracking it. Returns `false` when /// the kind has no deferred form (the caller tracks it as before). /// diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 544d188b..33014722 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -35975,9 +35975,11 @@ impl Interpreter { // Surrogate-aware printf: a `WStr` template (and any `WStr` // argument resolved via `%s`) bridges its lone surrogates into // the PUA window for the UTF-8 engine, then maps them back. - let template = match a { - Object::WStr(cps) => crate::builtins::bridge_encode_cps(cps), - _ => a.to_str(), + // An exact `str` template is borrowed, not copied per call. + let template: std::borrow::Cow<'_, str> = match a { + Object::Str(s) => std::borrow::Cow::Borrowed(s), + Object::WStr(cps) => crate::builtins::bridge_encode_cps(cps).into(), + _ => a.to_str().into(), }; let bridged = matches!(a, Object::WStr(_)) || percent_args_need_bridge(b); let mut resolve = |obj: &Object, kind: char| -> Result, RuntimeError> { @@ -54688,7 +54690,8 @@ pub(crate) fn str_subscript_slice(s: &SharedStr, slc: &PySlice) -> Result Result= bstop { return Ok(Object::from_static("")); } - return Ok(Object::from_str(s[bstart..bstop].to_string())); + // The whole string is the string itself (CPython's + // `PyUnicode_Substring` returns an exact `str` unchanged). + if bstop - bstart == blen { + return Ok(Object::Str(s.clone())); + } + let piece = &s[bstart..bstop]; + return Ok(Object::Str(if ascii { + SharedStr::from_ascii(piece) + } else { + SharedStr::from(piece) + })); } } // General path: negative bounds or non-unit step, over the code points. @@ -58451,9 +58464,12 @@ pub(crate) fn percent_format_with( return Err(value_error("incomplete format")); } } - // Codepoint index of the conversion char, for error messages. - let kind_index = template[..i].chars().count(); - let kind = bytes[i] as char; + // Byte offset of the conversion char; its code-point index (for + // error messages only) is counted when one is raised. + let kind_at = i; + // The whole character, so a non-ASCII one is reported by its + // code point rather than its first UTF-8 byte. + let kind = template[i..].chars().next().unwrap_or(char::from(bytes[i])); i += 1; // `%%` is a literal percent only when the two `%` are adjacent; // any intervening flag/width/precision/mapping makes the second @@ -58471,7 +58487,10 @@ pub(crate) fn percent_format_with( if mapping_key.is_none() && idx >= positional.len() { return Err(type_error("not enough arguments for format string")); } - return Err(unsupported_format_char('%', kind_index)); + return Err(unsupported_format_char( + '%', + template[..kind_at].chars().count(), + )); } out.push('%'); continue; @@ -58848,12 +58867,18 @@ pub(crate) fn percent_format_with( &spec.replace('c', "s"), )? } - _ => return Err(unsupported_format_char(kind, kind_index)), + _ => { + return Err(unsupported_format_char( + kind, + template[..kind_at].chars().count(), + )) + } }; out.push_str(&rendered); } else { - let ch_len = utf8_seq_len(bytes[i]); - let end = (i + ch_len).min(bytes.len()); + // The literal run up to the next `%` (ASCII, so always a + // character boundary), copied at once. + let end = memchr::memchr(b'%', &bytes[i..]).map_or(bytes.len(), |k| i + k); out.push_str(&template[i..end]); i = end; } @@ -62764,7 +62789,7 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result(); }; -/// Number of slots in the per-thread code-point-length cache. -const STR_LEN_CACHE_CAP: usize = 1024; - -thread_local! { - /// Direct-mapped cache of `SharedStr` heap identity -> code-point length. - /// - /// WeavePy stores `str` as UTF-8, so counting code points — needed by - /// `len(s)`, index/slice bounds, and `re` span clamping — is O(n). - /// Scanners and tokenisers touch the *same* large string repeatedly, so - /// without memoisation an otherwise-linear pass degrades to O(n^2) (the - /// cause of `test_json`'s deep-recursion cases hanging). Each slot holds - /// the source `Rc` so a freed allocation cannot be reused at the same - /// address and alias a stale length; collisions simply recompute and - /// overwrite, so the cache is always correct and bounded to CAP entries. - static STR_LEN_CACHE: std::cell::RefCell>> = - std::cell::RefCell::new(vec![None; STR_LEN_CACHE_CAP]); -} - -/// Code-point length of a UTF-8 `str`, memoised by heap identity. O(1) on a -/// cache hit; O(n) (and caches the result) on a miss. Strings of 0 or 1 -/// bytes are counted directly — they are necessarily 0 or 1 code points, so -/// caching them would only evict useful entries. +/// Code-point length of a `str`: O(1) from the count memoised in the +/// string itself (see [`SharedStr::char_count`]). WeavePy stores `str` as +/// UTF-8, so the first count is a scan; scanners and tokenisers that ask +/// for the length of the same large string repeatedly stay linear. +#[inline] pub(crate) fn str_char_len(s: &SharedStr) -> usize { - let bytes = s.len(); - if bytes <= 1 { - return bytes; - } - let ptr = SharedStr::as_ptr(s).cast::() as usize; - let slot = (ptr / 8) % STR_LEN_CACHE_CAP; - STR_LEN_CACHE.with(|c| { - let mut cache = c.borrow_mut(); - if let Some((p, _, n)) = &cache[slot] { - if *p == ptr { - return *n; - } - } - let n = s.chars().count(); - cache[slot] = Some((ptr, s.clone(), n)); - n - }) + SharedStr::char_count(s) } /// A Python value as seen by the interpreter. diff --git a/crates/weavepy-vm/src/rc.rs b/crates/weavepy-vm/src/rc.rs index c2289ee4..38f45863 100644 --- a/crates/weavepy-vm/src/rc.rs +++ b/crates/weavepy-vm/src/rc.rs @@ -119,6 +119,55 @@ pub(crate) unsafe fn try_release_shared(value: *const T) -> bool { true } +/// Free the `Arc` allocation holding `value` when this is its last owner +/// and no weak reference exists, with plain loads instead of `Arc`'s two +/// locked decrements (strong, then the implicit weak). Only for payloads +/// with no destructor, so nothing runs while the counts still read one. +/// Returns `false`, having changed nothing, when another owner or a weak +/// reference exists (or the bias is off); the caller must then drop an +/// `Arc` normally. +/// +/// While the bias holds, this thread is the only one that can touch the +/// counts, and with no weak reference nobody else can reach the payload. +/// +/// # Safety +/// +/// `value` must point at the payload of a live `Arc` allocation, and the +/// caller must own one of its strong references, which this consumes on +/// success. +#[inline(always)] +pub(crate) unsafe fn try_free_last(value: *const T) -> bool { + debug_assert!(!std::mem::needs_drop::()); + if !refcounts_biased() { + return false; + } + // SAFETY: live allocation, per the caller; the weak count is the + // word after the strong count (`ArcInner` is `repr(C)`). + let (strong, weak) = unsafe { + let strong = strong_word(value); + (&*strong, &*strong.add(1)) + }; + if strong.load(Ordering::Relaxed) != 1 || weak.load(Ordering::Relaxed) != 1 { + return false; + } + // `Arc` sizes its allocation as the two counters followed by the + // payload's layout, padded (`arcinner_layout_for_value_layout`). + // SAFETY: as above; the payload is live until the deallocation. + let payload = std::alloc::Layout::for_value(unsafe { &*value }); + let (inner, _) = std::alloc::Layout::new::<[usize; 2]>() + .extend(payload) + .expect("Arc layout"); + // SAFETY: the sole owner frees the allocation it owns, with the layout + // `Arc` allocated it with; the payload needs no drop. + unsafe { + std::alloc::dealloc( + ptr::from_ref(strong).cast::().cast_mut(), + inner.pad_to_align(), + ); + } + true +} + /// Convert an [`Rc`] to one of an unsized type, such as a trait object: /// `rc_unsize!(rc => dyn Trait)`. Stable Rust only coerces its own smart /// pointers, so the conversion passes through `Arc`. diff --git a/crates/weavepy-vm/src/shared_value.rs b/crates/weavepy-vm/src/shared_value.rs index e282dc88..dd2719c3 100644 --- a/crates/weavepy-vm/src/shared_value.rs +++ b/crates/weavepy-vm/src/shared_value.rs @@ -8,11 +8,11 @@ use std::borrow::Borrow; use std::fmt; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; -use std::mem::{align_of, size_of, ManuallyDrop}; +use std::mem::{align_of, size_of, ManuallyDrop, MaybeUninit}; use std::ops::Deref; use std::ptr::{self, NonNull}; use std::sync::{ - atomic::{AtomicI64, AtomicU64, Ordering}, + atomic::{AtomicI64, AtomicU64, AtomicUsize, Ordering}, Arc, Weak, }; @@ -134,9 +134,15 @@ impl Drop for ThinArc { // SAFETY: release this owner's one strong reference exactly once. // Arc drops the payload and handles remaining weak references normally. unsafe { - if !crate::rc::try_release_shared(self.raw()) { - T::release_last(Arc::from_raw(self.raw())); + if crate::rc::try_release_shared(self.raw()) { + return; } + // Text and byte payloads have no destructor: their last owner + // can free them without `Arc`'s locked decrements. + if !std::mem::needs_drop::() && crate::rc::try_free_last(self.raw()) { + return; + } + T::release_last(Arc::from_raw(self.raw())) } } } @@ -337,44 +343,148 @@ impl SharedSlice { } } } -impl SharedSlice { - /// A byte slice of `len` bytes written by `fill` (which receives a - /// zeroed buffer of exactly that length): one allocation, no - /// intermediate `Vec`. - // The backing allocation comes from `Arc::<[usize]>::new_zeroed_slice`, +impl From> for SharedSlice { + fn from(items: Vec) -> Self { + Self::from_vec_owned(items) + } +} +impl From> for SharedSlice { + fn from(items: Box<[T]>) -> Self { + Self::from_vec_owned(items.into_vec()) + } +} +impl From<&[T]> for SharedSlice { + fn from(items: &[T]) -> Self { + Self::from_copy_slice(items) + } +} +impl From<&[T; N]> for SharedSlice { + fn from(items: &[T; N]) -> Self { + Self::from_copy_slice(items) + } +} + +/// The code-point count of a [`StrStorage`] not yet counted. +const CHARS_UNKNOWN: usize = usize::MAX; + +/// UTF-8 text payload: [`SliceStorage`]'s length and memoised hash, plus +/// the memoised code-point count (`CHARS_UNKNOWN` until first needed). +/// CPython's PEP 393 strings know their length in code points, and with +/// it whether they are ASCII; this word gives `len(s)`, index and slice +/// bounds, and every ASCII fast path the same O(1) answer, and lets a +/// string derived from ASCII text (a split field, a case mapping, a join +/// of ASCII parts) be born knowing it. +#[repr(C)] +#[derive(Debug)] +pub struct StrStorage { + len: usize, + hash: AtomicI64, + chars: AtomicUsize, + bytes: [u8], +} +impl sealed::Sealed for StrStorage {} +// SAFETY: repr(C) places the immutable usize first; the byte tail carries +// len. The constructors below initialize the header and exactly len bytes. +unsafe impl ThinPayload for StrStorage { + type View = [u8]; + // `data` is the word-aligned header the byte view was cast from. + #[allow(clippy::cast_ptr_alignment)] + fn pointer(data: *const usize, len: usize) -> *const Self { + ptr::slice_from_raw_parts(data.cast::(), len) as *const Self + } + fn view(&self) -> &[u8] { + &self.bytes + } +} + +/// Copy `len` bytes from `src` to `dst`. Most strings are a few words +/// long (a split field, a dict key), and for them two overlapping word +/// moves beat a call to the platform `memcpy` through its stub. +/// +/// # Safety +/// +/// As [`ptr::copy_nonoverlapping`]. +#[inline(always)] +unsafe fn copy_text(src: *const u8, dst: *mut u8, len: usize) { + // SAFETY: every access below lies within the first `len` bytes of + // both buffers (the caller's contract). + unsafe { + if len >= 16 { + ptr::copy_nonoverlapping(src, dst, len); + } else if len >= 8 { + let head = src.cast::().read_unaligned(); + let tail = src.add(len - 8).cast::().read_unaligned(); + dst.cast::().write_unaligned(head); + dst.add(len - 8).cast::().write_unaligned(tail); + } else if len >= 4 { + let head = src.cast::().read_unaligned(); + let tail = src.add(len - 4).cast::().read_unaligned(); + dst.cast::().write_unaligned(head); + dst.add(len - 4).cast::().write_unaligned(tail); + } else if len > 0 { + *dst = *src; + *dst.add(len / 2) = *src.add(len / 2); + *dst.add(len - 1) = *src.add(len - 1); + } + } +} + +impl StrStorage { + /// The padded payload layout for `len` bytes of text: three header + /// words, then the bytes, rounded up to whole words. + fn layout(len: usize) -> Layout { + Layout::new::<[usize; 3]>() + .extend(Layout::array::(len).expect("string exceeds layout limit")) + .expect("string exceeds layout limit") + .0 + .pad_to_align() + } + + /// A payload of `len` bytes whose contents `init` writes, given the + /// (uninitialized) destination. One allocation, no intermediate `Vec`. + /// + /// # Safety + /// + /// `init` must initialize all `len` bytes with UTF-8 text, and + /// `chars`, when not `CHARS_UNKNOWN`, must be its code-point count. + // The backing allocation comes from `Arc::<[usize]>::new_uninit_slice`, // so it carries `usize` alignment by construction; the casts below // reinterpret it as the `usize`-headed DST it was sized for. #[allow(clippy::cast_ptr_alignment)] - pub fn build(len: usize, fill: impl FnOnce(&mut [u8])) -> Self { - let layout = Self::layout(len); - let words = layout.size() / size_of::(); - let storage = Arc::<[usize]>::new_zeroed_slice(words); - // SAFETY: an all-zero word slice is a valid `[usize]`; the header - // and bytes are (re)written below before any reader exists. - let storage = unsafe { storage.assume_init() }; + #[inline] + unsafe fn alloc(len: usize, chars: usize, init: impl FnOnce(*mut u8)) -> ThinArc { + let words = Self::layout(len).size() / size_of::(); + let storage = Arc::<[usize]>::new_uninit_slice(words); let data = Arc::into_raw(storage).cast::().cast_mut(); - let raw = ptr::slice_from_raw_parts_mut(data, len) as *mut SliceStorage; + let raw = ptr::slice_from_raw_parts_mut(data, len) as *mut Self; + // SAFETY: the word slice and the payload have identical padded + // size and alignment. The header is written here and the bytes by + // `init` (the caller's contract) before any reader can exist. unsafe { ptr::addr_of_mut!((*raw).len).write(len); ptr::addr_of_mut!((*raw).hash).write(AtomicI64::new(-1)); - let dst = ptr::addr_of_mut!((*raw).items).cast::(); - fill(std::slice::from_raw_parts_mut(dst, len)); - Self::from_arc(Arc::from_raw(raw)) + ptr::addr_of_mut!((*raw).chars).write(AtomicUsize::new(chars)); + init(ptr::addr_of_mut!((*raw).bytes).cast::()); + ThinArc::from_arc(Arc::from_raw(raw)) } } } -impl SharedSlice { - /// Append `extra` in place when this handle is the slice's only owner - /// (no other strong or weak reference): the allocation is resized - /// rather than copied, so repeated appends cost what `realloc` does - /// (CPython's `PyUnicode_Append` on a lone reference). `false` leaves - /// `this` untouched. - // The backing allocation comes from `Arc::<[usize]>::new_zeroed_slice`, - // so it carries `usize` alignment by construction; the casts below - // reinterpret it as the `usize`-headed DST it was sized for. +impl ThinArc { + #[inline] + fn storage(&self) -> &StrStorage { + // SAFETY: a live payload borrowed from `self`. + unsafe { &*self.raw() } + } + + /// Append `extra` (of `extra_chars` code points) in place when this + /// handle is the string's only owner (no other strong or weak + /// reference): the allocation is resized rather than copied, so + /// repeated appends cost what `realloc` does (CPython's + /// `PyUnicode_Append` on a lone reference). `false` leaves `this` + /// untouched. #[allow(clippy::cast_ptr_alignment)] - fn try_extend_unique(this: &mut Self, extra: &[u8]) -> bool { + fn try_extend_unique(this: &mut Self, extra: &[u8], extra_chars: usize) -> bool { if ThinArc::get_mut(this).is_none() { return false; } @@ -383,22 +493,29 @@ impl SharedSlice { let Some(new_len) = old_len.checked_add(extra.len()) else { return false; }; + let old_chars = this.storage().chars.load(Ordering::Relaxed); + let new_chars = if old_chars == CHARS_UNKNOWN { + CHARS_UNKNOWN + } else { + old_chars + extra_chars + }; // The `Arc` allocation is its two counters followed by the payload // (`ArcInner` is `repr(C)`, and std sizes it from the value layout // exactly this way), so the payload's offset and both sizes follow. let counters = Layout::new::<[usize; 2]>(); let (Ok((old_inner, offset)), Ok((new_inner, _))) = ( - counters.extend(Self::layout(old_len)), - counters.extend(Self::layout(new_len)), + counters.extend(StrStorage::layout(old_len)), + counters.extend(StrStorage::layout(new_len)), ) else { return false; }; let (old_inner, new_inner) = (old_inner.pad_to_align(), new_inner.pad_to_align()); // SAFETY: the sole owner (checked above) moves the allocation it owns // through the global allocator with the layout it was allocated - // with; the new length word and hash are written before any reader - // can exist, and the handle then points at the moved payload, whose - // eventual `Arc` drop computes the new layout from the new length. + // with; the new length, hash, and count are written before any + // reader can exist, and the handle then points at the moved + // payload, whose eventual `Arc` drop computes the new layout from + // the new length. unsafe { let base = this.data.as_ptr().cast::().sub(offset); let grown = std::alloc::realloc(base, old_inner, new_inner.size()); @@ -406,11 +523,11 @@ impl SharedSlice { return false; } let data = grown.add(offset).cast::(); - let raw = - ptr::slice_from_raw_parts_mut(data.cast::(), new_len) as *mut SliceStorage; + let raw = ptr::slice_from_raw_parts_mut(data.cast::(), new_len) as *mut StrStorage; ptr::addr_of_mut!((*raw).len).write(new_len); (*raw).hash.store(-1, Ordering::Relaxed); - let dst = ptr::addr_of_mut!((*raw).items).cast::(); + (*raw).chars.store(new_chars, Ordering::Relaxed); + let dst = ptr::addr_of_mut!((*raw).bytes).cast::(); ptr::copy_nonoverlapping(extra.as_ptr(), dst.add(old_len), extra.len()); this.data = NonNull::new_unchecked(data); } @@ -418,63 +535,11 @@ impl SharedSlice { } } -impl SharedStr { - /// Append `suffix` in place when this handle is the string's only - /// owner (see `SharedSlice::try_extend_unique`); `false` leaves it - /// untouched. - pub fn try_append(this: &mut Self, suffix: &str) -> bool { - SharedSlice::try_extend_unique(&mut this.0, suffix.as_bytes()) - } - - /// The concatenation of `parts`, in one allocation. - pub fn concat(parts: &[&str]) -> Self { - let len = parts.iter().map(|p| p.len()).sum(); - Self(SharedSlice::build(len, |buf| { - let mut at = 0; - for p in parts { - buf[at..at + p.len()].copy_from_slice(p.as_bytes()); - at += p.len(); - } - })) - } - - /// `s` repeated `times` times, in one allocation. - pub fn repeat(s: &str, times: usize) -> Self { - let len = s.len() * times; - Self(SharedSlice::build(len, |buf| { - for chunk in buf.chunks_exact_mut(s.len().max(1)) { - chunk.copy_from_slice(s.as_bytes()); - } - })) - } -} - -impl From> for SharedSlice { - fn from(items: Vec) -> Self { - Self::from_vec_owned(items) - } -} -impl From> for SharedSlice { - fn from(items: Box<[T]>) -> Self { - Self::from_vec_owned(items.into_vec()) - } -} -impl From<&[T]> for SharedSlice { - fn from(items: &[T]) -> Self { - Self::from_copy_slice(items) - } -} -impl From<&[T; N]> for SharedSlice { - fn from(items: &[T; N]) -> Self { - Self::from_copy_slice(items) - } -} - /// Shared UTF-8 text. Its private byte owner can only be constructed from str. #[derive(Clone, Eq, PartialEq, Ord, PartialOrd)] -pub struct SharedStr(SharedSlice); +pub struct SharedStr(ThinArc); #[derive(Debug)] -pub struct WeakStr(WeakSlice); +pub struct WeakStr(ThinWeak); impl SharedStr { pub fn as_ptr(this: &Self) -> *const str { ptr::from_ref(&**this) @@ -482,13 +547,43 @@ impl SharedStr { pub fn ptr_eq(this: &Self, other: &Self) -> bool { ThinArc::ptr_eq(&this.0, &other.0) } - /// The memoised `hash(s)` (see [`SharedSlice::hash_cached`]). + /// The memoised `hash(s)`, computed on first use the way CPython's + /// `str` caches its hash: a dict key hashes once per object, not once + /// per lookup. #[inline] pub fn hash_cached(this: &Self) -> i64 { - SharedSlice::hash_cached(&this.0, |bytes| { - // SAFETY: the private owner is only ever built from `str`. - crate::object::py_str_hash(unsafe { std::str::from_utf8_unchecked(bytes) }) - }) + let storage = this.0.storage(); + let h = storage.hash.load(Ordering::Relaxed); + if h != -1 { + return h; + } + let h = crate::object::py_str_hash(this); + storage.hash.store(h, Ordering::Relaxed); + h + } + /// `len(s)` in code points, memoised in the string itself. + #[inline] + pub fn char_count(this: &Self) -> usize { + let storage = this.0.storage(); + let n = storage.chars.load(Ordering::Relaxed); + if n != CHARS_UNKNOWN { + return n; + } + let n = this.chars().count(); + storage.chars.store(n, Ordering::Relaxed); + n + } + /// The code-point count if it is already known, without counting. + #[inline] + pub fn known_char_count(this: &Self) -> Option { + let n = this.0.storage().chars.load(Ordering::Relaxed); + (n != CHARS_UNKNOWN).then_some(n) + } + /// Whether the text is pure ASCII (one byte per code point), from the + /// memoised count. + #[inline] + pub fn is_ascii(this: &Self) -> bool { + Self::char_count(this) == this.len() } pub fn strong_count(this: &Self) -> usize { ThinArc::strong_count(&this.0) @@ -499,23 +594,178 @@ impl SharedStr { pub fn downgrade(this: &Self) -> WeakStr { WeakStr(ThinArc::downgrade(&this.0)) } + + /// A copy of `text`, known to be pure ASCII (`debug_assert`ed): it is + /// born with its code-point count. + #[inline] + pub fn from_ascii(text: &str) -> Self { + debug_assert!(text.is_ascii()); + Self::with_count(text, text.len()) + } + + /// A copy of `text`, which the caller knows to hold `chars` code points. + #[inline] + pub fn with_count(text: &str, chars: usize) -> Self { + debug_assert_eq!(text.chars().count(), chars); + let len = text.len(); + // SAFETY: exactly `len` bytes of UTF-8 are copied, and `chars` + // counts them (the caller's word, checked in debug builds). + Self(unsafe { + StrStorage::alloc(len, chars, |dst| { + copy_text(text.as_ptr(), dst, len); + }) + }) + } + + /// A string of `len` bytes that `fill` writes, in order, through a + /// [`TextWriter`] over its final allocation: one allocation, no + /// intermediate `String`, no zeroing. `chars` is its code-point count + /// when the caller knows it. Panics unless `fill` writes exactly `len` + /// bytes. + /// + /// # Safety + /// + /// The bytes `fill` writes must be valid UTF-8, of `chars` code points + /// when `chars` is given. + #[inline] + pub unsafe fn build_unchecked( + len: usize, + chars: Option, + fill: impl FnOnce(&mut TextWriter<'_>), + ) -> Self { + // SAFETY: the writer initializes every byte it accounts for, and + // the assertion checks that it accounted for all `len`; the + // caller vouches for the text and its count. + let out = Self(unsafe { + StrStorage::alloc(len, chars.unwrap_or(CHARS_UNKNOWN), |dst| { + let mut writer = TextWriter { + buf: std::slice::from_raw_parts_mut(dst.cast::>(), len), + at: 0, + }; + fill(&mut writer); + assert_eq!(writer.at, len, "string builder wrote the wrong length"); + }) + }); + debug_assert!(std::str::from_utf8(out.0.as_ref()).is_ok()); + debug_assert!(chars.is_none_or(|n| n == out.chars().count())); + out + } + + /// Append `suffix` in place when this handle is the string's only + /// owner (see `ThinArc::::try_extend_unique`); `false` + /// leaves it untouched. + pub fn try_append(this: &mut Self, suffix: &str) -> bool { + // Keep a known count known: `len(s)` after `s += t` in a loop would + // otherwise recount the whole string each time. + let extra_chars = if this.0.storage().chars.load(Ordering::Relaxed) == CHARS_UNKNOWN { + 0 + } else { + suffix.chars().count() + }; + ThinArc::::try_extend_unique(&mut this.0, suffix.as_bytes(), extra_chars) + } + + /// The concatenation of `parts`, in one allocation. + pub fn concat(parts: &[&str]) -> Self { + let len = parts.iter().map(|p| p.len()).sum(); + // SAFETY: a concatenation of `str`s is UTF-8. + unsafe { + Self::build_unchecked(len, None, |out| { + for p in parts { + out.push(p.as_bytes()); + } + }) + } + } + + /// `s` repeated `times` times, in one allocation. + pub fn repeat(s: &str, times: usize) -> Self { + let len = s.len() * times; + // SAFETY: a repetition of a `str` is UTF-8. + unsafe { + Self::build_unchecked(len, None, |out| { + if !s.is_empty() { + for _ in 0..times { + out.push(s.as_bytes()); + } + } + }) + } + } +} + +/// The sequential writer [`SharedStr::build_unchecked`] hands its `fill`: +/// each write lands after the last, and the builder checks that the +/// writes covered the whole string. +#[derive(Debug)] +pub struct TextWriter<'a> { + buf: &'a mut [MaybeUninit], + at: usize, +} +impl TextWriter<'_> { + /// Append `bytes`. Panics past the end of the string. + #[inline] + pub fn push(&mut self, bytes: &[u8]) { + let end = self.at + bytes.len(); + assert!(end <= self.buf.len(), "string builder overflow"); + // SAFETY: `at..end` lies within the buffer (checked above), which + // can't overlap the borrowed `bytes`. + unsafe { + copy_text( + bytes.as_ptr(), + self.buf.as_mut_ptr().add(self.at).cast(), + bytes.len(), + ) + }; + self.at = end; + } + + /// Append one byte. Panics past the end of the string. + #[inline] + pub fn push_byte(&mut self, byte: u8) { + self.buf[self.at] = MaybeUninit::new(byte); + self.at += 1; + } + + /// Append `n` bytes that `write` initializes, given exactly those + /// bytes. Panics past the end of the string. + /// + /// # Safety + /// + /// `write` must initialize all `n` bytes it is given. + #[inline] + pub unsafe fn push_with(&mut self, n: usize, write: impl FnOnce(&mut [MaybeUninit])) { + let end = self.at + n; + write(&mut self.buf[self.at..end]); + self.at = end; + } } impl Deref for SharedStr { type Target = str; + #[inline] fn deref(&self) -> &str { - // SAFETY: only From can create the private byte owner. - // No mutable byte view is exposed; clones and weak upgrades preserve it. + // SAFETY: only the UTF-8 constructors above create the private + // byte owner. No mutable byte view is exposed; clones and weak + // upgrades preserve it. unsafe { std::str::from_utf8_unchecked(&self.0) } } } impl From<&str> for SharedStr { + #[inline] fn from(value: &str) -> Self { - Self(SharedSlice::from(value.as_bytes())) + let len = value.len(); + // SAFETY: exactly `len` bytes of UTF-8 are copied; the count is + // left for first use. + Self(unsafe { + StrStorage::alloc(len, if len == 0 { 0 } else { CHARS_UNKNOWN }, |dst| { + copy_text(value.as_ptr(), dst, len); + }) + }) } } impl From for SharedStr { fn from(value: String) -> Self { - Self(SharedSlice::from(value.into_bytes())) + Self::from(value.as_str()) } } impl From<&String> for SharedStr { @@ -525,7 +775,7 @@ impl From<&String> for SharedStr { } impl From> for SharedStr { fn from(value: Box) -> Self { - Self::from(String::from(value)) + Self::from(&*value) } } impl AsRef for SharedStr { @@ -665,6 +915,58 @@ mod tests { } } + #[test] + fn strings_memoise_code_point_counts_through_appends() { + for text in [ + "", + "a", + "plain ascii", + "mañana", + "日本語", + "🧶🙂", + "e\u{301}", + ] { + let value = SharedStr::from(text); + assert_eq!( + SharedStr::known_char_count(&value), + text.is_empty().then_some(0) + ); + assert_eq!(SharedStr::char_count(&value), text.chars().count()); + assert_eq!( + SharedStr::known_char_count(&value), + Some(text.chars().count()) + ); + assert_eq!(SharedStr::is_ascii(&value), text.is_ascii()); + let mut grown = value.clone(); + drop(value); + assert!(SharedStr::try_append(&mut grown, "ñx")); + let expected = format!("{text}ñx"); + assert_eq!(&*grown, expected); + assert_eq!( + SharedStr::known_char_count(&grown), + Some(expected.chars().count()) + ); + assert_eq!( + SharedStr::hash_cached(&grown), + crate::object::py_str_hash(&expected) + ); + } + let ascii = SharedStr::from_ascii("abc"); + assert_eq!(SharedStr::known_char_count(&ascii), Some(3)); + // SAFETY: ASCII parts and their count. + let built = unsafe { + SharedStr::build_unchecked(7, Some(7), |out| { + out.push(b"ab"); + out.push_byte(b'-'); + out.push(b"cdef"); + }) + }; + assert_eq!(&*built, "ab-cdef"); + assert_eq!(SharedStr::concat(&["x", "ñ", ""]).chars().count(), 2); + assert_eq!(&*SharedStr::repeat("ab", 3), "ababab"); + assert_eq!(&*SharedStr::repeat("", 3), ""); + } + #[test] fn unique_access_rejects_strong_and_weak_aliases() { let mut value = SharedSlice::from(&[11_u8, 22, 33]); diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index 3aa35a92..7259fe52 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -1939,7 +1939,7 @@ pub(crate) fn sizeof_estimate(o: &Object) -> i64 { // wide struct, plus len+1 units of the kind width // (test_str.test_raiseMemError pins all four kinds). Object::Str(s) => { - let len = crate::builtins::str_char_len(s) as i64; + let len = crate::object::str_char_len(s) as i64; match s.chars().map(u32::from).max().unwrap_or(0) { 0..=0x7f => 40 + len + 1, 0x80..=0xff => 56 + (len + 1), diff --git a/crates/weavepy-vm/src/unicode_case.rs b/crates/weavepy-vm/src/unicode_case.rs index bc01b413..f0b03296 100644 --- a/crates/weavepy-vm/src/unicode_case.rs +++ b/crates/weavepy-vm/src/unicode_case.rs @@ -6,6 +6,7 @@ //! (multi-code-point) SpecialCasing expansions. use crate::stdlib::ucd; +use std::mem::MaybeUninit; const CAPITAL_SIGMA: u32 = 0x3A3; const FINAL_SIGMA: u32 = 0x3C2; @@ -61,7 +62,7 @@ fn lower_at(cps: &[char], i: usize) -> ucd::CaseMap { pub fn lower(s: &str) -> String { if s.is_ascii() { - return s.to_ascii_lowercase(); + return ascii_case_map(s, AsciiCase::Lower); } let cps: Vec = s.chars().collect(); let mut out = String::with_capacity(s.len()); @@ -73,7 +74,7 @@ pub fn lower(s: &str) -> String { pub fn upper(s: &str) -> String { if s.is_ascii() { - return s.to_ascii_uppercase(); + return ascii_case_map(s, AsciiCase::Upper); } let mut out = String::with_capacity(s.len()); for c in s.chars() { @@ -85,7 +86,7 @@ pub fn upper(s: &str) -> String { /// `str.casefold()` — context-free full fold (no Final_Sigma). pub fn casefold(s: &str) -> String { if s.is_ascii() { - return s.to_ascii_lowercase(); + return ascii_case_map(s, AsciiCase::Lower); } let mut out = String::with_capacity(s.len()); for c in s.chars() { @@ -96,16 +97,7 @@ pub fn casefold(s: &str) -> String { pub fn title(s: &str) -> String { if s.is_ascii() { - let mut previous_is_cased = false; - return ascii_case_map(s, |b| { - let mapped = if previous_is_cased { - b.to_ascii_lowercase() - } else { - b.to_ascii_uppercase() - }; - previous_is_cased = b.is_ascii_alphabetic(); - mapped - }); + return ascii_case_map(s, AsciiCase::Title); } let cps: Vec = s.chars().collect(); let mut out = String::with_capacity(s.len()); @@ -125,11 +117,7 @@ pub fn title(s: &str) -> String { /// lowered with the sigma rule. pub fn capitalize(s: &str) -> String { if s.is_ascii() { - let mut out = s.to_ascii_lowercase(); - if let Some(first) = out.get_mut(..1) { - first.make_ascii_uppercase(); - } - return out; + return ascii_case_map(s, AsciiCase::Capitalize); } let cps: Vec = s.chars().collect(); let mut out = String::with_capacity(s.len()); @@ -144,13 +132,7 @@ pub fn capitalize(s: &str) -> String { pub fn swapcase(s: &str) -> String { if s.is_ascii() { - return ascii_case_map(s, |b| { - if b.is_ascii_lowercase() { - b.to_ascii_uppercase() - } else { - b.to_ascii_lowercase() - } - }); + return ascii_case_map(s, AsciiCase::Swap); } let cps: Vec = s.chars().collect(); let mut out = String::with_capacity(s.len()); @@ -168,10 +150,87 @@ pub fn swapcase(s: &str) -> String { out } -/// ASCII case mappings preserve both length and UTF-8 validity. Keep this -/// safe even if a future caller passes a non-ASCII byte through the mapper. -fn ascii_case_map(s: &str, map: impl FnMut(u8) -> u8) -> String { - String::from_utf8(s.bytes().map(map).collect()).expect("ASCII case mapping") +/// The case operations' ASCII forms, which [`ascii_case_into`] applies +/// to a whole buffer at once. (`casefold` of ASCII is `Lower`.) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AsciiCase { + Upper, + Lower, + Title, + Capitalize, + Swap, +} + +/// Write the `case` mapping of ASCII `src` into `dst` (of equal length), +/// initializing all of it. +/// +/// Each output byte depends only on its input byte and, for `Title`, the +/// one before it (a letter follows a cased letter exactly when the byte +/// before it is alphabetic), so every loop below is branch-free and +/// vectorizes, where the per-character case-table lookups of the general +/// path don't. Each mapping flips the case bit (`0x20`) of selected +/// letters only, so ASCII stays ASCII; a non-ASCII byte is never a letter +/// here and passes through unchanged. +pub fn ascii_case_into(case: AsciiCase, src: &[u8], dst: &mut [MaybeUninit]) { + #[inline(always)] + fn is_lower(b: u8) -> bool { + b.wrapping_sub(b'a') < 26 + } + #[inline(always)] + fn is_upper(b: u8) -> bool { + b.wrapping_sub(b'A') < 26 + } + #[inline(always)] + fn is_alpha(b: u8) -> bool { + (b | 0x20).wrapping_sub(b'a') < 26 + } + #[inline(always)] + fn flip(b: u8, when: bool) -> MaybeUninit { + MaybeUninit::new(b ^ (u8::from(when) << 5)) + } + assert_eq!(src.len(), dst.len()); + let (Some(&first), Some(head)) = (src.first(), dst.first_mut()) else { + return; + }; + match case { + AsciiCase::Upper => { + for (d, &c) in dst.iter_mut().zip(src) { + *d = flip(c, is_lower(c)); + } + } + AsciiCase::Lower => { + for (d, &c) in dst.iter_mut().zip(src) { + *d = flip(c, is_upper(c)); + } + } + AsciiCase::Swap => { + for (d, &c) in dst.iter_mut().zip(src) { + *d = flip(c, is_alpha(c)); + } + } + AsciiCase::Capitalize => { + *head = flip(first, is_lower(first)); + for (d, &c) in dst[1..].iter_mut().zip(&src[1..]) { + *d = flip(c, is_upper(c)); + } + } + AsciiCase::Title => { + *head = flip(first, is_lower(first)); + for ((d, &c), &prev) in dst[1..].iter_mut().zip(&src[1..]).zip(src) { + let cased = is_alpha(prev); + *d = flip(c, (cased && is_upper(c)) || (!cased && is_lower(c))); + } + } + } +} + +/// [`ascii_case_into`] into a new `String`. +fn ascii_case_map(s: &str, case: AsciiCase) -> String { + let mut out = Vec::with_capacity(s.len()); + ascii_case_into(case, s.as_bytes(), &mut out.spare_capacity_mut()[..s.len()]); + // SAFETY: `ascii_case_into` initialized all `s.len()` bytes. + unsafe { out.set_len(s.len()) }; + String::from_utf8(out).expect("ASCII case mapping") } /// `Py_UNICODE_ISSPACE` — differs from Rust's `char::is_whitespace` (e.g. From 153232493d6f7277e252ad2dd9dab240f0093b8e Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:41:01 -0700 Subject: [PATCH 16/96] perf: decode native pickles in one pass The decoder no longer parses each stream twice (a preflight probe, then the object build). It builds objects directly and checks the stream's shape as it goes: a node table records each container's height and whether a container, tuple, or BUILD already holds it. Only a container that nothing holds may receive items, so an item can never reach its own container and heights never propagate. Any reason to return to the Python unpickler drops what was built; only classes proven to have no finalizer or other hook are instantiated, so that runs no Python code. Short streams still check just the dispatch entries they use, after decoding. A BUILD state that only the memo can still reach gives its storage to the instance (its memo slot is spent, and reading it falls back), and the state's keys are interned in place through a per-stream cache. Reachable containers join the collector's deferred set once the stream is accepted, the decoder's vectors are reused across calls, and the reader checks one limit per read. pickle_bench (macOS, JIT off): about 2.58M to 1.98M retired instructions per iteration. --- crates/weavepy-vm/src/gc_trace.rs | 11 + crates/weavepy-vm/src/stdlib/pickle_accel.rs | 1323 ++++++++---------- 2 files changed, 628 insertions(+), 706 deletions(-) diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index a44cb4c5..837d5b34 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -2330,6 +2330,17 @@ pub fn track(obj: &Object) { with_state(|s| s.track(obj)); } +/// Track a list or dict that native code filled before publishing it (the +/// unpickler's results) as if it had been tracked at birth, while still +/// empty: it joins the deferred containers. Every collection promotes the +/// deferred containers that can anchor a cycle before its mark phase, so +/// the collector still sees every cycle. +pub fn track_built(obj: &Object) { + if !with_state(|s| s.defer_container(obj)) { + track(obj); + } +} + /// `counter += n` (wrapping) for a collector counter: a plain load and /// store while the GIL serializes every writer, a locked read-modify-write /// only in free-threaded mode. diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs index e116bcd3..829576dc 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs @@ -1,9 +1,10 @@ //! Guarded protocol-4/5 encoding and decoding of built-in pickle data and of //! instances of plain classes. //! -//! A first pass checks the complete supported stream and its container graph -//! without creating Python objects. Cycles, deep graphs, custom reconstruction, -//! and unsupported or malformed input return to the existing unpickler. +//! Decoding builds the result in one pass and publishes it only once the +//! whole stream has been accepted. Cycles, deep graphs, custom +//! reconstruction, and unsupported or malformed input drop everything built +//! so far, which runs no Python code, and return to the existing unpickler. //! //! Classes are resolved from `sys.modules` by plain dictionary probes, and //! `NEWOBJ` and `BUILD` are accepted only for classes whose type metadata @@ -19,7 +20,7 @@ use weavepy_compiler::CodeObject; use crate::error::RuntimeError; use crate::import::ModuleCache; -use crate::object::{BuiltinFn, DictData, DictKey, Object, PyFunction, PyModule, StrKey}; +use crate::object::{BuiltinFn, DictData, DictKey, DictMap, Object, PyFunction, PyModule, StrKey}; use crate::sync::{Rc, RefCell, Weak}; use crate::types::TypeObject; @@ -569,46 +570,17 @@ fn make_dumps(args: &[Object]) -> Result { }))) } -#[derive(Clone, Copy, PartialEq, Eq)] -enum Container { - List, - Dict, - Tuple, -} - -enum Scalar<'a> { - None, - Bool(bool), - Int(i64), - Long(&'a [u8]), - Float(f64), - Str(&'a str), - Bytes(&'a [u8]), -} - const STACK_GLOBAL: u8 = 0x93; const NEWOBJ: u8 = 0x81; const BUILD: u8 = b'b'; +/// The deepest container nesting that is decoded natively. +const MAX_DEPTH: u32 = 128; /// `sys.modules`, produced on demand once a stream names a class. struct DecodeContext { modules: Rc>, } -trait Sink<'a> { - type Value: Clone; - - fn opcode(&mut self, _value: u8) {} - fn scalar(&mut self, value: Scalar<'a>) -> Self::Value; - fn empty(&mut self, kind: Container) -> Option; - fn tuple(&mut self, items: Vec) -> Option; - fn extend(&mut self, target: &Self::Value, items: Batch<'_, Self::Value>) -> Option<()>; - fn setitems(&mut self, target: &Self::Value, items: Batch<'_, Self::Value>) -> Option<()>; - fn global(&mut self, module: &Self::Value, name: &Self::Value) -> Option; - fn newobj(&mut self, class: &Self::Value, args: &Self::Value) -> Option; - fn build(&mut self, target: &Self::Value, state: Self::Value) -> Option<()>; -} - #[inline(always)] fn push(items: &mut Vec, value: T) -> Option<()> { // (The fallible reservation only when the vector is full.) @@ -622,25 +594,54 @@ fn push(items: &mut Vec, value: T) -> Option<()> { struct Reader<'a> { data: &'a [u8], pos: usize, - frame_end: Option, + /// The end of the current frame, or of the data outside a frame. + limit: usize, + framed: bool, } impl<'a> Reader<'a> { + fn new(data: &'a [u8], pos: usize) -> Self { + Self { + data, + pos, + limit: data.len(), + framed: false, + } + } + + #[inline(always)] fn read(&mut self, size: usize) -> Option<&'a [u8]> { - if self.frame_end == Some(self.pos) { - self.frame_end = None; + if size <= self.limit - self.pos { + let value = self.data.get(self.pos..self.pos + size)?; + self.pos += size; + return Some(value); } - let end = self.pos.checked_add(size)?; - if self.frame_end.is_some_and(|limit| end > limit) { + self.read_past_limit(size) + } + + /// A read beyond the current limit, which leaves a frame that ends + /// where the read begins, and fails otherwise. + #[cold] + fn read_past_limit(&mut self, size: usize) -> Option<&'a [u8]> { + if !self.framed || self.pos != self.limit { return None; } + self.framed = false; + self.limit = self.data.len(); + let end = self.pos.checked_add(size)?; let value = self.data.get(self.pos..end)?; self.pos = end; Some(value) } + #[inline(always)] fn byte(&mut self) -> Option { - Some(self.read(1)?[0]) + if self.pos < self.limit { + let value = *self.data.get(self.pos)?; + self.pos += 1; + return Some(value); + } + Some(self.read_past_limit(1)?[0]) } fn u32(&mut self) -> Option { @@ -654,7 +655,7 @@ impl<'a> Reader<'a> { fn frame(&mut self) -> Option<()> { // FRAME is valid only after the preceding frame has ended. In // particular, its argument cannot be read out of an active frame. - if self.frame_end.is_some() { + if self.framed { return None; } let size = self.size64()?; @@ -662,22 +663,12 @@ impl<'a> Reader<'a> { if end > self.data.len() { return None; } - self.frame_end = Some(end); + self.framed = true; + self.limit = end; Some(()) } } -/// The items an APPEND(S) or SETITEM(S) moves into their container. -type Batch<'s, V> = std::vec::Drain<'s, V>; - -fn take(stack: &mut Vec, count: usize, marks: &[usize]) -> Option> { - let begin = stack.len().checked_sub(count)?; - if marks.last().is_some_and(|mark| begin < *mark) { - return None; - } - Some(stack.split_off(begin)) -} - fn pop(stack: &mut Vec, marks: &[usize]) -> Option { let begin = stack.len().checked_sub(1)?; if marks.last().is_some_and(|mark| begin < *mark) { @@ -693,194 +684,41 @@ fn top<'a, V>(stack: &'a [V], marks: &[usize]) -> Option<&'a V> { stack.last() } -/// The container below the items from `begin` on, and those items. Both -/// must lie above the innermost mark. -fn batch<'s, V: Clone>( - stack: &'s mut Vec, - begin: usize, - marks: &[usize], -) -> Option<(V, Batch<'s, V>)> { - if begin == 0 || begin > stack.len() || marks.last().is_some_and(|mark| begin <= *mark) { - return None; - } - let target = stack[begin - 1].clone(); - Some((target, stack.drain(begin..))) -} - -#[allow(clippy::too_many_lines)] -fn parse<'a, S: Sink<'a>>(data: &'a [u8], sink: &mut S) -> Option { - if data.len() < 3 || data[0] != 0x80 || !matches!(data[1], 4 | 5) { - return None; - } - sink.opcode(0x80); - let mut reader = Reader { - data, - pos: 2, - frame_end: None, - }; - let mut stack = Vec::new(); - stack.try_reserve(64).ok()?; - let mut marks = Vec::new(); - let mut memo: Vec = Vec::new(); - memo.try_reserve(data.len() / 16).ok()?; - loop { - let opcode = reader.byte()?; - sink.opcode(opcode); - let value = match opcode { - b'.' => { - return (marks.is_empty() && stack.len() == 1) - .then(|| stack.pop()) - .flatten(); - } - 0x95 => { - reader.frame()?; - continue; - } - b'(' => { - push(&mut marks, stack.len())?; - continue; - } - 0x94 => { - push(&mut memo, top(&stack, &marks)?.clone())?; - continue; - } - b'h' => memo.get(usize::from(reader.byte()?))?.clone(), - b'j' => memo.get(usize::try_from(reader.u32()?).ok()?)?.clone(), - b'N' => sink.scalar(Scalar::None), - 0x88 => sink.scalar(Scalar::Bool(true)), - 0x89 => sink.scalar(Scalar::Bool(false)), - b'K' => sink.scalar(Scalar::Int(i64::from(reader.byte()?))), - b'M' => sink.scalar(Scalar::Int(i64::from(u16::from_le_bytes( - reader.read(2)?.try_into().ok()?, - )))), - b'J' => sink.scalar(Scalar::Int(i64::from(i32::from_le_bytes( - reader.read(4)?.try_into().ok()?, - )))), - 0x8a | 0x8b => { - let size = if data[reader.pos - 1] == 0x8a { - usize::from(reader.byte()?) - } else { - let size = i32::from_le_bytes(reader.read(4)?.try_into().ok()?); - usize::try_from(size).ok()? - }; - sink.scalar(Scalar::Long(reader.read(size)?)) - } - b'G' => sink.scalar(Scalar::Float(f64::from_be_bytes( - reader.read(8)?.try_into().ok()?, - ))), - opcode @ (0x8c | b'X' | 0x8d | b'C' | b'B' | 0x8e) => { - let size = match opcode { - 0x8c | b'C' => usize::from(reader.byte()?), - b'X' | b'B' => usize::try_from(reader.u32()?).ok()?, - _ => reader.size64()?, - }; - let bytes = reader.read(size)?; - sink.scalar(if matches!(opcode, 0x8c | b'X' | 0x8d) { - // Surrogate-pass strings stay on the existing WStr path. - Scalar::Str(std::str::from_utf8(bytes).ok()?) - } else { - Scalar::Bytes(bytes) - }) - } - b']' => sink.empty(Container::List)?, - b'}' => sink.empty(Container::Dict)?, - b')' => sink.empty(Container::Tuple)?, - 0x85..=0x87 => { - let count = usize::from(data[reader.pos - 1] - 0x84); - sink.tuple(take(&mut stack, count, &marks)?)? - } - b't' => { - let mark = marks.pop()?; - sink.tuple(stack.split_off(mark))? - } - b'a' => { - let begin = stack.len().checked_sub(1)?; - let (target, items) = batch(&mut stack, begin, &marks)?; - sink.extend(&target, items)?; - continue; - } - b'e' => { - let mark = marks.pop()?; - let (target, items) = batch(&mut stack, mark, &marks)?; - sink.extend(&target, items)?; - continue; - } - b's' => { - let begin = stack.len().checked_sub(2)?; - let (target, items) = batch(&mut stack, begin, &marks)?; - sink.setitems(&target, items)?; - continue; - } - b'u' => { - let mark = marks.pop()?; - let (target, items) = batch(&mut stack, mark, &marks)?; - sink.setitems(&target, items)?; - continue; - } - STACK_GLOBAL => { - let name = pop(&mut stack, &marks)?; - let module = pop(&mut stack, &marks)?; - sink.global(&module, &name)? - } - NEWOBJ => { - let args = pop(&mut stack, &marks)?; - let class = pop(&mut stack, &marks)?; - sink.newobj(&class, &args)? - } - BUILD => { - let state = pop(&mut stack, &marks)?; - sink.build(top(&stack, &marks)?, state)?; - continue; - } - // GLOBAL, REDUCE, NEWOBJ_EX, persistent IDs, external buffers, - // legacy protocols, and all other operations retain the full - // unpickler. - _ => return None, - }; - push(&mut stack, value)?; - } -} - -#[derive(Clone, Copy)] -enum Kind<'a> { - None, - Key, - Str(&'a str), - Float, - /// An index into [`Probe::classes`]. Only NEWOBJ may consume it. - Class(u32), - /// An index into [`Probe::nodes`]: a container or an instance. - Container(u32), -} - -/// What BUILD needs to know about one item of a two-item state tuple. -#[derive(Clone, Copy)] -enum StateItem { - None, - Node(u32), - Other, -} - -enum Shape<'a> { - List, - Tuple { - len: usize, - pair: [StateItem; 2], - }, - /// `keys` lists the string keys; `only_str` reports that there are no - /// others, so the keys are complete. - Dict { - keys: Vec<&'a str>, - only_str: bool, - }, - Instance { - class: u32, - }, +/// Whether the items from `begin` on and the container below them all lie +/// above the innermost mark. +fn is_batch(stack: &[V], begin: usize, marks: &[usize]) -> bool { + begin != 0 && begin <= stack.len() && marks.last().is_none_or(|mark| begin > *mark) } -struct Node<'a> { - shape: Shape<'a>, - children: Vec, +/// A value on the stack or in the memo, with the number of the [`Node`] +/// that describes it, [`LEAF`], or [`CLASS`]. +type Entry = (Object, u32); + +/// The node number of a value that isn't a container. +const LEAF: u32 = 0; +/// The node number of a class, which only NEWOBJ may consume. +const CLASS: u32 = u32::MAX; +/// The node number of a memo slot whose container's storage moved into an +/// instance. Reading it returns to the full unpickler. +const SPENT: u32 = u32::MAX - 1; +/// [`Node::memo`] for a container that no memo slot holds. +const UNMEMOIZED: u32 = u32::MAX; +/// [`Node::memo`] for a container that more than one memo slot holds. +const MEMOIZED_TWICE: u32 = u32::MAX - 1; + +/// What the decoder knows about one container or instance it built. +struct Node { + /// One more than the height of the tallest container inside it. + height: u32, + /// Whether a container, tuple, or BUILD has taken it. Only a container + /// that nothing holds yet may receive items, so an item can never reach + /// its own container (a cycle), and no height ever has to propagate + /// to an enclosing container. + held: bool, + /// The memo slot that holds it, or one of the markers above. + memo: u32, + /// For a two-item tuple, the node numbers of its items. + pair: [u32; 2], } /// A class that NEWOBJ and BUILD can handle without calling Python code. @@ -911,7 +749,7 @@ struct Opcodes([bool; 256]); impl Opcodes { fn insert(&mut self, opcode: u8) { // One indexed store avoids a shift and a read-modify-write on each - // decoded instruction. This scratch is local to one preflight. + // decoded instruction. This scratch is local to one decode. self.0[usize::from(opcode)] = true; } @@ -920,302 +758,257 @@ impl Opcodes { } } -/// One BUILD's state containers: the `__dict__` state and the slot state. -#[derive(Clone, Copy)] -struct StateNodes { - dict: Option, - slots: Option, -} - -/// What the first pass tells the second, all in stream order. -struct Plan { - classes: Vec>, - /// Per node: whether the result can reach it. An unreachable container - /// can never be mutated, so it needs no cycle-collector registration. - reachable: Vec, - /// Per BUILD: which state containers only that BUILD consumes, so that - /// their storage can move into the instance. - builds: Vec, -} - -struct Probe<'a, 'c, const RECORD_OPCODES: bool> { - nodes: Vec>, +/// Builds the result in one pass over the stream. The objects stay private +/// until the whole stream has been accepted: any reason to return to the +/// full unpickler drops them, and dropping them runs no Python code (an +/// instance's class is proven to have no finalizer). +struct Decoder<'c, const RECORD_OPCODES: bool> { opcodes: Opcodes, - /// Every class that STACK_GLOBAL resolved, in stream order. The second - /// pass consumes the same list instead of repeating the lookups. + nodes: Vec, + memo: Vec, + /// Every class that STACK_GLOBAL resolved. classes: Vec, - /// The state of every BUILD, in stream order, with the pair tuple. - builds: Vec<(StateNodes, Option)>, context: &'c dyn Fn() -> Option, resolved: Option, /// The `__slots__` members already verified in this stream (see /// `classes::is_member_slot`): every instance of a class names the /// same ones. - member_slots: Vec<(*const TypeObject, &'a str)>, + member_slots: Vec<(*const TypeObject, SharedStr)>, + /// State keys and their interned strings (see [`Self::intern`]). + interned: Vec<(SharedStr, SharedStr)>, + /// Every list and dictionary, and every instance tracked from birth, in + /// creation order. Those the result can reach join the cycle collector + /// once the stream is accepted. + containers: Vec, +} + +/// A decoder's vectors, kept empty between calls with their capacity, so a +/// `loads` loop neither reallocates nor regrows them. +#[derive(Default)] +struct Scratch { + stack: Vec, + marks: Vec, + memo: Vec, + nodes: Vec, + containers: Vec, +} + +impl Scratch { + /// The largest vector kept between calls. + const MAX_KEPT: usize = 1 << 14; + + fn take() -> Self { + SPARE_SCRATCH + .with(std::cell::Cell::take) + .unwrap_or_default() + } + + /// Empty the vectors (releasing what they hold, which runs no Python + /// code) and keep them for the next call. + fn keep(mut self) { + self.stack.clear(); + self.marks.clear(); + self.memo.clear(); + self.nodes.clear(); + self.containers.clear(); + if self.stack.capacity().max(self.marks.capacity()) <= Self::MAX_KEPT + && self.memo.capacity().max(self.nodes.capacity()) <= Self::MAX_KEPT + && self.containers.capacity() <= Self::MAX_KEPT + { + SPARE_SCRATCH.with(|spare| spare.set(Some(self))); + } + } } -impl<'a, 'c, const RECORD_OPCODES: bool> Probe<'a, 'c, RECORD_OPCODES> { - fn new(context: &'c dyn Fn() -> Option) -> Self { +thread_local! { + static SPARE_SCRATCH: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { + fn new(context: &'c dyn Fn() -> Option, scratch: &mut Scratch) -> Self { Self { - nodes: Vec::new(), opcodes: Opcodes([!RECORD_OPCODES; 256]), + nodes: std::mem::take(&mut scratch.nodes), + memo: std::mem::take(&mut scratch.memo), classes: Vec::new(), - builds: Vec::new(), context, resolved: None, member_slots: Vec::new(), + interned: Vec::new(), + containers: std::mem::take(&mut scratch.containers), } } - /// [`classes::is_member_slot`], remembered for this stream. (No Python - /// runs during the probe, so the class can't change in between.) - fn is_member_slot(&mut self, class: &Rc, name: &'a str) -> bool { - let key = Rc::as_ptr(class); - if self - .member_slots - .iter() - .any(|&(c, n)| c == key && n == name) - { - return true; - } - let yes = classes::is_member_slot(class, name); - if yes && self.member_slots.len() < 64 { - self.member_slots.push((key, name)); - } - yes + /// Return the vectors to `scratch`. + fn recycle(self, scratch: &mut Scratch) { + scratch.nodes = self.nodes; + scratch.memo = self.memo; + scratch.containers = self.containers; } - /// Count the references to each node that the result keeps: the root, - /// container items, and the state pair's items when the pair itself is - /// kept. BUILD copies its state, so its edge keeps nothing. - fn plan(self, root: Kind<'_>) -> Option { - let mut references = Vec::new(); - references.try_reserve(self.nodes.len()).ok()?; - references.resize(self.nodes.len(), 0u32); - if let Kind::Container(root) = root { - references[root as usize] = 1; - } - let mut is_pair = Vec::new(); - is_pair.try_reserve(self.nodes.len()).ok()?; - is_pair.resize(self.nodes.len(), false); - for pair in self.builds.iter().filter_map(|(_, pair)| *pair) { - is_pair[pair as usize] = true; - } - for (index, node) in self.nodes.iter().enumerate() { - if matches!(node.shape, Shape::Instance { .. }) || is_pair[index] { - continue; - } - for &child in &node.children { - references[child as usize] += 1; - } - } - // A tuple's items precede it, so a pair inside a kept pair is seen - // after the reference to it has been counted. - for (index, node) in self.nodes.iter().enumerate().rev() { - if is_pair[index] && references[index] > 0 { - for &child in &node.children { - references[child as usize] += 1; - } - } - } - let mut uses = Vec::new(); - uses.try_reserve(self.nodes.len()).ok()?; - uses.resize(self.nodes.len(), 0u32); - for (state, _) in &self.builds { - for index in [state.dict, state.slots].into_iter().flatten() { - uses[index as usize] += 1; - } + #[inline(always)] + fn opcode(&mut self, value: u8) { + if RECORD_OPCODES { + self.opcodes.insert(value); } - let movable = |index: Option| { - index.filter(|&i| references[i as usize] == 0 && uses[i as usize] == 1) - }; - let mut builds = Vec::new(); - builds.try_reserve(self.builds.len()).ok()?; - builds.extend(self.builds.iter().map(|(state, _)| StateNodes { - dict: movable(state.dict), - slots: movable(state.slots), - })); - Some(Plan { - classes: self.classes.into_iter().map(|c| c.class).collect(), - reachable: references.into_iter().map(|n| n > 0).collect(), - builds, - }) } - fn node(&mut self, shape: Shape<'a>, items: &[Kind<'a>]) -> Option> { - let index = u32::try_from(self.nodes.len()).ok()?; - let mut children = Vec::new(); - for item in items { - match item { - Kind::Container(child) => push(&mut children, *child)?, - // A class is only ever the operand of NEWOBJ. - Kind::Class(_) => return None, - _ => {} - } - } - push(&mut self.nodes, Node { shape, children })?; - Some(Kind::Container(index)) - } - - /// The string keys of a state dictionary, or `None` when the node is - /// not a dictionary keyed only by strings. - fn state_keys(&self, index: u32) -> Option<&[&'a str]> { - match &self.nodes.get(index as usize)?.shape { - Shape::Dict { - keys, - only_str: true, - } => Some(keys), - _ => None, + fn node(&mut self, height: u32, pair: [u32; 2]) -> Option { + if height > MAX_DEPTH { + return None; } + let number = u32::try_from(self.nodes.len()) + .ok()? + .checked_add(1) + .filter(|&number| number < SPENT)?; + push( + &mut self.nodes, + Node { + height, + held: false, + memo: UNMEMOIZED, + pair, + }, + )?; + Some(number) } - fn acyclic_and_bounded(&self) -> Option<()> { - const MAX_DEPTH: usize = 128; - let mut colors = Vec::new(); - colors.try_reserve(self.nodes.len()).ok()?; - colors.resize(self.nodes.len(), 0u8); - let mut heights = Vec::new(); - heights.try_reserve(self.nodes.len()).ok()?; - heights.resize(self.nodes.len(), 0usize); - let mut stack = Vec::new(); - for root in 0..self.nodes.len() { - if colors[root] != 0 { - continue; - } - colors[root] = 1; - push(&mut stack, (root, 0usize))?; - while let Some((index, next)) = stack.last_mut() { - let node = &self.nodes[*index]; - if let Some(&child) = node.children.get(*next) { - *next += 1; - let child = child as usize; - match colors[child] { - 1 => return None, - 0 => { - if stack.len() >= MAX_DEPTH { - return None; - } - colors[child] = 1; - push(&mut stack, (child, 0))?; - } - _ => {} - } - } else { - let height = 1 + node - .children - .iter() - .map(|&child| heights[child as usize]) - .max() - .unwrap_or(0); - if height > MAX_DEPTH { - return None; - } - heights[*index] = height; - colors[*index] = 2; - stack.pop(); - } - } - } - Some(()) + /// A new, empty list or dictionary. + fn container(&mut self, value: Object) -> Option { + let node = self.node(1, [LEAF; 2])?; + push(&mut self.containers, value.clone())?; + Some((value, node)) } -} -impl<'a, const RECORD_OPCODES: bool> Sink<'a> for Probe<'a, '_, RECORD_OPCODES> { - type Value = Kind<'a>; + /// Mark the value of node `node` as taken by a container, and return its + /// height. A class can't be taken. + #[inline] + fn hold(&mut self, node: u32) -> Option { + if node == LEAF { + return Some(0); + } + // (CLASS lies beyond every node.) + let node = self.nodes.get_mut(node as usize - 1)?; + node.held = true; + Some(node.height) + } - fn opcode(&mut self, value: u8) { - if RECORD_OPCODES { - self.opcodes.insert(value); + /// Let the container of node `node`, which nothing may hold yet, take + /// items no taller than `height`. + fn grow(&mut self, node: u32, height: u32) -> Option<()> { + let node = self.nodes.get_mut((node as usize).checked_sub(1)?)?; + if node.held { + return None; } + node.height = node.height.max(height + 1); + (node.height <= MAX_DEPTH).then_some(()) } - fn scalar(&mut self, value: Scalar<'a>) -> Kind<'a> { - match value { - Scalar::None => Kind::None, - Scalar::Float(_) => Kind::Float, - Scalar::Str(text) => Kind::Str(text), - _ => Kind::Key, + fn memoize(&mut self, entry: &Entry) -> Option<()> { + let slot = u32::try_from(self.memo.len()).ok()?; + if let Some(node) = (entry.1 as usize) + .checked_sub(1) + .and_then(|index| self.nodes.get_mut(index)) + { + node.memo = if node.memo == UNMEMOIZED && slot < MEMOIZED_TWICE { + slot + } else { + MEMOIZED_TWICE + }; } + push(&mut self.memo, entry.clone()) } - fn empty(&mut self, kind: Container) -> Option> { - let shape = match kind { - Container::List => Shape::List, - Container::Dict => Shape::Dict { - keys: Vec::new(), - only_str: true, - }, - Container::Tuple => Shape::Tuple { - len: 0, - pair: [StateItem::Other; 2], - }, - }; - self.node(shape, &[]) + fn get(&self, slot: usize) -> Option { + let entry = self.memo.get(slot)?; + (entry.1 != SPENT).then(|| entry.clone()) } - fn tuple(&mut self, items: Vec>) -> Option> { - let item = |kind: &Kind<'a>| match kind { - Kind::None => StateItem::None, - Kind::Container(index) => StateItem::Node(*index), - _ => StateItem::Other, + /// The tallest of `items`, after marking them taken. + fn hold_all(&mut self, items: &[Entry]) -> Option { + let mut height = 0; + for &(_, node) in items { + height = height.max(self.hold(node)?); + } + Some(height) + } + + fn tuple(&mut self, stack: &mut Vec, begin: usize) -> Option { + let height = self.hold_all(&stack[begin..])?; + let pair = match &stack[begin..] { + [(_, first), (_, second)] => [*first, *second], + _ => [LEAF; 2], }; - let pair = match items.as_slice() { - [first, second] => [item(first), item(second)], - _ => [StateItem::Other; 2], + let value = if begin == stack.len() { + Object::new_tuple(Vec::new()) + } else { + Object::Tuple(crate::tuple_storage::TupleStorage::from_exact_iter( + stack.drain(begin..).map(|(value, _)| value), + )) }; - let len = items.len(); - self.node(Shape::Tuple { len, pair }, &items) + Some((value, self.node(height + 1, pair)?)) } - fn extend(&mut self, target: &Kind<'a>, items: Batch<'_, Kind<'a>>) -> Option<()> { - let Kind::Container(index) = target else { - return None; - }; - let node = self.nodes.get_mut(*index as usize)?; - if !matches!(node.shape, Shape::List) { + /// APPEND(S): the items from `begin` on join the list below them. + fn extend(&mut self, stack: &mut Vec, begin: usize, marks: &[usize]) -> Option<()> { + if !is_batch(stack, begin, marks) { return None; } - for item in items { - match item { - Kind::Container(child) => push(&mut node.children, child)?, - Kind::Class(_) => return None, - _ => {} - } - } + // Taking the items first rejects a list that receives itself. + let height = self.hold_all(&stack[begin..])?; + let (Object::List(list), target) = &stack[begin - 1] else { + return None; + }; + let list = list.clone(); + self.grow(*target, height)?; + let mut list = list.borrow_mut(); + list.try_reserve(stack.len() - begin).ok()?; + list.extend(stack.drain(begin..).map(|(value, _)| value)); Some(()) } - fn setitems(&mut self, target: &Kind<'a>, mut items: Batch<'_, Kind<'a>>) -> Option<()> { - let Kind::Container(index) = target else { - return None; - }; - let node = self.nodes.get_mut(*index as usize)?; - let Shape::Dict { keys, only_str } = &mut node.shape else { - return None; - }; - if !items.len().is_multiple_of(2) { + /// SETITEM(S): the pairs from `begin` on join the dictionary below them. + fn setitems(&mut self, stack: &mut Vec, begin: usize, marks: &[usize]) -> Option<()> { + if !is_batch(stack, begin, marks) || !(stack.len() - begin).is_multiple_of(2) { return None; } - while let Some(key) = items.next() { + let mut height = 0; + for pair in stack[begin..].chunks_exact(2) { // Tuple and float keys retain the full unpickler, including // arbitrary nesting and NaN key-identity behavior. So do // instances, whose hashing can call Python code. - match key { - Kind::Str(key) => push(keys, key)?, - Kind::None | Kind::Key => *only_str = false, - _ => return None, - } - match items.next()? { - Kind::Container(child) => push(&mut node.children, child)?, - Kind::Class(_) => return None, - _ => {} + if pair[0].1 != LEAF + || !matches!( + pair[0].0, + Object::Str(_) + | Object::None + | Object::Bool(_) + | Object::Int(_) + | Object::Long(_) + | Object::Bytes(_) + ) + { + return None; } + height = height.max(self.hold(pair[1].1)?); + } + let (Object::Dict(dict), target) = &stack[begin - 1] else { + return None; + }; + let dict = dict.clone(); + self.grow(*target, height)?; + let mut dict = dict.borrow_mut(); + let map: &mut DictMap = &mut dict; + map.try_reserve((stack.len() - begin) / 2).ok()?; + let mut items = stack.drain(begin..); + while let Some((key, _)) = items.next() { + map.insert(DictKey(key), items.next()?.0); } Some(()) } - fn global(&mut self, module: &Kind<'a>, name: &Kind<'a>) -> Option> { - let (Kind::Str(module), Kind::Str(name)) = (module, name) else { + fn global(&mut self, module: &Object, name: &Object) -> Option { + let (Object::Str(module), Object::Str(name)) = (module, name) else { return None; }; if self.resolved.is_none() { @@ -1223,142 +1016,115 @@ impl<'a, const RECORD_OPCODES: bool> Sink<'a> for Probe<'a, '_, RECORD_OPCODES> } // An absent module keeps the full unpickler, which imports it. let class = PlainClass::resolve(self.resolved.as_ref()?, module, name)?; - let index = u32::try_from(self.classes.len()).ok()?; + let value = Object::Type(class.class.clone()); push(&mut self.classes, class)?; - Some(Kind::Class(index)) + Some((value, CLASS)) } - fn newobj(&mut self, class: &Kind<'a>, args: &Kind<'a>) -> Option> { - let (Kind::Class(class), Kind::Container(args)) = (class, args) else { + fn newobj(&mut self, class: Entry, args: Entry) -> Option { + let ((Object::Type(class), CLASS), (Object::Tuple(args), _)) = (class, args) else { return None; }; // `object.__new__(cls)` accepts no further arguments. - if !matches!( - self.nodes.get(*args as usize)?.shape, - Shape::Tuple { len: 0, .. } - ) { + if !args.is_empty() { return None; } - self.node(Shape::Instance { class: *class }, &[]) + // `object.__new__(cls)`, including its cycle-collector registration: + // deferred, as for a plain class's ordinary construction, until the + // instance could hold a non-atomic value (the state paths below + // track it then). + let value = if class.native_kind.get() == 0 + && !class.flags.is_builtin + && !class.instances_need_finalize() + { + Object::Instance(crate::types::PyInstance::new_deferred(class)) + } else { + let value = Object::Instance(Rc::new(crate::types::PyInstance::new(class))); + push(&mut self.containers, value.clone())?; + value + }; + Some((value, self.node(1, [LEAF; 2])?)) } - fn build(&mut self, target: &Kind<'a>, state: Kind<'a>) -> Option<()> { - let (Kind::Container(target), Kind::Container(state)) = (*target, state) else { - return None; - }; - let Shape::Instance { class } = &self.nodes.get(target as usize)?.shape else { - return None; - }; - let class = self.classes.get(*class as usize)?; - // The default state: a dictionary for `__dict__`, or the pair - // `(dictionary or None, {slot: value})`. - let (nodes, pair) = match &self.nodes.get(state as usize)?.shape { - Shape::Dict { .. } => ( - StateNodes { - dict: Some(state), - slots: None, - }, - None, - ), - Shape::Tuple { - len: 2, - pair: [first, StateItem::Node(slots)], - } => { - let dict = match first { - StateItem::None => None, - StateItem::Node(dict) => Some(*dict), - StateItem::Other => return None, - }; - ( - StateNodes { - dict, - slots: Some(*slots), - }, - Some(state), - ) - } - _ => return None, - }; - if let Some(dict) = nodes.dict { - if !self.state_keys(dict)?.is_empty() && !class.has_dict { - return None; - } + /// [`classes::is_member_slot`], remembered for this stream. (No Python + /// runs during decoding, so the class can't change in between.) + fn is_member_slot(&mut self, class: &Rc, name: &SharedStr) -> bool { + let key = Rc::as_ptr(class); + if self + .member_slots + .iter() + .any(|(c, n)| *c == key && n.as_ref() == name.as_ref()) + { + return true; } - if let Some(slots) = nodes.slots { - // `setattr` must reach a member descriptor of `__slots__`. - let names = self.state_keys(slots)?.to_vec(); - let class = class.class.clone(); - if !names - .into_iter() - .all(|name| self.is_member_slot(&class, name)) - { - return None; - } + let yes = classes::is_member_slot(class, name); + if yes && self.member_slots.len() < 64 { + self.member_slots.push((key, name.clone())); } - push(&mut self.builds, (nodes, pair))?; - push(&mut self.nodes.get_mut(target as usize)?.children, state) + yes } -} -/// The second pass, driven by the first pass's [`Plan`]. -struct Objects { - classes: std::vec::IntoIter>, - reachable: Vec, - builds: std::vec::IntoIter, - /// The index of the next node, in the first pass's numbering. - next_node: usize, -} + /// Whether a container of node `node`, with `count` strong references, + /// is reachable only through the `holders` references the caller + /// accounts for and its one memo slot. Then its storage may move, and + /// the memo slot (returned, when there is one) must be spent. + fn movable(&self, count: usize, node: u32, holders: usize) -> Option> { + let node = self.nodes.get((node as usize).checked_sub(1)?)?; + match node.memo { + UNMEMOIZED => (count == holders).then_some(None), + MEMOIZED_TWICE => None, + slot => (count == holders + 1).then_some(Some(slot)), + } + } -impl Objects { - /// Register a container with the cycle collector when the result can - /// reach it. Even an acyclic decoded container can acquire a cycle - /// through later mutation, so every reachable mutable container is - /// registered, including those nested below the returned object. - fn created(&mut self, value: &Object) { - let index = self.next_node; - self.next_node += 1; - if self.reachable.get(index).copied().unwrap_or(true) - && matches!( - value, - Object::List(_) | Object::Dict(_) | Object::Instance(_) - ) + /// `sys.intern(name)`, remembered for this stream: every instance's + /// state usually names the same memoized strings. + fn intern(&mut self, name: &SharedStr) -> SharedStr { + if let Some((_, interned)) = self + .interned + .iter() + .find(|(seen, _)| SharedStr::ptr_eq(seen, name)) { - crate::gc_trace::track(&value); + return interned.clone(); + } + let interned = crate::stdlib::sys::intern_shared(name); + if self.interned.len() < 64 { + self.interned.push((name.clone(), interned.clone())); } + interned } - /// `inst.__dict__[sys.intern(key)] = value` for every item, moving - /// the state's storage when nothing else can reach it. + /// `inst.__dict__[sys.intern(key)] = value` for every item, moving the + /// state's storage when nothing else can reach it. fn apply_dict_state( + &mut self, instance: &crate::types::PyInstance, state: &RefCell, movable: bool, ) -> Option<()> { - if movable && instance.dict.get().is_none() { + if movable { // The unpickled state holds arbitrary objects, so the // instance can no longer be left untracked; track it before - // publishing a dictionary that would bypass the write - // barrier (the record is retired with it). + // publishing a table that bypasses the write barrier (the + // record is retired with it). instance.ensure_gc_tracked(); - let data = std::mem::take(&mut *state.borrow_mut()); - let interned = data.keys().all(|key| match &key.0 { - Object::Str(name) => { - SharedStr::ptr_eq(&crate::stdlib::sys::intern_shared(name), name) - } - _ => false, - }); - if interned { - instance.dict.get_or_init(|| Rc::new(RefCell::new(data))); - return Some(()); - } - let mut attributes = instance.dict_cell().borrow_mut(); - attributes.try_reserve(data.len()).ok()?; - for (key, value) in data { + let mut table: DictMap = std::mem::take(&mut **state.borrow_mut()); + // An interned key is equal to the key it replaces and has the + // same hash, so the table stays valid. + for (key, _) in indexmap::map::MutableKeys::iter_mut2(&mut table) { let Object::Str(name) = &key.0 else { return None; }; - let name = crate::stdlib::sys::intern_shared(name); - attributes.insert(DictKey(Object::Str(name)), value); + key.0 = Object::Str(self.intern(name)); + } + let mut attributes = instance.dict_cell().borrow_mut(); + if attributes.is_empty() { + **attributes = table; + return Some(()); + } + attributes.try_reserve(table.len()).ok()?; + for (key, value) in table { + attributes.insert(key, value); } return Some(()); } @@ -1369,100 +1135,25 @@ impl Objects { let Object::Str(name) = &key.0 else { return None; }; - let name = crate::stdlib::sys::intern_shared(name); + let name = self.intern(name); attributes.insert(DictKey(Object::Str(name)), value.clone()); } Some(()) } -} - -impl<'a> Sink<'a> for Objects { - type Value = Object; - - fn scalar(&mut self, value: Scalar<'a>) -> Object { - match value { - Scalar::None => Object::None, - Scalar::Bool(value) => Object::Bool(value), - Scalar::Int(value) => Object::Int(value), - Scalar::Long(bytes) => Object::int_from_bigint(BigInt::from_signed_bytes_le(bytes)), - Scalar::Float(value) => Object::Float(value), - Scalar::Str(value) => Object::Str(SharedStr::from(value)), - Scalar::Bytes(value) => Object::new_bytes(value), - } - } - - fn empty(&mut self, kind: Container) -> Option { - let value = match kind { - Container::List => Object::new_list(Vec::new()), - Container::Dict => Object::new_dict(), - Container::Tuple => Object::new_tuple(Vec::new()), - }; - // Tuples are traced through their owners. - self.created(&value); - Some(value) - } - - fn tuple(&mut self, items: Vec) -> Option { - let value = Object::new_tuple(items); - self.created(&value); - Some(value) - } - - fn extend(&mut self, target: &Object, items: Batch<'_, Object>) -> Option<()> { - let Object::List(list) = target else { - return None; - }; - let mut list = list.borrow_mut(); - list.try_reserve(items.len()).ok()?; - list.extend(items); - Some(()) - } - - fn setitems(&mut self, target: &Object, mut items: Batch<'_, Object>) -> Option<()> { - let Object::Dict(dict) = target else { - return None; - }; - if !items.len().is_multiple_of(2) { - return None; - } - let mut dict = dict.borrow_mut(); - dict.try_reserve(items.len() / 2).ok()?; - while let Some(key) = items.next() { - dict.insert(DictKey(key), items.next()?); - } - Some(()) - } - - fn global(&mut self, _module: &Object, _name: &Object) -> Option { - Some(Object::Type(self.classes.next()?)) - } - fn newobj(&mut self, class: &Object, _args: &Object) -> Option { - let Object::Type(class) = class else { - return None; - }; - // `object.__new__(cls)`, including its cycle-collector registration: - // deferred, as for a plain class's ordinary construction, until the - // instance could hold a non-atomic value (the state paths below - // track it then). - if class.native_kind.get() == 0 - && !class.flags.is_builtin - && !class.instances_need_finalize() - { - self.next_node += 1; - return Some(Object::Instance(crate::types::PyInstance::new_deferred( - class.clone(), - ))); - } - let instance = Object::Instance(Rc::new(crate::types::PyInstance::new(class.clone()))); - self.created(&instance); - Some(instance) - } - - fn build(&mut self, target: &Object, state: Object) -> Option<()> { - let Object::Instance(instance) = target else { + fn build(&mut self, target: &Entry, state: Entry) -> Option<()> { + let (Object::Instance(instance), target) = target else { return None; }; + let (state, state_node) = state; + let class = instance.cls(); + let has_dict = self + .classes + .iter() + .find(|plain| Rc::ptr_eq(&plain.class, &class))? + .has_dict; + // The default state: a dictionary for `__dict__`, or the pair + // `(dictionary or None, {slot: value})`. let (dict, slots) = match &state { Object::Dict(dict) => (Some(dict), None), Object::Tuple(items) if items.len() == 2 => { @@ -1477,10 +1168,66 @@ impl<'a> Sink<'a> for Objects { } _ => return None, }; - let movable = self.builds.next()?; + if let Some(dict) = dict { + let dict = dict.borrow(); + if !dict.keys().all(|key| matches!(key.0, Object::Str(_))) + || (!dict.is_empty() && !has_dict) + { + return None; + } + } + if let Some(slots) = slots { + // `setattr` must reach a member descriptor of `__slots__`. + for key in slots.borrow().keys() { + let Object::Str(name) = &key.0 else { + return None; + }; + if !self.is_member_slot(&class, name) { + return None; + } + } + } + // The instance takes the state's items, so it must be free to + // change, and the state can't reach it. + let height = self.hold(state_node)?; + self.grow(*target, height)?; + + // A state container that nothing else can reach gives its storage + // to the instance. Its memo slot is spent, and so is the pair's: + // reading either would see the emptied container. + let mut spent = [None; 3]; + let (move_dict, move_slots) = match &state { + Object::Tuple(pair) => { + let items = self.nodes.get((state_node as usize).checked_sub(1)?)?.pair; + match self.movable( + crate::shared_value::ThinArc::strong_count(pair), + state_node, + 1, + ) { + None => (false, false), + Some(slot) => { + // Each item is held by the pair and by `containers`. + let dict = + dict.and_then(|d| self.movable(Rc::strong_count(d), items[0], 2)); + let slots = + slots.and_then(|s| self.movable(Rc::strong_count(s), items[1], 2)); + if dict.is_some() || slots.is_some() { + spent = [slot, dict.flatten(), slots.flatten()]; + } + (dict.is_some(), slots.is_some()) + } + } + } + _ => { + // Held by `state` and by `containers`. + let dict = dict.and_then(|d| self.movable(Rc::strong_count(d), state_node, 2)); + spent[0] = dict.flatten(); + (dict.is_some(), false) + } + }; if let Some(dict) = dict { if !dict.borrow().is_empty() { - Self::apply_dict_state(instance, dict, movable.dict.is_some())?; + self.apply_dict_state(instance, dict, move_dict)?; } } if let Some(slots) = slots { @@ -1491,7 +1238,7 @@ impl<'a> Sink<'a> for Objects { instance.ensure_gc_tracked(); } let mut storage = instance.slots.borrow_mut(); - if movable.slots.is_some() { + if move_slots { let data = std::mem::take(&mut *slots.borrow_mut()); for (key, value) in data { let Object::Str(name) = &key.0 else { @@ -1500,8 +1247,7 @@ impl<'a> Sink<'a> for Objects { storage.insert_shared(name, value); } } else { - let slots = slots.borrow(); - for (key, value) in slots.iter() { + for (key, value) in slots.borrow().iter() { let Object::Str(name) = &key.0 else { return None; }; @@ -1509,11 +1255,184 @@ impl<'a> Sink<'a> for Objects { } } } + for slot in spent.into_iter().flatten() { + *self.memo.get_mut(slot as usize)? = (Object::None, SPENT); + } Some(()) } + + #[allow(clippy::too_many_lines)] + fn parse( + &mut self, + data: &[u8], + stack: &mut Vec, + marks: &mut Vec, + ) -> Option { + self.opcode(0x80); + let mut reader = Reader::new(data, 2); + stack.try_reserve(64).ok()?; + self.memo.try_reserve(data.len() / 16).ok()?; + self.nodes.try_reserve(data.len() / 16).ok()?; + loop { + let opcode = reader.byte()?; + self.opcode(opcode); + let entry = match opcode { + b'.' => { + return (marks.is_empty() && stack.len() == 1) + .then(|| stack.pop()) + .flatten() + // A class is only ever the operand of NEWOBJ. + .filter(|(_, node)| *node != CLASS); + } + 0x95 => { + reader.frame()?; + continue; + } + b'(' => { + push(&mut *marks, stack.len())?; + continue; + } + 0x94 => { + self.memoize(top(&*stack, &*marks)?)?; + continue; + } + b'h' => self.get(usize::from(reader.byte()?))?, + b'j' => self.get(usize::try_from(reader.u32()?).ok()?)?, + b'N' => (Object::None, LEAF), + 0x88 => (Object::Bool(true), LEAF), + 0x89 => (Object::Bool(false), LEAF), + b'K' => (Object::Int(i64::from(reader.byte()?)), LEAF), + b'M' => ( + Object::Int(i64::from(u16::from_le_bytes( + reader.read(2)?.try_into().ok()?, + ))), + LEAF, + ), + b'J' => ( + Object::Int(i64::from(i32::from_le_bytes( + reader.read(4)?.try_into().ok()?, + ))), + LEAF, + ), + 0x8a | 0x8b => { + let size = if opcode == 0x8a { + usize::from(reader.byte()?) + } else { + let size = i32::from_le_bytes(reader.read(4)?.try_into().ok()?); + usize::try_from(size).ok()? + }; + let bytes = reader.read(size)?; + ( + Object::int_from_bigint(BigInt::from_signed_bytes_le(bytes)), + LEAF, + ) + } + b'G' => ( + Object::Float(f64::from_be_bytes(reader.read(8)?.try_into().ok()?)), + LEAF, + ), + 0x8c | b'X' | 0x8d | b'C' | b'B' | 0x8e => { + let size = match opcode { + 0x8c | b'C' => usize::from(reader.byte()?), + b'X' | b'B' => usize::try_from(reader.u32()?).ok()?, + _ => reader.size64()?, + }; + let bytes = reader.read(size)?; + let value = if matches!(opcode, 0x8c | b'X' | 0x8d) { + // Surrogate-pass strings stay on the existing WStr path. + let text = if bytes.is_ascii() { + // SAFETY: ASCII text is valid UTF-8. + unsafe { std::str::from_utf8_unchecked(bytes) } + } else { + std::str::from_utf8(bytes).ok()? + }; + Object::Str(SharedStr::from(text)) + } else { + Object::new_bytes(bytes) + }; + (value, LEAF) + } + b']' => self.container(Object::new_list(Vec::new()))?, + b'}' => self.container(Object::new_dict())?, + b')' => (Object::new_tuple(Vec::new()), self.node(1, [LEAF; 2])?), + 0x85..=0x87 => { + let count = usize::from(opcode - 0x84); + let begin = stack.len().checked_sub(count)?; + if marks.last().is_some_and(|mark| begin < *mark) { + return None; + } + self.tuple(&mut *stack, begin)? + } + b't' => { + let mark = marks.pop()?; + self.tuple(&mut *stack, mark)? + } + b'a' => { + let begin = stack.len().checked_sub(1)?; + self.extend(&mut *stack, begin, &*marks)?; + continue; + } + b'e' => { + let mark = marks.pop()?; + self.extend(&mut *stack, mark, &*marks)?; + continue; + } + b's' => { + let begin = stack.len().checked_sub(2)?; + self.setitems(&mut *stack, begin, &*marks)?; + continue; + } + b'u' => { + let mark = marks.pop()?; + self.setitems(&mut *stack, mark, &*marks)?; + continue; + } + STACK_GLOBAL => { + let (name, _) = pop(&mut *stack, &*marks)?; + let (module, _) = pop(&mut *stack, &*marks)?; + self.global(&module, &name)? + } + NEWOBJ => { + let args = pop(&mut *stack, &*marks)?; + let class = pop(&mut *stack, &*marks)?; + self.newobj(class, args)? + } + BUILD => { + let state = pop(&mut *stack, &*marks)?; + self.build(top(&*stack, &*marks)?, state)?; + continue; + } + // GLOBAL, REDUCE, NEWOBJ_EX, persistent IDs, external buffers, + // legacy protocols, and all other operations retain the full + // unpickler. + _ => return None, + }; + push(&mut *stack, entry)?; + } + } + + /// Publish the accepted result. Every container that the result can + /// reach is registered with the cycle collector: even an acyclic decoded + /// container can acquire a cycle through later mutation. Releasing the + /// memo first leaves the result's references as the only ones beyond + /// `containers`' own. + fn finish(&mut self, root: Object) -> Object { + self.memo.clear(); + for value in self.containers.drain(..) { + let reachable = match &value { + Object::List(list) => Rc::strong_count(list) > 1, + Object::Dict(dict) => Rc::strong_count(dict) > 1, + Object::Instance(instance) => Rc::strong_count(instance) > 1, + _ => false, + }; + if reachable { + crate::gc_trace::track_built(&value); + } + } + root + } } -/// Decode only after the preflight succeeds without creating Python objects. fn decode( data: &[u8], validate: impl FnOnce(&Opcodes) -> bool, @@ -1526,7 +1445,7 @@ fn decode( // streams, avoid recording every opcode and pay that fixed cost once. // Both specializations still validate the entire supported stream. if data.len() <= 512 { - decode_with_probe::(data, validate, context) + decode_with::(data, validate, context) } else { // A replaced reader may ignore the input or reject it immediately. // Check the full table before scanning a large stream so that such @@ -1534,38 +1453,27 @@ fn decode( if !validate(&Opcodes([true; 256])) { return None; } - decode_with_probe::(data, |_| true, context) + decode_with::(data, |_| true, context) } } -fn decode_with_probe( +fn decode_with( data: &[u8], validate: impl FnOnce(&Opcodes) -> bool, context: &dyn Fn() -> Option, ) -> Option { - let mut probe = Probe::::new(context); - let root = parse(data, &mut probe)?; - // A class is only ever the operand of NEWOBJ, never a result. - if matches!(root, Kind::Class(_)) { - return None; - } - probe.acyclic_and_bounded()?; - // Unsupported inputs return before checking Python dispatch functions. - // Guard supported instructions before any Python objects are constructed. - if !validate(&probe.opcodes) { - return None; - } - // Drop the graph scratch before allocating the returned Python objects. - let plan = probe.plan(root)?; - parse( - data, - &mut Objects { - classes: plan.classes.into_iter(), - reachable: plan.reachable, - builds: plan.builds.into_iter(), - next_node: 0, - }, - ) + let mut scratch = Scratch::take(); + let mut decoder = Decoder::::new(context, &mut scratch); + let result = decoder + .parse(data, &mut scratch.stack, &mut scratch.marks) + // A short stream's Python dispatch functions are checked only once + // the stream proves supported. Rejecting it then discards what was + // built. + .filter(|_| validate(&decoder.opcodes)) + .map(|(root, _)| decoder.finish(root)); + decoder.recycle(&mut scratch); + scratch.keep(); + result } #[cfg(test)] @@ -1615,6 +1523,9 @@ mod tests { &b"\x80\x05]\x94(h\x00\x85e."[..], &b"\x80\x05}]K\x01s."[..], &b"\x80\x05})K\x01s."[..], + // A list that another list holds can't change (conservatively, + // although this one would not form a cycle). + &b"\x80\x05]\x94]\x94ah\x01K\x01a\x86."[..], ] { assert!(decode(data).is_none(), "{data:?}"); } From b1c9d668fe4865ee038565700e9054bb32ff778e Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:41:01 -0700 Subject: [PATCH 17/96] perf: encode lists and dicts without snapshots, and probe the memo once Lists and dictionaries are written in place under short borrows: leaves are saved directly, and only a nested container is cloned so that the borrow can be released before recursing. A dictionary that changes meanwhile (a new mutation stamp) returns to the Python pickler, as a resized list already did. Strings, bytes, lists, and dicts look up and claim their memo slot with one table probe. While one thread owns every object, memoized values are no longer pinned, since nothing else can release them. Identities are computed in line, frames commit through an in-line check, the empty argument tuple of NEWOBJ is written directly, slot state reuses one vector, and the output reservation follows the larger of the last two outputs. pickle_bench (macOS, JIT off): about 1.98M to 1.69M retired instructions per iteration. --- .../src/stdlib/pickle_accel/encode.rs | 356 +++++++++++++----- 1 file changed, 264 insertions(+), 92 deletions(-) diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs index 2d11af98..dfa948d9 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs @@ -19,6 +19,7 @@ use crate::fasthash::FxHashMap; use crate::object::{DictData, DictKey, Object, StrKey}; use crate::sync::{Rc, RefCell}; use crate::types::{PyInstance, TypeObject}; +use std::collections::hash_map::Entry; const FRAME_TARGET: usize = 65_536; const MAX_DEPTH: usize = 128; @@ -40,25 +41,36 @@ struct Framer { } impl Framer { - #[inline] + #[inline(always)] fn write(&mut self, data: &[u8]) -> Option<()> { if self.frame_start.is_none() { - let start = self.output.len(); - extend(&mut self.output, &[0x95, 0, 0, 0, 0, 0, 0, 0, 0])?; - self.frame_start = Some(start); + self.open()?; } extend(&mut self.output, data) } + #[cold] + fn open(&mut self) -> Option<()> { + let start = self.output.len(); + extend(&mut self.output, &[0x95, 0, 0, 0, 0, 0, 0, 0, 0])?; + self.frame_start = Some(start); + Some(()) + } + + #[inline] fn commit(&mut self, force: bool) { let Some(start) = self.frame_start else { return; }; + if force || self.output.len() - (start + 9) >= FRAME_TARGET { + self.end_frame(start); + } + } + + #[cold] + fn end_frame(&mut self, start: usize) { let body = start + 9; let len = self.output.len() - body; - if !force && len < FRAME_TARGET { - return; - } if len >= 4 { self.output[start + 1..body].copy_from_slice(&(len as u64).to_le_bytes()); } else { @@ -113,9 +125,13 @@ struct ClassPlan { struct Encoder<'a> { writer: Framer, + // Memo positions by object identity. + memo: FxHashMap, // Pins prevent another thread's container mutation from releasing an // already memoized value and reusing its address during this operation. - memo: FxHashMap, + // While one thread owns every object, nothing else can mutate the graph, + // so the values stay alive and unpinned. + pins: Option>, // Memo positions taken by the default state's temporary tuple and // dictionary, to which no later object can refer. anonymous: u32, @@ -126,6 +142,25 @@ struct Encoder<'a> { resolved: Option, classes: FxHashMap>, slot_name_caches: Vec<(Rc, Vec)>, + // An empty vector for the next instance's slot state. + spare_slots: Vec<(Object, Object)>, +} + +/// `id(value)` for the kinds of value the encoder memoizes. +#[inline] +fn identity(value: &Object) -> i64 { + match value { + Object::Str(text) => text.as_ptr() as usize as i64, + Object::Bytes(bytes) => bytes.as_ptr() as usize as i64, + Object::Tuple(items) => { + crate::shared_value::ThinArc::as_ptr(items).cast::<()>() as usize as i64 + } + Object::List(items) => Rc::as_ptr(items) as usize as i64, + Object::Dict(items) => Rc::as_ptr(items) as usize as i64, + Object::Instance(instance) => Rc::as_ptr(instance) as usize as i64, + Object::Type(class) => Rc::as_ptr(class) as usize as i64, + _ => object_identity(value), + } } impl Encoder<'_> { @@ -135,14 +170,63 @@ impl Encoder<'_> { .checked_add(self.anonymous) } - fn memoize(&mut self, value: &Object) -> Option<()> { + fn pin(&mut self, value: &Object) -> Option<()> { + if let Some(pins) = &mut self.pins { + pins.try_reserve(1).ok()?; + pins.push(value.clone()); + } + Some(()) + } + + /// MEMOIZE for `value`, just written, whose identity is `id`. + fn memoize(&mut self, value: &Object, id: i64) -> Option<()> { let index = self.next_index()?; self.memo.try_reserve(1).ok()?; - self.memo - .insert(object_identity(value), (index, value.clone())); + self.memo.insert(id, index); + self.pin(value)?; self.writer.write(&[0x94]) } + /// GET for a memoized `value` (reporting `true`), or else `write` it + /// and memoize it, with one table probe for both. + #[inline] + fn save_memoized( + &mut self, + value: &Object, + id: i64, + write: impl FnOnce(&mut Framer) -> Option<()>, + ) -> Option { + let next = self.next_index()?; + self.memo.try_reserve(1).ok()?; + match self.memo.entry(id) { + Entry::Occupied(entry) => { + let index = *entry.get(); + self.get(index)?; + Some(true) + } + Entry::Vacant(entry) => { + write(&mut self.writer)?; + entry.insert(next); + self.pin(value)?; + self.writer.write(&[0x94])?; + Some(false) + } + } + } + + /// The checks and the frame commit with which every `save` starts. + #[inline] + fn enter(&mut self, depth: usize) -> Option<()> { + // Leave ample room for the Python pickler's constructor/descriptor + // calls and save/handler/batch expansion at every graph level. A + // low recursion budget retains its existing fallback behavior. + if depth > MAX_DEPTH || depth.saturating_mul(8).saturating_add(32) >= self.python_headroom { + return None; + } + self.writer.commit(false); + Some(()) + } + fn memoize_anonymous(&mut self) -> Option<()> { self.next_index()?; self.anonymous += 1; @@ -191,34 +275,22 @@ impl Encoder<'_> { } } - fn save(&mut self, value: &Object, depth: usize) -> Option<()> { - // Leave ample room for the Python pickler's constructor/descriptor - // calls and save/handler/batch expansion at every graph level. A - // low recursion budget retains its existing fallback behavior. - if depth > MAX_DEPTH || depth.saturating_mul(8).saturating_add(32) >= self.python_headroom { + /// Whether `id`, a container's identity, is memoized (written as GET) + /// or must be written. A container that is still being written means + /// a cycle: recursive tuples require a POP/GET repair, so every cyclic + /// graph retains the existing pickler. + fn memoized_container(&mut self, id: i64) -> Option { + if self.active.contains(&id) { return None; } - self.writer.commit(false); - let memoized = matches!( - value, - Object::Str(_) - | Object::Bytes(_) - | Object::List(_) - | Object::Dict(_) - | Object::Tuple(_) - | Object::Instance(_) - ); - let id = if memoized { object_identity(value) } else { 0 }; - if memoized { - // Recursive tuples require a POP/GET repair. Retain the existing - // pickler for every cyclic graph in this first implementation. - if !matches!(value, Object::Str(_) | Object::Bytes(_)) && self.active.contains(&id) { - return None; - } - if let Some(&(index, _)) = self.memo.get(&id) { - return self.get(index); - } + match self.memo.get(&id) { + Some(&index) => self.get(index).map(|()| true), + None => Some(false), } + } + + fn save(&mut self, value: &Object, depth: usize) -> Option<()> { + self.enter(depth)?; match value { Object::None => self.writer.write(b"N"), Object::Bool(value) => self.writer.write(&[if *value { 0x88 } else { 0x89 }]), @@ -231,16 +303,21 @@ impl Encoder<'_> { self.writer.write(b"G")?; self.writer.write(&value.to_bits().to_be_bytes()) } - Object::Str(text) => { - self.writer.data(text.as_bytes(), true)?; - self.memoize(value) - } - Object::Bytes(bytes) => { - self.writer.data(bytes, false)?; - self.memoize(value) - } + Object::Str(text) => self + .save_memoized(value, identity(value), |writer| { + writer.data(text.as_bytes(), true) + }) + .map(drop), + Object::Bytes(bytes) => self + .save_memoized(value, identity(value), |writer| writer.data(bytes, false)) + .map(drop), + // The empty tuple is never memoized. Object::Tuple(items) if items.is_empty() => self.writer.write(b")"), Object::Tuple(items) => { + let id = identity(value); + if self.memoized_container(id)? { + return Some(()); + } self.active.try_reserve(1).ok()?; self.active.push(id); if items.len() > 3 { @@ -255,70 +332,84 @@ impl Encoder<'_> { 3 => 0x87, _ => b't', }])?; - self.memoize(value)?; + self.memoize(value, id)?; self.active.pop(); Some(()) } Object::List(items) => { - // A bounded snapshot releases the container lock before - // recursion without duplicating the entire list's storage. + // (See `memoized_container`.) + let id = identity(value); + if self.active.contains(&id) { + return None; + } + if self.save_memoized(value, id, |writer| writer.write(b"]"))? { + return Some(()); + } // Keep the original length as a bound; observed resizing // returns to the Python pickler before any callbacks run. let length = items.borrow().len(); - let mut batch = Vec::new(); - batch.try_reserve_exact(length.min(1000)).ok()?; self.active.try_reserve(1).ok()?; self.active.push(id); - self.writer.write(b"]")?; - self.memoize(value)?; for start in (0..length).step_by(1000) { - batch.clear(); - { - let items = items.borrow(); - if items.len() != length { - return None; - } - let end = start.saturating_add(1000).min(length); - batch.extend(items[start..end].iter().cloned()); - } - if batch.len() != 1 { + let end = start.saturating_add(1000).min(length); + if end - start != 1 { self.writer.write(b"(")?; } - for item in &batch { - self.save(item, depth + 1)?; + let mut index = start; + while index < end { + // Leaves are written under one borrow. The lock is + // released before recursing into a container, which + // a clone keeps alive meanwhile. + let next = { + let items = items.borrow(); + if items.len() != length { + return None; + } + loop { + let Some(item) = items[..end].get(index) else { + break None; + }; + index += 1; + if !self.save_leaf(item, depth + 1)? { + break Some(item.clone()); + } + } + }; + let Some(item) = next else { + break; + }; + self.save(&item, depth + 1)?; } self.writer - .write(if batch.len() == 1 { b"a" } else { b"e" })?; + .write(if end - start == 1 { b"a" } else { b"e" })?; } self.active.pop(); Some(()) } Object::Dict(items) => { - let items = { - let items = items.borrow(); - let mut snapshot = Vec::new(); - snapshot.try_reserve_exact(items.len()).ok()?; - snapshot.extend( - items - .iter() - .map(|(key, value)| (key.0.clone(), value.clone())), - ); - snapshot - }; + let id = identity(value); + if self.active.contains(&id) { + return None; + } + if self.save_memoized(value, id, |writer| writer.write(b"}"))? { + return Some(()); + } self.active.try_reserve(1).ok()?; self.active.push(id); - self.writer.write(b"}")?; - self.memoize(value)?; - self.save_items(&items, depth + 1)?; + self.save_dict_items(items, depth + 1)?; self.active.pop(); Some(()) } Object::Instance(instance) => { + let id = identity(value); + if self.memoized_container(id)? { + return Some(()); + } // The instance is memoized before its state, but a graph // that returns to it is cyclic and keeps the full pickler. self.active.try_reserve(1).ok()?; self.active.push(id); - self.save_instance(value, instance, depth)?; + self.save_instance(value, id, instance, depth)?; self.active.pop(); Some(()) } @@ -326,6 +417,73 @@ impl Encoder<'_> { } } + /// `save` for a value that needs no recursion. `Some(false)` reports a + /// value that isn't such a leaf, leaving it unsaved. + #[inline] + fn save_leaf(&mut self, value: &Object, depth: usize) -> Option { + match value { + Object::None + | Object::Bool(_) + | Object::Int(_) + | Object::Long(_) + | Object::Float(_) + | Object::Str(_) + | Object::Bytes(_) => { + self.save(value, depth)?; + Some(true) + } + _ => Some(false), + } + } + + /// `_batch_setitems` over a live dictionary, batched as lists are in + /// [`Self::save`]. Any mutable access to the dictionary meanwhile, + /// which draws a new stamp, returns to the Python pickler. + fn save_dict_items(&mut self, dict: &RefCell, depth: usize) -> Option<()> { + let (length, stamp) = { + let dict = dict.borrow(); + (dict.len(), dict.mutation_stamp()) + }; + for start in (0..length).step_by(1000) { + let end = start.saturating_add(1000).min(length); + if end - start != 1 { + self.writer.write(b"(")?; + } + let mut index = start; + while index < end { + let next = { + let dict = dict.borrow(); + if dict.mutation_stamp() != stamp || dict.len() != length { + return None; + } + loop { + if index == end { + break None; + } + let (key, value) = dict.get_index(index)?; + index += 1; + if !self.save_leaf(&key.0, depth)? { + break Some((Some(key.0.clone()), value.clone())); + } + if !self.save_leaf(value, depth)? { + break Some((None, value.clone())); + } + } + }; + let Some((key, value)) = next else { + break; + }; + if let Some(key) = key { + self.save(&key, depth)?; + } + self.save(&value, depth)?; + } + self.writer + .write(if end - start == 1 { b"s" } else { b"u" })?; + } + Some(()) + } + fn save_items(&mut self, items: &[(Object, Object)], depth: usize) -> Option<()> { for batch in items.chunks(1000) { if batch.len() != 1 { @@ -346,6 +504,7 @@ impl Encoder<'_> { fn save_instance( &mut self, value: &Object, + id: i64, instance: &Rc, depth: usize, ) -> Option<()> { @@ -375,7 +534,8 @@ impl Encoder<'_> { if dict.is_some() && !plan.has_dict { return None; } - let mut slots: Vec<(Object, Object)> = Vec::new(); + // (Nested instances take a new vector.) + let mut slots = std::mem::take(&mut self.spare_slots); if !plan.slot_names.is_empty() { let storage = instance.slots.borrow(); slots.try_reserve_exact(plan.slot_names.len()).ok()?; @@ -393,11 +553,16 @@ impl Encoder<'_> { } self.save_class(&class, &plan, depth + 1)?; - self.save(&Object::new_tuple(Vec::new()), depth + 1)?; + // `save(())` + self.enter(depth + 1)?; + self.writer.write(b")")?; self.writer.write(&[0x81])?; - self.memoize(value)?; + self.memoize(value, id)?; match (dict, slots.is_empty()) { - (None, true) => return Some(()), + (None, true) => { + self.spare_slots = slots; + return Some(()); + } (Some(dict), true) => self.save(&Object::Dict(dict), depth + 1)?, (dict, false) => { // The state is the temporary `(dict or None, {slot: value})`. @@ -411,6 +576,8 @@ impl Encoder<'_> { self.memoize_anonymous()?; } } + slots.clear(); + self.spare_slots = slots; self.writer.write(b"b") } @@ -418,13 +585,14 @@ impl Encoder<'_> { fn save_class(&mut self, class: &Rc, plan: &ClassPlan, depth: usize) -> Option<()> { self.writer.commit(false); let value = Object::Type(class.clone()); - if let Some(&(index, _)) = self.memo.get(&object_identity(&value)) { + let id = identity(&value); + if let Some(&index) = self.memo.get(&id) { return self.get(index); } self.save(&plan.module, depth + 1)?; self.save(&plan.qualname, depth + 1)?; self.writer.write(&[0x93])?; - self.memoize(&value) + self.memoize(&value, id) } fn class_plan(&mut self, class: &Rc) -> Option> { @@ -555,17 +723,20 @@ fn encode_with_context( if !matches!(protocol, 4 | 5) { return None; } - // The previous call's memo table (emptied, its capacity kept) and - // output size: a `dumps` loop then neither regrows the table nor - // copies the output as it doubles. + // The previous call's memo table (emptied, its capacity kept) and the + // last two output sizes: a `dumps` loop, even one that alternates + // between two values, then neither regrows the table nor copies the + // output as it doubles. let memo = SPARE_MEMO.with(std::cell::Cell::take).unwrap_or_default(); - let hint = OUTPUT_HINT.with(std::cell::Cell::get); + let (last, before) = OUTPUT_HINT.with(std::cell::Cell::get); + let hint = last.max(before); let mut encoder = Encoder { writer: Framer { output: Vec::new(), frame_start: None, }, memo, + pins: (!crate::sync::bias_held()).then(Vec::new), anonymous: 0, active: Vec::new(), python_headroom, @@ -573,6 +744,7 @@ fn encode_with_context( resolved: None, classes: FxHashMap::default(), slot_name_caches: Vec::new(), + spare_slots: Vec::new(), }; let done = (|| { encoder @@ -593,7 +765,7 @@ fn encode_with_context( SPARE_MEMO.with(|spare| spare.set(Some(memo))); } done?; - OUTPUT_HINT.with(|h| h.set(encoder.writer.output.len())); + OUTPUT_HINT.with(|h| h.set((encoder.writer.output.len(), last))); Some(encoder.writer.output) } @@ -602,9 +774,9 @@ const MAX_SPARE_MEMO: usize = 1 << 14; const MAX_OUTPUT_HINT: usize = 1 << 20; thread_local! { - static SPARE_MEMO: std::cell::Cell>> = + static SPARE_MEMO: std::cell::Cell>> = const { std::cell::Cell::new(None) }; - static OUTPUT_HINT: std::cell::Cell = const { std::cell::Cell::new(0) }; + static OUTPUT_HINT: std::cell::Cell<(usize, usize)> = const { std::cell::Cell::new((0, 0)) }; } #[cfg(test)] From f7b65a10aa774b0b6bb6b40e106ff7fe80a1976d Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:59:46 -0700 Subject: [PATCH 18/96] perf: decode without store-forwarding stalls, and reuse state tables A value built in a temporary, with a byte store for its variant, and then copied with word loads stalls store forwarding, and the decoder did that on almost every instruction. Each opcode arm now writes its entry straight into the stack's slot, node numbers are word-sized like the rest of an entry, a GET copies the memo's stored words, and MEMOIZE rebuilds the value from its fields. A MEMOIZE that follows the instruction that pushed the value is taken without a dispatch, and TUPLE2, MEMOIZE, BUILD (a state with __slots__) runs without creating the pair, whose memo slot is spent from the start. A moved state dictionary is reused, with its table's capacity, by the next EMPTY_DICT: a __dict__ state trades tables with the instance's empty dictionary, and slot state is stored in one step. SETITEMS inserts string keys with their cached hash. The dispatch table is now always checked after decoding, for the instructions the stream used, and the unchanged class version stands in for looking up __init__, load, and find_class. loads of pickle_bench's two values, interleaved runs on a loaded host: 0.91 to 0.72 ms per pair (CPython 3.14: 0.56 ms). --- crates/weavepy-vm/src/stdlib/pickle_accel.rs | 619 ++++++++++++------- 1 file changed, 406 insertions(+), 213 deletions(-) diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs index 829576dc..ed4bac03 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs @@ -14,7 +14,10 @@ mod classes; mod encode; +use std::hash::BuildHasher; + use crate::shared_value::{SharedSlice, SharedStr}; +use indexmap::map::raw_entry_v1::{RawEntryApiV1, RawEntryMut}; use num_bigint::BigInt; use weavepy_compiler::CodeObject; @@ -293,13 +296,14 @@ impl UnpicklerGuard { Rc::as_ptr(class) == self.class.as_ptr() && class.attr_version.get() == self.version } - fn holds(&self, class: &Rc, opcodes: &Opcodes) -> bool { - if !self.matches_class(class) - || !class - .lookup("__init__") - .is_some_and(|f| self.init.holds(&f)) - || !class.lookup("load").is_some_and(|f| self.load.holds(&f)) - { + /// Whether the functions that a stream of `opcodes` would run are still + /// the guarded ones, with their guarded code and defaults. Checked once + /// the stream proves supported, after [`Self::matches_class`]: an + /// unchanged class version (which every change to the class or its + /// bases advances) proves that `__init__`, `load`, and `find_class` + /// still resolve to the guarded functions. + fn holds(&self, opcodes: &Opcodes) -> bool { + if !self.init.holds_live() || !self.load.holds_live() { return false; } // STACK_GLOBAL is the only supported instruction that calls a @@ -308,7 +312,7 @@ impl UnpicklerGuard { && !self .find_class .as_ref() - .is_some_and(|guard| class.lookup("find_class").is_some_and(|f| guard.holds(&f))) + .is_some_and(FunctionGuard::holds_live) { return false; } @@ -392,15 +396,11 @@ fn make_loads(args: &[Object]) -> Result { // request to use the existing unpickler. let context = || instances.as_ref()?.context(); let decoded = match data { - Object::Bytes(data) => { - decode(data, |opcodes| guard.holds(class, opcodes), &context) - } + Object::Bytes(data) => decode(data, |opcodes| guard.holds(opcodes), &context), // Decoding runs no Python code, so the borrow is safe. - Object::ByteArray(data) => decode( - &data.borrow(), - |opcodes| guard.holds(class, opcodes), - &context, - ), + Object::ByteArray(data) => { + decode(&data.borrow(), |opcodes| guard.holds(opcodes), &context) + } _ => None, }; Ok(decoded.map_or(Object::None, |value| Object::new_tuple_array([value]))) @@ -581,6 +581,34 @@ struct DecodeContext { modules: Rc>, } +/// Push `$value` (with node number `$node`, by default [`LEAF`]) onto the +/// decoder stack, written straight into its slot. A value built in a +/// temporary (with a byte store for its variant) and then copied with word +/// loads stalls store forwarding, and the decoder's opcode arms otherwise +/// share such a temporary. +macro_rules! push_entry { + ($stack:expr, $value:expr) => { + push_entry!($stack, $value, LEAF) + }; + ($stack:expr, $value:expr, $node:expr) => { + push_in_place!($stack, ($value, $node)) + }; +} + +/// Push `$value` onto the vector `$items`, written straight into its slot +/// (see [`push_entry`]). +macro_rules! push_in_place { + ($items:expr, $value:expr) => {{ + let items = &mut *$items; + if items.len() == items.capacity() { + items.try_reserve(1).ok()?; + } + items.spare_capacity_mut().first_mut()?.write($value); + // SAFETY: the slot past the end has just been initialized. + unsafe { items.set_len(items.len() + 1) }; + }}; +} + #[inline(always)] fn push(items: &mut Vec, value: T) -> Option<()> { // (The fallible reservation only when the vector is full.) @@ -634,6 +662,14 @@ impl<'a> Reader<'a> { Some(value) } + /// Whether the current frame (or the data outside one) continues with + /// `bytes`. + #[inline] + fn next_is(&self, bytes: &[u8]) -> bool { + self.limit - self.pos >= bytes.len() + && self.data.get(self.pos..self.pos + bytes.len()) == Some(bytes) + } + #[inline(always)] fn byte(&mut self) -> Option { if self.pos < self.limit { @@ -692,15 +728,56 @@ fn is_batch(stack: &[V], begin: usize, marks: &[usize]) -> bool { /// A value on the stack or in the memo, with the number of the [`Node`] /// that describes it, [`LEAF`], or [`CLASS`]. -type Entry = (Object, u32); +type Entry = (Object, NodeId); + +/// A node number. It is word-sized, like the rest of an [`Entry`], so an +/// entry is copied with the same word stores that wrote it. +type NodeId = u64; + +/// `value.clone()`, in line for the kinds that streams memoize, as a copy +/// of the existing value's words: a value rebuilt in a temporary would be +/// copied with word loads that stall on the byte store of its variant. +#[inline(always)] +fn clone_value(value: &Object) -> Object { + match value { + Object::Str(text) => std::mem::forget(text.clone()), + Object::Tuple(items) => std::mem::forget(items.clone()), + Object::List(items) => std::mem::forget(items.clone()), + Object::Dict(items) => std::mem::forget(items.clone()), + Object::Instance(instance) => std::mem::forget(instance.clone()), + Object::Type(class) => std::mem::forget(class.clone()), + Object::None | Object::Bool(_) | Object::Int(_) | Object::Float(_) => {} + _ => return value.clone(), + } + // SAFETY: each kind above is a plain value or a single counted + // reference, whose clone is a copy of the same words holding one more + // reference: the one just taken and forgotten, which the copy owns. + unsafe { std::ptr::read(value) } +} + +/// The dictionary (or None) and the slot dictionary of a BUILD state pair. +#[allow(clippy::type_complexity)] +fn state_dicts<'s>( + first: &'s Object, + second: &'s Object, +) -> Option<(Option<&'s Rc>>, &'s Rc>)> { + let Object::Dict(slots) = second else { + return None; + }; + match first { + Object::None => Some((None, slots)), + Object::Dict(dict) => Some((Some(dict), slots)), + _ => None, + } +} /// The node number of a value that isn't a container. -const LEAF: u32 = 0; +const LEAF: NodeId = 0; /// The node number of a class, which only NEWOBJ may consume. -const CLASS: u32 = u32::MAX; +const CLASS: NodeId = NodeId::MAX; /// The node number of a memo slot whose container's storage moved into an /// instance. Reading it returns to the full unpickler. -const SPENT: u32 = u32::MAX - 1; +const SPENT: NodeId = NodeId::MAX - 1; /// [`Node::memo`] for a container that no memo slot holds. const UNMEMOIZED: u32 = u32::MAX; /// [`Node::memo`] for a container that more than one memo slot holds. @@ -718,7 +795,7 @@ struct Node { /// The memo slot that holds it, or one of the markers above. memo: u32, /// For a two-item tuple, the node numbers of its items. - pair: [u32; 2], + pair: [NodeId; 2], } /// A class that NEWOBJ and BUILD can handle without calling Python code. @@ -762,7 +839,7 @@ impl Opcodes { /// until the whole stream has been accepted: any reason to return to the /// full unpickler drops them, and dropping them runs no Python code (an /// instance's class is proven to have no finalizer). -struct Decoder<'c, const RECORD_OPCODES: bool> { +struct Decoder<'c> { opcodes: Opcodes, nodes: Vec, memo: Vec, @@ -776,6 +853,9 @@ struct Decoder<'c, const RECORD_OPCODES: bool> { member_slots: Vec<(*const TypeObject, SharedStr)>, /// State keys and their interned strings (see [`Self::intern`]). interned: Vec<(SharedStr, SharedStr)>, + /// Emptied state dictionaries that nothing else can reach, with their + /// tables' capacity, for the next EMPTY_DICT. + spare_dicts: Vec>>, /// Every list and dictionary, and every instance tracked from birth, in /// creation order. Those the result can reach join the cycle collector /// once the stream is accepted. @@ -824,10 +904,10 @@ thread_local! { static SPARE_SCRATCH: std::cell::Cell> = const { std::cell::Cell::new(None) }; } -impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { +impl<'c> Decoder<'c> { fn new(context: &'c dyn Fn() -> Option, scratch: &mut Scratch) -> Self { Self { - opcodes: Opcodes([!RECORD_OPCODES; 256]), + opcodes: Opcodes([false; 256]), nodes: std::mem::take(&mut scratch.nodes), memo: std::mem::take(&mut scratch.memo), classes: Vec::new(), @@ -835,6 +915,7 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { resolved: None, member_slots: Vec::new(), interned: Vec::new(), + spare_dicts: Vec::new(), containers: std::mem::take(&mut scratch.containers), } } @@ -848,16 +929,14 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { #[inline(always)] fn opcode(&mut self, value: u8) { - if RECORD_OPCODES { - self.opcodes.insert(value); - } + self.opcodes.insert(value); } - fn node(&mut self, height: u32, pair: [u32; 2]) -> Option { + fn node(&mut self, height: u32, pair: [NodeId; 2]) -> Option { if height > MAX_DEPTH { return None; } - let number = u32::try_from(self.nodes.len()) + let number = NodeId::try_from(self.nodes.len()) .ok()? .checked_add(1) .filter(|&number| number < SPENT)?; @@ -873,30 +952,50 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { Some(number) } - /// A new, empty list or dictionary. - fn container(&mut self, value: Object) -> Option { + /// Push a new, empty dictionary, reusing a moved state's when one waits. + fn empty_dict(&mut self, stack: &mut Vec) -> Option<()> { let node = self.node(1, [LEAF; 2])?; - push(&mut self.containers, value.clone())?; - Some((value, node)) + let dict = match self.spare_dicts.pop() { + // (A spare is already in `containers`.) + Some(dict) => dict, + None => { + let dict = Rc::new(RefCell::new(DictData::default())); + push_in_place!(&mut self.containers, Object::Dict(dict.clone())); + dict + } + }; + push_entry!(stack, Object::Dict(dict), node); + Some(()) + } + + /// Push a new, empty list. + fn empty_list(&mut self, stack: &mut Vec) -> Option<()> { + let node = self.node(1, [LEAF; 2])?; + let list = Rc::new(RefCell::new(Vec::new())); + push_in_place!(&mut self.containers, Object::List(list.clone())); + push_entry!(stack, Object::List(list), node); + Some(()) } /// Mark the value of node `node` as taken by a container, and return its /// height. A class can't be taken. #[inline] - fn hold(&mut self, node: u32) -> Option { + fn hold(&mut self, node: NodeId) -> Option { if node == LEAF { return Some(0); } // (CLASS lies beyond every node.) - let node = self.nodes.get_mut(node as usize - 1)?; + let node = self.nodes.get_mut(usize::try_from(node - 1).ok()?)?; node.held = true; Some(node.height) } /// Let the container of node `node`, which nothing may hold yet, take /// items no taller than `height`. - fn grow(&mut self, node: u32, height: u32) -> Option<()> { - let node = self.nodes.get_mut((node as usize).checked_sub(1)?)?; + fn grow(&mut self, node: NodeId, height: u32) -> Option<()> { + let node = self + .nodes + .get_mut(usize::try_from(node.checked_sub(1)?).ok()?)?; if node.held { return None; } @@ -906,8 +1005,8 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { fn memoize(&mut self, entry: &Entry) -> Option<()> { let slot = u32::try_from(self.memo.len()).ok()?; - if let Some(node) = (entry.1 as usize) - .checked_sub(1) + if let Some(node) = (entry.1.checked_sub(1)) + .and_then(|index| usize::try_from(index).ok()) .and_then(|index| self.nodes.get_mut(index)) { node.memo = if node.memo == UNMEMOIZED && slot < MEMOIZED_TWICE { @@ -916,12 +1015,47 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { MEMOIZED_TWICE }; } - push(&mut self.memo, entry.clone()) + // The value usually was just pushed, its variant with a byte store: + // each kind is rebuilt from its fields (not copied by words) into + // the memo's slot. + let node = entry.1; + let memo = &mut self.memo; + match &entry.0 { + Object::Str(text) => push_entry!(memo, Object::Str(text.clone()), node), + Object::Tuple(items) => push_entry!(memo, Object::Tuple(items.clone()), node), + Object::List(items) => push_entry!(memo, Object::List(items.clone()), node), + Object::Dict(items) => push_entry!(memo, Object::Dict(items.clone()), node), + Object::Instance(instance) => { + push_entry!(memo, Object::Instance(instance.clone()), node); + } + Object::Type(class) => push_entry!(memo, Object::Type(class.clone()), node), + other => push_entry!(memo, other.clone(), node), + } + Some(()) } - fn get(&self, slot: usize) -> Option { - let entry = self.memo.get(slot)?; - (entry.1 != SPENT).then(|| entry.clone()) + /// A MEMOIZE that follows the instruction that pushed the value, taken + /// without a dispatch: the pickler memoizes nearly every value it + /// writes. + #[inline(always)] + fn memoize_next(&mut self, reader: &mut Reader<'_>, stack: &[Entry]) -> Option<()> { + if reader.next_is(&[0x94]) { + reader.pos += 1; + self.opcode(0x94); + self.memoize(stack.last()?)?; + } + Some(()) + } + + /// GET: push the value of memo slot `slot`. + #[inline] + fn push_memo(&self, stack: &mut Vec, slot: usize) -> Option<()> { + let (value, node) = self.memo.get(slot)?; + if *node == SPENT { + return None; + } + push_entry!(stack, clone_value(value), *node); + Some(()) } /// The tallest of `items`, after marking them taken. @@ -933,20 +1067,23 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { Some(height) } - fn tuple(&mut self, stack: &mut Vec, begin: usize) -> Option { + /// Replace the items from `begin` on with a tuple of them. + fn tuple(&mut self, stack: &mut Vec, begin: usize) -> Option<()> { let height = self.hold_all(&stack[begin..])?; let pair = match &stack[begin..] { [(_, first), (_, second)] => [*first, *second], _ => [LEAF; 2], }; - let value = if begin == stack.len() { - Object::new_tuple(Vec::new()) + let node = self.node(height + 1, pair)?; + if begin == stack.len() { + push_entry!(stack, Object::new_tuple(Vec::new()), node); } else { - Object::Tuple(crate::tuple_storage::TupleStorage::from_exact_iter( + let items = crate::tuple_storage::TupleStorage::from_exact_iter( stack.drain(begin..).map(|(value, _)| value), - )) - }; - Some((value, self.node(height + 1, pair)?)) + ); + push_entry!(stack, Object::Tuple(items), node); + } + Some(()) } /// APPEND(S): the items from `begin` on join the list below them. @@ -1002,12 +1139,29 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { map.try_reserve((stack.len() - begin) / 2).ok()?; let mut items = stack.drain(begin..); while let Some((key, _)) = items.next() { - map.insert(DictKey(key), items.next()?.0); + let (value, _) = items.next()?; + let Object::Str(name) = key else { + map.insert(DictKey(key), value); + continue; + }; + // `insert` for a string key, which equals only another string: + // the cached hash and a direct comparison, in line. + let hash = crate::fasthash::FxBuildHasher.hash_one(SharedStr::hash_cached(&name)); + match map.raw_entry_mut_v1().from_hash(hash, |key| { + matches!(&key.0, Object::Str(other) if SharedStr::ptr_eq(other, &name) || *other == name) + }) { + RawEntryMut::Occupied(mut entry) => { + entry.insert(value); + } + RawEntryMut::Vacant(entry) => { + entry.insert_hashed_nocheck(hash, DictKey(Object::Str(name)), value); + } + } } Some(()) } - fn global(&mut self, module: &Object, name: &Object) -> Option { + fn global(&mut self, module: &Object, name: &Object) -> Option { let (Object::Str(module), Object::Str(name)) = (module, name) else { return None; }; @@ -1018,10 +1172,10 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { let class = PlainClass::resolve(self.resolved.as_ref()?, module, name)?; let value = Object::Type(class.class.clone()); push(&mut self.classes, class)?; - Some((value, CLASS)) + Some(value) } - fn newobj(&mut self, class: Entry, args: Entry) -> Option { + fn newobj(&mut self, stack: &mut Vec, class: Entry, args: Entry) -> Option<()> { let ((Object::Type(class), CLASS), (Object::Tuple(args), _)) = (class, args) else { return None; }; @@ -1029,21 +1183,23 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { if !args.is_empty() { return None; } + let node = self.node(1, [LEAF; 2])?; // `object.__new__(cls)`, including its cycle-collector registration: // deferred, as for a plain class's ordinary construction, until the // instance could hold a non-atomic value (the state paths below // track it then). - let value = if class.native_kind.get() == 0 + let instance = if class.native_kind.get() == 0 && !class.flags.is_builtin && !class.instances_need_finalize() { - Object::Instance(crate::types::PyInstance::new_deferred(class)) + crate::types::PyInstance::new_deferred(class) } else { - let value = Object::Instance(Rc::new(crate::types::PyInstance::new(class))); - push(&mut self.containers, value.clone())?; - value + let instance = Rc::new(crate::types::PyInstance::new(class)); + push_in_place!(&mut self.containers, Object::Instance(instance.clone())); + instance }; - Some((value, self.node(1, [LEAF; 2])?)) + push_entry!(stack, Object::Instance(instance), node); + Some(()) } /// [`classes::is_member_slot`], remembered for this stream. (No Python @@ -1068,8 +1224,10 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { /// is reachable only through the `holders` references the caller /// accounts for and its one memo slot. Then its storage may move, and /// the memo slot (returned, when there is one) must be spent. - fn movable(&self, count: usize, node: u32, holders: usize) -> Option> { - let node = self.nodes.get((node as usize).checked_sub(1)?)?; + fn movable(&self, count: usize, node: NodeId, holders: usize) -> Option> { + let node = self + .nodes + .get(usize::try_from(node.checked_sub(1)?).ok()?)?; match node.memo { UNMEMOIZED => (count == holders).then_some(None), MEMOIZED_TWICE => None, @@ -1108,24 +1266,26 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { // publishing a table that bypasses the write barrier (the // record is retired with it). instance.ensure_gc_tracked(); - let mut table: DictMap = std::mem::take(&mut **state.borrow_mut()); + let mut state = state.borrow_mut(); + let table: &mut DictMap = &mut state; // An interned key is equal to the key it replaces and has the // same hash, so the table stays valid. - for (key, _) in indexmap::map::MutableKeys::iter_mut2(&mut table) { + for (key, _) in indexmap::map::MutableKeys::iter_mut2(&mut *table) { let Object::Str(name) = &key.0 else { return None; }; key.0 = Object::Str(self.intern(name)); } let mut attributes = instance.dict_cell().borrow_mut(); + let attributes: &mut DictMap = &mut attributes; if attributes.is_empty() { - **attributes = table; + // The state keeps the instance's empty table and its + // capacity, for reuse (see `spare_dicts`). + std::mem::swap(attributes, table); return Some(()); } attributes.try_reserve(table.len()).ok()?; - for (key, value) in table { - attributes.insert(key, value); - } + attributes.extend(table.drain(..)); return Some(()); } let state = state.borrow(); @@ -1142,32 +1302,80 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { } fn build(&mut self, target: &Entry, state: Entry) -> Option<()> { + let (state, state_node) = state; + // The instance takes the state's items, so it must be free to + // change, and the state can't reach it. + let height = self.hold(state_node)?; + match &state { + Object::Dict(dict) => { + // Held by `state` and by `containers`. + let movable = self.movable(Rc::strong_count(dict), state_node, 2); + self.build_from(target, height, Some(dict), None, [movable, None], None) + } + Object::Tuple(pair) if pair.len() == 2 => { + let [first, second] = self + .nodes + .get(usize::try_from(state_node.checked_sub(1)?).ok()?)? + .pair; + let (dict, slots) = state_dicts(&pair[0], &pair[1])?; + // The items move only when nothing else can reach the pair. + let count = crate::shared_value::ThinArc::strong_count(pair); + let Some(pair_slot) = self.movable(count, state_node, 1) else { + return self.build_from(target, height, dict, Some(slots), [None, None], None); + }; + // Each item is held by the pair and by `containers`. + let moves = [ + dict.and_then(|d| self.movable(Rc::strong_count(d), first, 2)), + self.movable(Rc::strong_count(slots), second, 2), + ]; + self.build_from(target, height, dict, Some(slots), moves, pair_slot) + } + _ => None, + } + } + + /// TUPLE2, MEMOIZE, and BUILD, the way the pickler writes a state with + /// `__slots__`, without creating the pair. Its memo slot is spent from + /// the start, so reading it returns to the full unpickler. + fn build_pair(&mut self, stack: &mut Vec, marks: &[usize]) -> Option<()> { + let second = pop(stack, marks)?; + let first = pop(stack, marks)?; + // The pair takes its items. + let height = 1 + self.hold(first.1)?.max(self.hold(second.1)?); + if height > MAX_DEPTH { + return None; + } + push(&mut self.memo, (Object::None, SPENT))?; + let (dict, slots) = state_dicts(&first.0, &second.0)?; + // Each item is held by `first` or `second` and by `containers`. + let moves = [ + dict.and_then(|d| self.movable(Rc::strong_count(d), first.1, 2)), + self.movable(Rc::strong_count(slots), second.1, 2), + ]; + self.build_from(top(stack, marks)?, height, dict, Some(slots), moves, None) + } + + /// BUILD for the state dictionaries `dict` and `slots`, whose state is + /// `height` high. `moves` tells, for each, whether its storage may move + /// (and its memo slot, which is then spent, as is `pair_slot`). + fn build_from( + &mut self, + target: &Entry, + height: u32, + dict: Option<&Rc>>, + slots: Option<&Rc>>, + moves: [Option>; 2], + pair_slot: Option, + ) -> Option<()> { let (Object::Instance(instance), target) = target else { return None; }; - let (state, state_node) = state; let class = instance.cls(); let has_dict = self .classes .iter() .find(|plain| Rc::ptr_eq(&plain.class, &class))? .has_dict; - // The default state: a dictionary for `__dict__`, or the pair - // `(dictionary or None, {slot: value})`. - let (dict, slots) = match &state { - Object::Dict(dict) => (Some(dict), None), - Object::Tuple(items) if items.len() == 2 => { - let Object::Dict(slots) = &items[1] else { - return None; - }; - match &items[0] { - Object::None => (None, Some(slots)), - Object::Dict(dict) => (Some(dict), Some(slots)), - _ => return None, - } - } - _ => return None, - }; if let Some(dict) = dict { let dict = dict.borrow(); if !dict.keys().all(|key| matches!(key.0, Object::Str(_))) @@ -1187,43 +1395,16 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { } } } - // The instance takes the state's items, so it must be free to - // change, and the state can't reach it. - let height = self.hold(state_node)?; self.grow(*target, height)?; // A state container that nothing else can reach gives its storage // to the instance. Its memo slot is spent, and so is the pair's: // reading either would see the emptied container. - let mut spent = [None; 3]; - let (move_dict, move_slots) = match &state { - Object::Tuple(pair) => { - let items = self.nodes.get((state_node as usize).checked_sub(1)?)?.pair; - match self.movable( - crate::shared_value::ThinArc::strong_count(pair), - state_node, - 1, - ) { - None => (false, false), - Some(slot) => { - // Each item is held by the pair and by `containers`. - let dict = - dict.and_then(|d| self.movable(Rc::strong_count(d), items[0], 2)); - let slots = - slots.and_then(|s| self.movable(Rc::strong_count(s), items[1], 2)); - if dict.is_some() || slots.is_some() { - spent = [slot, dict.flatten(), slots.flatten()]; - } - (dict.is_some(), slots.is_some()) - } - } - } - _ => { - // Held by `state` and by `containers`. - let dict = dict.and_then(|d| self.movable(Rc::strong_count(d), state_node, 2)); - spent[0] = dict.flatten(); - (dict.is_some(), false) - } + let [move_dict, move_slots] = moves.map(|movable| movable.is_some()); + let spent = if move_dict || move_slots { + [pair_slot, moves[0].flatten(), moves[1].flatten()] + } else { + [None; 3] }; if let Some(dict) = dict { if !dict.borrow().is_empty() { @@ -1238,7 +1419,17 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { instance.ensure_gc_tracked(); } let mut storage = instance.slots.borrow_mut(); - if move_slots { + if move_slots && storage.get_index(0).is_none() { + // Into empty storage, the distinct keys in order: the + // result of setting each in turn, built in one step. The + // emptied dictionary keeps its capacity for reuse. + let mut data = slots.borrow_mut(); + let data: &mut DictMap = &mut data; + let mut entries = Vec::new(); + entries.try_reserve_exact(data.len()).ok()?; + entries.extend(data.drain(..)); + *storage = crate::types::SlotStorage::from_entries(entries); + } else if move_slots { let data = std::mem::take(&mut *slots.borrow_mut()); for (key, value) in data { let Object::Str(name) = &key.0 else { @@ -1258,6 +1449,15 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { for slot in spent.into_iter().flatten() { *self.memo.get_mut(slot as usize)? = (Object::None, SPENT); } + // A moved state dictionary is now empty and out of reach, so the + // next EMPTY_DICT can reuse it. (It remains in `containers`.) + for (moved, dict) in [(move_dict, dict), (move_slots, slots)] { + if let (true, Some(dict)) = (moved, dict) { + if self.spare_dicts.len() < 64 { + self.spare_dicts.push(dict.clone()); + } + } + } Some(()) } @@ -1276,7 +1476,10 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { loop { let opcode = reader.byte()?; self.opcode(opcode); - let entry = match opcode { + // Every arm pushes its own result: a value that passes through a + // shared temporary (or an `Option`) is copied with unaligned + // loads that stall on the stores that just wrote it. + match opcode { b'.' => { return (marks.is_empty() && stack.len() == 1) .then(|| stack.pop()) @@ -1284,36 +1487,26 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { // A class is only ever the operand of NEWOBJ. .filter(|(_, node)| *node != CLASS); } - 0x95 => { - reader.frame()?; - continue; + 0x95 => reader.frame()?, + b'(' => push(&mut *marks, stack.len())?, + 0x94 => self.memoize(top(&*stack, &*marks)?)?, + b'h' => self.push_memo(stack, usize::from(reader.byte()?))?, + b'j' => self.push_memo(stack, usize::try_from(reader.u32()?).ok()?)?, + b'N' => push_entry!(stack, Object::None), + 0x88 => push_entry!(stack, Object::Bool(true)), + 0x89 => push_entry!(stack, Object::Bool(false)), + b'K' => { + let value = i64::from(reader.byte()?); + push_entry!(stack, Object::Int(value)); } - b'(' => { - push(&mut *marks, stack.len())?; - continue; + b'M' => { + let value = u16::from_le_bytes(reader.read(2)?.try_into().ok()?); + push_entry!(stack, Object::Int(i64::from(value))); } - 0x94 => { - self.memoize(top(&*stack, &*marks)?)?; - continue; + b'J' => { + let value = i32::from_le_bytes(reader.read(4)?.try_into().ok()?); + push_entry!(stack, Object::Int(i64::from(value))); } - b'h' => self.get(usize::from(reader.byte()?))?, - b'j' => self.get(usize::try_from(reader.u32()?).ok()?)?, - b'N' => (Object::None, LEAF), - 0x88 => (Object::Bool(true), LEAF), - 0x89 => (Object::Bool(false), LEAF), - b'K' => (Object::Int(i64::from(reader.byte()?)), LEAF), - b'M' => ( - Object::Int(i64::from(u16::from_le_bytes( - reader.read(2)?.try_into().ok()?, - ))), - LEAF, - ), - b'J' => ( - Object::Int(i64::from(i32::from_le_bytes( - reader.read(4)?.try_into().ok()?, - ))), - LEAF, - ), 0x8a | 0x8b => { let size = if opcode == 0x8a { usize::from(reader.byte()?) @@ -1322,102 +1515,123 @@ impl<'c, const RECORD_OPCODES: bool> Decoder<'c, RECORD_OPCODES> { usize::try_from(size).ok()? }; let bytes = reader.read(size)?; - ( - Object::int_from_bigint(BigInt::from_signed_bytes_le(bytes)), - LEAF, - ) + let value = Object::int_from_bigint(BigInt::from_signed_bytes_le(bytes)); + push_entry!(stack, value); } - b'G' => ( - Object::Float(f64::from_be_bytes(reader.read(8)?.try_into().ok()?)), - LEAF, - ), - 0x8c | b'X' | 0x8d | b'C' | b'B' | 0x8e => { + b'G' => { + let value = f64::from_be_bytes(reader.read(8)?.try_into().ok()?); + push_entry!(stack, Object::Float(value)); + } + 0x8c | b'X' | 0x8d => { let size = match opcode { - 0x8c | b'C' => usize::from(reader.byte()?), - b'X' | b'B' => usize::try_from(reader.u32()?).ok()?, + 0x8c => usize::from(reader.byte()?), + b'X' => usize::try_from(reader.u32()?).ok()?, _ => reader.size64()?, }; let bytes = reader.read(size)?; - let value = if matches!(opcode, 0x8c | b'X' | 0x8d) { - // Surrogate-pass strings stay on the existing WStr path. - let text = if bytes.is_ascii() { - // SAFETY: ASCII text is valid UTF-8. - unsafe { std::str::from_utf8_unchecked(bytes) } - } else { - std::str::from_utf8(bytes).ok()? - }; - Object::Str(SharedStr::from(text)) + // Surrogate-pass strings stay on the existing WStr path. + let text = if bytes.is_ascii() { + // SAFETY: ASCII text is valid UTF-8. + unsafe { std::str::from_utf8_unchecked(bytes) } } else { - Object::new_bytes(bytes) + std::str::from_utf8(bytes).ok()? }; - (value, LEAF) + let text = SharedStr::from(text); + push_entry!(stack, Object::Str(text)); + self.memoize_next(&mut reader, stack)?; + } + b'C' | b'B' | 0x8e => { + let size = match opcode { + b'C' => usize::from(reader.byte()?), + b'B' => usize::try_from(reader.u32()?).ok()?, + _ => reader.size64()?, + }; + let value = Object::new_bytes(reader.read(size)?); + push_entry!(stack, value); + self.memoize_next(&mut reader, stack)?; + } + b']' => { + self.empty_list(stack)?; + self.memoize_next(&mut reader, stack)?; + } + b'}' => { + self.empty_dict(stack)?; + self.memoize_next(&mut reader, stack)?; + } + b')' => { + let node = self.node(1, [LEAF; 2])?; + push(stack, (Object::new_tuple(Vec::new()), node))?; + } + // The pickler writes a state with `__slots__` this way. + 0x86 if reader.next_is(&[0x94, BUILD]) => { + reader.pos += 2; + self.opcode(0x94); + self.opcode(BUILD); + self.build_pair(stack, &*marks)?; } - b']' => self.container(Object::new_list(Vec::new()))?, - b'}' => self.container(Object::new_dict())?, - b')' => (Object::new_tuple(Vec::new()), self.node(1, [LEAF; 2])?), 0x85..=0x87 => { let count = usize::from(opcode - 0x84); let begin = stack.len().checked_sub(count)?; if marks.last().is_some_and(|mark| begin < *mark) { return None; } - self.tuple(&mut *stack, begin)? + self.tuple(stack, begin)?; + self.memoize_next(&mut reader, stack)?; } b't' => { let mark = marks.pop()?; - self.tuple(&mut *stack, mark)? + self.tuple(stack, mark)?; + self.memoize_next(&mut reader, stack)?; } b'a' => { let begin = stack.len().checked_sub(1)?; - self.extend(&mut *stack, begin, &*marks)?; - continue; + self.extend(stack, begin, &*marks)?; } b'e' => { let mark = marks.pop()?; - self.extend(&mut *stack, mark, &*marks)?; - continue; + self.extend(stack, mark, &*marks)?; } b's' => { let begin = stack.len().checked_sub(2)?; - self.setitems(&mut *stack, begin, &*marks)?; - continue; + self.setitems(stack, begin, &*marks)?; } b'u' => { let mark = marks.pop()?; - self.setitems(&mut *stack, mark, &*marks)?; - continue; + self.setitems(stack, mark, &*marks)?; } STACK_GLOBAL => { let (name, _) = pop(&mut *stack, &*marks)?; let (module, _) = pop(&mut *stack, &*marks)?; - self.global(&module, &name)? + let class = self.global(&module, &name)?; + push(stack, (class, CLASS))?; + self.memoize_next(&mut reader, stack)?; } NEWOBJ => { let args = pop(&mut *stack, &*marks)?; let class = pop(&mut *stack, &*marks)?; - self.newobj(class, args)? + self.newobj(stack, class, args)?; + self.memoize_next(&mut reader, stack)?; } BUILD => { let state = pop(&mut *stack, &*marks)?; self.build(top(&*stack, &*marks)?, state)?; - continue; } // GLOBAL, REDUCE, NEWOBJ_EX, persistent IDs, external buffers, // legacy protocols, and all other operations retain the full // unpickler. _ => return None, - }; - push(&mut *stack, entry)?; + } } } /// Publish the accepted result. Every container that the result can /// reach is registered with the cycle collector: even an acyclic decoded /// container can acquire a cycle through later mutation. Releasing the - /// memo first leaves the result's references as the only ones beyond - /// `containers`' own. + /// memo and the spare dictionaries first leaves the result's references + /// as the only ones beyond `containers`' own. fn finish(&mut self, root: Object) -> Object { self.memo.clear(); + self.spare_dicts.clear(); for value in self.containers.drain(..) { let reachable = match &value { Object::List(list) => Rc::strong_count(list) > 1, @@ -1441,34 +1655,13 @@ fn decode( if data.len() < 3 || data[0] != 0x80 || !matches!(data[1], 4 | 5) { return None; } - // Checking the whole dispatch table has a fixed cost. For longer - // streams, avoid recording every opcode and pay that fixed cost once. - // Both specializations still validate the entire supported stream. - if data.len() <= 512 { - decode_with::(data, validate, context) - } else { - // A replaced reader may ignore the input or reject it immediately. - // Check the full table before scanning a large stream so that such - // readers keep their existing fallback cost. - if !validate(&Opcodes([true; 256])) { - return None; - } - decode_with::(data, |_| true, context) - } -} - -fn decode_with( - data: &[u8], - validate: impl FnOnce(&Opcodes) -> bool, - context: &dyn Fn() -> Option, -) -> Option { let mut scratch = Scratch::take(); - let mut decoder = Decoder::::new(context, &mut scratch); + let mut decoder = Decoder::new(context, &mut scratch); let result = decoder .parse(data, &mut scratch.stack, &mut scratch.marks) - // A short stream's Python dispatch functions are checked only once - // the stream proves supported. Rejecting it then discards what was - // built. + // The code of the Python functions that the stream's instructions + // would run is checked once the stream proves supported. Rejecting + // it then discards what was built. .filter(|_| validate(&decoder.opcodes)) .map(|(root, _)| decoder.finish(root)); decoder.recycle(&mut scratch); From 7b6f1134998e9b811c2c39cae4fb6825eb75230f Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:59:47 -0700 Subject: [PATCH 19/96] perf: give the encoder's memo an open-addressing table Memo positions live in a linear-probing table keyed by address, whose entries carry the generation of the call that wrote them, so the next `dumps` clears it by starting a new generation. A slot is read by its declaration position first, which is where a plain class keeps it. dumps of pickle_bench's two values, interleaved runs on a loaded host: 0.51 to 0.44 ms per pair (CPython 3.14: 1.05 ms). --- .../src/stdlib/pickle_accel/encode.rs | 157 ++++++++++++++++-- 1 file changed, 139 insertions(+), 18 deletions(-) diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs index dfa948d9..9159f658 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs @@ -19,7 +19,6 @@ use crate::fasthash::FxHashMap; use crate::object::{DictData, DictKey, Object, StrKey}; use crate::sync::{Rc, RefCell}; use crate::types::{PyInstance, TypeObject}; -use std::collections::hash_map::Entry; const FRAME_TARGET: usize = 65_536; const MAX_DEPTH: usize = 128; @@ -126,7 +125,7 @@ struct ClassPlan { struct Encoder<'a> { writer: Framer, // Memo positions by object identity. - memo: FxHashMap, + memo: MemoTable, // Pins prevent another thread's container mutation from releasing an // already memoized value and reusing its address during this operation. // While one thread owns every object, nothing else can mutate the graph, @@ -146,6 +145,120 @@ struct Encoder<'a> { spare_slots: Vec<(Object, Object)>, } +/// The memo: memo positions by object identity (an address, never 0), in an +/// open-addressing table with linear probing. Each entry records the +/// generation of the call that wrote it, so the next call clears the table +/// by starting a new generation. +#[derive(Default)] +struct MemoTable { + entries: Vec, + len: usize, + generation: u32, +} + +#[derive(Clone, Copy, Default)] +struct MemoEntry { + id: i64, + index: u32, + generation: u32, +} + +impl MemoTable { + const MIN_CAPACITY: usize = 64; + + /// The first slot to probe for `id`, in a table of `capacity` slots (a + /// power of two): a multiplicative hash spreads aligned addresses. + #[inline] + fn home(id: i64, capacity: usize) -> usize { + let hash = (id as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15); + (hash >> (64 - capacity.trailing_zeros())) as usize + } + + #[inline] + fn get(&self, id: i64) -> Option { + if self.entries.is_empty() { + return None; + } + let mask = self.entries.len() - 1; + let mut slot = Self::home(id, self.entries.len()); + loop { + let entry = &self.entries[slot]; + if entry.generation != self.generation { + return None; + } + if entry.id == id { + return Some(entry.index); + } + slot = (slot + 1) & mask; + } + } + + /// `id`'s memo position, or else the slot at which to insert it. Makes + /// room for one more entry first; `None` only when that fails. + #[inline] + fn find(&mut self, id: i64) -> Option> { + if (self.len + 1) * 2 > self.entries.len() { + self.grow()?; + } + let mask = self.entries.len() - 1; + let mut slot = Self::home(id, self.entries.len()); + loop { + let entry = &self.entries[slot]; + if entry.generation != self.generation { + return Some(Err(slot)); + } + if entry.id == id { + return Some(Ok(entry.index)); + } + slot = (slot + 1) & mask; + } + } + + /// Insert at a slot that [`Self::find`] returned. + #[inline] + fn insert(&mut self, slot: usize, id: i64, index: u32) { + self.entries[slot] = MemoEntry { + id, + index, + generation: self.generation, + }; + self.len += 1; + } + + #[cold] + fn grow(&mut self) -> Option<()> { + if self.generation == 0 { + // A fresh table: generation 0 marks every slot empty. + self.generation = 1; + } + let capacity = (self.entries.len() * 2).max(Self::MIN_CAPACITY); + let mut entries = Vec::new(); + entries.try_reserve_exact(capacity).ok()?; + entries.resize(capacity, MemoEntry::default()); + let old = std::mem::replace(&mut self.entries, entries); + let mask = capacity - 1; + for entry in old.into_iter().filter(|e| e.generation == self.generation) { + let mut slot = Self::home(entry.id, capacity); + while self.entries[slot].generation == self.generation { + slot = (slot + 1) & mask; + } + self.entries[slot] = entry; + } + Some(()) + } + + /// Empty the table, keeping its storage. + fn clear(&mut self) { + self.len = 0; + self.generation = self.generation.wrapping_add(1); + if self.generation == 0 { + // The generations wrapped: every stale entry must go. + self.entries.fill(MemoEntry::default()); + self.generation = 1; + } + } +} + /// `id(value)` for the kinds of value the encoder memoizes. #[inline] fn identity(value: &Object) -> i64 { @@ -165,11 +278,12 @@ fn identity(value: &Object) -> i64 { impl Encoder<'_> { fn next_index(&self) -> Option { - u32::try_from(self.memo.len()) + u32::try_from(self.memo.len) .ok()? .checked_add(self.anonymous) } + #[inline] fn pin(&mut self, value: &Object) -> Option<()> { if let Some(pins) = &mut self.pins { pins.try_reserve(1).ok()?; @@ -181,8 +295,9 @@ impl Encoder<'_> { /// MEMOIZE for `value`, just written, whose identity is `id`. fn memoize(&mut self, value: &Object, id: i64) -> Option<()> { let index = self.next_index()?; - self.memo.try_reserve(1).ok()?; - self.memo.insert(id, index); + if let Err(slot) = self.memo.find(id)? { + self.memo.insert(slot, id, index); + } self.pin(value)?; self.writer.write(&[0x94]) } @@ -197,16 +312,14 @@ impl Encoder<'_> { write: impl FnOnce(&mut Framer) -> Option<()>, ) -> Option { let next = self.next_index()?; - self.memo.try_reserve(1).ok()?; - match self.memo.entry(id) { - Entry::Occupied(entry) => { - let index = *entry.get(); + match self.memo.find(id)? { + Ok(index) => { self.get(index)?; Some(true) } - Entry::Vacant(entry) => { + Err(slot) => { write(&mut self.writer)?; - entry.insert(next); + self.memo.insert(slot, id, next); self.pin(value)?; self.writer.write(&[0x94])?; Some(false) @@ -283,8 +396,8 @@ impl Encoder<'_> { if self.active.contains(&id) { return None; } - match self.memo.get(&id) { - Some(&index) => self.get(index).map(|()| true), + match self.memo.get(id) { + Some(index) => self.get(index).map(|()| true), None => Some(false), } } @@ -539,14 +652,22 @@ impl Encoder<'_> { if !plan.slot_names.is_empty() { let storage = instance.slots.borrow(); slots.try_reserve_exact(plan.slot_names.len()).ok()?; - for name in &plan.slot_names { + for (index, name) in plan.slot_names.iter().enumerate() { let Object::Str(text) = name else { return None; }; let repeated = slots .iter() .any(|(seen, _)| matches!(seen, Object::Str(seen) if seen == text)); - if let (false, Some(value)) = (repeated, storage.get(text)) { + if repeated { + continue; + } + // Storage usually holds the slots in declaration order. + let value = match storage.get_index(index) { + Some((DictKey(Object::Str(stored)), value)) if stored == text => Some(value), + _ => storage.get(text), + }; + if let Some(value) = value { slots.push((name.clone(), value.clone())); } } @@ -586,7 +707,7 @@ impl Encoder<'_> { self.writer.commit(false); let value = Object::Type(class.clone()); let id = identity(&value); - if let Some(&index) = self.memo.get(&id) { + if let Some(index) = self.memo.get(id) { return self.get(index); } self.save(&plan.module, depth + 1)?; @@ -760,7 +881,7 @@ fn encode_with_context( Some(()) })(); let mut memo = std::mem::take(&mut encoder.memo); - if memo.capacity() <= MAX_SPARE_MEMO { + if memo.entries.len() <= MAX_SPARE_MEMO { memo.clear(); SPARE_MEMO.with(|spare| spare.set(Some(memo))); } @@ -774,7 +895,7 @@ const MAX_SPARE_MEMO: usize = 1 << 14; const MAX_OUTPUT_HINT: usize = 1 << 20; thread_local! { - static SPARE_MEMO: std::cell::Cell>> = + static SPARE_MEMO: std::cell::Cell> = const { std::cell::Cell::new(None) }; static OUTPUT_HINT: std::cell::Cell<(usize, usize)> = const { std::cell::Cell::new((0, 0)) }; } From 199ff25bab1c7705463d40c9c84eaf0aa0fd1ba2 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:46:50 -0700 Subject: [PATCH 20/96] perf: peek at the next instruction without memcmp The fused MEMOIZE and the slot-state BUILD peek at the next one or two bytes of the stream. A comparison against a constant array, byte by byte and in line, replaces a slice comparison that called memcmp on every pushed value. --- crates/weavepy-vm/src/stdlib/pickle_accel.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs index ed4bac03..446bfa2e 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs @@ -663,11 +663,14 @@ impl<'a> Reader<'a> { } /// Whether the current frame (or the data outside one) continues with - /// `bytes`. - #[inline] - fn next_is(&self, bytes: &[u8]) -> bool { - self.limit - self.pos >= bytes.len() - && self.data.get(self.pos..self.pos + bytes.len()) == Some(bytes) + /// `bytes`, compared byte by byte (not with a call to `memcmp`). + #[inline(always)] + fn next_is(&self, bytes: [u8; N]) -> bool { + self.limit - self.pos >= N + && bytes + .iter() + .enumerate() + .all(|(i, byte)| self.data.get(self.pos + i) == Some(byte)) } #[inline(always)] @@ -1039,7 +1042,7 @@ impl<'c> Decoder<'c> { /// writes. #[inline(always)] fn memoize_next(&mut self, reader: &mut Reader<'_>, stack: &[Entry]) -> Option<()> { - if reader.next_is(&[0x94]) { + if reader.next_is([0x94]) { reader.pos += 1; self.opcode(0x94); self.memoize(stack.last()?)?; @@ -1563,7 +1566,7 @@ impl<'c> Decoder<'c> { push(stack, (Object::new_tuple(Vec::new()), node))?; } // The pickler writes a state with `__slots__` this way. - 0x86 if reader.next_is(&[0x94, BUILD]) => { + 0x86 if reader.next_is([0x94, BUILD]) => { reader.pos += 2; self.opcode(0x94); self.opcode(BUILD); From 0338165c1a82cd8086ef876b17fe275f1f7a4995 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 23:03:42 -0700 Subject: [PATCH 21/96] perf: compile hot loops the tier-2 compiler leaves behind to native code A baseline compiler for the core loop: a code object whose hot loop runs mostly in instructions it handles is compiled with Cranelift into one function over its basic blocks, entered at any block start. It keeps a virtual operand stack (locals and constants are referenced, not copied; int, float and bool arithmetic and compares stay unboxed), reads cached instance fields in place, runs local method calls the core loop would run in place through a helper, and hands every other instruction back to the core loop with the frame exactly as the loop expects it. Admission keeps compile costs off code that wouldn't gain: only a loop that got hot compiles, and only when its instructions run natively at least eight to one against hand-overs, counting a local method call as native only when its site resolved to a builtin or a leaf. Cranelift runs without its optimizer and with the single-pass allocator, since the lowering does the optimizing that pays. Whole-run instructions, frame JIT off vs on: deque_ops -15%, dict_ops -8%, datetime_ops -4%, generators -3%, deltablue +1%, the rest within noise. --- crates/weavepy-compiler/src/cpython_code.rs | 2 +- crates/weavepy-compiler/src/lib.rs | 7 + crates/weavepy-vm/src/frame_jit.rs | 2805 +++++++++++++++++++ crates/weavepy-vm/src/gen_fast.rs | 50 + crates/weavepy-vm/src/hot_gates.rs | 7 + crates/weavepy-vm/src/lib.rs | 425 +-- tests/regrtest/test_frame_jit.py | 191 ++ 7 files changed, 3312 insertions(+), 175 deletions(-) create mode 100644 crates/weavepy-vm/src/frame_jit.rs create mode 100644 tests/regrtest/test_frame_jit.py diff --git a/crates/weavepy-compiler/src/cpython_code.rs b/crates/weavepy-compiler/src/cpython_code.rs index ca02b7f6..0a2e1d7b 100644 --- a/crates/weavepy-compiler/src/cpython_code.rs +++ b/crates/weavepy-compiler/src/cpython_code.rs @@ -1480,7 +1480,7 @@ fn encode_exception_table(code: &CodeObject, starts: &[usize]) -> Vec { /// depths for inlined comprehensions; entries the walk never reaches /// stay -1. Handlers whose depth is still the sentinel do not seed /// the walk (their depth is exactly what's being computed). -pub(crate) fn compute_startdepths(code: &CodeObject) -> Vec { +pub fn compute_startdepths(code: &CodeObject) -> Vec { use OpCode as O; let n = code.instructions.len(); let mut startdepth: Vec = vec![-1; n]; diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index 6f840348..64b87217 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -280,6 +280,13 @@ impl JitHint { true } + /// Whether the tier-2 state has no further interest in this code's + /// back edges (see [`Self::set_backedge_quiet`]). + #[inline] + pub fn is_backedge_quiet(&self) -> bool { + self.backedge_quiet.load(std::sync::atomic::Ordering::Relaxed) + } + pub fn set_backedge_quiet(&self) { self.backedge_quiet .store(true, std::sync::atomic::Ordering::Relaxed); diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs new file mode 100644 index 00000000..62bb9b8b --- /dev/null +++ b/crates/weavepy-vm/src/frame_jit.rs @@ -0,0 +1,2805 @@ +//! Native code for hot frames: a baseline compiler for the core loop. +//! +//! The core loop ([`Interpreter::leaf_core`]) decodes and dispatches one +//! instruction at a time, and every operand it handles makes a round trip +//! through the frame's operand stack. A code object that keeps running is +//! compiled here into one function over its instructions, entered at the +//! start of any basic block. The function keeps a *virtual* operand stack +//! while it lowers a block: a local or constant load is just a note of +//! where the value lives, scalar arithmetic and compares produce unboxed +//! values in registers, and a branch tests them directly, so a typical +//! `i = i + 1` or `if a < b:` touches neither the stack nor a reference +//! count. Values reach the frame's stack (copied, with their references +//! taken) only where something else must see them: at the end of a block, +//! and before any instruction the native code leaves to the core loop. +//! +//! An instruction the native code doesn't take, and any shape an in-line +//! one doesn't settle (an overflow, a heap operand, a cache miss), returns +//! to the core loop with the frame exactly as the loop would have it +//! before that instruction: the loop runs it, and enters the native code +//! again at the next block start it reaches. The core loop's fused shapes +//! (a local receiver's method call or attribute store) start at their +//! `LOAD_FAST`, so the native code hands those over at it. +//! +//! Compiling costs far more than interpreting a few iterations, and every +//! hand-over to the core loop costs a round trip, so a code object +//! compiles only once a loop of it is hot and that loop runs mostly in +//! native code (see `worth_compiling`); a loop of calls and global loads +//! stays with the core loop. Code a tier-2 loop may still take waits for +//! tier 2 to settle first. +//! +//! The code reads and writes [`Object`]s in place, through the layout +//! `repr(u8)` fixes: a tag byte at offset 0, a `bool` payload at offset 1, +//! and a word payload at offset 8. Instance fields are read through the +//! object layout tier 2 measures (see `tier2::obj_layout`). + +use std::cell::Cell; +use std::sync::atomic::{AtomicU32, Ordering}; + +use cranelift_codegen::ir::{ + condcodes::{FloatCC, IntCC}, + types, AbiParam, Block, InstBuilder, JumpTableData, MemFlags, SigRef, Signature, Type, Value, +}; +use cranelift_codegen::settings::{self, Configurable}; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use cranelift_jit::{JITBuilder, JITModule}; +use cranelift_module::{Linkage, Module}; +use weavepy_compiler::{BinOpKind, CodeObject, CompareKind, OpCode, COMPARE_OP_TO_BOOL_FLAG}; + +use crate::error::RuntimeError; +use crate::object::{Object, PyIterator}; +use crate::sync::Rc; +use crate::{CodeConstObjects, FieldSlot, Interpreter}; + +/// The core loop runs the instruction at `pc` itself. +pub(crate) const INTERP: u32 = 0; +/// A release queued a finalizer: the core loop stops (`LeafStop::Marked`) +/// with `pc` past the releasing instruction. +pub(crate) const MARKED: u32 = 1; +/// A call raised `State::err`, with `pc` past it. +pub(crate) const RAISED: u32 = 2; + +/// Activations and back edges before a code object is compiled +/// (`WEAVEPY_FRAME_JIT_HOT` overrides it, a tuning aid). +fn hot() -> u32 { + static HOT: std::sync::OnceLock = std::sync::OnceLock::new(); + *HOT.get_or_init(|| { + std::env::var("WEAVEPY_FRAME_JIT_HOT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(1000) + }) +} + +/// The longest code compiled. +const MAX_INSTRUCTIONS: usize = 4096; + +/// The running activation's state, as the core loop holds it: the native +/// code reads it on entry and writes it back on exit. +#[repr(C)] +pub(crate) struct State { + pub(crate) locals: *mut Object, + pub(crate) stack: *mut Object, + pub(crate) len: usize, + pub(crate) cap: usize, + pub(crate) pc: usize, + pub(crate) last: usize, + /// The interpreter's GIL countdown (the back edges' eval breaker). + pub(crate) countdown: *mut u32, + /// The loop generation the burst started under. + pub(crate) snap_gen: u64, + /// The running thread's queued-finalizer flag. + pub(crate) maybe_dead: *const Cell, + pub(crate) interp: *const Interpreter, + /// A helper's scalar result. + pub(crate) out: u64, + /// The thread's recursion-depth cell. + pub(crate) depth_cell: *const Cell, + /// What a `RAISED` exit raised. + pub(crate) err: Option, +} + +const S_LOCALS: i32 = 0; +const S_STACK: i32 = 8; +const S_LEN: i32 = 16; +const S_CAP: i32 = 24; +const S_PC: i32 = 32; +const S_LAST: i32 = 40; +const S_COUNTDOWN: i32 = 48; +const S_SNAP: i32 = 56; +const S_DEAD: i32 = 64; +const S_OUT: i32 = 80; +const _: () = { + assert!(std::mem::offset_of!(State, locals) == S_LOCALS as usize); + assert!(std::mem::offset_of!(State, stack) == S_STACK as usize); + assert!(std::mem::offset_of!(State, len) == S_LEN as usize); + assert!(std::mem::offset_of!(State, cap) == S_CAP as usize); + assert!(std::mem::offset_of!(State, pc) == S_PC as usize); + assert!(std::mem::offset_of!(State, last) == S_LAST as usize); + assert!(std::mem::offset_of!(State, countdown) == S_COUNTDOWN as usize); + assert!(std::mem::offset_of!(State, snap_gen) == S_SNAP as usize); + assert!(std::mem::offset_of!(State, maybe_dead) == S_DEAD as usize); + assert!(std::mem::offset_of!(State, out) == S_OUT as usize); +}; + +type NativeFn = unsafe extern "C" fn(*mut State) -> u32; + +/// A code object's native code. +pub(crate) struct Native { + func: NativeFn, + /// The locals vector's length the code was compiled for. + nlocals: usize, + /// The pcs worth entering at: block starts whose first instruction + /// the native code runs. + entries: Box<[bool]>, + /// The code's name and opcodes (for `WEAVEPY_FRAME_JIT_STATS`). + name: String, + ops: Box<[OpCode]>, +} + +// SAFETY: the code only touches the state it is handed; it is only +// entered with the GIL held (never in the free-threaded build). +unsafe impl Send for Native {} +// SAFETY: as above. +unsafe impl Sync for Native {} + +impl Native { + /// Run from `st.pc` until an instruction the core loop must run. + /// + /// # Safety + /// + /// `st` is the running activation of the code this was compiled + /// from, as the core loop holds it, with `nlocals` locals. + /// Whether entering at `pc` gets anywhere. + #[inline(always)] + pub(crate) fn enters_at(&self, pc: usize) -> bool { + self.entries.get(pc).copied().unwrap_or(false) + } + + #[inline(always)] + pub(crate) unsafe fn run(&self, st: &mut State) -> u32 { + let from = st.pc; + // SAFETY: the caller's contract. + let status = unsafe { (self.func)(st) }; + if stats::enabled() { + stats::note(&self.name, &self.ops, from, st.pc, status); + } + status + } +} + +/// A code object's compilation state, kept in its extension. +#[derive(Default)] +pub(crate) struct Slot { + native: std::sync::OnceLock>>, + heat: AtomicU32, +} + +impl Slot { + /// The native code for an activation with `nlocals` locals, if + /// compiled. + #[inline(always)] + pub(crate) fn get(&self, nlocals: usize) -> Option<&Native> { + match self.native.get() { + Some(Some(n)) if n.nlocals == nlocals => Some(n), + _ => None, + } + } + + /// Count one activation or back edge of code without native code, + /// compiling it once it is hot. + #[inline(always)] + /// Count a call (`at` 0) or a back edge (`at` its pc) toward + /// compiling `code`. + pub(crate) fn warm(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize, at: usize) { + if self.native.get().is_some() { + return; + } + // (A racing thread losing a count is harmless.) + let h = self.heat.load(Ordering::Relaxed) + 1; + self.heat.store(h, Ordering::Relaxed); + if h >= hot() { + self.heat.store(0, Ordering::Relaxed); + self.try_compile(code, ext, nlocals, at); + } + } + + #[cold] + #[inline(never)] + fn try_compile(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize, at: usize) { + if !enabled() { + let _ = self.native.set(None); + return; + } + // Loops the tier-2 compiler may still take keep their back edges' + // consultation in the core loop. + let hint = &code.jit_hint; + if !(hint.is_not_jitable() + || crate::tier2::jit_off_for_process() + || hint.loop_free(code) + || hint.is_backedge_quiet()) + { + return; + } + if !name_admitted(&code.qualname) { + let _ = self.native.set(None); + return; + } + if !worth_compiling(code, ext, at) { + // (Another loop may yet get hot.) + return; + } + let t0 = stats::enabled().then(std::time::Instant::now); + let n = self + .native + .get_or_init(|| compile(code, ext, nlocals).map(Box::new)); + if let Some(t0) = t0 { + eprintln!( + "frame jit: {} compiled: {} ({} instructions, {:?})", + code.qualname, + n.is_some(), + code.instructions.len(), + t0.elapsed() + ); + } + } +} + +/// Whether the loop of `code` whose back edge is at `at` (any loop, for +/// a call) runs mostly in native code: each +/// instruction the core loop runs costs a round trip out of the native +/// code and back, so a loop of calls and global loads gains nothing from +/// compiling (nor does a loop-free body, whose entry costs as much as the +/// little it would save). +fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: usize) -> bool { + use weavepy_compiler::OpCode; + let ins = &code.instructions; + // A method call on a local runs through a helper without leaving, when + // its callee runs in place. + let mut helped = vec![false; ins.len()]; + for pc in 0..ins.len().saturating_sub(1) { + if ins[pc].op == OpCode::LoadFast + && ins[pc + 1].op == OpCode::LoadMethodAttr + && crate::method_site_in_place(ext, pc + 1) + { + helped[pc + 1] = true; + if let Some(call) = (pc + 2..ins.len().min(pc + 12)).find(|&k| ins[k].op == OpCode::Call) { + helped[call] = true; + } + } + } + let at_loop = ins.get(at).is_some_and(|i| i.op == OpCode::JumpBackward); + ins.iter().enumerate().any(|(pc, i)| { + if i.op != OpCode::JumpBackward || (at_loop && pc != at) { + return false; + } + let top = (pc + 1).saturating_sub(i.arg as usize); + let (mut native, mut exits) = (0usize, 0usize); + for k in top..=pc { + if native_op(ins[k].op) || helped[k] { + native += 1; + } else { + exits += 1; + } + } + if stats::enabled() { + eprintln!("frame jit: {} loop at {top}: {native} native, {exits} exits", code.qualname); + } + exits * EXIT_WEIGHT <= native + }) +} + +/// How many natively run instructions a loop needs per instruction that +/// leaves to the core loop before compiling it pays. +const EXIT_WEIGHT: usize = 8; + +/// Whether a code object of this qualified name may compile: +/// `WEAVEPY_FRAME_JIT_ONLY` / `WEAVEPY_FRAME_JIT_SKIP` admit only, or +/// skip, names containing their value (a debugging aid). +fn name_admitted(name: &str) -> bool { + static FILTER: std::sync::OnceLock<(Option, Option)> = + std::sync::OnceLock::new(); + let (only, skip) = FILTER.get_or_init(|| { + let var = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty()); + (var("WEAVEPY_FRAME_JIT_ONLY"), var("WEAVEPY_FRAME_JIT_SKIP")) + }); + only.as_deref().is_none_or(|o| name.contains(o)) + && skip.as_deref().is_none_or(|s| !name.contains(s)) +} + +/// Whether frames compile at all (`WEAVEPY_FRAME_JIT=0` turns them off). +fn enabled() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + std::env::var_os("WEAVEPY_FRAME_JIT").is_none_or(|v| v != "0") + && !crate::gil::free_threading_enabled() + }) +} + +/// `WEAVEPY_FRAME_JIT_STATS=1`: entries and the instructions native code +/// returned at, by code object, printed at exit (a tuning aid). +mod stats { + use std::collections::HashMap; + use std::sync::Mutex; + use weavepy_compiler::OpCode; + + type Counts = HashMap<(String, String), [u64; 2]>; + + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + static COUNTS: Mutex> = Mutex::new(None); + + #[inline(always)] + pub(super) fn enabled() -> bool { + *ON.get_or_init(|| { + let on = std::env::var_os("WEAVEPY_FRAME_JIT_STATS").is_some(); + if on { + extern "C" fn dump() { + let counts = COUNTS.lock().unwrap_or_else(|e| e.into_inner()); + let mut rows: Vec<_> = counts.iter().flat_map(|c| c.iter()).collect(); + rows.sort_by_key(|(_, n)| std::cmp::Reverse(n[0])); + for ((name, op), n) in rows.iter().take(40) { + eprintln!("{:>10} {:>10} {name}: {op}", n[0], n[1]); + } + } + // SAFETY: registering a plain function. + unsafe { libc::atexit(dump) }; + } + on + }) + } + + #[cold] + pub(super) fn note(name: &str, ops: &[OpCode], from: usize, at: usize, status: u32) { + let op = match (status, ops.get(at)) { + (0, Some(op)) => format!("{op:?}"), + (0, None) => "".to_owned(), + _ => "".to_owned(), + }; + let mut counts = COUNTS.lock().unwrap_or_else(|e| e.into_inner()); + let e = counts + .get_or_insert_with(HashMap::new) + .entry((name.to_owned(), op)) + .or_default(); + e[0] += 1; + e[1] += u64::from(at != from); + } +} + +// ---- the object layout ---------------------------------------------------- + +/// The tags of the variants the code handles in line. +#[derive(Clone, Copy)] +struct Tags { + none: u8, + unbound: u8, + boolean: u8, + int: u8, + float: u8, + cell: u8, + instance: u8, + /// The heap variants whose identity is their payload pointer (and so + /// `is` compares payload words): instances, strings, lists, dicts, + /// tuples, classes. + by_pointer: i64, +} + +impl Tags { + /// `None`, `bool`, `int` and `float`: copied by value, dropped by + /// forgetting. + fn scalar_mask(self) -> i64 { + (1i64 << self.none) | (1 << self.boolean) | (1 << self.int) | (1 << self.float) + } +} + +/// The tags, read off sample values, if the layout is the one the code +/// assumes. +fn tags() -> Option { + let tag = |v: &Object| { + // SAFETY: `Object` is `repr(u8)`: its first byte is the tag. + unsafe { *std::ptr::from_ref(v).cast::() } + }; + let word = |v: &Object| { + // SAFETY: every `Object` is 16 bytes; the second word is + // initialized for the word payloads read here. + unsafe { *std::ptr::from_ref(v).cast::().add(1) } + }; + let byte1 = |v: &Object| { + // SAFETY: as above (a `bool` payload's byte). + unsafe { *std::ptr::from_ref(v).cast::().add(1) } + }; + let cell = Object::Cell(Rc::new(crate::sync::RefCell::new(Object::None))); + let inst = Object::Instance(Rc::new(crate::types::PyInstance::new( + crate::builtin_types::builtin_types().object_.clone(), + ))); + let samples = [ + tag(&inst), + tag(&Object::from_str("")), + tag(&Object::new_list(Vec::new())), + tag(&Object::new_dict()), + tag(&Object::new_tuple_array([Object::None])), + tag(&Object::Type(crate::builtin_types::builtin_types().object_.clone())), + ]; + let t = Tags { + none: tag(&Object::None), + unbound: tag(&Object::Unbound), + boolean: tag(&Object::Bool(true)), + int: tag(&Object::Int(1)), + float: tag(&Object::Float(1.0)), + cell: tag(&cell), + instance: tag(&inst), + by_pointer: samples + .iter() + .fold(0, |m, &t| if t < 64 { m | (1i64 << t) } else { m }), + }; + if samples.iter().any(|&t| t >= 64) { + return None; + } + let all = [t.none, t.unbound, t.boolean, t.int, t.float, t.cell, t.instance]; + let distinct = all + .iter() + .enumerate() + .all(|(i, a)| all[i + 1..].iter().all(|b| a != b)); + (distinct + && all.iter().all(|&t| t < 64) + && tag(&Object::Bool(false)) == t.boolean + && byte1(&Object::Bool(true)) == 1 + && byte1(&Object::Bool(false)) == 0 + && word(&Object::Int(0x0123_4567_89ab_cdef)) == 0x0123_4567_89ab_cdef + && word(&Object::Float(1.5)) == 1.5f64.to_bits() + && std::mem::size_of::() == 16) + .then_some(t) +} + +// ---- helpers (the core loop's arms, for native code) ---------------------- + +#[inline(always)] +fn droppable(v: &Object) -> bool { + Interpreter::core_droppable(v) +} + +/// `*dst = (*src).clone()`. +unsafe extern "C" fn h_clone(dst: *mut Object, src: *const Object) { + // SAFETY: the code passes an initialized value and a free slot. + unsafe { dst.write(crate::clone_hot(&*src)) } +} + +/// Release the heap value at `slot` (it leaves the stack): `0` released, +/// anything else declined with the slot untouched. +unsafe extern "C" fn h_pop(slot: *mut Object) -> u32 { + // SAFETY: the code passes an initialized stack slot. + unsafe { + if !droppable(&*slot) { + return 1; + } + crate::drop_hot(slot.read()); + } + 0 +} + +/// `STORE_FAST i` of the value at stack `slot` over a heap value: `0` +/// stored (the value left the stack), anything else declined untouched. +unsafe extern "C" fn h_store(st: *mut State, slot: *mut Object, i: u64) -> u32 { + // SAFETY: the code passes its live state, an initialized stack slot, + // and a local index in range. + unsafe { + let st = &mut *st; + let local = st.locals.add(i as usize); + if !droppable(&*local) { + return 1; + } + let v = slot.read(); + (*st.interp).release(std::mem::replace(&mut *local, v)); + } + 0 +} + +/// `FOR_ITER` over a range, list or tuple iterator at `it` (the next value +/// goes to `out`, the slot above): `0` an `int` in `st.out`, `1` another +/// value written to `out`, `2` the exhausted iterator retired (it left the +/// stack), anything else declined. +unsafe extern "C" fn h_for_iter(st: *mut State, it: *mut Object, out: *mut Object) -> u32 { + // SAFETY: the code passes its live state and an initialized slot with + // a free one above it. + let (st, top) = unsafe { (&mut *st, &*it) }; + let Object::Iter(rc) = top else { + return 3; + }; + let unique = Rc::strong_count(rc) == 1; + // SAFETY: nothing below runs code while the iterator is borrowed. + let Some(iter) = (unsafe { rc.peek_mut() }) else { + return 3; + }; + let v = match iter { + PyIterator::Range { + current, + stop, + step, + } => { + let live = if *step > 0 { + *current < *stop + } else { + *step < 0 && *current > *stop + }; + if live { + st.out = *current as u64; + *current = current.wrapping_add(*step); + return 0; + } + if !unique { + return 3; + } + // SAFETY: the iterator leaves the stack. + drop(unsafe { it.read() }); + return 2; + } + PyIterator::List { + items, + index, + owner, + } => { + // SAFETY: as above. + let Some(xs) = (unsafe { items.peek() }) else { + return 3; + }; + match xs.get(*index) { + Some(v) => { + let v = crate::clone_hot(v); + *index += 1; + v + } + None if unique && owner.is_none() => { + // SAFETY: as above. + drop(unsafe { it.read() }); + return 2; + } + None => return 3, + } + } + PyIterator::Tuple { items, index } => { + let Some(v) = items.get(*index).cloned() else { + return 3; + }; + *index += 1; + v + } + _ => return 3, + }; + if let Object::Int(i) = v { + st.out = i as u64; + return 0; + } + // SAFETY: the slot above the iterator is free. + unsafe { out.write(v) }; + 1 +} + +/// A container instruction (`BINARY_SUBSCR`, `BINARY_SLICE`, `STORE_SUBSCR`, +/// `LIST_APPEND`, `UNPACK_SEQUENCE`; the one at `ins`, the `pc`th of +/// `code`) on the top of a `len`-deep stack, as the core loop's arm runs +/// it: the new depth, or `u64::MAX` declined untouched. +unsafe extern "C" fn h_container( + st: *mut State, + ins: *const weavepy_compiler::Instruction, + code: *const CodeObject, + pc: u64, + len: u64, +) -> u64 { + // SAFETY: the code passes its live state, one of its own instructions + // and its code, and its stack depth. + unsafe { + let st = &mut *st; + let (ins, len) = (*ins, len as usize); + if let Some(n) = Interpreter::core_container_op(ins, st.stack, len, st.cap) { + return n as u64; + } + // An instance whose class's `__getitem__` is a native fast + // subscript the site cached. + if ins.op != OpCode::BinarySubscr || len < 2 { + return u64::MAX; + } + let ops = std::slice::from_raw_parts(st.stack.add(len - 2), 2); + let Some(fast) = (*st.interp).core_native_subscript(&*code, pc as usize, ops) else { + return u64::MAX; + }; + // (An error leaves the subscript to the core loop, which raises it.) + let Some(Ok(v)) = fast(ops) else { + return u64::MAX; + }; + #[cfg(test)] + crate::NATIVE_FAST_SUBSCRIPTS.with(|calls| calls.set(calls.get() + 1)); + crate::drop_hot(st.stack.add(len - 1).read()); + crate::drop_hot(st.stack.add(len - 2).read()); + st.stack.add(len - 2).write(v); + (len - 1) as u64 + } +} + +/// `CONTAINS_OP` of the item at `at` and the container above it: `0` or +/// `1` (found; both left the stack), anything else declined untouched. +unsafe extern "C" fn h_contains(at: *mut Object) -> u32 { + // SAFETY: the code passes two initialized stack slots. + unsafe { + let (item, container) = (&*at, &*at.add(1)); + if !droppable(item) || !droppable(container) { + return 2; + } + let Some(found) = Interpreter::leaf_contains(container, item) else { + return 2; + }; + crate::drop_hot(at.add(1).read()); + crate::drop_hot(at.read()); + u32::from(found) + } +} + +/// `BINARY_OP` of `kind` over the values at `at` and above it, for the +/// shapes the core loop's arm runs (two numbers; strings' concatenation, +/// repetition and `%` formatting; a natively served instance's operator): +/// `0` the result at `at` (the operands released), anything else declined +/// untouched. +unsafe extern "C" fn h_binop(at: *mut Object, kind: u32) -> u32 { + // SAFETY: the code passes two initialized stack slots, and a kind the + // compiler emitted (`BinOpKind` is `repr(u8)`). + unsafe { + let (a, b) = (&*at, &*at.add(1)); + // Two numbers (the core loop's in-line arm). + let num = |o: &Object| match *o { + Object::Int(i) => Some(crate::leaf_plan::V::I(i)), + Object::Float(x) => Some(crate::leaf_plan::V::F(x)), + _ => None, + }; + if let (Some(x), Some(y)) = (num(a), num(b)) { + let r = match crate::leaf_plan::binary(x, y, kind as u8) { + Some(crate::leaf_plan::V::I(i)) => Object::Int(i), + Some(crate::leaf_plan::V::F(f)) => Object::Float(f), + _ => return 1, + }; + at.write(r); + return 0; + } + let kind: BinOpKind = std::mem::transmute(kind as u8); + let r = match (a, b) { + (Object::Str(_), _) | (Object::Int(_), Object::Str(_)) => { + Interpreter::core_str_binop(a, b, kind) + } + (Object::Instance(i), _) if i.cls_raw().native_kind.get() != 0 => { + match Interpreter::core_native_binop(kind, a, b) { + Some(Ok(r)) if droppable(a) && droppable(b) => Some(r), + _ => None, + } + } + _ => None, + }; + let Some(r) = r else { + return 1; + }; + crate::drop_hot(at.add(1).read()); + crate::drop_hot(at.read()); + at.write(r); + } + 0 +} + +/// `LOAD_FAST x` (the `pc`th instruction of `code`) and the call +/// `x.m()` after it, as the core loop's fused arm runs it +/// (see `Interpreter::core_local_method`), on a stack `len` deep: `0` the +/// result pushed and `st.pc` / `st.last` past the call, `RAISED` the call +/// raised, anything else declined untouched. +unsafe extern "C" fn h_local_method( + st: *mut State, + code: *const CodeObject, + ext: *const CodeConstObjects, + pc: u64, + nlocals: u64, + len: u64, +) -> u32 { + // SAFETY: the code passes its live state, its own code and extension, + // a `LOAD_FAST` of an in-range local, and its stack depth (with room + // for a push). + unsafe { + let st = &mut *st; + let (code, ext, pc) = (&*code, &*ext, pc as usize); + let local = &*st.locals.add(code.instructions[pc].arg as usize); + let Some((r, call_pc)) = (*st.interp).core_local_method( + code, + Some(ext), + local, + pc, + st.locals, + nlocals as usize, + &ext.objects, + st.depth_cell, + ) else { + return 1; + }; + st.last = call_pc; + st.pc = call_pc + 1; + match r { + Ok(v) => { + st.stack.add(len as usize).write(v); + st.len = len as usize + 1; + 0 + } + Err(e) => { + st.len = len as usize; + st.err = Some(e); + RAISED + } + } + } +} + +/// `LOAD_FAST x` (the `pc`th instruction of `code`) and `x.attr = v` +/// after it, with `v` at stack `slot`, as the core loop's fused arm runs +/// it: `0` stored (the value left the stack), anything else declined +/// untouched. +unsafe extern "C" fn h_store_attr( + st: *mut State, + code: *const CodeObject, + pc: u64, + slot: *mut Object, +) -> u32 { + // SAFETY: the code passes its live state, its own code, a `LOAD_FAST` + // of an in-range local followed by `STORE_ATTR`, and the value's slot. + unsafe { + let st = &*st; + let (code, pc) = (&*code, pc as usize); + let local = &*st.locals.add(code.instructions[pc].arg as usize); + let name = code.instructions[pc + 1].arg; + u32::from(!Interpreter::core_store_local_attr(code, local, pc + 1, name, &*slot)) + } +} + +// ---- the compiler --------------------------------------------------------- + +/// The process's frame compiler. Code it emits lives as long as the +/// process (code objects are shared between threads, and their native +/// code is never freed). +struct Engine { + module: JITModule, + ctx: cranelift_codegen::Context, + fbctx: FunctionBuilderContext, + ptr: types::Type, + tags: Tags, + next: u32, +} + +// SAFETY: the engine is only used under its mutex. +unsafe impl Send for Engine {} + +static ENGINE: std::sync::Mutex>> = std::sync::Mutex::new(None); + +impl Engine { + fn new() -> Option { + let tags = tags()?; + let mut flags = settings::builder(); + flags.set("use_colocated_libcalls", "false").ok()?; + flags.set("is_pic", "false").ok()?; + // The lowering already does the optimizing that pays here (operands + // kept unboxed and in registers); Cranelift's own passes cost more + // to run than they save. `WEAVEPY_FRAME_JIT_OPT` turns them on. + if std::env::var_os("WEAVEPY_FRAME_JIT_OPT").is_some() { + flags.set("opt_level", "speed").ok()?; + } else { + flags.set("opt_level", "none").ok()?; + flags.set("regalloc_algorithm", "single_pass").ok()?; + } + if !cfg!(debug_assertions) && !verify() { + flags.set("enable_verifier", "false").ok()?; + } + let isa = cranelift_native::builder() + .ok()? + .finish(settings::Flags::new(flags)) + .ok()?; + let module = JITModule::new(JITBuilder::with_isa( + isa, + cranelift_module::default_libcall_names(), + )); + let ptr = module.target_config().pointer_type(); + let ctx = module.make_context(); + Some(Engine { + module, + ctx, + fbctx: FunctionBuilderContext::new(), + ptr, + tags, + next: 0, + }) + } +} + +/// Whether to verify the IR of every compiled frame and report failures +/// (`WEAVEPY_FRAME_JIT_VERIFY`, a debugging aid). +fn verify() -> bool { + static VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); + *VERIFY.get_or_init(|| std::env::var_os("WEAVEPY_FRAME_JIT_VERIFY").is_some()) +} + +/// Compile `code`; `None` leaves it to the core loop. A compiler panic +/// turns frame compilation off for the process rather than taking the +/// interpreter down. +fn compile(code: &CodeObject, ext: &CodeConstObjects, nlocals: usize) -> Option { + if code.instructions.is_empty() || code.instructions.len() > MAX_INSTRUCTIONS { + return None; + } + crate::tier2::ensure_obj_layout(); + let depths = weavepy_compiler::cpython_code::compute_startdepths(code); + let mut guard = ENGINE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let engine = guard.get_or_insert_with(Engine::new).as_mut()?; + let built = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + compile_with(engine, code, ext, nlocals, &depths) + })); + match built { + Ok(native) => native, + Err(_) => { + *guard = Some(None); + None + } + } +} + +fn compile_with( + engine: &mut Engine, + code: &CodeObject, + ext: &CodeConstObjects, + nlocals: usize, + depths: &[i64], +) -> Option { + engine.module.clear_context(&mut engine.ctx); + let ptr = engine.ptr; + let sig = &mut engine.ctx.func.signature; + sig.params.push(AbiParam::new(ptr)); + sig.returns.push(AbiParam::new(types::I32)); + let name = format!("wpframe_{}", engine.next); + engine.next += 1; + let id = engine + .module + .declare_function(&name, Linkage::Local, &engine.ctx.func.signature) + .ok()?; + let ninstrs = code.instructions.len(); + // The attribute sites' field caches (allocated here if no read has + // recorded one yet: the code addresses them). + let field_slots: &[FieldSlot] = ext + .field_slots + .get_or_init(|| (0..ninstrs).map(|_| FieldSlot::empty()).collect()); + let entries: Vec; + let built = { + let b = FunctionBuilder::new(&mut engine.ctx.func, &mut engine.fbctx); + let mut lower = Lower::new( + b, + ptr, + engine.tags, + code, + ext, + nlocals, + depths, + field_slots, + ); + let done = lower.lower(); + entries = (0..ninstrs).map(|pc| lower.enters_at(pc)).collect(); + if done { + lower.b.seal_all_blocks(); + lower.b.finalize(); + } + done + }; + if !built { + // An unfinished function leaves the builder's state behind. + engine.fbctx = FunctionBuilderContext::new(); + engine.module.clear_context(&mut engine.ctx); + return None; + } + if std::env::var("WEAVEPY_FRAME_JIT_DUMP").is_ok_and(|n| code.qualname.contains(&n)) { + eprintln!("{}", engine.ctx.func.display()); + } + let defined = engine.module.define_function(id, &mut engine.ctx); + if let (Err(e), true) = (&defined, verify()) { + eprintln!( + "weavepy: frame {:?} failed to compile: {e:?}", + code.qualname + ); + } + engine.module.clear_context(&mut engine.ctx); + defined.ok()?; + engine.module.finalize_definitions().ok()?; + let func = engine.module.get_finalized_function(id); + // SAFETY: `func` is a finalized function with exactly the signature + // declared above, and the module (never dropped) keeps it alive. + let func = unsafe { std::mem::transmute::<*const u8, NativeFn>(func) }; + Some(Native { + func, + nlocals, + entries: entries.into_boxed_slice(), + name: code.qualname.clone(), + ops: code.instructions.iter().map(|i| i.op).collect(), + }) +} + +const FLAGS: MemFlags = MemFlags::trusted(); + +/// A value on the virtual operand stack: where it lives until something +/// needs it on the frame's stack. +#[derive(Clone, Copy)] +enum Item { + /// On the frame's stack, at this slot (owned there). + Mem(usize), + /// The value of this local, not copied yet (no reference taken). + Local(u32), + /// This constant, not copied yet. + Const(u32), + /// An `int`, unboxed. + Int(Value), + /// A `float`, unboxed. + Float(Value), + /// A `bool`, unboxed (an `i8`, `0` or `1`). + Bool(Value), + /// A scalar (`None`, `bool`, `int` or `float`) known only at run time: + /// its tag, word payload, and `bool` byte. + Dyn(Value, Value, Value), +} + +/// An operand's type, as far as the code knows it. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + Int, + Float, + Bool, + None, + /// Read at run time. + Unknown, +} + +/// An operand ready for a scalar operation: its kind, and its tag and +/// payload values (the tag only for an unknown kind). +#[derive(Clone, Copy)] +struct Opnd { + kind: Kind, + tag: Option, + /// The `i64` word (an `int`, a `float`'s bits), or a `bool`'s `i8`. + word: Value, + /// A `bool`'s byte (unknown kinds). + byte: Option, +} + +struct Lower<'a> { + b: FunctionBuilder<'a>, + ptr: types::Type, + tags: Tags, + layout: Option<&'static weavepy_jit::ObjLayout>, + code: &'a CodeObject, + ext: &'a CodeConstObjects, + nlocals: usize, + depths: &'a [i64], + field_slots: &'a [FieldSlot], + st: Value, + locals: Value, + stack: Value, + snap: Value, + countdown: Value, + dead: Value, + /// The instruction that last ran, when known here; `None` at a block's + /// start, whose `st.last` the entry or the jump in wrote. + last_pc: Option, + /// Locals that may be unbound when a block starts (parameters no + /// `DELETE_FAST` names never are), and the ones this block has seen + /// bound. + maybe_unbound: Vec, + bound: Vec, + /// Each block start's block (instruction index → block). + blocks: Vec>, + /// Dispatches on the state's pc and depth (the entry, and helpers that + /// move the pc by a run-time amount). + dispatch: Block, + /// Whether copies are on an exit path (one helper call each, not the + /// scalar test in line: exits are rare, and code size costs compile + /// time). + cold: bool, + /// The virtual stack above the frame's, and the logical depth. + vs: Vec, + depth: usize, + sigs: Vec<(Vec, Option, SigRef)>, + /// Side exits waiting for their blocks to be filled (after the body: + /// the builder fills one block at a time). + side_exits: Vec, +} + +/// An exit block of [`Lower::exit_with`], filled after the body from the +/// lowering state at its branch. +struct SideExit { + block: Block, + vs: Vec, + depth: usize, + last_pc: Option, + restore: Vec, + status: u32, + pc: usize, +} + +/// Whether the native code runs `op` itself (the rest are handed to the +/// core loop: their successors start blocks). +fn native_op(op: OpCode) -> bool { + matches!( + op, + OpCode::Nop + | OpCode::NotTaken + | OpCode::Resume + | OpCode::CopyFreeVars + | OpCode::LoadFast + | OpCode::LoadFastBorrow + | OpCode::LoadConst + | OpCode::LoadSmallInt + | OpCode::StoreFast + | OpCode::PopTop + | OpCode::BinaryOp + | OpCode::CompareOp + | OpCode::ToBool + | OpCode::PopJumpIfFalse + | OpCode::PopJumpIfTrue + | OpCode::PopJumpIfNone + | OpCode::PopJumpIfNotNone + | OpCode::JumpForward + | OpCode::JumpBackward + | OpCode::ForIter + | OpCode::LoadAttr + | OpCode::IsOp + | OpCode::CopyTop + | OpCode::Swap + | OpCode::BinarySubscr + | OpCode::BinarySlice + | OpCode::StoreSubscr + | OpCode::ListAppend + | OpCode::UnpackSequence + | OpCode::ContainsOp + ) +} + +impl<'a> Lower<'a> { + #[allow(clippy::too_many_arguments)] + fn new( + mut b: FunctionBuilder<'a>, + ptr: types::Type, + tags: Tags, + code: &'a CodeObject, + ext: &'a CodeConstObjects, + nlocals: usize, + depths: &'a [i64], + field_slots: &'a [FieldSlot], + ) -> Self { + let entry = b.create_block(); + b.append_block_params_for_function_params(entry); + b.switch_to_block(entry); + let st = b.block_params(entry)[0]; + let load = |b: &mut FunctionBuilder<'_>, ty, off| b.ins().load(ty, FLAGS, st, off); + let locals = load(&mut b, ptr, S_LOCALS); + let stack = load(&mut b, ptr, S_STACK); + let snap = load(&mut b, types::I64, S_SNAP); + let countdown = load(&mut b, ptr, S_COUNTDOWN); + let dead = load(&mut b, ptr, S_DEAD); + Lower { + b, + ptr, + tags, + layout: crate::tier2::published_obj_layout(), + code, + ext, + nlocals, + depths, + field_slots, + st, + locals, + stack, + snap, + countdown, + dead, + last_pc: None, + maybe_unbound: { + let params = code.arg_count as usize + + code.kwonly_count as usize + + usize::from(code.has_varargs) + + usize::from(code.has_varkeywords); + let mut m: Vec = (0..nlocals).map(|i| i >= params).collect(); + for ins in &code.instructions { + if ins.op == OpCode::DeleteFast { + if let Some(x) = m.get_mut(ins.arg as usize) { + *x = true; + } + } + } + m + }, + bound: vec![false; nlocals], + blocks: Vec::new(), + dispatch: Block::from_u32(0), + cold: false, + vs: Vec::new(), + depth: 0, + sigs: Vec::new(), + side_exits: Vec::new(), + } + } + + // ---- building blocks ---- + + fn sig(&mut self, params: &[Type], ret: Option) -> SigRef { + if let Some((_, _, s)) = self + .sigs + .iter() + .find(|(p, r, _)| p == params && *r == ret) + { + return *s; + } + let mut sig = Signature::new(self.b.func.signature.call_conv); + for &t in params { + sig.params.push(AbiParam::new(t)); + } + if let Some(t) = ret { + sig.returns.push(AbiParam::new(t)); + } + let s = self.b.import_signature(sig); + self.sigs.push((params.to_vec(), ret, s)); + s + } + + /// Call the helper at `addr`: its `u32` result, if it has one. + fn call(&mut self, addr: usize, args: &[Value], ret: bool) -> Option { + self.call_typed(addr, args, ret.then_some(types::I32)) + } + + /// Call the helper at `addr`: its result of type `ret`, if any. + fn call_typed(&mut self, addr: usize, args: &[Value], ret: Option) -> Option { + let params: Vec = args + .iter() + .map(|&a| self.b.func.dfg.value_type(a)) + .collect(); + let sig = self.sig(¶ms, ret); + let callee = self.b.ins().iconst(self.ptr, addr as i64); + let inst = self.b.ins().call_indirect(sig, callee, args); + ret.map(|_| self.b.inst_results(inst)[0]) + } + + fn set_last(&mut self, pc: usize) { + self.last_pc = Some(pc); + } + + /// Write the last instruction to the state (unless the state has it). + fn store_last(&mut self) { + if let Some(pc) = self.last_pc { + let v = self.b.ins().iconst(types::I64, pc as i64); + self.b.ins().store(FLAGS, v, self.st, S_LAST); + } + } + + fn slot_addr(&mut self, slot: usize) -> Value { + self.b.ins().iadd_imm(self.stack, 16 * slot as i64) + } + + fn local_addr(&mut self, i: u32) -> Value { + self.b.ins().iadd_imm(self.locals, 16 * i64::from(i)) + } + + fn const_addr(&mut self, k: u32) -> Value { + let at = std::ptr::from_ref(&self.ext.objects[k as usize]) as i64; + self.b.ins().iconst(self.ptr, at) + } + + fn tag_at(&mut self, addr: Value) -> Value { + self.b.ins().uload8(types::I64, FLAGS, addr, 0) + } + + fn is_scalar(&mut self, tag: Value) -> Value { + let one = self.b.ins().iconst(types::I64, 1); + let bit = self.b.ins().ishl(one, tag); + let m = self.b.ins().band_imm(bit, self.tags.scalar_mask()); + self.b.ins().icmp_imm(IntCC::NotEqual, m, 0) + } + + fn write_tag(&mut self, dst: Value, t: u8) { + let t = self.b.ins().iconst(types::I8, i64::from(t)); + self.b.ins().store(FLAGS, t, dst, 0); + } + + fn copy16(&mut self, dst: Value, src: Value) { + let a = self.b.ins().load(types::I64, FLAGS, src, 0); + let w = self.b.ins().load(types::I64, FLAGS, src, 8); + self.b.ins().store(FLAGS, a, dst, 0); + self.b.ins().store(FLAGS, w, dst, 8); + } + + /// Branch to `out` when `cond`; continue in a fresh block otherwise. + fn branch_out(&mut self, cond: Value, out: Block) { + let ok = self.b.create_block(); + self.b.ins().brif(cond, out, &[], ok, &[]); + self.b.switch_to_block(ok); + } + + // ---- the virtual stack ---- + + fn push(&mut self, item: Item) { + self.vs.push(item); + self.depth += 1; + } + + fn pop(&mut self) -> Item { + self.depth -= 1; + self.vs.pop().unwrap_or(Item::Mem(self.depth)) + } + + /// Write `item` to the frame's stack at `slot` (taking a reference for + /// a heap value it copies). + fn materialize(&mut self, item: Item, slot: usize) { + match item { + Item::Mem(s) => debug_assert_eq!(s, slot), + Item::Local(i) => { + let src = self.local_addr(i); + let dst = self.slot_addr(slot); + self.copy_value(dst, src); + } + Item::Const(k) => { + let dst = self.slot_addr(slot); + match self.ext.objects[k as usize] { + Object::Int(v) => { + let v = self.b.ins().iconst(types::I64, v); + self.write_tag(dst, self.tags.int); + self.b.ins().store(FLAGS, v, dst, 8); + } + Object::Float(x) => { + let v = self.b.ins().iconst(types::I64, x.to_bits() as i64); + self.write_tag(dst, self.tags.float); + self.b.ins().store(FLAGS, v, dst, 8); + } + Object::Bool(x) => { + let v = self.b.ins().iconst(types::I8, i64::from(x)); + self.write_tag(dst, self.tags.boolean); + self.b.ins().store(FLAGS, v, dst, 1); + } + Object::None => self.write_tag(dst, self.tags.none), + _ => { + let src = self.const_addr(k); + self.call(h_clone as *const () as usize, &[dst, src], false); + } + } + } + Item::Int(v) => { + let dst = self.slot_addr(slot); + self.write_tag(dst, self.tags.int); + self.b.ins().store(FLAGS, v, dst, 8); + } + Item::Float(v) => { + let dst = self.slot_addr(slot); + self.write_tag(dst, self.tags.float); + let bits = self.b.ins().bitcast(types::I64, MemFlags::new(), v); + self.b.ins().store(FLAGS, bits, dst, 8); + } + Item::Bool(v) => { + let dst = self.slot_addr(slot); + self.write_tag(dst, self.tags.boolean); + self.b.ins().store(FLAGS, v, dst, 1); + } + Item::Dyn(tag, word, byte) => { + let dst = self.slot_addr(slot); + let t = self.b.ins().ireduce(types::I8, tag); + let b8 = self.b.ins().ireduce(types::I8, byte); + self.b.ins().store(FLAGS, t, dst, 0); + self.b.ins().store(FLAGS, b8, dst, 1); + self.b.ins().store(FLAGS, word, dst, 8); + } + } + } + + /// `*dst = (*src).clone()`: a scalar by value, anything else through + /// its clone. + fn copy_value(&mut self, dst: Value, src: Value) { + if self.cold { + self.call(h_clone as *const () as usize, &[dst, src], false); + return; + } + let tag = self.tag_at(src); + let scalar = self.is_scalar(tag); + let by_value = self.b.create_block(); + let by_clone = self.b.create_block(); + let done = self.b.create_block(); + self.b.ins().brif(scalar, by_value, &[], by_clone, &[]); + self.b.switch_to_block(by_value); + self.copy16(dst, src); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(by_clone); + self.call(h_clone as *const () as usize, &[dst, src], false); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + + /// Write the whole virtual stack to the frame's. + fn flush(&mut self) { + let base = self.depth - self.vs.len(); + let items = std::mem::take(&mut self.vs); + for (k, item) in items.into_iter().enumerate() { + self.materialize(item, base + k); + } + } + + /// Write the virtual stack's copies of local `i` to the frame's stack + /// (before the local changes). + fn flush_local(&mut self, i: u32) { + let base = self.depth - self.vs.len(); + for k in 0..self.vs.len() { + if let Item::Local(j) = self.vs[k] { + if j == i { + self.materialize(Item::Local(j), base + k); + self.vs[k] = Item::Mem(base + k); + } + } + } + } + + /// Return `status` with the virtual stack written out and `pc` next. + fn exit(&mut self, status: u32, pc: usize) { + let cold = std::mem::replace(&mut self.cold, true); + self.flush(); + self.cold = cold; + let l = self.b.ins().iconst(types::I64, self.depth as i64); + self.b.ins().store(FLAGS, l, self.st, S_LEN); + self.store_last(); + let pcv = self.b.ins().iconst(types::I64, pc as i64); + self.b.ins().store(FLAGS, pcv, self.st, S_PC); + let s = self.b.ins().iconst(types::I32, i64::from(status)); + self.b.ins().return_(&[s]); + } + + /// Leave for the core loop at `pc` (the instruction about to run), + /// with `restore` (the operands it popped) back on the stack, from a + /// side block; lowering continues in the current block afterwards. + fn exit_with(&mut self, pc: usize, restore: &[Item], status: u32) -> Block { + let block = self.b.create_block(); + self.side_exits.push(SideExit { + block, + vs: self.vs.clone(), + depth: self.depth, + last_pc: self.last_pc, + restore: restore.to_vec(), + status, + pc, + }); + block + } + + /// Fill the side exits' blocks. + fn side_exits(&mut self) { + while let Some(x) = self.side_exits.pop() { + self.b.switch_to_block(x.block); + self.vs = x.vs; + self.depth = x.depth; + self.last_pc = x.last_pc; + for item in x.restore { + self.push(item); + } + self.exit(x.status, x.pc); + } + } + + /// Stop for a queued finalizer (with `pc` next), from the current + /// state. + fn check_released(&mut self, next: usize) { + let f = self.b.ins().uload8(types::I32, FLAGS, self.dead, 0); + let out = self.exit_with(next, &[], MARKED); + self.branch_out(f, out); + } + + /// Jump to the block starting at `pc`, the virtual stack written out. + fn goto(&mut self, pc: usize) { + self.flush(); + match self.blocks.get(pc).copied().flatten() { + Some(b) => { + // The block finds the last instruction in the state. + self.store_last(); + self.b.ins().jump(b, &[]); + } + None => self.exit(INTERP, pc), + } + } + + // ---- operands ---- + + /// `item` as a scalar operand. An unknown-kind operand's tag still has + /// to be checked. + fn operand(&mut self, item: Item) -> Option { + let at = |s: &mut Self, addr: Value| -> Opnd { + let tag = s.tag_at(addr); + let word = s.b.ins().load(types::I64, FLAGS, addr, 8); + let byte = s.b.ins().uload8(types::I64, FLAGS, addr, 1); + Opnd { + kind: Kind::Unknown, + tag: Some(tag), + word, + byte: Some(byte), + } + }; + Some(match item { + Item::Int(v) => Opnd { + kind: Kind::Int, + tag: None, + word: v, + byte: None, + }, + Item::Float(v) => Opnd { + kind: Kind::Float, + tag: None, + word: self.b.ins().bitcast(types::I64, MemFlags::new(), v), + byte: None, + }, + Item::Bool(v) => Opnd { + kind: Kind::Bool, + tag: None, + word: v, + byte: None, + }, + Item::Const(k) => match self.ext.objects[k as usize] { + Object::Int(v) => Opnd { + kind: Kind::Int, + tag: None, + word: self.b.ins().iconst(types::I64, v), + byte: None, + }, + Object::Float(x) => Opnd { + kind: Kind::Float, + tag: None, + word: self.b.ins().iconst(types::I64, x.to_bits() as i64), + byte: None, + }, + Object::Bool(x) => Opnd { + kind: Kind::Bool, + tag: None, + word: self.b.ins().iconst(types::I8, i64::from(x)), + byte: None, + }, + Object::None => Opnd { + kind: Kind::None, + tag: None, + word: self.b.ins().iconst(types::I64, 0), + byte: None, + }, + _ => return None, + }, + Item::Local(i) => { + let a = self.local_addr(i); + at(self, a) + } + Item::Mem(s) => { + let a = self.slot_addr(s); + at(self, a) + } + Item::Dyn(tag, word, byte) => Opnd { + kind: Kind::Unknown, + tag: Some(tag), + word, + byte: Some(byte), + }, + }) + } + + /// Whether `o` is of `kind`: a static answer, or a run-time test. + fn is_kind(&mut self, o: Opnd, kind: Kind, tag: u8) -> Result { + match o.kind { + Kind::Unknown => { + let t = o.tag.expect("unknown kinds carry a tag"); + Err(self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tag))) + } + k => Ok(k == kind), + } + } + + /// `o`'s value as an `f64`, given it is an `int` or a `float` + /// (`is_float` says which, when unknown). + fn as_f64(&mut self, o: Opnd, is_float: Result) -> Value { + match is_float { + Ok(true) => self.b.ins().bitcast(types::F64, MemFlags::new(), o.word), + Ok(false) => self.b.ins().fcvt_from_sint(types::F64, o.word), + Err(f) => { + let x = self.b.ins().bitcast(types::F64, MemFlags::new(), o.word); + let c = self.b.ins().fcvt_from_sint(types::F64, o.word); + self.b.ins().select(f, x, c) + } + } + } + + /// The truth of a scalar operand (`None`, `bool`, `int`; and `float` + /// with `floats`), as an `i8`; anything else branches to `other`. + fn truth(&mut self, o: Opnd, floats: bool, other: Block) -> Value { + match o.kind { + Kind::Bool => o.word, + Kind::Int => self.b.ins().icmp_imm(IntCC::NotEqual, o.word, 0), + Kind::None => self.b.ins().iconst(types::I8, 0), + Kind::Float if floats => { + let x = self.b.ins().bitcast(types::F64, MemFlags::new(), o.word); + let z = self.b.ins().f64const(0.0); + self.b.ins().fcmp(FloatCC::NotEqual, x, z) + } + Kind::Float => { + self.b.ins().jump(other, &[]); + let dead = self.b.create_block(); + self.b.switch_to_block(dead); + self.b.ins().iconst(types::I8, 0) + } + Kind::Unknown => { + let t = o.tag.expect("unknown kinds carry a tag"); + let byte = o.byte.expect("unknown kinds carry a byte"); + let tb = self.tags; + let is_bool = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.boolean)); + let is_int = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.int)); + let is_none = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.none)); + let mut ok = self.b.ins().bor(is_bool, is_int); + ok = self.b.ins().bor(ok, is_none); + let fnz = if floats { + let is_float = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.float)); + ok = self.b.ins().bor(ok, is_float); + let x = self.b.ins().bitcast(types::F64, MemFlags::new(), o.word); + let z = self.b.ins().f64const(0.0); + let nz = self.b.ins().fcmp(FloatCC::NotEqual, x, z); + Some((is_float, nz)) + } else { + None + }; + let bad = self.b.ins().bxor_imm(ok, 1); + self.branch_out(bad, other); + let bnz = self.b.ins().icmp_imm(IntCC::NotEqual, byte, 0); + let wnz = self.b.ins().icmp_imm(IntCC::NotEqual, o.word, 0); + let mut r = self.b.ins().select(is_bool, bnz, wnz); + let not_none = self.b.ins().bxor_imm(is_none, 1); + r = self.b.ins().band(r, not_none); + if let Some((is_float, nz)) = fnz { + r = self.b.ins().select(is_float, nz, r); + } + r + } + } + } + + // ---- the instructions ---- + + /// Whether entering at `pc` gets anywhere: a block start whose first + /// instruction the native code runs (not one it hands straight back). + fn enters_at(&self, pc: usize) -> bool { + let instrs = &self.code.instructions; + if !self.blocks.get(pc).is_some_and(Option::is_some) { + return false; + } + native_op(instrs[pc].op) + } + + /// Lower the whole function; `false` when it can't be compiled. + fn lower(&mut self) -> bool { + let code = self.code; + let n = code.instructions.len(); + if self.depths.len() != n { + return false; + } + // Block starts: the entry, jump targets, and every instruction + // after one that ends a block or that the core loop runs. + let mut starts = vec![false; n]; + starts[0] = true; + for (pc, ins) in code.instructions.iter().enumerate() { + let next = pc + 1; + let target = match ins.op { + OpCode::JumpForward + | OpCode::PopJumpIfFalse + | OpCode::PopJumpIfTrue + | OpCode::PopJumpIfNone + | OpCode::PopJumpIfNotNone + | OpCode::ForIter => Some(next + ins.arg as usize), + OpCode::JumpBackward => Some(next.saturating_sub(ins.arg as usize)), + _ => None, + }; + if let Some(t) = target { + if t < n { + starts[t] = true; + } + if ins.op == OpCode::ForIter { + // The loop exit past `END_FOR` / `POP_ITER`. + let mut to = t; + if code.instructions.get(to).map(|i| i.op) == Some(OpCode::EndFor) { + to += 1; + if matches!( + code.instructions.get(to).map(|i| i.op), + Some(OpCode::PopIter | OpCode::PopTop) + ) { + to += 1; + } + } + if to < n { + starts[to] = true; + } + } + } + if next < n && (target.is_some() || !native_op(ins.op)) { + starts[next] = true; + } + // A fused attribute store continues after its `STORE_ATTR`. + if ins.op == OpCode::LoadFast + && code.instructions.get(next).map(|i| i.op) == Some(OpCode::StoreAttr) + && pc + 2 < n + { + starts[pc + 2] = true; + } + // A fused method call continues after its `CALL`. + if ins.op == OpCode::LoadFast + && code.instructions.get(next).map(|i| i.op) == Some(OpCode::LoadMethodAttr) + { + if let Some(call) = (pc + 2..n.min(pc + 12)) + .find(|&k| code.instructions[k].op == OpCode::Call) + { + if call + 1 < n { + starts[call + 1] = true; + } + } + } + } + for (pc, s) in starts.iter_mut().enumerate() { + if *s && self.depths[pc] < 0 { + // Unreachable on the ordinary paths (a handler's): no entry. + *s = false; + } + } + self.blocks = starts + .iter() + .map(|&s| s.then(|| self.b.create_block())) + .collect(); + // The entry: to the block at the state's pc, when the stack has the + // block's depth. + self.dispatch = self.b.create_block(); + self.b.ins().jump(self.dispatch, &[]); + self.b.switch_to_block(self.dispatch); + let pc = self.b.ins().load(types::I64, FLAGS, self.st, S_PC); + let len = self.b.ins().load(types::I64, FLAGS, self.st, S_LEN); + let pc32 = self.b.ins().ireduce(types::I32, pc); + let out = self.b.create_block(); + // The code writes stack slots at fixed offsets: the frame's stack + // must have room for the deepest point plus an instruction's own + // pushes (a pooled stack can be smaller; the core loop grows it). + let need = self.depths.iter().copied().max().unwrap_or(0).max(0) + 4; + let cap = self.b.ins().load(types::I64, FLAGS, self.st, S_CAP); + let short = self.b.ins().icmp_imm(IntCC::SignedLessThan, cap, need); + let roomy = self.b.create_block(); + self.b.ins().brif(short, out, &[], roomy, &[]); + self.b.switch_to_block(roomy); + let mut tramps = Vec::new(); + let mut calls = Vec::with_capacity(n); + for pc in 0..n { + let target = match self.blocks[pc] { + Some(blk) => { + let t = self.b.create_block(); + tramps.push((t, blk, self.depths[pc])); + t + } + None => out, + }; + calls.push(self.b.func.dfg.block_call(target, &[])); + } + let def = self.b.func.dfg.block_call(out, &[]); + let jt = self.b.create_jump_table(JumpTableData::new(def, &calls)); + self.b.ins().br_table(pc32, jt); + self.b.switch_to_block(out); + let s = self.b.ins().iconst(types::I32, i64::from(INTERP)); + self.b.ins().return_(&[s]); + for (t, blk, depth) in tramps { + self.b.switch_to_block(t); + let wrong = self.b.ins().icmp_imm(IntCC::NotEqual, len, depth); + self.b.ins().brif(wrong, out, &[], blk, &[]); + } + // The blocks: each from its start until it ends or the next block + // starts. + for start in 0..n { + let Some(blk) = self.blocks[start] else { + continue; + }; + self.b.switch_to_block(blk); + self.vs.clear(); + self.depth = self.depths[start] as usize; + self.last_pc = None; + self.bound.iter_mut().for_each(|b| *b = false); + let mut pc = start; + loop { + if !self.instruction(pc) { + break; + } + pc += 1; + if pc >= n { + self.exit(INTERP, pc); + break; + } + if self.blocks[pc].is_some() { + self.goto(pc); + break; + } + } + } + self.side_exits(); + true + } + + /// Lower the instruction at `pc` into the current block: `true` when + /// the block continues with the next instruction, `false` when it ended + /// (a jump, an exit). + fn instruction(&mut self, pc: usize) -> bool { + let ins = self.code.instructions[pc]; + let next = pc + 1; + let cont = match ins.op { + OpCode::Nop | OpCode::NotTaken | OpCode::Resume | OpCode::CopyFreeVars => true, + OpCode::LoadFast | OpCode::LoadFastBorrow => self.load_fast(pc, ins.arg), + OpCode::LoadConst if (ins.arg as usize) < self.ext.objects.len() => { + self.push(Item::Const(ins.arg)); + true + } + OpCode::LoadSmallInt => { + let v = self.b.ins().iconst(types::I64, i64::from(ins.arg)); + self.push(Item::Int(v)); + true + } + OpCode::StoreFast if (ins.arg as usize) < self.nlocals => { + self.store_fast(pc, ins.arg) + } + OpCode::PopTop => { + match self.pop() { + Item::Mem(s) => { + let at = self.slot_addr(s); + let tag = self.tag_at(at); + let scalar = self.is_scalar(tag); + let heap = self.b.create_block(); + let done = self.b.create_block(); + self.b.ins().brif(scalar, done, &[], heap, &[]); + self.b.switch_to_block(heap); + let r = self + .call(h_pop as *const () as usize, &[at], true) + .expect("returns"); + let out = self.exit_with(pc, &[Item::Mem(s)], INTERP); + self.branch_out(r, out); + self.set_last(pc); + self.check_released(next); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + // A copy never taken has nothing to release. + _ => {} + } + true + } + OpCode::BinaryOp => return self.binary(pc, ins.arg), + OpCode::CompareOp => return self.compare(pc, ins.arg), + OpCode::ToBool => { + let item = self.pop(); + let Some(o) = self.operand(item) else { + self.push(item); + self.exit(INTERP, pc); + return false; + }; + if o.kind == Kind::Bool { + self.push(item); + } else { + let other = self.exit_with(pc, &[item], INTERP); + let t = self.truth(o, true, other); + self.push(Item::Bool(t)); + } + true + } + OpCode::PopJumpIfFalse | OpCode::PopJumpIfTrue => { + let item = self.pop(); + let Some(o) = self.operand(item) else { + self.push(item); + self.exit(INTERP, pc); + return false; + }; + let other = self.exit_with(pc, &[item], INTERP); + let t = self.truth(o, false, other); + self.set_last(pc); + let taken = next + ins.arg as usize; + self.branch(t, ins.op == OpCode::PopJumpIfTrue, taken, next); + return false; + } + OpCode::PopJumpIfNone | OpCode::PopJumpIfNotNone => { + let item = self.pop(); + let is_none = match item { + Item::Mem(s) => { + let at = self.slot_addr(s); + let tag = self.tag_at(at); + let none = self.b.ins().icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)); + let scalar = self.is_scalar(tag); + let heap = self.b.create_block(); + let done = self.b.create_block(); + self.b.ins().brif(scalar, done, &[], heap, &[]); + self.b.switch_to_block(heap); + let r = self + .call(h_pop as *const () as usize, &[at], true) + .expect("returns"); + let out = self.exit_with(pc, &[Item::Mem(s)], INTERP); + self.branch_out(r, out); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + none + } + Item::Local(i) => { + let at = self.local_addr(i); + let tag = self.tag_at(at); + self.b.ins().icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)) + } + Item::Const(k) => { + let none = matches!(self.ext.objects[k as usize], Object::None); + self.b.ins().iconst(types::I8, i64::from(none)) + } + Item::Dyn(tag, ..) => { + self.b.ins().icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)) + } + Item::Int(_) | Item::Float(_) | Item::Bool(_) => { + self.b.ins().iconst(types::I8, 0) + } + }; + self.set_last(pc); + let taken = next + ins.arg as usize; + self.branch(is_none, ins.op == OpCode::PopJumpIfNone, taken, next); + return false; + } + OpCode::IsOp => return self.is_op(pc, ins.arg), + OpCode::CopyTop => { + let n = (ins.arg as usize).max(1); + if n > self.depth { + self.exit(INTERP, pc); + return false; + } + let k = self.vs.len(); + let item = if n <= k { + self.vs[k - n] + } else { + Item::Mem(self.depth - n) + }; + match item { + Item::Mem(s) => { + // A second reference to a value on the stack. + let dst = self.slot_addr(self.depth); + let src = self.slot_addr(s); + self.copy_value(dst, src); + let d = self.depth; + self.push(Item::Mem(d)); + } + other => self.push(other), + } + true + } + OpCode::Swap => { + let n = ins.arg as usize; + if n < 2 || n > self.depth { + self.exit(INTERP, pc); + return false; + } + let k = self.vs.len(); + let in_registers = n <= k + && !matches!(self.vs[k - 1], Item::Mem(_)) + && !matches!(self.vs[k - n], Item::Mem(_)); + if in_registers { + self.vs.swap(k - 1, k - n); + } else { + // A stack value's slot can't move on the virtual stack: + // swap on the frame's. + self.flush(); + let a = self.slot_addr(self.depth - 1); + let b = self.slot_addr(self.depth - n); + let (a0, a1) = ( + self.b.ins().load(types::I64, FLAGS, a, 0), + self.b.ins().load(types::I64, FLAGS, a, 8), + ); + self.copy16(a, b); + self.b.ins().store(FLAGS, a0, b, 0); + self.b.ins().store(FLAGS, a1, b, 8); + } + true + } + + OpCode::JumpForward => { + self.set_last(pc); + self.goto(next + ins.arg as usize); + return false; + } + OpCode::JumpBackward => { + self.flush(); + // The back edge is the eval breaker (the core loop's arm, + // which the exits below hand the instruction to). + let c = self.b.ins().load(types::I32, FLAGS, self.countdown, 0); + let low = self.b.ins().icmp_imm(IntCC::UnsignedLessThanOrEqual, c, 1); + let gen_addr = self + .b + .ins() + .iconst(self.ptr, crate::hot_gates::loop_gen_ptr() as i64); + let g = self.b.ins().load(types::I64, FLAGS, gen_addr, 0); + let moved = self.b.ins().icmp(IntCC::NotEqual, g, self.snap); + let trip = self.b.ins().bor(low, moved); + let out = self.exit_with(pc, &[], INTERP); + self.branch_out(trip, out); + let c1 = self.b.ins().iadd_imm(c, -1); + self.b.ins().store(FLAGS, c1, self.countdown, 0); + self.set_last(pc); + self.goto(next.saturating_sub(ins.arg as usize)); + return false; + } + OpCode::ForIter => return self.for_iter(pc, ins.arg), + OpCode::BinarySubscr + | OpCode::BinarySlice + | OpCode::StoreSubscr + | OpCode::ListAppend + | OpCode::UnpackSequence => return self.container(pc), + OpCode::ContainsOp => return self.contains(pc, ins.arg), + OpCode::LoadAttr => return self.load_attr(pc, ins.arg), + _ => { + self.exit(INTERP, pc); + return false; + } + }; + if cont { + self.set_last(pc); + } + cont + } + + /// Branch to `taken` when `cond` (an `i8` truth) is `want`. + fn branch(&mut self, cond: Value, want: bool, taken: usize, fall: usize) { + self.flush(); + let (t, f) = if want { (taken, fall) } else { (fall, taken) }; + let tb = self.b.create_block(); + let fb = self.b.create_block(); + self.b.ins().brif(cond, tb, &[], fb, &[]); + let depth = self.depth; + self.b.switch_to_block(tb); + self.goto(t); + self.depth = depth; + self.b.switch_to_block(fb); + self.goto(f); + } + + fn load_fast(&mut self, pc: usize, i: u32) -> bool { + if i as usize >= self.nlocals { + self.exit(INTERP, pc); + return false; + } + // The core loop's fused receiver shapes (`x.m(...)`, `x.attr = v`) + // start here. + match self.code.instructions.get(pc + 1).map(|i| i.op) { + Some(OpCode::LoadMethodAttr) => return self.method_call(pc), + Some(OpCode::StoreAttr) => return self.store_attr(pc), + _ => {} + } + // An unbound local raises in the core loop (a parameter nothing + // deletes is always bound, as is a local this block bound or tested). + let ix = i as usize; + if self.maybe_unbound[ix] && !self.bound[ix] { + let at = self.local_addr(i); + let tag = self.tag_at(at); + let unbound = self + .b + .ins() + .icmp_imm(IntCC::Equal, tag, i64::from(self.tags.unbound)); + let out = self.exit_with(pc, &[], INTERP); + self.branch_out(unbound, out); + self.bound[ix] = true; + } + self.push(Item::Local(i)); + true + } + + /// `LOAD_FAST x; x.m()` through [`h_local_method`], + /// continuing at whatever follows the call (through the dispatch). A + /// generator body's resume can't raise from here: its calls are the + /// general loop's. + fn method_call(&mut self, pc: usize) -> bool { + if self.code.is_generator || self.code.is_coroutine || self.code.is_async_generator { + self.exit(INTERP, pc); + return false; + } + self.flush(); + let code = self.b.ins().iconst(self.ptr, std::ptr::from_ref(self.code) as i64); + let ext = self.b.ins().iconst(self.ptr, std::ptr::from_ref(self.ext) as i64); + let pcv = self.b.ins().iconst(types::I64, pc as i64); + let nl = self.b.ins().iconst(types::I64, self.nlocals as i64); + let len = self.b.ins().iconst(types::I64, self.depth as i64); + self.store_last(); + let r = self + .call( + h_local_method as *const () as usize, + &[self.st, code, ext, pcv, nl, len], + true, + ) + .expect("returns"); + let ran = self.b.create_block(); + let other = self.b.create_block(); + self.b.ins().brif(r, other, &[], ran, &[]); + // Ran: the state holds the pc, depth and last instruction after the + // call. + self.b.switch_to_block(ran); + let saved = self.last_pc.take(); + self.check_released_dynamic(); + self.b.ins().jump(self.dispatch, &[]); + // Raised: the core loop raises it. + self.b.switch_to_block(other); + let raised = self.b.ins().icmp_imm(IntCC::Equal, r, i64::from(RAISED)); + let raise_b = self.b.create_block(); + let declined = self.b.create_block(); + self.b.ins().brif(raised, raise_b, &[], declined, &[]); + self.b.switch_to_block(raise_b); + let s = self.b.ins().iconst(types::I32, i64::from(RAISED)); + self.b.ins().return_(&[s]); + self.b.switch_to_block(declined); + self.last_pc = saved; + self.exit(INTERP, pc); + false + } + + /// Stop for a queued finalizer with the state as a helper left it (pc + /// and depth already stored). + fn check_released_dynamic(&mut self) { + let f = self.b.ins().uload8(types::I32, FLAGS, self.dead, 0); + let out = self.b.create_block(); + let ok = self.b.create_block(); + self.b.ins().brif(f, out, &[], ok, &[]); + self.b.switch_to_block(out); + let s = self.b.ins().iconst(types::I32, i64::from(MARKED)); + self.b.ins().return_(&[s]); + self.b.switch_to_block(ok); + } + + /// `LOAD_FAST x; x.attr = v` through [`h_store_attr`] (the value on + /// the stack's top). + fn store_attr(&mut self, pc: usize) -> bool { + if self.depth == 0 { + self.exit(INTERP, pc); + return false; + } + self.flush(); + let slot = self.slot_addr(self.depth - 1); + let code = self.b.ins().iconst(self.ptr, std::ptr::from_ref(self.code) as i64); + let pcv = self.b.ins().iconst(types::I64, pc as i64); + let r = self + .call(h_store_attr as *const () as usize, &[self.st, code, pcv, slot], true) + .expect("returns"); + let out = self.exit_with(pc, &[], INTERP); + self.branch_out(r, out); + self.depth -= 1; + self.set_last(pc + 1); + self.check_released(pc + 2); + self.goto(pc + 2); + false + } + + fn store_fast(&mut self, pc: usize, i: u32) -> bool { + self.flush_local(i); + self.bound[i as usize] = true; + let item = self.pop(); + if let Item::Local(j) = item { + if j == i { + return true; + } + } + let local = self.local_addr(i); + // The new value's words, for a store that needs no release. + let words: Option<(Value, Value, Value)> = match item { + Item::Int(v) => Some(( + self.b.ins().iconst(types::I64, i64::from(self.tags.int)), + v, + self.b.ins().iconst(types::I8, 0), + )), + Item::Float(v) => Some(( + self.b.ins().iconst(types::I64, i64::from(self.tags.float)), + self.b.ins().bitcast(types::I64, MemFlags::new(), v), + self.b.ins().iconst(types::I8, 0), + )), + Item::Bool(v) => Some(( + self.b.ins().iconst(types::I64, i64::from(self.tags.boolean)), + self.b.ins().iconst(types::I64, 0), + v, + )), + Item::Dyn(t, w, b) => Some((t, w, self.b.ins().ireduce(types::I8, b))), + Item::Const(k) if matches!( + self.ext.objects[k as usize], + Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None + ) => + { + let o = self.operand(item).expect("a scalar constant"); + let tag = match o.kind { + Kind::Int => self.tags.int, + Kind::Float => self.tags.float, + Kind::Bool => self.tags.boolean, + _ => self.tags.none, + }; + let (w, b) = if o.kind == Kind::Bool { + (self.b.ins().iconst(types::I64, 0), o.word) + } else { + (o.word, self.b.ins().iconst(types::I8, 0)) + }; + Some((self.b.ins().iconst(types::I64, i64::from(tag)), w, b)) + } + _ => None, + }; + let old = self.tag_at(local); + let plain_old = { + let s = self.is_scalar(old); + let u = self + .b + .ins() + .icmp_imm(IntCC::Equal, old, i64::from(self.tags.unbound)); + self.b.ins().bor(s, u) + }; + // The value's slot (it left the virtual stack). + let slot = self.depth; + match words { + Some((tag, word, byte)) => { + // A scalar over a scalar (or nothing): three stores. + let slow = self.b.create_block(); + let done = self.b.create_block(); + let fast = self.b.create_block(); + self.b.ins().brif(plain_old, fast, &[], slow, &[]); + self.b.switch_to_block(fast); + let t8 = self.b.ins().ireduce(types::I8, tag); + self.b.ins().store(FLAGS, t8, local, 0); + self.b.ins().store(FLAGS, byte, local, 1); + self.b.ins().store(FLAGS, word, local, 8); + self.b.ins().jump(done, &[]); + // Over a heap value: through the stack slot, released by + // the helper. + self.b.switch_to_block(slow); + self.materialize(item, slot); + self.store_from_slot(pc, i, slot); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + None => { + // The value into its slot (a copy taken), then moved in. + self.materialize(item, slot); + let src = self.slot_addr(slot); + let vt = self.tag_at(src); + // The PEP 709 cell restore stays on the full path. + let cell = self + .b + .ins() + .icmp_imm(IntCC::Equal, vt, i64::from(self.tags.cell)); + let out = self.exit_with(pc, &[Item::Mem(slot)], INTERP); + self.branch_out(cell, out); + let slow = self.b.create_block(); + let done = self.b.create_block(); + let fast = self.b.create_block(); + self.b.ins().brif(plain_old, fast, &[], slow, &[]); + self.b.switch_to_block(fast); + self.copy16(local, src); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(slow); + self.store_from_slot(pc, i, slot); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + } + true + } + + /// `STORE_FAST i` of the value at stack `slot` (just above the + /// virtual stack) over a heap value, through the helper. + fn store_from_slot(&mut self, pc: usize, i: u32, slot: usize) { + let src = self.slot_addr(slot); + let iv = self.b.ins().iconst(types::I64, i64::from(i)); + let r = self + .call(h_store as *const () as usize, &[self.st, src, iv], true) + .expect("returns"); + // Declined: the value is on the stack, as the core loop expects. + let out = self.exit_with(pc, &[Item::Mem(slot)], INTERP); + self.branch_out(r, out); + let saved = self.last_pc; + self.set_last(pc); + self.check_released(pc + 1); + self.last_pc = saved; + } + + fn binary(&mut self, pc: usize, arg: u32) -> bool { + let kind = arg as u8; + let is = |k: BinOpKind| kind == k as u8; + let bi = self.pop(); + let ai = self.pop(); + // A value on the stack (a call's result, an element) is as likely + // a heap value as a number: the helper takes every shape. + if matches!(ai, Item::Mem(_)) || matches!(bi, Item::Mem(_)) { + return self.binary_helper(pc, kind, ai, bi); + } + let (Some(a), Some(b)) = (self.operand(ai), self.operand(bi)) else { + // A string or other heap constant: the core loop's out-of-line + // shapes, through the helper. + return self.binary_helper(pc, kind, ai, bi); + }; + let slow = self.exit_with(pc, &[ai, bi], INTERP); + if matches!(a.kind, Kind::Bool | Kind::None) || matches!(b.kind, Kind::Bool | Kind::None) { + self.b.ins().jump(slow, &[]); + return false; + } + let int_kind = [ + BinOpKind::Add, + BinOpKind::Sub, + BinOpKind::Mult, + BinOpKind::BitAnd, + BinOpKind::BitOr, + BinOpKind::BitXor, + BinOpKind::RShift, + BinOpKind::LShift, + BinOpKind::FloorDiv, + BinOpKind::Mod, + ] + .into_iter() + .any(|k| is(k)); + let float_kind = is(BinOpKind::Add) || is(BinOpKind::Sub) || is(BinOpKind::Mult) || is(BinOpKind::Div); + let ia = self.is_kind(a, Kind::Int, self.tags.int); + let ib = self.is_kind(b, Kind::Int, self.tags.int); + let fa = self.is_kind(a, Kind::Float, self.tags.float); + let fb = self.is_kind(b, Kind::Float, self.tags.float); + // Both statically known: one path. + let static_int = matches!((ia, ib), (Ok(true), Ok(true))); + let static_num = matches!( + (ia, ib, fa, fb), + (Ok(x), Ok(y), Ok(p), Ok(q)) if (x || p) && (y || q) + ); + if static_int { + let Some(r) = self.int_op(kind, a.word, b.word, slow) else { + self.b.ins().jump(slow, &[]); + return false; + }; + self.push(Item::Int(r)); + self.set_last(pc); + return true; + } + if static_num { + if !float_kind { + self.b.ins().jump(slow, &[]); + return false; + } + let x = self.as_f64(a, fa); + let y = self.as_f64(b, fb); + let r = self.float_op(kind, x, y, slow); + self.push(Item::Float(r)); + self.set_last(pc); + return true; + } + if matches!((ia, ib), (Ok(false), _) | (_, Ok(false))) && !float_kind { + self.b.ins().jump(slow, &[]); + return false; + } + // At run time: two ints, or two numbers with a float. + let to_v = |s: &mut Self, r: Result| match r { + Ok(x) => s.b.ins().iconst(types::I8, i64::from(x)), + Err(v) => v, + }; + let (iav, ibv, fav, fbv) = (to_v(self, ia), to_v(self, ib), to_v(self, fa), to_v(self, fb)); + let done = self.b.create_block(); + self.b.append_block_param(done, types::I64); + self.b.append_block_param(done, types::I64); + let both_int = self.b.ins().band(iav, ibv); + let int_b = self.b.create_block(); + let not_int = self.b.create_block(); + self.b.ins().brif(both_int, int_b, &[], not_int, &[]); + self.b.switch_to_block(int_b); + if int_kind { + match self.int_op(kind, a.word, b.word, slow) { + Some(r) => { + let t = self.b.ins().iconst(types::I64, i64::from(self.tags.int)); + self.b.ins().jump(done, &[t.into(), r.into()]); + } + None => { + self.b.ins().jump(slow, &[]); + } + } + } else { + self.b.ins().jump(slow, &[]); + } + self.b.switch_to_block(not_int); + if float_kind { + let na = self.b.ins().bor(iav, fav); + let nb = self.b.ins().bor(ibv, fbv); + let nums = self.b.ins().band(na, nb); + let bad = self.b.ins().bxor_imm(nums, 1); + self.branch_out(bad, slow); + let x = self.as_f64(a, Err(fav)); + let y = self.as_f64(b, Err(fbv)); + let r = self.float_op(kind, x, y, slow); + let bits = self.b.ins().bitcast(types::I64, MemFlags::new(), r); + let t = self.b.ins().iconst(types::I64, i64::from(self.tags.float)); + self.b.ins().jump(done, &[t.into(), bits.into()]); + } else { + self.b.ins().jump(slow, &[]); + } + self.b.switch_to_block(done); + let (t, w) = (self.b.block_params(done)[0], self.b.block_params(done)[1]); + let zero = self.b.ins().iconst(types::I64, 0); + self.push(Item::Dyn(t, w, zero)); + self.set_last(pc); + true + } + + /// `BINARY_OP` through [`h_binop`]: the operands onto the stack, and + /// the result in the lower one's slot. + fn binary_helper(&mut self, pc: usize, kind: u8, ai: Item, bi: Item) -> bool { + let slot = self.depth; + self.materialize(ai, slot); + self.materialize(bi, slot + 1); + let at = self.slot_addr(slot); + let k = self.b.ins().iconst(types::I32, i64::from(kind)); + let r = self + .call(h_binop as *const () as usize, &[at, k], true) + .expect("returns"); + let out = self.exit_with(pc, &[Item::Mem(slot), Item::Mem(slot + 1)], INTERP); + self.branch_out(r, out); + self.push(Item::Mem(slot)); + self.set_last(pc); + true + } + + /// A container instruction through [`h_container`] (the operands on + /// the stack; the depth after is the static one). + fn container(&mut self, pc: usize) -> bool { + let after = self.depths.get(pc + 1).copied().unwrap_or(-1); + if after < 0 { + self.exit(INTERP, pc); + return false; + } + self.flush(); + let ins = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(&self.code.instructions[pc]) as i64); + let code = self.b.ins().iconst(self.ptr, std::ptr::from_ref(self.code) as i64); + let pcv = self.b.ins().iconst(types::I64, pc as i64); + let len = self.b.ins().iconst(types::I64, self.depth as i64); + let n = self + .call_typed( + h_container as *const () as usize, + &[self.st, ins, code, pcv, len], + Some(types::I64), + ) + .expect("returns"); + let declined = self.b.ins().icmp_imm(IntCC::Equal, n, -1); + let out = self.exit_with(pc, &[], INTERP); + self.branch_out(declined, out); + self.depth = after as usize; + self.set_last(pc); + self.check_released(pc + 1); + true + } + + /// `CONTAINS_OP` through [`h_contains`]: an unboxed `bool`. + fn contains(&mut self, pc: usize, arg: u32) -> bool { + let bi = self.pop(); + let ai = self.pop(); + let slot = self.depth; + self.materialize(ai, slot); + self.materialize(bi, slot + 1); + let at = self.slot_addr(slot); + let r = self + .call(h_contains as *const () as usize, &[at], true) + .expect("returns"); + let declined = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThan, r, 1); + let out = self.exit_with(pc, &[Item::Mem(slot), Item::Mem(slot + 1)], INTERP); + self.branch_out(declined, out); + let found = self.b.ins().icmp_imm(IntCC::NotEqual, r, 0); + let found = if arg == 1 { + self.b.ins().bxor_imm(found, 1) + } else { + found + }; + self.push(Item::Bool(found)); + self.set_last(pc); + self.check_released(pc + 1); + true + } + + /// Two ints, by `kind`; a result that doesn't fit (or an operand the + /// core loop's arm declines) branches to `slow`. `None` for a kind + /// with no `int` arm. + fn int_op(&mut self, kind: u8, x: Value, y: Value, slow: Block) -> Option { + let is = |k: BinOpKind| kind == k as u8; + Some(if is(BinOpKind::Add) { + let r = self.b.ins().iadd(x, y); + let axr = self.b.ins().bxor(x, r); + let bxr = self.b.ins().bxor(y, r); + let and = self.b.ins().band(axr, bxr); + let ovf = self.b.ins().icmp_imm(IntCC::SignedLessThan, and, 0); + self.branch_out(ovf, slow); + r + } else if is(BinOpKind::Sub) { + let r = self.b.ins().isub(x, y); + let axb = self.b.ins().bxor(x, y); + let axr = self.b.ins().bxor(x, r); + let and = self.b.ins().band(axb, axr); + let ovf = self.b.ins().icmp_imm(IntCC::SignedLessThan, and, 0); + self.branch_out(ovf, slow); + r + } else if is(BinOpKind::Mult) { + let lo = self.b.ins().imul(x, y); + let hi = self.b.ins().smulhi(x, y); + let sign = self.b.ins().sshr_imm(lo, 63); + let ovf = self.b.ins().icmp(IntCC::NotEqual, hi, sign); + self.branch_out(ovf, slow); + lo + } else if is(BinOpKind::BitAnd) { + self.b.ins().band(x, y) + } else if is(BinOpKind::BitOr) { + self.b.ins().bor(x, y) + } else if is(BinOpKind::BitXor) { + self.b.ins().bxor(x, y) + } else if is(BinOpKind::RShift) { + let out = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, y, 64); + self.branch_out(out, slow); + self.b.ins().sshr(x, y) + } else if is(BinOpKind::LShift) { + let out = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, y, 63); + self.branch_out(out, slow); + let r = self.b.ins().ishl(x, y); + let back = self.b.ins().sshr(r, y); + let lost = self.b.ins().icmp(IntCC::NotEqual, back, x); + self.branch_out(lost, slow); + r + } else if is(BinOpKind::FloorDiv) || is(BinOpKind::Mod) { + let yneg = self.b.ins().icmp_imm(IntCC::SignedLessThanOrEqual, y, 0); + let xneg = self.b.ins().icmp_imm(IntCC::SignedLessThan, x, 0); + let bad = self.b.ins().bor(yneg, xneg); + self.branch_out(bad, slow); + if is(BinOpKind::FloorDiv) { + self.b.ins().udiv(x, y) + } else { + self.b.ins().urem(x, y) + } + } else { + return None; + }) + } + + /// Two floats, by `kind` (one of add, subtract, multiply, divide); a + /// zero divisor or a NaN result branches to `slow` (the core loop + /// raises, or tags the NaN). + fn float_op(&mut self, kind: u8, x: Value, y: Value, slow: Block) -> Value { + let is = |k: BinOpKind| kind == k as u8; + let r = if is(BinOpKind::Add) { + self.b.ins().fadd(x, y) + } else if is(BinOpKind::Sub) { + self.b.ins().fsub(x, y) + } else if is(BinOpKind::Mult) { + self.b.ins().fmul(x, y) + } else { + let zero = self.b.ins().f64const(0.0); + let z = self.b.ins().fcmp(FloatCC::Equal, y, zero); + self.branch_out(z, slow); + self.b.ins().fdiv(x, y) + }; + let nan = self.b.ins().fcmp(FloatCC::Unordered, r, r); + self.branch_out(nan, slow); + r + } + + fn compare(&mut self, pc: usize, arg: u32) -> bool { + let kind = (arg & !COMPARE_OP_TO_BOOL_FLAG) as u8; + let (icc, fcc) = match kind { + k if k == CompareKind::Lt as u8 => (IntCC::SignedLessThan, FloatCC::LessThan), + k if k == CompareKind::LtE as u8 => { + (IntCC::SignedLessThanOrEqual, FloatCC::LessThanOrEqual) + } + k if k == CompareKind::Eq as u8 => (IntCC::Equal, FloatCC::Equal), + k if k == CompareKind::NotEq as u8 => (IntCC::NotEqual, FloatCC::NotEqual), + k if k == CompareKind::Gt as u8 => (IntCC::SignedGreaterThan, FloatCC::GreaterThan), + k if k == CompareKind::GtE as u8 => { + (IntCC::SignedGreaterThanOrEqual, FloatCC::GreaterThanOrEqual) + } + _ => { + self.exit(INTERP, pc); + return false; + } + }; + let bi = self.pop(); + let ai = self.pop(); + let (Some(a), Some(b)) = (self.operand(ai), self.operand(bi)) else { + self.push(ai); + self.push(bi); + self.exit(INTERP, pc); + return false; + }; + let slow = self.exit_with(pc, &[ai, bi], INTERP); + if matches!(a.kind, Kind::Bool | Kind::None) || matches!(b.kind, Kind::Bool | Kind::None) { + self.b.ins().jump(slow, &[]); + return false; + } + let ia = self.is_kind(a, Kind::Int, self.tags.int); + let ib = self.is_kind(b, Kind::Int, self.tags.int); + let fa = self.is_kind(a, Kind::Float, self.tags.float); + let fb = self.is_kind(b, Kind::Float, self.tags.float); + let int_cmp = |s: &mut Self| s.b.ins().icmp(icc, a.word, b.word); + let float_cmp = |s: &mut Self| { + let x = s.b.ins().bitcast(types::F64, MemFlags::new(), a.word); + let y = s.b.ins().bitcast(types::F64, MemFlags::new(), b.word); + // An unordered pair takes the core loop's arm. + let uno = s.b.ins().fcmp(FloatCC::Unordered, x, y); + s.branch_out(uno, slow); + s.b.ins().fcmp(fcc, x, y) + }; + let r = match (ia, ib, fa, fb) { + (Ok(true), Ok(true), ..) => int_cmp(self), + (_, _, Ok(true), Ok(true)) => float_cmp(self), + _ => { + let to_v = |s: &mut Self, r: Result| match r { + Ok(x) => s.b.ins().iconst(types::I8, i64::from(x)), + Err(v) => v, + }; + let (iav, ibv, fav, fbv) = + (to_v(self, ia), to_v(self, ib), to_v(self, fa), to_v(self, fb)); + let done = self.b.create_block(); + self.b.append_block_param(done, types::I8); + let both_int = self.b.ins().band(iav, ibv); + let int_b = self.b.create_block(); + let not_int = self.b.create_block(); + self.b.ins().brif(both_int, int_b, &[], not_int, &[]); + self.b.switch_to_block(int_b); + let r = int_cmp(self); + self.b.ins().jump(done, &[r.into()]); + self.b.switch_to_block(not_int); + let both_float = self.b.ins().band(fav, fbv); + let bad = self.b.ins().bxor_imm(both_float, 1); + self.branch_out(bad, slow); + let r = float_cmp(self); + self.b.ins().jump(done, &[r.into()]); + self.b.switch_to_block(done); + self.b.block_params(done)[0] + } + }; + self.push(Item::Bool(r)); + self.set_last(pc); + true + } + + /// An item's tag, word and `bool` byte, for an identity test (`None` + /// for a stack value, whose release the test would owe). + fn identity(&mut self, item: Item) -> Option<(Value, Value, Value)> { + let c = |s: &mut Self, v: i64| s.b.ins().iconst(types::I64, v); + Some(match item { + Item::Mem(_) => return None, + Item::Local(i) => { + let at = self.local_addr(i); + let t = self.tag_at(at); + let w = self.b.ins().load(types::I64, FLAGS, at, 8); + let b = self.b.ins().uload8(types::I64, FLAGS, at, 1); + (t, w, b) + } + Item::Const(k) => { + let obj = &self.ext.objects[k as usize]; + let p = std::ptr::from_ref(obj).cast::(); + // SAFETY: a constant's tag, payload byte and word (the + // constants live as long as the code). + let (t, b, w) = unsafe { + (*p, *p.add(1), p.add(8).cast::().read_unaligned()) + }; + let (t, w, b) = (i64::from(t), w as i64, i64::from(b)); + (c(self, t), c(self, w), c(self, b)) + } + Item::Int(v) => { + let t = c(self, i64::from(self.tags.int)); + let z = c(self, 0); + (t, v, z) + } + Item::Float(v) => { + let t = c(self, i64::from(self.tags.float)); + let w = self.b.ins().bitcast(types::I64, MemFlags::new(), v); + let z = c(self, 0); + (t, w, z) + } + Item::Bool(v) => { + let t = c(self, i64::from(self.tags.boolean)); + let z = c(self, 0); + let b = self.b.ins().uextend(types::I64, v); + (t, z, b) + } + Item::Dyn(t, w, b) => (t, w, b), + }) + } + + /// `IS_OP` of two values the code holds no reference to: the + /// variants' identity rules (`Object::is_same`) in line, for the + /// scalars and the pointer-identity heap variants. + fn is_op(&mut self, pc: usize, arg: u32) -> bool { + let bi = self.pop(); + let ai = self.pop(); + let (Some((ta, wa, ba)), Some((tb, wb, bb))) = (self.identity(ai), self.identity(bi)) + else { + self.push(ai); + self.push(bi); + self.exit(INTERP, pc); + return false; + }; + let slow = self.exit_with(pc, &[ai, bi], INTERP); + let tags = self.tags; + let same_tag = self.b.ins().icmp(IntCC::Equal, ta, tb); + // Which rule the (shared) tag takes. + let one = self.b.ins().iconst(types::I64, 1); + let bit = self.b.ins().ishl(one, ta); + let words_mask = tags.by_pointer | (1i64 << tags.int) | (1i64 << tags.float); + let by_word = self.b.ins().band_imm(bit, words_mask); + let by_word = self.b.ins().icmp_imm(IntCC::NotEqual, by_word, 0); + let is_bool = self.b.ins().icmp_imm(IntCC::Equal, ta, i64::from(tags.boolean)); + let unit_mask = (1i64 << tags.none) | (1i64 << tags.unbound); + let unit = self.b.ins().band_imm(bit, unit_mask); + let unit = self.b.ins().icmp_imm(IntCC::NotEqual, unit, 0); + // A tag no rule here covers (a long, a complex, …) is the core + // loop's, when the tags match. + let known = self.b.ins().bor(by_word, is_bool); + let known = self.b.ins().bor(known, unit); + let unknown = self.b.ins().bxor_imm(known, 1); + let bad = self.b.ins().band(same_tag, unknown); + self.branch_out(bad, slow); + let weq = self.b.ins().icmp(IntCC::Equal, wa, wb); + let beq = self.b.ins().icmp(IntCC::Equal, ba, bb); + let r = self.b.ins().select(is_bool, beq, weq); + let r = self.b.ins().select(unit, same_tag, r); + let r = self.b.ins().band(r, same_tag); + let r = if arg == 1 { + self.b.ins().bxor_imm(r, 1) + } else { + r + }; + self.push(Item::Bool(r)); + self.set_last(pc); + true + } + + fn for_iter(&mut self, pc: usize, arg: u32) -> bool { + let before = self.last_pc; + self.flush(); + let it_slot = self.depth - 1; + let it = self.slot_addr(it_slot); + let out_slot = self.slot_addr(self.depth); + let r = self + .call(h_for_iter as *const () as usize, &[self.st, it, out_slot], true) + .expect("returns"); + let int_b = self.b.create_block(); + let other = self.b.create_block(); + self.b.ins().brif(r, other, &[], int_b, &[]); + // An int: unboxed. + self.b.switch_to_block(int_b); + let v = self.b.ins().load(types::I64, FLAGS, self.st, S_OUT); + let depth = self.depth; + self.push(Item::Int(v)); + self.set_last(pc); + self.goto(pc + 1); + self.depth = depth; + // Another value, on the stack; or the end; or the core loop's. + self.b.switch_to_block(other); + let pushed = self.b.ins().icmp_imm(IntCC::Equal, r, 1); + let pushed_b = self.b.create_block(); + let rest = self.b.create_block(); + self.b.ins().brif(pushed, pushed_b, &[], rest, &[]); + self.b.switch_to_block(pushed_b); + self.depth += 1; + self.set_last(pc); + self.goto(pc + 1); + self.depth = depth; + self.b.switch_to_block(rest); + let retired = self.b.ins().icmp_imm(IntCC::Equal, r, 2); + let ret_b = self.b.create_block(); + let decl = self.b.create_block(); + self.b.ins().brif(retired, ret_b, &[], decl, &[]); + self.b.switch_to_block(ret_b); + // Past the loop's `END_FOR` / `POP_ITER` pair. + let mut to = pc + 1 + arg as usize; + let op = |s: &Self, pc: usize| s.code.instructions.get(pc).map(|i| i.op); + if op(self, to) == Some(OpCode::EndFor) { + to += 1; + if matches!(op(self, to), Some(OpCode::PopIter | OpCode::PopTop)) { + to += 1; + } + } + self.depth -= 1; + self.set_last(pc); + self.goto(to); + self.depth = depth; + self.b.switch_to_block(decl); + self.last_pc = before; + self.exit(INTERP, pc); + false + } + + /// `x.attr` of an instance (a local's, or one on the stack): the + /// site's field shortcut in line. The value goes onto the stack (a + /// scalar copied, anything else with a reference taken); a stack + /// receiver is released after. Anything else, and a miss, is the core + /// loop's. + fn load_attr(&mut self, pc: usize, _arg: u32) -> bool { + let (Some(l), Some(cache_slot)) = (self.layout, self.field_slots.get(pc)) else { + self.exit(INTERP, pc); + return false; + }; + let l = *l; + let item = self.pop(); + let at = match item { + Item::Local(i) => self.local_addr(i), + Item::Mem(s) => self.slot_addr(s), + _ => { + self.push(item); + self.exit(INTERP, pc); + return false; + } + }; + let miss = self.exit_with(pc, &[item], INTERP); + let otag = self.tag_at(at); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(self.tags.instance)); + self.branch_out(not_inst, miss); + let ptr = self.ptr; + let inst = self.b.ins().load(ptr, FLAGS, at, 8); + let cls = self.b.ins().load(ptr, FLAGS, inst, l.inst_class); + let ver = self.b.ins().load(types::I64, FLAGS, cls, l.type_attr_version); + let cache = self + .b + .ins() + .iconst(ptr, std::ptr::from_ref(cache_slot).cast::() as i64); + let want = self.b.ins().load(types::I64, FLAGS, cache, FIELD_VER); + let idx = self.b.ins().uload32(FLAGS, cache, FIELD_IDX); + let lazy = self.b.ins().load(ptr, FLAGS, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, FLAGS, flag, 0); + let borrow = self.b.ins().sload32(FLAGS, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, FLAGS, inst, l.inst_split_block); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.branch_out(bad, miss); + let keys = self.b.ins().load(ptr, FLAGS, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, FLAGS, cls, l.type_shared_keys); + let len = self.b.ins().uload32(FLAGS, block, l.split_len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let bad = self.b.ins().bor(foreign, absent); + self.branch_out(bad, miss); + let off = self.b.ins().ishl_imm(idx, 4); + let v = self.b.ins().iadd(block, off); + let v = self.b.ins().iadd_imm(v, i64::from(l.split_values)); + let slot = self.depth; + let dst = self.slot_addr(slot); + match item { + Item::Local(_) => { + // The receiver is the local's: the value just takes a copy. + self.copy_value(dst, v); + } + _ => { + // The receiver is the stack's: the value goes above it + // (copied) until the receiver is released, then down. + let above = self.slot_addr(slot + 1); + self.copy_value(above, v); + let r = self + .call(h_pop as *const () as usize, &[dst], true) + .expect("returns"); + let undo = self.b.create_block(); + let released = self.b.create_block(); + self.b.ins().brif(r, undo, &[], released, &[]); + // Declined: the copy goes, and the core loop takes the + // instruction from the start. + self.b.switch_to_block(undo); + // (A fresh copy has another owner: its release is a plain + // decrement.) + self.call(h_pop as *const () as usize, &[above], true); + self.b.ins().jump(miss, &[]); + self.b.switch_to_block(released); + self.copy16(dst, above); + } + } + self.push(Item::Mem(slot)); + self.set_last(pc); + if matches!(item, Item::Mem(_)) { + self.check_released(pc + 1); + } + true + } +} + +/// Where a field cache's version and position sit. +const FIELD_VER: i32 = std::mem::offset_of!((u64, u32), 0) as i32; +const FIELD_IDX: i32 = std::mem::offset_of!((u64, u32), 1) as i32; diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs index b9ed6706..a56c311d 100644 --- a/crates/weavepy-vm/src/gen_fast.rs +++ b/crates/weavepy-vm/src/gen_fast.rs @@ -372,12 +372,62 @@ impl Interpreter { if pc >= ninstrs { return GenStep::Bail; } + // The body's native form (see `frame_jit`), run from every pc the + // arms below leave it at; a hot body compiles. + #[cfg(feature = "jit")] + let native = { + let nlocals = locals.len(); + if ext.frame_jit.get(nlocals).is_none() { + ext.frame_jit.warm(code, ext, nlocals, 0); + } + ext.frame_jit.get(nlocals) + }; + #[cfg(feature = "jit")] + let mut nst = crate::frame_jit::State { + locals: lbase, + stack: base, + len, + cap, + pc, + last: pc, + countdown: std::ptr::addr_of_mut!(self.gil_countdown), + snap_gen, + // (Only read when the native form runs.) + maybe_dead: if native.is_some() { + crate::gc_trace::maybe_dead_flag() + } else { + std::ptr::null() + }, + interp: std::ptr::from_ref(self), + out: 0, + depth_cell: std::ptr::null(), + err: None, + }; + #[cfg(feature = "jit")] + let mut handed = usize::MAX; // SAFETY (throughout): `base` indexes only below `len` (initialized) // or `cap` as checked; `lbase`, `cbase` and `instrs` only at the // indices and pcs the eligibility scan proved in range (`in_bounds`: // every jump lands on an instruction and the last can't fall // through, so `pc < ninstrs` whenever an instruction is read). let yielded = loop { + #[cfg(feature = "jit")] + if pc != handed { + if let Some(native) = native.filter(|n| n.enters_at(pc)) { + nst.len = len; + nst.pc = pc; + // SAFETY: the body's activation state, as this loop + // holds it (its locals count checked above). A queued + // finalizer waits for the general loop's next check. + let _ = unsafe { native.run(&mut nst) }; + len = nst.len; + pc = nst.pc; + handed = pc; + if pc >= ninstrs { + break None; + } + } + } let ins = unsafe { *instrs.add(pc) }; match ins.op { OpCode::Nop | OpCode::NotTaken | OpCode::Resume => pc += 1, diff --git a/crates/weavepy-vm/src/hot_gates.rs b/crates/weavepy-vm/src/hot_gates.rs index 396e54fd..aea4bde6 100644 --- a/crates/weavepy-vm/src/hot_gates.rs +++ b/crates/weavepy-vm/src/hot_gates.rs @@ -92,6 +92,13 @@ pub fn loop_gen() -> u64 { LOOP_GEN.load(Ordering::Relaxed) } +/// The loop generation's address, for native code that compares it at +/// its back edges. +#[cfg(feature = "jit")] +pub(crate) fn loop_gen_ptr() -> *const u64 { + LOOP_GEN.as_ptr() +} + /// Invalidate every dispatch loop's cached prologue snapshot /// (RFC 0065 WS1). Cheap (one relaxed RMW); call from any mutation /// site whose state the loop prologue consults. diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 33014722..1a7927bc 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -61,6 +61,8 @@ pub mod hot_gates; pub mod import; pub mod import_time; pub mod inst_dict; +#[cfg(feature = "jit")] +mod frame_jit; mod lazy_arc; mod leaf_plan; pub mod linejump; @@ -9808,8 +9810,25 @@ impl Interpreter { /// the full leaf arms and hands back at the next core instruction. #[inline(never)] fn leaf_burst(&mut self, sw: &mut CoreSwitch, snap_gen: u64) -> LeafStop { + // Which copy of the core loop runs: the native one drives the + // running code's native form (see `frame_jit`). + #[cfg(feature = "jit")] + let mut native = false; loop { - match self.leaf_core(sw, snap_gen) { + #[cfg(feature = "jit")] + let exit = if native { + self.leaf_core::(sw, snap_gen) + } else { + self.leaf_core::(sw, snap_gen) + }; + #[cfg(not(feature = "jit"))] + let exit = self.leaf_core::(sw, snap_gen); + match exit { + #[cfg(feature = "jit")] + CoreExit::Switch => { + native = !native; + continue; + } CoreExit::Stop(stop) => return stop, CoreExit::Helper => { // SAFETY: see `CoreSwitch` (the running activation's @@ -9953,6 +9972,104 @@ impl Interpreter { done } + /// The core loop's `LOAD_FAST x` (at `pc`, of the heap local `other`) + /// followed by `x.m()`: a native method the site + /// cached for the receiver's class, a leaf method the site verified, + /// a pure leaf method, or a container's site-cached leaf method, + /// called on the borrowed operands. Returns the call's result and the + /// `CALL`'s pc; `None` touches nothing (the instructions run one by + /// one). + /// + /// # Safety + /// + /// `lbase` is the running activation's `nlocals` locals; nothing else + /// borrows them. + #[inline(always)] + #[allow(clippy::too_many_arguments)] + unsafe fn core_local_method( + &self, + code: &CodeObject, + ext: Option<&CodeConstObjects>, + other: &Object, + pc: usize, + lbase: *mut Object, + nlocals: usize, + consts: &[Object], + depth_cell: *const std::cell::Cell, + ) -> Option<(Result, usize)> { + let next = *code.instructions.get(pc + 1)?; + let mslots: &[MethodSlot] = ext + .and_then(|e| e.method_slots.get()) + .map_or(&[][..], |s| &s[..]); + match other { + Object::Instance(_) | Object::Module(_) => { + // `x.m(...)` of a pure leaf method with simple arguments: its + // result, evaluated on the borrowed operands. + if !simple_args_prefix(&code.instructions, pc + 2) { + return None; + } + // A native method the site cached for this class version: + // straight to its fused call (the Python leaf probes below + // would miss). + if let (Object::Instance(i), Some(ms)) = (other, mslots.get(pc + 1)) { + if ms + .peek_inst_builtin(i.cls_raw().attr_version.get()) + .is_some() + { + if let Some(hit) = self.core_native_method( + code, other, pc + 1, next.arg, mslots, lbase, nlocals, consts, + ) { + return Some(hit); + } + } + } + // A site that verified its callee for this class version. + let mut missed = false; + if let (Object::Instance(i), Some(ext)) = (other, ext) { + if let Some(site) = leaf_site_hit(ext, pc + 1, i.cls_raw().attr_version.get()) { + match self.core_leaf_site_call( + code, + i, + other, + site, + pc + 1, + next.arg, + lbase, + nlocals, + consts, + depth_cell, + ) { + SiteCall::Done(v, call_pc) => return Some((Ok(v), call_pc)), + SiteCall::Declined => {} + SiteCall::Missed => missed = true, + } + } + } + let pure_site = !missed + && match (other, mslots.get(pc + 1)) { + (Object::Instance(i), Some(ms)) => ms + .peek_fn(i.cls_raw().attr_version.get()) + // SAFETY: the slot's function is alive (its class + // version matched). + .is_some_and(|fp| fn_is_leaf(unsafe { &*fp })), + _ => false, + }; + if pure_site { + let (v, call_pc) = self.core_pure_method( + code, other, pc + 1, next.arg, mslots, lbase, nlocals, consts, depth_cell, + )?; + return Some((Ok(v), call_pc)); + } + self.core_native_method(code, other, pc + 1, next.arg, mslots, lbase, nlocals, consts) + } + // A container's site-cached leaf method. + Object::List(_) | Object::Dict(_) | Object::Set(_) | Object::Str(_) => { + self.core_builtin_method(code, other, pc + 1, mslots, lbase, nlocals, consts) + } + _ => None, + } + } + /// [`Self::leaf_fused_len`] for the `LOAD_GLOBAL` at `pc` of `frame`, /// when the four-instruction `len(x)` shape is there. #[inline] @@ -9995,7 +10112,7 @@ impl Interpreter { /// the frame first and reload everything after (see /// [`Self::core_call`], [`Self::core_return`]). #[inline(always)] - fn leaf_core(&mut self, sw: &mut CoreSwitch, snap_gen: u64) -> CoreExit { + fn leaf_core(&mut self, sw: &mut CoreSwitch, snap_gen: u64) -> CoreExit { #[inline(always)] fn scalar(v: &Object) -> bool { matches!( @@ -10081,7 +10198,63 @@ impl Interpreter { } }; } + // The code's native form (see `frame_jit`): the native copy of + // this loop runs it from every pc the arms below leave it at, + // and the arms only run the instructions it returns at. + #[cfg(feature = "jit")] + let native = ext.and_then(|e| e.frame_jit.get(nlocals)); + #[cfg(feature = "jit")] + if NATIVE != native.is_some() { + // (The activation is synced at every reload.) + return CoreExit::Switch; + } + #[cfg(feature = "jit")] + if pc == 0 && !NATIVE { + if let Some(ext) = ext { + ext.frame_jit.warm(code, ext, nlocals, 0); + } + } + #[cfg(feature = "jit")] + let mut nst = frame_jit::State { + locals: lbase, + stack: base, + len, + cap, + pc, + last, + countdown: std::ptr::addr_of_mut!(self.gil_countdown), + snap_gen, + maybe_dead: pending_work, + interp: std::ptr::from_ref(self), + out: 0, + depth_cell: sw.depth_cell, + err: None, + }; + // The pc the native code last returned at (the arms run it). + #[cfg(feature = "jit")] + let mut handed = usize::MAX; let stop = loop { + #[cfg(feature = "jit")] + if NATIVE && pc != handed { + if let Some(native) = native.filter(|n| n.enters_at(pc)) { + nst.len = len; + nst.pc = pc; + nst.last = last; + // SAFETY: the running activation's state, as this + // loop holds it (its locals count checked above). + let status = unsafe { native.run(&mut nst) }; + len = nst.len; + pc = nst.pc; + last = nst.last; + if status == frame_jit::MARKED { + break Some(CoreExit::Stop(LeafStop::Marked)); + } + if let Some(e) = nst.err.take() { + break Some(CoreExit::Stop(LeafStop::Raised(e))); + } + handed = pc; + } + } if pc >= ninstrs { break Some(CoreExit::Stop(LeafStop::Step)); } @@ -10238,148 +10411,42 @@ impl Interpreter { // off the local (no receiver clone pushed and // released); a miss loads the receiver and // leaves `LOAD_ATTR` to its own arm. - if matches!(other, Object::Instance(_) | Object::Module(_)) - && pc + 1 < ninstrs + // `x.m(...)` of a leaf or native method on the + // borrowed local (see `core_local_method`). + if pc + 1 < ninstrs + && len < cap + && (*instrs.add(pc + 1)).op == OpCode::LoadMethodAttr { - let next = *instrs.add(pc + 1); - // `x.m(...)` of a pure leaf method with - // simple arguments: its result, - // evaluated on the borrowed operands. - if next.op == OpCode::LoadMethodAttr - && len < cap - && simple_args_prefix(&code.instructions, pc + 2) - { - // A native method the site cached - // for this class version: straight - // to its fused call (the Python - // leaf probes below would miss). - if let (Object::Instance(i), Some(ms)) = - (other, mslots!(cold_mslots, ext).get(pc + 1)) - { - if ms - .peek_inst_builtin( - i.cls_raw().attr_version.get(), - ) - .is_some() - { - if let Some((r, call_pc)) = self - .core_native_method( - code, - other, - pc + 1, - next.arg, - mslots!(cold_mslots, ext), - lbase, - nlocals, - consts, - ) - { - last = call_pc; - pc = call_pc + 1; - match r { - Ok(v) => { - base.add(len).write(v); - len += 1; - continue; - } - Err(e) => { - break Some(CoreExit::Stop( - LeafStop::Raised(e), - )); - } - } - } - } - } - // A site that verified its callee - // for this class version. - let mut missed = false; - if let (Object::Instance(i), Some(ext)) = (other, ext) { - if let Some(site) = leaf_site_hit( - ext, - pc + 1, - i.cls_raw().attr_version.get(), - ) { - match self.core_leaf_site_call( - code, - i, - other, - site, - pc + 1, - next.arg, - lbase, - nlocals, - consts, - sw.depth_cell, - ) { - SiteCall::Done(v, call_pc) => { - base.add(len).write(v); - len += 1; - last = call_pc; - pc = call_pc + 1; - continue; - } - SiteCall::Declined => {} - SiteCall::Missed => missed = true, - } - } - } - let pure_site = !missed - && match ( - other, - mslots!(cold_mslots, ext).get(pc + 1), - ) { - (Object::Instance(i), Some(ms)) => ms - .peek_fn(i.cls_raw().attr_version.get()) - .is_some_and(|fp| fn_is_leaf(&*fp)), - _ => false, - }; - if pure_site { - if let Some((v, call_pc)) = self.core_pure_method( - code, - other, - pc + 1, - next.arg, - mslots!(cold_mslots, ext), - lbase, - nlocals, - consts, - sw.depth_cell, - ) { + if let Some((r, call_pc)) = self.core_local_method( + code, + ext, + other, + pc, + lbase, + nlocals, + consts, + sw.depth_cell, + ) { + last = call_pc; + pc = call_pc + 1; + match r { + Ok(v) => { base.add(len).write(v); len += 1; - last = call_pc; - pc = call_pc + 1; continue; } - } else if let Some((r, call_pc)) = self - .core_native_method( - code, - other, - pc + 1, - next.arg, - mslots!(cold_mslots, ext), - lbase, - nlocals, - consts, - ) - { - last = call_pc; - pc = call_pc + 1; - match r { - Ok(v) => { - base.add(len).write(v); - len += 1; - continue; - } - Err(e) => { - break Some(CoreExit::Stop( - LeafStop::Raised(e), - )); - } + Err(e) => { + break Some(CoreExit::Stop(LeafStop::Raised( + e, + ))); } } } + } + if matches!(other, Object::Instance(_) | Object::Module(_)) + && pc + 1 < ninstrs + { + let next = *instrs.add(pc + 1); if next.op == OpCode::LoadAttr { if code .instructions @@ -10442,43 +10509,6 @@ impl Interpreter { continue; } } - // `x.m(...)` of a container's site-cached - // leaf method on the borrowed local. - if matches!( - other, - Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::Str(_) - ) && pc + 1 < ninstrs - && len < cap - && (*instrs.add(pc + 1)).op == OpCode::LoadMethodAttr - { - if let Some((r, call_pc)) = self.core_builtin_method( - code, - other, - pc + 1, - mslots!(cold_mslots, ext), - lbase, - nlocals, - consts, - ) { - last = call_pc; - pc = call_pc + 1; - match r { - Ok(v) => { - base.add(len).write(v); - len += 1; - continue; - } - Err(e) => { - break Some(CoreExit::Stop(LeafStop::Raised( - e, - ))); - } - } - } - } clone_hot(other) } }; @@ -11085,6 +11115,17 @@ impl Interpreter { self.gil_countdown -= 1; last = pc; pc = (pc + 1).saturating_sub(ins.arg as usize); + // A hot loop compiles its code, and switches to the + // native copy of this loop on the spot. + #[cfg(feature = "jit")] + if !NATIVE { + if let Some(ext) = ext { + ext.frame_jit.warm(code, ext, nlocals, last); + if ext.frame_jit.get(nlocals).is_some() { + break Some(CoreExit::Switch); + } + } + } } OpCode::ForIter => { // A range iterator's next value; anything else (another @@ -55986,7 +56027,7 @@ enum QuietShell<'a> { #[cfg(test)] thread_local! { - static NATIVE_FAST_SUBSCRIPTS: std::cell::Cell = const { std::cell::Cell::new(0) }; + pub(crate) static NATIVE_FAST_SUBSCRIPTS: std::cell::Cell = const { std::cell::Cell::new(0) }; static LEAF_SET_REMOVALS: std::cell::Cell<[u64; 2]> = const { std::cell::Cell::new([0; 2]) }; static SAVED_NATIVE_LEAF_CALLS: std::cell::Cell = const { std::cell::Cell::new(0) }; static SAVED_NATIVE_REJECTED_CALLS: std::cell::Cell = const { std::cell::Cell::new(0) }; @@ -56372,6 +56413,11 @@ enum CoreExit { /// (Inside [`Interpreter::leaf_core`] only.) An in-loop call or /// return synced and moved the running activation: reload. Reload, + /// The running activation (synced) has native code and the loop is + /// its interpreted copy, or the other way round: run the other copy + /// (see `frame_jit`). + #[cfg(feature = "jit")] + Switch, } /// How [`Interpreter::leaf_core_attr`] ran its instruction. @@ -60827,6 +60873,9 @@ struct CodeConstObjects { /// Whether every return is `return None` (see /// [`code_returns_only_none`]): `0` not yet decided, `1` no, `2` yes. returns_none: std::sync::atomic::AtomicU8, + /// The code's native form for the core loop (see [`frame_jit`]). + #[cfg(feature = "jit")] + frame_jit: frame_jit::Slot, } /// A `CALL` site's inline-call shape (see `Interpreter::core_call`): the @@ -61129,6 +61178,32 @@ fn leaf_site_hit( .then(|| (site.func.as_ptr(), site.code, site.effect)) } +/// Whether the method call whose `LOAD_ATTR` is at `pc` last resolved to +/// a callee the core loop runs in place (a builtin, or a leaf function), +/// without an activation of its own. +#[cfg(feature = "jit")] +pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool { + // SAFETY: GIL-serialized; the borrows end before any refill. + let leaf = ext + .leaf_sites + .get() + .and_then(|s| s.get(pc)) + .and_then(|s| unsafe { &*s.0.get() }.as_ref()) + .is_some_and(|site| site.func.strong_count() > 0); + leaf || ext + .method_slots + .get() + .and_then(|s| s.get(pc)) + .is_some_and(|slot| match unsafe { &(*slot.0.get()).1 } { + MethodSlotFn::Builtin(_) | MethodSlotFn::Leaf(..) => true, + MethodSlotFn::Py(w) | MethodSlotFn::Static(w) => { + // SAFETY: the function is alive while the upgrade holds it. + w.upgrade().is_some_and(|f| fn_is_leaf(&f)) + } + _ => false, + }) +} + /// Record (or, with `data` `None`, forget) the site at `pc`'s callee. #[inline(never)] fn leaf_site_set(ext: &CodeConstObjects, ninstrs: usize, pc: usize, data: Option) { @@ -62248,6 +62323,8 @@ fn code_vm_ext_build( leaf_sites: std::sync::OnceLock::new(), leaf_plan: std::sync::OnceLock::new(), returns_none: std::sync::atomic::AtomicU8::new(0), + #[cfg(feature = "jit")] + frame_jit: frame_jit::Slot::default(), }) }) } diff --git a/tests/regrtest/test_frame_jit.py b/tests/regrtest/test_frame_jit.py new file mode 100644 index 00000000..e434f22d --- /dev/null +++ b/tests/regrtest/test_frame_jit.py @@ -0,0 +1,191 @@ +"""The core loop's native code keeps the interpreter's semantics. + +Each function runs long enough to be compiled, then meets the shapes the +native code hands back to the core loop: overflows, NaNs, unbound +locals, heap values where it expects scalars, identity of every kind of +value, finalizers released by stores, and attribute caches that go +stale. +""" + +import gc +import unittest + +N = 2000 + + +def ints(n): + t = 0 + for i in range(n): + t += i * 3 - (i >> 2) + (i & 7) ^ (i | 1) + if i % 5 == 0: + t -= i // 3 + t = t % 1000003 + return t + + +def floats(n): + x = 0.5 + for i in range(n): + x = x * 1.0001 + i / 7.0 - (x / 3.0) + if x > 1e6: + x = x - 1e6 + return round(x, 6) + + +def grows(n): + t = 1 + for i in range(n): + t = t * 3 + i + return t + + +class Box: + def __init__(self, v): + self.v = v + self.next = None + + +class FrameJitTests(unittest.TestCase): + def test_arithmetic(self): + self.assertEqual(ints(N), 363158) + self.assertEqual(floats(N), floats(N)) + + def test_overflow_promotes(self): + self.assertEqual(grows(N), grows(N)) + self.assertGreater(grows(200), 2**64) + + def test_nan_and_zero_division(self): + def f(n, d): + t = 0.0 + for i in range(n): + t = t + float("inf") - float("inf") if i == n - 1 else t + i / d + return t + + r = f(N, 2.0) + self.assertNotEqual(r, r) + with self.assertRaises(ZeroDivisionError): + f(N, 0.0) + + def test_unbound_local(self): + def f(n, flag): + for i in range(n): + x = i + if flag and i == n - 1: + del x + return x + + self.assertEqual(f(N, False), N - 1) + with self.assertRaises(UnboundLocalError): + f(N, True) + + def test_identity(self): + a = object() + s = "x" * 3 + big = 2**70 + + def f(n, x, y): + c = 0 + for _ in range(n): + if x is y: + c += 1 + if x is not None: + c += 2 + return c + + self.assertEqual(f(N, a, a), 3 * N) + self.assertEqual(f(N, a, object()), 2 * N) + self.assertEqual(f(N, s, s), 3 * N) + self.assertEqual(f(N, 5, 5), 3 * N) + self.assertEqual(f(N, 5, 5.0), 2 * N) + self.assertEqual(f(N, 1.5, 1.5), 3 * N) + self.assertEqual(f(N, 0.0, -0.0), 2 * N) + self.assertEqual(f(N, True, True), 3 * N) + self.assertEqual(f(N, True, 1), 2 * N) + self.assertEqual(f(N, None, None), N) + self.assertEqual(f(N, big, big), 3 * N) + + def test_finalizer_on_store(self): + log = [] + + class D: + def __del__(self): + log.append(1) + + def f(n): + x = None + for i in range(n): + x = D() + x = i + return x + + self.assertEqual(f(N), N - 1) + gc.collect() + self.assertEqual(len(log), N) + + def test_fields(self): + def total(b, n): + t = 0 + for _ in range(n): + t += b.v + return t + + b = Box(3) + self.assertEqual(total(b, N), 3 * N) + b.v = 2.5 + self.assertEqual(total(b, 4), 10.0) + b.v = "s" + with self.assertRaises(TypeError): + total(b, 1) + + def test_field_chains(self): + def walk(head, n): + t = 0 + for _ in range(n): + node = head + while node is not None: + t += node.v + node = node.next + return t + + head = Box(1) + head.next = Box(2) + head.next.next = Box(3) + self.assertEqual(walk(head, N), 6 * N) + Box.v = property(lambda self: 10) + try: + self.assertEqual(walk(head, 2), 60) + finally: + del Box.v + + def test_copy_and_swap(self): + def f(n): + a, b = 1, 2 + t = 0 + for i in range(n): + a, b = b, a + i + a, b = b % 997, a % 991 + t += a + return t, a, b + + self.assertEqual(f(N), f(N)) + + def test_truth(self): + def f(n, x): + c = 0 + for _ in range(n): + if x: + c += 1 + if not x: + c += 2 + return c + + self.assertEqual(f(N, 0), 2 * N) + self.assertEqual(f(N, 3), N) + self.assertEqual(f(N, 0.0), 2 * N) + self.assertEqual(f(N, None), 2 * N) + self.assertEqual(f(N, [1]), N) + self.assertEqual(f(N, ""), 2 * N) + + +if __name__ == "__main__": + unittest.main() From 545af0f9812ef290291eefbe7b30fa7024da9791 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 00:08:22 -0700 Subject: [PATCH 22/96] perf: compile leaf plans only once interpreting them has cost a compile A plan compiled after 64 evaluations: each compile costs Cranelift millions of instructions, and a small method's native code saves tens per call, so short runs paid for code they never amortized (deltablue compiled a dozen plans for about 110M instructions). Plans now compile after 20,000 evaluations, the rent-or-buy point. deltablue: -12% at the default work, -2% at 4x; long runs of every fixture within noise. --- crates/weavepy-vm/src/leaf_plan/jit.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index a9b54053..799de7b5 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -33,8 +33,12 @@ use cranelift_jit::{JITBuilder, JITModule}; use cranelift_module::{Linkage, Module}; use weavepy_compiler::{BinOpKind, CodeObject, CompareKind}; -/// Evaluations of a plan (per effect mode) before it is compiled. -const WARM_RUNS: u32 = 64; +/// Evaluations of a plan (per effect mode) before it is compiled. A +/// compile costs millions of instructions and saves tens to hundreds per +/// evaluation, so a plan compiles once interpreting it has cost about +/// what compiling would (the rent-or-buy rule): a short run doesn't pay +/// for code it never amortizes, and a long one loses little. +const WARM_RUNS: u32 = 20_000; /// How deep callee plans are compiled in line. const INLINE_DEPTH: u8 = 2; From 9cf597ea2ae570d8430e4846ae9086dafd8d0790 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:20:36 -0700 Subject: [PATCH 23/96] perf: run field-update methods in line and reuse element pins in tier 2 A method whose body is `self.n += k; return self.n` now runs in compiled code without a call once wpjit_call_method has run it through the helper: the helper arms the method entry with the field's split index, the most split values a receiver can hold without shadowing the method's name, the increment's form and the function's code pointer, and compiled code checks those, the receiver's class version and split names, and the observer, dict-watcher and exotic-key gates in line before the store. Anything else takes the helper as before. A list loop, a list read or an object-lane method result that pins an object an activation already pinned reuses that pin (a 16-entry memo by address, checked against the pin it names), so a loop over the same few objects no longer grows the pin table until the pin-pressure exit. Per-unit instructions: richards 4,465 -> 1,851 (the memo alone: 4,010), call_overhead 6,473 -> 6,162, attr_access 3,023 -> 2,783. --- crates/weavepy-jit/src/lower.rs | 157 ++++++++++++++ crates/weavepy-jit/src/runtime.rs | 26 +++ crates/weavepy-vm/src/capi_watchers.rs | 6 + crates/weavepy-vm/src/inst_dict.rs | 8 + crates/weavepy-vm/src/object.rs | 6 + crates/weavepy-vm/src/tier2.rs | 252 +++++++++++++++++++--- crates/weavepy-vm/src/trace.rs | 7 + tests/regrtest/test_jit_inline_methods.py | 182 ++++++++++++++++ 8 files changed, 617 insertions(+), 27 deletions(-) create mode 100644 tests/regrtest/test_jit_inline_methods.py diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 99c4d400..548db074 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -2507,10 +2507,34 @@ impl<'a, 'b> Lowerer<'a, 'b> { let tagv = self.b.ins().iconst(types::I32, Self::tag(ty)); self.b.ins().store(trusted, tagv, self.call_tags_base, toff); } + // What a field update would add: the `int` argument, or (none + // passed) the method's literal. + let update_arg = match &self.vstack[base..] { + [] => Some(None), + [(v, JitType::Int)] => Some(Some(*v)), + _ => None, + }; self.vstack.truncate(base); let (pin, _) = self.pop(); let snapshot = self.vstack.clone(); + // A method whose body is a field update, run in line once the + // helper has armed it; anything else takes the helper below. + let inline = match (runtime::obj_layout(), ret, update_arg) { + (Some(l), MethodRet::Scalar(JitType::Int), Some(arg)) => Some((*l, arg)), + _ => None, + }; + let mut join = None; + if let Some((l, arg)) = inline { + let miss = self.b.create_block(); + let done = self.b.create_block(); + self.b.append_block_param(done, types::I64); + let v = self.inline_method_update(&l, pin, token, arg, miss); + self.b.ins().jump(done, &[BlockArg::from(v)]); + self.b.switch_to_block(miss); + join = Some(done); + } + self.writeback_locals(); self.store_call_site_pc(pc); @@ -2562,10 +2586,143 @@ impl<'a, 'b> Lowerer<'a, 'b> { .b .ins() .load(Self::cl_ty(t), trusted, self.frame_ptr, OFF_RET_BITS); + let res = match join { + Some(done) => { + self.b.ins().jump(done, &[BlockArg::from(res)]); + self.b.switch_to_block(done); + self.b.block_params(done)[0] + } + None => res, + }; self.vstack.push((res, t)); } } + /// The armed field update of method token `token` on the pinned + /// receiver `pin` (see [`runtime::ObjLayout::method_upd_idx`]): the + /// updated field's new value, with the current block continuing past + /// the store. A receiver of another class version, values not split + /// over the class's names or numerous enough to shadow the method, a + /// swapped `__code__`, a non-`int` field, an overflow, or an active + /// observer, dict watcher or exotic key branches to `miss` (the helper, + /// which runs the call exactly) before anything is stored. + fn inline_method_update( + &mut self, + l: &runtime::ObjLayout, + pin: Value, + token: u32, + arg: Option, + miss: Block, + ) -> Value { + let t = MemFlags::trusted(); + let ptr = self.ptr_ty; + let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); + // The entry: armed, for this call's increment form. + let methods = self.b.ins().load(ptr, t, ctx, l.ctx_methods); + let n = self.b.ins().load(types::I64, t, methods, l.methods_len); + let out = self + .b + .ins() + .icmp_imm(IntCC::UnsignedLessThanOrEqual, n, i64::from(token)); + self.miss_if(out, miss); + let mbuf = self.b.ins().load(ptr, t, methods, l.methods_buf); + let e = self + .b + .ins() + .iadd_imm(mbuf, i64::from(token) * i64::from(l.method_size)); + let idx = self.b.ins().uload32(t, e, l.method_upd_idx); + let from_arg = self.b.ins().uload32(t, e, l.method_upd_from_arg); + let unarmed = self.b.ins().icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); + let form = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, from_arg, i64::from(arg.is_some())); + // The function's code, and the gates. + let code_at = self.b.ins().load(ptr, t, e, l.method_upd_code_at); + let want = self.b.ins().load(ptr, t, e, l.method_upd_code); + let observers = self.b.ins().iconst(ptr, l.observers as i64); + let observers = self.b.ins().load(types::I64, t, observers, 0); + let watchers = self.b.ins().iconst(ptr, l.dict_watchers as i64); + let watchers = self.b.ins().uload8(types::I64, t, watchers, 0); + let exotic = self.b.ins().iconst(ptr, l.exotic_keys as i64); + let exotic = self.b.ins().load(types::I64, t, exotic, 0); + let gates = self.b.ins().bor(observers, watchers); + let gates = self.b.ins().bor(gates, exotic); + let gated = self.b.ins().icmp_imm(IntCC::NotEqual, gates, 0); + let bad = self.b.ins().bor(unarmed, form); + let bad = self.b.ins().bor(bad, gated); + self.miss_if(bad, miss); + let code = self.b.ins().load(ptr, t, code_at, 0); + let swapped = self.b.ins().icmp(IntCC::NotEqual, code, want); + self.miss_if(swapped, miss); + // The pin: an instance. + let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); + self.miss_if(out, miss); + let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); + let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); + let p = self.b.ins().iadd(buf, off); + let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); + let otag = self.b.ins().uload8(types::I32, t, p, l.pin_obj); + let not_obj = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + let bad = self.b.ins().bor(not_obj, not_inst); + self.miss_if(bad, miss); + let inst = self.b.ins().load(ptr, t, p, l.pin_obj + 8); + // Its class and split values, as `split_field_addr` checks them + // for a write, against the entry's version. + let ver = self.b.ins().load(types::I64, t, e, l.method_ver); + let cls = self.b.ins().load(ptr, t, inst, l.inst_class); + let cver = self.b.ins().load(types::I64, t, cls, l.type_attr_version); + let lazy = self.b.ins().load(ptr, t, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, t, flag, 0); + let borrow = self.b.ins().sload32(t, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, t, inst, l.inst_split_block); + let stale = self.b.ins().icmp(IntCC::NotEqual, cver, ver); + let busy = self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(stale, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, miss); + // The class's names, holding the field but not the method's name. + let keys = self.b.ins().load(ptr, t, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, t, cls, l.type_shared_keys); + let len = self.b.ins().uload32(t, block, l.split_len); + let shadow = self.b.ins().uload32(t, e, l.method_upd_shadow); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let shadowed = self.b.ins().icmp(IntCC::UnsignedGreaterThan, len, shadow); + let bad = self.b.ins().bor(foreign, absent); + let bad = self.b.ins().bor(bad, shadowed); + self.miss_if(bad, miss); + let off = self.b.ins().ishl_imm(idx, 4); + let at = self.b.ins().iadd(block, off); + let at = self.b.ins().iadd_imm(at, i64::from(l.split_values)); + // An `int` field, plus the increment without overflow. + let tag = self.b.ins().uload8(types::I32, t, at, 0); + let not_int = self.b.ins().icmp_imm(IntCC::NotEqual, tag, i64::from(l.tag_int)); + self.miss_if(not_int, miss); + let old = self.b.ins().load(types::I64, t, at, 8); + let inc = match arg { + Some(a) => a, + None => self.b.ins().load(types::I64, t, e, l.method_upd_inc), + }; + let (new, overflow) = self.checked_add(old, inc); + self.miss_if(overflow, miss); + self.b.ins().store(t, new, at, 8); + new + } + /// RFC 0073 WS3 — burned-in native `str`-method call via the /// registered `wpjit_str_method` helper (the [`Self::emit_call_method`] /// discipline, sharing its signature). The helper receives the diff --git a/crates/weavepy-jit/src/runtime.rs b/crates/weavepy-jit/src/runtime.rs index 5b122b28..b50af8ee 100644 --- a/crates/weavepy-jit/src/runtime.rs +++ b/crates/weavepy-jit/src/runtime.rs @@ -212,6 +212,32 @@ pub struct ObjLayout { pub list_ptr: i32, pub list_len: i32, pub list_cap: i32, + /// [`JitFrame::ctx`] → the method entries' shared pointer; that + /// pointer → the entry array and its length. + pub ctx_methods: i32, + pub methods_buf: i32, + pub methods_len: i32, + /// A method entry: its size, its class version (`u64`), and its + /// in-line field update: the field's split index (`u32`, `u32::MAX` + /// while unarmed), the most split values a receiver may hold without + /// shadowing the method (`u32`), whether the increment is the call's + /// argument (`u32`: `1`, else `0`), the literal increment (`i64`), + /// where the function keeps its code pointer and the code pointer that + /// must be there (words). + pub method_size: i32, + pub method_ver: i32, + pub method_upd_idx: i32, + pub method_upd_shadow: i32, + pub method_upd_from_arg: i32, + pub method_upd_inc: i32, + pub method_upd_code_at: i32, + pub method_upd_code: i32, + /// Words (or a byte, for `dict_watchers`) that are nonzero while a + /// call can't be skipped: an observer may be active, a dict watcher + /// is, or a class may hold an exotic key. + pub observers: usize, + pub dict_watchers: usize, + pub exotic_keys: usize, } static OBJ_LAYOUT: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/crates/weavepy-vm/src/capi_watchers.rs b/crates/weavepy-vm/src/capi_watchers.rs index 521225b2..508d319b 100644 --- a/crates/weavepy-vm/src/capi_watchers.rs +++ b/crates/weavepy-vm/src/capi_watchers.rs @@ -105,6 +105,12 @@ pub fn dicts_active() -> bool { DICTS_ACTIVE.load(Ordering::Relaxed) } +/// The flag [`dicts_active`] reads, for native code that reads it in line +/// (a byte: nonzero while any dict watcher is active). +pub(crate) fn dicts_active_flag() -> *const bool { + DICTS_ACTIVE.as_ptr() +} + pub fn watch_dict(watcher_id: u8, d: &Rc>) { let mut w = WATCHED_DICTS.lock(); let ptr = Rc::as_ptr(d) as usize; diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs index 6a521f4f..cd98d69f 100644 --- a/crates/weavepy-vm/src/inst_dict.rs +++ b/crates/weavepy-vm/src/inst_dict.rs @@ -107,6 +107,14 @@ impl SharedKeys { }) } + /// How many leading names are not `name` (Python hash `hash`): its + /// position, or all of them when none is. An instance holding no more + /// values than this has no attribute `name`. + pub(crate) fn names_before(&self, name: &str, hash: i64) -> usize { + let n = self.len(); + self.position_hashed(n, name, hash).unwrap_or(n) + } + /// Publish the `str` name `name` as the next name; `None` when the /// table is full. The caller holds the GIL outside free-threaded mode. fn push(&self, name: &SharedStr) -> Option { diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 84eeab5d..4acf6af2 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -3127,6 +3127,12 @@ pub fn exotic_str_keys_possible() -> bool { EXOTIC_CLASS_KEYS.load(std::sync::atomic::Ordering::Acquire) != 0 } +/// The counter [`exotic_str_keys_possible`] reads, for native code that +/// reads it in line (a `usize`: nonzero once exotic keys are possible). +pub(crate) fn exotic_str_keys_flag() -> *const usize { + EXOTIC_CLASS_KEYS.as_ptr() +} + /// Reach for the running interpreter to compute a user instance's Python /// `__hash__`. `DictKey`'s `Hash`/`Eq` impls have no interpreter handle, so /// they borrow the thread's published interpreter pointer — the same bridge diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 9b8e0bca..efff05af 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -131,6 +131,47 @@ struct MethodEntry { /// A globally unique resolution token. The live receiver owns the class /// checked at access; this cached method must not keep that class alive. ver: u64, + /// What compiled code needs to run this method's callback-free field + /// update (`self.n += k; return self.n`) in line, armed by + /// `wpjit_call_method` once the helper has run it (see [`arm_update`]). + update: InlineUpdate, +} + +/// [`MethodEntry::update`]: compiled code reads these in place (see +/// [`obj_layout`]) and calls the helper unless the receiver's class still +/// has the entry's version, its values are split over the class's names +/// and are too few to shadow the method's name, the function still wears +/// the entry's code, and no observer or exotic key could tell. +#[repr(C)] +struct InlineUpdate { + /// The split index of the field the update adds to (`u32::MAX` while + /// unarmed). + idx: Cell, + /// A receiver holding more split values than this may shadow the + /// method's name. + shadow: Cell, + /// `1` when the increment is the call's argument; `0` when it's + /// [`Self::inc`]. + from_arg: Cell, + /// The literal increment. + inc: Cell, + /// Where the function keeps its code pointer, and the entry's code + /// pointer it must equal. + code_at: Cell, + code: Cell, +} + +impl InlineUpdate { + fn unarmed() -> Self { + Self { + idx: Cell::new(u32::MAX), + shadow: Cell::new(0), + from_arg: Cell::new(0), + inc: Cell::new(0), + code_at: Cell::new(0), + code: Cell::new(0), + } + } } /// One slot per method token (parallel to `cf.method_sites`). @@ -2866,6 +2907,19 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option = StdRc::new(Vec::with_capacity(3)); + let methods_box = { + // SAFETY: as for the guards above. + unsafe { std::mem::transmute_copy::, usize>(&methods) } + }; + let methods_vec = std::ptr::from_ref::(&methods) as usize - methods_box; + if word(methods_box as *const u8, methods_vec + vec_ptr) != methods.as_ptr() as usize + || word(methods_box as *const u8, methods_vec + vec_len) != 0 + { + return None; + } + let update = std::mem::offset_of!(MethodEntry, update); let ctx_pins = std::mem::offset_of!(CallCtx, pins); Some(weavepy_jit::ObjLayout { ctx_pins_ptr: i32_of(ctx_pins + vec_ptr)?, @@ -2901,6 +2955,20 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option())?, + method_ver: i32_of(std::mem::offset_of!(MethodEntry, ver))?, + method_upd_idx: i32_of(update + std::mem::offset_of!(InlineUpdate, idx))?, + method_upd_shadow: i32_of(update + std::mem::offset_of!(InlineUpdate, shadow))?, + method_upd_from_arg: i32_of(update + std::mem::offset_of!(InlineUpdate, from_arg))?, + method_upd_inc: i32_of(update + std::mem::offset_of!(InlineUpdate, inc))?, + method_upd_code_at: i32_of(update + std::mem::offset_of!(InlineUpdate, code_at))?, + method_upd_code: i32_of(update + std::mem::offset_of!(InlineUpdate, code))?, + observers: crate::trace::observer_count_flag() as usize, + dict_watchers: crate::capi_watchers::dicts_active_flag() as usize, + exotic_keys: crate::object::exotic_str_keys_flag() as usize, }) } @@ -2978,6 +3046,7 @@ fn probe_method_entry( min_args, ret, ver, + update: InlineUpdate::unarmed(), }) } @@ -3449,6 +3518,74 @@ struct CallCtx { /// a local) re-enters without re-resolving or rebuilding the handle. /// Taken out for the call's duration and put back after. dyn_callee: Option<(usize, usize, bool, NativeCallee)>, + /// Pin indices recently handed out for instances and strings, by + /// object address (see [`pin_reusing`]): a loop over the same few + /// objects reuses their pins instead of growing the table. Advisory; + /// every hit is checked against the pin it names. + pin_memo: [u32; PIN_MEMO], +} + +/// The size of [`CallCtx::pin_memo`]. +const PIN_MEMO: usize = 16; + +/// The identity [`pin_reusing`] keys an object by (`0` for objects whose +/// pins aren't reused). +#[inline] +fn pin_identity(v: &Object) -> usize { + match v { + Object::Instance(i) => Rc::as_ptr(i) as usize, + Object::Str(s) => SharedStr::as_ptr(s).cast::() as usize, + _ => 0, + } +} + +/// The [`CallCtx::pin_memo`] entry for an object identity. +#[inline] +fn pin_memo_slot(id: usize) -> usize { + (id >> 4 ^ id >> 9) % PIN_MEMO +} + +/// The pin that already holds this very object, when `memo` remembers +/// one. A pin only owns its object, so two sites that pin the same object +/// can share one. +#[inline] +fn pin_memo_hit(v: &Object, pins: &PinTable, memo: &[u32; PIN_MEMO]) -> Option { + let id = pin_identity(v); + if id == 0 { + return None; + } + let hint = memo[pin_memo_slot(id)] as usize; + match pins.get(hint) { + Some(Pin::Obj(p)) if pin_identity(p) == id => Some(hint as u64), + _ => None, + } +} + +/// Remember that pin `ix` will hold `p`'s object. +#[inline] +fn pin_memo_note(p: &Pin, ix: usize, memo: &mut [u32; PIN_MEMO]) { + if let Pin::Obj(v) = p { + let id = pin_identity(v); + if id != 0 { + memo[pin_memo_slot(id)] = ix as u32; + } + } +} + +/// Pin `v` in `pins`, reusing the pin that already holds this very object +/// (see [`pin_memo_hit`]); `None` at the pin cap. +#[inline] +fn pin_reusing(v: &Object, pins: &mut PinTable, memo: &mut [u32; PIN_MEMO]) -> Option { + if let Some(bits) = pin_memo_hit(v, pins, memo) { + return Some(bits); + } + if pins.len() >= RUNTIME_PIN_CAP { + return None; + } + let p = Pin::Obj(v.clone()); + pin_memo_note(&p, pins.len(), memo); + pins.push(p); + Some((pins.len() - 1) as u64) } impl CallCtx { @@ -3811,14 +3948,12 @@ fn lane_tag(t: JitType) -> u32 { /// RFC 0071 WS1 — pack a call result for an `ObjPin` return lane: the /// `None` singleton is the nullable lane's `-1`; an instance pins into -/// the caller's table (capped). Anything else can't ride the lane. -fn obj_ret_bits(v: &Object, pins: &mut PinTable) -> Option { +/// the caller's table (capped), reusing the pin that already holds it +/// (a method returning `self`). Anything else can't ride the lane. +fn obj_ret_bits(v: &Object, pins: &mut PinTable, memo: &mut [u32; PIN_MEMO]) -> Option { match v { Object::None => Some(u64::MAX), - Object::Instance(_) if pins.len() < RUNTIME_PIN_CAP => { - pins.push(Pin::Obj(v.clone())); - Some((pins.len() - 1) as u64) - } + Object::Instance(_) => pin_reusing(v, pins, memo), _ => None, } } @@ -4172,6 +4307,49 @@ unsafe fn native_scalar_field_update( Some(value) } +/// Arm `entry`'s in-line update (see [`InlineUpdate`]) after the helper +/// ran it on `receiver`: the method guard and [`native_scalar_field_update`] +/// have just held, so what compiled code still checks per call (the +/// receiver's class version and split names, the shadow bound, the code +/// identity, the observer and key gates) is all that can change. A split +/// field and the plain increments only; anything else stays unarmed. +fn arm_update(entry: &MethodEntry, nc: &NativeCallee, receiver: &Object, argc: u32) { + let Some(plan) = nc.scalar_update.as_deref() else { + return; + }; + let Some(guard) = nc.attr_guards.get(plan.store_token) else { + return; + }; + let Object::Instance(inst) = receiver else { + return; + }; + let cls = inst.cls_raw(); + let Some(keys) = cls.shared_keys.get() else { + return; + }; + let (from_arg, inc) = match (plan.increment, argc) { + (Some(inc), 0) => (0, inc), + (None, 1) => (1, 0), + _ => return, + }; + if guard.split_idx == u32::MAX + || guard.ver != entry.ver + || cls.attr_version.get() != entry.ver + || !matches!(guard.storage, AttrStorage::Indexed(i) if i == guard.split_idx) + { + return; + } + let shadow = keys.names_before(&entry.name, entry.name_hash); + let u = &entry.update; + u.shadow.set(u32::try_from(shadow).unwrap_or(0)); + u.from_arg.set(from_arg); + u.inc.set(inc); + u.code_at.set(entry.func.code.as_ptr() as usize); + // SAFETY: `Rc` is one pointer, compared and never dereferenced. + u.code.set(unsafe { std::mem::transmute_copy::, usize>(&entry.code) }); + u.idx.set(guard.split_idx); +} + /// Bind the slots a keyword call skipped (tagged [`SlotTag::Default`] /// under [`weavepy_jit::CALL_GAPS`]) to the callee's current /// `__defaults__` when those are scalars, so the call is a positional @@ -4724,6 +4902,7 @@ unsafe fn try_native_call( // callee — keep it observable to callee-side stack walkers. frameless_code: Some(nc.code.clone()), dyn_callee: None, + pin_memo: [u32::MAX; PIN_MEMO], }) }; child.interp = ctx.interp; @@ -4905,7 +5084,7 @@ unsafe fn try_native_call( // RFC 0071 WS1 — an object-lane return pins into the // *caller's* table (`None` rides as the nullable `-1`). if guards_ok && expect_tag == SlotTag::ObjPin as u32 { - return Some(match obj_ret_bits(&v, &mut ctx.pins) { + return Some(match obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo) { Some(bits) => { jf.ret_bits = bits; jf.ret_tag = expect_tag; @@ -5004,7 +5183,7 @@ unsafe fn try_native_ctor( // `__init__` completed and returned `None` with guards intact; // the call site's value is the fresh instance. if expect_tag == SlotTag::ObjPin as u32 { - if let Some(bits) = obj_ret_bits(&inst, &mut ctx.pins) { + if let Some(bits) = obj_ret_bits(&inst, &mut ctx.pins, &mut ctx.pin_memo) { jf.ret_bits = bits; jf.ret_tag = expect_tag; return Some(CallStatus::Ok as i64); @@ -5343,7 +5522,7 @@ unsafe extern "C" fn wpjit_call_py( // RFC 0071 WS1 — an object-lane result pins into this // activation's table. if expect_tag == SlotTag::ObjPin as u32 { - if let Some(bits) = obj_ret_bits(&v, &mut ctx.pins) { + if let Some(bits) = obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo) { jf.ret_bits = bits; jf.ret_tag = expect_tag; return CallStatus::Ok as i64; @@ -5447,7 +5626,7 @@ fn deliver_call_result(jf: &mut JitFrame, ctx: &mut CallCtx, v: Object, expect_t // RFC 0071 WS1 — an object-lane result pins into this // activation's table. SlotTag::ObjPin => { - if let Some(bits) = obj_ret_bits(&v, &mut ctx.pins) { + if let Some(bits) = obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo) { jf.ret_bits = bits; jf.ret_tag = expect_tag; return CallStatus::Ok as i64; @@ -5981,6 +6160,8 @@ unsafe extern "C" fn wpjit_call_method( if let Some(value) = unsafe { native_scalar_field_update(jf, ctx, nc, &recv, argc as usize) } { + // Compiled code runs the next ones in line. + arm_update(entry, nc, &recv, argc); if expect_tag == SlotTag::Int as u32 { jf.ret_bits = value as u64; jf.ret_tag = SlotTag::Int as u32; @@ -6319,7 +6500,10 @@ unsafe extern "C" fn wpjit_list_get(frame: *mut JitFrame, pin: i64, idx: i64) -> // RFC 0073 WS3 — exact-`str` elements pin on the object // lane (indexing into a `split` result). (v @ (Object::Instance(_) | Object::Str(_) | Object::Tuple(_)), JitType::Obj) => { - Err(Pin::Obj(v.clone())) + match pin_memo_hit(v, &ctx.pins, &ctx.pin_memo) { + Some(bits) => Ok(bits), + None => Err(Pin::Obj(v.clone())), + } } // The nested lane: a list element pins on the float-list // lane (its reads re-validate each float). @@ -6339,6 +6523,7 @@ unsafe extern "C" fn wpjit_list_get(frame: *mut JitFrame, pin: i64, idx: i64) -> return 1; } jf.ret_bits = ctx.pins.len() as u64; + pin_memo_note(&p, ctx.pins.len(), &mut ctx.pin_memo); ctx.pins.push(p); 0 } @@ -7127,8 +7312,12 @@ unsafe extern "C" fn wpjit_list_next(frame: *mut JitFrame, pin: i64, idx: i64) - (Object::None, JitType::Obj) => Ok(u64::MAX), // RFC 0073 WS3 — exact-`str` elements pin on the object // lane (a `split` result's `ForList` consumer). + // A loop over the same few objects reuses their pins. (v @ (Object::Instance(_) | Object::Str(_) | Object::Tuple(_)), JitType::Obj) => { - Err(Pin::Obj(v.clone())) + match pin_memo_hit(v, &ctx.pins, &ctx.pin_memo) { + Some(bits) => Ok(bits), + None => Err(Pin::Obj(v.clone())), + } } // The nested lane: a list element pins on the float-list // lane (see `wpjit_list_get`). @@ -7138,19 +7327,20 @@ unsafe extern "C" fn wpjit_list_next(frame: *mut JitFrame, pin: i64, idx: i64) - _ => return 2, } }; - match outcome { - Ok(bits) => { - jf.ret_bits = bits; - 0 - } - Err(p) => { - if ctx.pins.len() >= RUNTIME_PIN_CAP { - return 2; - } - jf.ret_bits = ctx.pins.len() as u64; + let pinned = match outcome { + Ok(bits) => Some(bits), + Err(p) => (ctx.pins.len() < RUNTIME_PIN_CAP).then(|| { + pin_memo_note(&p, ctx.pins.len(), &mut ctx.pin_memo); ctx.pins.push(p); + (ctx.pins.len() - 1) as u64 + }), + }; + match pinned { + Some(bits) => { + jf.ret_bits = bits; 0 } + None => 2, } } @@ -7259,7 +7449,7 @@ unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i SlotTag::Int => pack(&v, JitType::Int), SlotTag::Float => pack(&v, JitType::Float), SlotTag::Bool => pack(&v, JitType::Bool), - SlotTag::ObjPin => obj_ret_bits(&v, &mut ctx.pins), + SlotTag::ObjPin => obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo), _ => None, } }; @@ -9805,7 +9995,12 @@ unsafe extern "C" fn wpjit_iter_new(frame: *mut JitFrame, pin: i64) -> i64 { /// Pack one yielded element into a compiled loop-variable lane /// (RFC 0074 WS3 — the [`weavepy_jit::ITER_ELEM_STR`]-aware sibling /// of `wpjit_iter_next`'s packing). `None` = outside the lane. -fn pack_iter_elem(v: &Object, tag: i64, pins: &mut PinTable) -> Option { +fn pack_iter_elem( + v: &Object, + tag: i64, + pins: &mut PinTable, + memo: &mut [u32; PIN_MEMO], +) -> Option { if tag == weavepy_jit::ITER_ELEM_STR { return match v { Object::Str(_) if pins.len() < RUNTIME_PIN_CAP => { @@ -9831,7 +10026,7 @@ fn pack_iter_elem(v: &Object, tag: i64, pins: &mut PinTable) -> Option { | Object::Str(_) | Object::WStr(_) | Object::Bytes(_) => pin_any(v.clone(), pins), - _ => obj_ret_bits(v, pins), + _ => obj_ret_bits(v, pins, memo), }, _ => None, } @@ -10081,9 +10276,9 @@ unsafe extern "C" fn wpjit_iter_next_pair( // store-pc deopt and unpack generically). if let Object::Tuple(items) = &v { if items.len() == 2 { - let packed1 = pack_iter_elem(&items[0], tag1, &mut ctx.pins); + let packed1 = pack_iter_elem(&items[0], tag1, &mut ctx.pins, &mut ctx.pin_memo); let packed2 = - packed1.and_then(|_| pack_iter_elem(&items[1], tag2, &mut ctx.pins)); + packed1.and_then(|_| pack_iter_elem(&items[1], tag2, &mut ctx.pins, &mut ctx.pin_memo)); if let (Some(b1), Some(b2)) = (packed1, packed2) { jf.ret_bits = b1; // SAFETY: the marshal buffer is at least @@ -10409,6 +10604,7 @@ pub(crate) fn try_call_native_direct( // keep the activation observable to callee-side stack walkers. frameless_code: Some(code.clone()), dyn_callee: None, + pin_memo: [u32::MAX; PIN_MEMO], }; let mut jf = JitFrame { locals: locals_buf.as_mut_ptr(), @@ -11217,6 +11413,7 @@ fn enter_compiled( // spine already. frameless_code: None, dyn_callee: None, + pin_memo: [u32::MAX; PIN_MEMO], }; let mut jf = JitFrame { locals: locals_buf.as_mut_ptr(), @@ -12101,6 +12298,7 @@ fn resume_parked(interp: &mut super::Interpreter, frame: &mut super::Frame) -> J // is on the spine already. frameless_code: None, dyn_callee: None, + pin_memo: [u32::MAX; PIN_MEMO], }; let mut jf = JitFrame { locals: act.locals_buf.as_mut_ptr(), diff --git a/crates/weavepy-vm/src/trace.rs b/crates/weavepy-vm/src/trace.rs index 13a06750..17d15409 100644 --- a/crates/weavepy-vm/src/trace.rs +++ b/crates/weavepy-vm/src/trace.rs @@ -445,6 +445,13 @@ pub fn any_observers_active() -> bool { any_observers_active_slow() } +/// The observer count [`any_observers_active`] tests first, for native +/// code that tests it in line (a `usize`: nonzero while any observer may +/// be active). +pub(crate) fn observer_count_flag() -> *const usize { + OBSERVER_COUNT.as_ptr() +} + #[cold] #[inline(never)] fn any_observers_active_slow() -> bool { diff --git a/tests/regrtest/test_jit_inline_methods.py b/tests/regrtest/test_jit_inline_methods.py new file mode 100644 index 00000000..98554baf --- /dev/null +++ b/tests/regrtest/test_jit_inline_methods.py @@ -0,0 +1,182 @@ +"""Compiled code runs a method that only updates a field in line. + +A method whose body is `self.n += k; return self.n` runs without a call +once the tier-2 helper has run it: each loop below runs long enough for +that, then the receiver, its class, the method or the field changes under +the compiled code, and every change must reach the generic path with the +interpreter's result. +""" + +import sys +import unittest + + +class Counter: + def __init__(self): + self.n = 0 + + def bump(self, by): + self.n += by + return self.n + + def tick(self): + self.n += 1 + return self.n + + +def bump_loop(c, n): + t = 0 + for _ in range(n): + t += c.bump(2) + return t + + +def tick_loop(c, n): + t = 0 + for _ in range(n): + t += c.tick() + return t + + +def tick_all(items, n): + for _ in range(n): + for c in items: + c.tick() + return [c.n for c in items] + + +def triangle(k, n): + # The sum of k, 2k, ..., nk. + return k * n * (n + 1) // 2 + + +class InlineMethodTests(unittest.TestCase): + N = 3000 + + def test_updates(self): + c = Counter() + self.assertEqual(bump_loop(c, self.N), triangle(2, self.N)) + self.assertEqual(c.n, 2 * self.N) + d = Counter() + self.assertEqual(tick_loop(d, self.N), triangle(1, self.N)) + self.assertEqual(tick_all([Counter() for _ in range(5)], self.N), [self.N] * 5) + + def test_instance_shadows_the_method(self): + c = Counter() + self.assertEqual(bump_loop(c, self.N), triangle(2, self.N)) + c.bump = lambda by: 1000 + self.assertEqual(bump_loop(c, 10), 10000) + self.assertEqual(c.n, 2 * self.N) + del c.bump + self.assertEqual(bump_loop(c, 1), 2 * self.N + 2) + + def test_other_instance_shadows_the_method(self): + c = Counter() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + d = Counter() + d.tick = lambda: -1 + self.assertEqual(tick_loop(d, 10), -10) + self.assertEqual(tick_loop(c, 10), 10 * self.N + triangle(1, 10)) + e = Counter() + self.assertEqual(tick_loop(e, self.N), triangle(1, self.N)) + + def test_class_rebinds_the_method(self): + class C(Counter): + pass + + c = C() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + + def tick(self): + self.n += 10 + return -self.n + + C.tick = tick + self.assertEqual(tick_loop(c, 1), -(self.N + 10)) + del C.tick + self.assertEqual(tick_loop(c, 1), self.N + 11) + + def test_code_reassignment(self): + class C: + def __init__(self): + self.n = 0 + + def tick(self): + self.n += 1 + return self.n + + def other(self): + self.n += 100 + return 0 + + c = C() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + saved = C.tick.__code__ + C.tick.__code__ = other.__code__ + self.assertEqual(tick_loop(c, 5), 0) + self.assertEqual(c.n, self.N + 500) + C.tick.__code__ = saved + self.assertEqual(tick_loop(c, 1), self.N + 501) + + def test_overflow_and_field_types(self): + c = Counter() + self.assertEqual(bump_loop(c, self.N), triangle(2, self.N)) + c.n = sys.maxsize - 3 + self.assertEqual(bump_loop(c, 4), 4 * sys.maxsize + 8) + self.assertEqual(c.n, sys.maxsize + 5) + c.n = 0.5 + self.assertEqual(bump_loop(c, 2), 7.0) + c.n = True + self.assertEqual(tick_loop(c, 3), 2 + 3 + 4) + c.n = 0 + self.assertEqual(bump_loop(c, self.N), triangle(2, self.N)) + + def test_receiver_changes_class(self): + class A: + def __init__(self): + self.n = 0 + + def tick(self): + self.n += 1 + return self.n + + class B: + def __init__(self): + self.n = 0 + + def tick(self): + self.n += 2 + return self.n + + a = A() + self.assertEqual(tick_loop(a, self.N), triangle(1, self.N)) + a.__class__ = B + self.assertEqual(tick_loop(a, 3), 3 * self.N + triangle(2, 3)) + self.assertEqual(tick_all([A(), B(), A()], self.N), [self.N, 2 * self.N, self.N]) + + def test_materialized_dict(self): + c = Counter() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + vars(c)["n"] = 10 + self.assertEqual(tick_loop(c, 3), 11 + 12 + 13) + + def test_profiler_sees_every_call(self): + c = Counter() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + calls = [] + + def profile(frame, event, arg): + if event == "call" and frame.f_code.co_name == "tick": + calls.append(1) + + sys.setprofile(profile) + try: + tick_loop(c, 50) + finally: + sys.setprofile(None) + self.assertEqual(len(calls), 50) + self.assertEqual(c.n, self.N + 50) + + +if __name__ == "__main__": + unittest.main() From fce80f787622c54c5391055fb37976e26de2b36d Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:47:33 -0700 Subject: [PATCH 24/96] perf: trim the native call path's guard, pin and exit work guards_hold settles the namespaces' stamps in line and leaves the name-by-name revalidation, the callee table and the class-constructor probes out of line; try_native_call fills the cached child context's pin table in place instead of moving it in and out, reads the pure-leaf hint before calling the classifier, and note_callee_exit keeps its round-trip accounting out of line. A list built from a short range is deferred with the collector without scanning its integers first. Per-unit instructions: fannkuch 1,759 -> 1,562, float_math 7,563 -> 7,347, attr_access 2,783 -> 2,713. --- crates/weavepy-vm/src/gc_trace.rs | 13 ++ crates/weavepy-vm/src/tier2.rs | 212 +++++++++++++++++------------- 2 files changed, 134 insertions(+), 91 deletions(-) diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 837d5b34..9d9de59f 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -2341,6 +2341,19 @@ pub fn track_built(obj: &Object) { } } +/// [`track`] for a fresh list, dict or set its builder knows holds only +/// atomic values (a `list(range(..))`): such a container is deferred +/// whatever it holds, so the element scan [`container_can_cycle`] would +/// make first is skipped. +pub fn track_inert(obj: &Object) { + debug_assert!(!container_can_cycle(obj)); + with_state(|s| { + if !s.defer_container(obj) { + s.track_now(obj); + } + }); +} + /// `counter += n` (wrapping) for a collector counter: a plain load and /// store while the GIL serializes every writer, a locked read-modify-write /// only in free-threaded mode. diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index efff05af..b7c3c331 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -3615,6 +3615,8 @@ impl CallCtx { /// code-rebindable; a swap invalidates arity/lane assumptions), and /// each burned-in math intrinsic's `name.attr` still resolves to the /// snapshotted function (RFC 0069 WS2 — module dicts are mutable). +/// In line up to the namespaces' stamps, which settle the common case. +#[inline] fn guards_hold( interp: &super::Interpreter, globals: &Rc>, @@ -3635,16 +3637,38 @@ fn guards_hold( // A rebinding in place leaves the stamps alone (see `STORE_GLOBAL`). crate::object::global_value_epoch(), ); - if guard_snapshot.last_ok.get() != key || interp.globals_missing_any.get() { - for (name, expected) in guard_snapshot.entries.iter() { - let ok = resolve_plain_dicts(interp, globals, builtins, name) - .is_some_and(|cur| cur.is_same(expected)); - if !ok { - return false; - } + if (guard_snapshot.last_ok.get() != key || interp.globals_missing_any.get()) + && !global_guards_hold(interp, globals, builtins, guard_snapshot, key) + { + return false; + } + (callees.is_empty() || callee_guards_hold(interp, callees)) && math.iter().all(MathGuard::holds) +} + +/// [`guards_hold`]'s globals, resolved name by name (and remembered as +/// holding at `key` when they do). +#[inline(never)] +fn global_guards_hold( + interp: &super::Interpreter, + globals: &Rc>, + builtins: &Rc>, + guard_snapshot: &GuardSnapshot, + key: (usize, u64, usize, u64, u64), +) -> bool { + for (name, expected) in guard_snapshot.entries.iter() { + let ok = resolve_plain_dicts(interp, globals, builtins, name) + .is_some_and(|cur| cur.is_same(expected)); + if !ok { + return false; } - guard_snapshot.last_ok.set(key); } + guard_snapshot.last_ok.set(key); + true +} + +/// [`guards_hold`]'s burned-in callees. +#[inline(never)] +fn callee_guards_hold(interp: &super::Interpreter, callees: &CalleeTable) -> bool { for (f, code_snap) in callees { match f { Object::Function(pf) => { @@ -3676,11 +3700,6 @@ fn guards_hold( _ => return false, } } - for guard in math { - if !guard.holds() { - return false; - } - } true } @@ -4651,9 +4670,11 @@ unsafe fn try_native_call( // callee whatever its shape. A tiny *method* that stores — the // attribute fixtures' `tick` — is not, and its loop keeps its // native lane. - if crate::code_is_pure_leaf_pub(&nc.code) - || (recv.is_none() && nc.code.instructions.len() <= TINY_CALLEE_OPS) - { + let pure_leaf = match nc.code.jit_hint.pure_leaf() { + Some(known) => known, + None => crate::code_is_pure_leaf_pub(&nc.code), + }; + if pure_leaf || (recv.is_none() && nc.code.instructions.len() <= TINY_CALLEE_OPS) { ctx.dyn_py_calls = ctx.dyn_py_calls.saturating_add(1); } // The framed native call below is itself expensive (buffers, pins, a @@ -4741,73 +4762,6 @@ unsafe fn try_native_call( let (locals_buf, rest) = u64s.split_at_mut(n_locals); let (spill, call_args) = rest.split_at_mut(cap); let (tags, call_tags) = u32s.split_at_mut(cap); - // The cached child's (drained) pin table keeps its capacity. - let mut pins: PinTable = match ctx.child.as_deref_mut() { - Some(c) => std::mem::take(&mut c.pins), - None => Vec::new(), - }; - if !pins.is_empty() { - pins.clear(); - } - if let Some(r) = recv { - // The receiver slot carries pin index 0 (`take_u64` zeroed it). - pins.push(Pin::Obj(r.clone())); - } - for j in 0..argc_usize { - // SAFETY: as above — `argc` marshaled entries are live. - let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; - // RFC 0071 WS1 — a caller pin index means nothing to the - // callee: re-pin the object in the callee's own table (the - // nullable `-1` passes through unchanged). Validated above. - locals_buf[j + offset] = if tag == SlotTag::ObjPin as u32 && bits != u64::MAX { - match ctx.pins.get(bits as usize) { - Some(Pin::Obj(o)) => { - let idx = pins.len() as u64; - pins.push(Pin::Obj(o.clone())); - idx - } - // Unreachable per the validation pass; the nullable - // `None` is the safe stand-in. - _ => u64::MAX, - } - } else if tag == SlotTag::ListPin as u32 { - match ctx.pins.get(bits as usize) { - Some(Pin::List(l, e)) => { - let idx = pins.len() as u64; - pins.push(Pin::List(l.clone(), *e)); - idx - } - // Unreachable per the validation pass. - _ => u64::MAX, - } - } else { - bits - }; - } - // RFC 0073 WS5 — splice the trailing defaults (lane-validated - // above) into the unsupplied parameter slots. - #[allow(clippy::needless_range_loop)] - for k in supplied..n_params { - let d = &nc.func.defaults[nc.func.defaults.len() - (n_params - k)]; - locals_buf[k] = match nc.cf.local_types.get(k).copied().flatten() { - Some(ty @ (JitType::Int | JitType::Bool | JitType::Float)) => pack(d, ty).unwrap_or(0), - Some(JitType::Obj) => match d { - Object::None => u64::MAX, - o => { - let idx = pins.len() as u64; - pins.push(Pin::Obj(o.clone())); - idx - } - }, - Some(JitType::Str | JitType::Bytes | JitType::Dict) => { - let idx = pins.len() as u64; - pins.push(Pin::Obj(d.clone())); - idx - } - _ => 0, - }; - } - let entry_pin_count = pins.len(); let callee_key = Rc::as_ptr(&nc.code).cast::(); // The callee's context: this activation's cached child when it was // built for this very callee (same code, artifact tables and @@ -4828,8 +4782,6 @@ unsafe fn try_native_call( if !c.const_pins.is_empty() { c.const_pins.clear(); } - c.pins = pins; - c.entry_pin_count = entry_pin_count; c.pin_pressure_exit = false; if !c.obj_global_pins.is_empty() { c.obj_global_pins.clear(); @@ -4879,8 +4831,8 @@ unsafe fn try_native_call( parked: None, raised: None, const_pins: Vec::new(), - pins, - entry_pin_count, + pins: Vec::new(), + entry_pin_count: 0, pin_pressure_exit: false, obj_globals: nc.obj_globals.clone(), obj_global_pins: Vec::new(), @@ -4905,6 +4857,71 @@ unsafe fn try_native_call( pin_memo: [u32::MAX; PIN_MEMO], }) }; + // The callee's pins: the receiver, then the arguments' objects (the + // reused child's drained table keeps its capacity). + let pins = &mut child.pins; + if !pins.is_empty() { + pins.clear(); + } + if let Some(r) = recv { + // The receiver slot carries pin index 0 (`take_u64` zeroed it). + pins.push(Pin::Obj(r.clone())); + } + for j in 0..argc_usize { + // SAFETY: as above — `argc` marshaled entries are live. + let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; + // RFC 0071 WS1 — a caller pin index means nothing to the + // callee: re-pin the object in the callee's own table (the + // nullable `-1` passes through unchanged). Validated above. + locals_buf[j + offset] = if tag == SlotTag::ObjPin as u32 && bits != u64::MAX { + match ctx.pins.get(bits as usize) { + Some(Pin::Obj(o)) => { + let idx = pins.len() as u64; + pins.push(Pin::Obj(o.clone())); + idx + } + // Unreachable per the validation pass; the nullable + // `None` is the safe stand-in. + _ => u64::MAX, + } + } else if tag == SlotTag::ListPin as u32 { + match ctx.pins.get(bits as usize) { + Some(Pin::List(l, e)) => { + let idx = pins.len() as u64; + pins.push(Pin::List(l.clone(), *e)); + idx + } + // Unreachable per the validation pass. + _ => u64::MAX, + } + } else { + bits + }; + } + // RFC 0073 WS5 — splice the trailing defaults (lane-validated + // above) into the unsupplied parameter slots. + #[allow(clippy::needless_range_loop)] + for k in supplied..n_params { + let d = &nc.func.defaults[nc.func.defaults.len() - (n_params - k)]; + locals_buf[k] = match nc.cf.local_types.get(k).copied().flatten() { + Some(ty @ (JitType::Int | JitType::Bool | JitType::Float)) => pack(d, ty).unwrap_or(0), + Some(JitType::Obj) => match d { + Object::None => u64::MAX, + o => { + let idx = pins.len() as u64; + pins.push(Pin::Obj(o.clone())); + idx + } + }, + Some(JitType::Str | JitType::Bytes | JitType::Dict) => { + let idx = pins.len() as u64; + pins.push(Pin::Obj(d.clone())); + idx + } + _ => 0, + }; + } + child.entry_pin_count = child.pins.len(); child.interp = ctx.interp; let nctx: &mut CallCtx = &mut child; let mut njf = JitFrame { @@ -5902,13 +5919,20 @@ unsafe extern "C" fn wpjit_self_slow( /// interpreter calls is a thin native driver around them. Each such call pays pin /// traffic, an activation shell and a generic call that the interpreter's /// inline call path avoids, so the callee retires to tier-1. -#[inline] +#[inline(always)] fn note_callee_exit(art: &Artifacts, code: &Rc, child: &CallCtx) { let entries = art.callee_entries.get().saturating_add(1); art.callee_entries.set(entries); - if child.dyn_py_calls == 0 { - return; + if child.dyn_py_calls != 0 { + note_callee_roundtrips(art, code, child, entries); } +} + +/// [`note_callee_exit`] for an activation that made interpreter +/// round-trips, out of line. +#[cold] +#[inline(never)] +fn note_callee_roundtrips(art: &Artifacts, code: &Rc, child: &CallCtx, entries: u32) { let trips = art .callee_roundtrips .get() @@ -7837,7 +7861,13 @@ unsafe extern "C" fn wpjit_list_from_range(frame: *mut JitFrame, start: i64, sto let items: Vec = (start..start + len).map(Object::Int).collect(); let list = Rc::new(crate::sync::RefCell::new(items)); let obj = Object::List(list.clone()); - crate::gc_trace::track(&obj); + // A short list of integers can't close a cycle (a long one is + // registered without the scan as well). + if len <= 32 { + crate::gc_trace::track_inert(&obj); + } else { + crate::gc_trace::track(&obj); + } if crate::stdlib::tracemalloc_real::is_tracking() { crate::stdlib::tracemalloc_real::track_new_object(&obj); } From 8a64deeaab8345e88ca90011ebbd3087273b1359 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 18:50:13 -0700 Subject: [PATCH 25/96] perf: call compiled scalar leaves directly when defaults fill the gap A call that leaves trailing or keyword-skipped parameters out of a compiled scalar leaf now enters it directly from native code too: the leaf carries the function's scalar defaults (each in its parameter's lane), keyword values land in the slots the site's permutation names, and the rest bind the burned-in defaults. The caller's guard snapshot lists every callee whose defaults it may have burned in, and guards_hold fails once one of them has its `__defaults__` replaced; a declined entry or a deopting leaf still runs the ordinary call from the start. call_overhead's per-unit instructions drop from 6,183 to 4,952. --- crates/weavepy-jit/src/engine.rs | 49 ++++++-- crates/weavepy-jit/src/lower.rs | 112 +++++++++++++++---- crates/weavepy-vm/src/tier2.rs | 62 +++++++++-- tests/regrtest/test_jit_leaf_defaults.py | 135 +++++++++++++++++++++++ 4 files changed, 319 insertions(+), 39 deletions(-) create mode 100644 tests/regrtest/test_jit_leaf_defaults.py diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index 66ded5a4..9aa2a191 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -133,6 +133,26 @@ pub struct DirectLeaf { max_stack: u32, params: Vec, ret: JitType, + /// Per parameter, the bits of the default a call that leaves it out + /// binds (in the parameter's lane), when the caller burns it in. + defaults: Vec>, +} + +impl DirectLeaf { + /// The lanes of the leaf's parameters. + #[must_use] + pub fn params(&self) -> &[JitType] { + &self.params + } + + /// Let calls that leave parameters out bind `defaults` (per + /// parameter, the default's bits in its lane): the embedder guards + /// that they stay the function's defaults. + #[must_use] + pub fn with_defaults(mut self, defaults: Vec>) -> Self { + self.defaults = defaults; + self + } } impl CompiledFrame { @@ -168,6 +188,7 @@ impl CompiledFrame { func_id: self.func_id, n_locals: self.n_locals, max_stack: self.max_stack, + defaults: vec![None; params.len()], params, ret, }) @@ -530,20 +551,29 @@ impl JitEngine { let mut leaves: Vec<(u32, FuncRef, DirectLeaf)> = Vec::new(); if runtime::self_call_helper_addrs().is_some() { for stmt in tfunc.blocks.iter().flat_map(|b| &b.stmts) { - let TOp::CallPy { - token, - argc, - ret, - is_self: false, - } = stmt.op - else { - continue; + // A site that leaves parameters out binds the defaults the + // leaf carries (checked per site by the lowering). + let (token, argc, ret) = match stmt.op { + TOp::CallPy { + token, + argc, + ret, + is_self: false, + } => (token, argc, ret), + TOp::CallPyKw { + token, + argc, + kwc, + ret, + .. + } => (token, argc + kwc, ret), + _ => continue, }; if leaves.iter().any(|(t, ..)| *t == token) { continue; } if let Some(leaf) = direct(token) { - if leaf.params.len() == argc as usize && leaf.ret == ret { + if leaf.params.len() >= argc as usize && leaf.ret == ret { let fref = self .module .declare_func_in_func(leaf.func_id, &mut self.ctx.func); @@ -567,6 +597,7 @@ impl JitEngine { n_locals: leaf.n_locals, max_stack: leaf.max_stack, params: leaf.params, + defaults: leaf.defaults, }) .collect(), ); diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 548db074..845fa96d 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -45,6 +45,26 @@ pub(crate) struct LeafTarget { pub n_locals: u32, pub max_stack: u32, pub params: Vec, + /// Per parameter, the default a call that leaves it out binds. + pub defaults: Vec>, +} + +/// Where a direct leaf call takes one parameter from: the call's `j`th +/// value (positionals, then keyword values), or a burned-in default. +#[derive(Clone, Copy)] +enum LeafArg { + Arg(usize), + Default(u64), +} + +/// A `CallPy`/`CallPyKw` site's operands, for its ordinary call. +#[derive(Clone, Copy)] +struct SiteCall { + token: u32, + argc: u8, + kwc: u8, + perm: u32, + gaps: u32, } /// Build the Cranelift function body for `tfunc` into `func`. @@ -1034,8 +1054,15 @@ impl<'a, 'b> Lowerer<'a, 'b> { } => { if is_self && self.self_func.is_some() { self.emit_call_self(token, argc, ret, stmt.pc); - } else if let Some(ix) = self.leaf_for(token, argc) { - self.emit_call_leaf(ix, token, argc, ret, stmt.pc); + } else if let Some((ix, from)) = self.leaf_for(token, argc, 0, 0) { + let call = SiteCall { + token, + argc, + kwc: 0, + perm: 0, + gaps: 0, + }; + self.emit_call_leaf(ix, &from, call, ret, stmt.pc); } else { self.emit_call_py(token, argc, 0, 0, 0, ret, stmt.pc); } @@ -1047,7 +1074,20 @@ impl<'a, 'b> Lowerer<'a, 'b> { perm, gaps, ret, - } => self.emit_call_py(token, argc, kwc, perm, gaps, ret, stmt.pc), + } => { + if let Some((ix, from)) = self.leaf_for(token, argc, kwc, perm) { + let call = SiteCall { + token, + argc, + kwc, + perm, + gaps, + }; + self.emit_call_leaf(ix, &from, call, ret, stmt.pc); + } else { + self.emit_call_py(token, argc, kwc, perm, gaps, ret, stmt.pc); + } + } TOp::ListGet { elem } => self.emit_list_get(elem, stmt.pc), TOp::ListSet => self.emit_list_set(stmt.pc), TOp::CellGet { idx, lane } => self.emit_cell_get(idx, lane, stmt.pc), @@ -3854,28 +3894,50 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.vstack.push((v, ret)); } - /// The direct leaf target of `token` when this site's argument lanes - /// are exactly the leaf's parameter lanes. - fn leaf_for(&self, token: u32, argc: u8) -> Option { + /// The direct leaf target of `token`, and where each of its + /// parameters comes from, when this site's `argc` positional and `kwc` + /// keyword values (keyword `j` binding slot `(perm >> 4j) & 0xF`) have + /// the leaf's parameter lanes and every parameter they leave out has a + /// default the leaf carries. + fn leaf_for(&self, token: u32, argc: u8, kwc: u8, perm: u32) -> Option<(usize, Vec)> { let ix = self.leaves.iter().position(|l| l.token == token)?; - let base = self.vstack.len().checked_sub(argc as usize)?; - let lanes = self.vstack[base..].iter().map(|&(_, ty)| ty); - lanes - .eq(self.leaves[ix].params.iter().copied()) - .then_some(ix) + let leaf = &self.leaves[ix]; + let n = argc as usize + kwc as usize; + let base = self.vstack.len().checked_sub(n)?; + let mut from: Vec> = vec![None; leaf.params.len()]; + for j in 0..n { + let slot = if j < argc as usize { + j + } else { + ((perm >> (4 * (j - argc as usize))) & 0xF) as usize + }; + let at = from.get_mut(slot)?; + if at.is_some() || self.vstack[base + j].1 != leaf.params[slot] { + return None; + } + *at = Some(LeafArg::Arg(j)); + } + let from = from + .iter() + .zip(&leaf.defaults) + .map(|(&f, &d)| f.or(d.map(LeafArg::Default))) + .collect::>>()?; + Some((ix, from)) } /// Lower a direct call of a compiled scalar leaf (see [`LeafTarget`]): /// charge the activation through the self-call enter helper, fill the - /// leaf's `JitFrame` on this function's native stack frame, call it, - /// and release the charge. The leaf only computes, so when the enter - /// helper declines or the leaf deopts (an overflow, a zero divisor), - /// the ordinary call helper runs the call from the start. - fn emit_call_leaf(&mut self, ix: usize, token: u32, argc: u8, ret: JitType, pc: u32) { + /// leaf's `JitFrame` on this function's native stack frame (the + /// parameters from the call's values and the leaf's burned-in + /// defaults, per `from`), call it, and release the charge. The leaf + /// only computes, so when the enter helper declines or the leaf deopts + /// (an overflow, a zero divisor), the ordinary call helper runs the + /// call from the start. + fn emit_call_leaf(&mut self, ix: usize, from: &[LeafArg], call: SiteCall, ret: JitType, pc: u32) { let trusted = MemFlags::trusted(); let (enter_addr, exit_addr, _) = runtime::self_call_helper_addrs().expect("checked by the engine"); - let n = argc as usize; + let n = call.argc as usize + call.kwc as usize; let base = self.vstack.len() - n; let args: Vec<(Value, JitType)> = self.vstack[base..].to_vec(); self.vstack.truncate(base); @@ -3884,8 +3946,8 @@ impl<'a, 'b> Lowerer<'a, 'b> { let sig = self.self_sig(); let enter = self.b.ins().iconst(self.ptr_ty, enter_addr as i64); - let call = self.b.ins().call_indirect(sig, enter, &[self.frame_ptr]); - let declined = self.b.inst_results(call)[0]; + let c = self.b.ins().call_indirect(sig, enter, &[self.frame_ptr]); + let declined = self.b.inst_results(c)[0]; let direct_b = self.b.create_block(); let generic_b = self.b.create_block(); @@ -3936,11 +3998,15 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.b.ins().store(trusted, null, fp, OFF_CALL_ARGS); self.b.ins().store(trusted, null, fp, OFF_CALL_TAGS); for slot_ix in 0..n_locals { - let v = args.get(slot_ix as usize).map_or(zero64, |&(v, _)| v); + let v = match from.get(slot_ix as usize) { + Some(&LeafArg::Arg(j)) => args[j].0, + Some(&LeafArg::Default(bits)) => self.b.ins().iconst(types::I64, bits as i64), + None => zero64, + }; self.b.ins().store(trusted, v, locals, slot_ix * 8); } - let call = self.b.ins().call(func, &[fp]); - let status = self.b.inst_results(call)[0]; + let c = self.b.ins().call(func, &[fp]); + let status = self.b.inst_results(c)[0]; let exit = self.b.ins().iconst(self.ptr_ty, exit_addr as i64); self.b.ins().call_indirect(sig, exit, &[self.frame_ptr]); let returned_b = self.b.create_block(); @@ -3959,7 +4025,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { // Declined or deopted: the ordinary call, from the start. self.b.switch_to_block(generic_b); self.vstack.extend(args.iter().copied()); - self.emit_call_py(token, argc, 0, 0, 0, ret, pc); + self.emit_call_py(call.token, call.argc, call.kwc, call.perm, call.gaps, ret, pc); let (v, _) = self.vstack.pop().expect("the call's result"); self.b.ins().jump(join_b, &[v.into()]); diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index b7c3c331..14b81119 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -253,17 +253,27 @@ struct GuardSnapshot { /// value epoch)` of the last full validation that held; all-zero /// until one has. last_ok: std::cell::Cell<(usize, u64, usize, u64, u64)>, + /// Callees whose defaults the compiled code may have burned in (a + /// direct leaf call that leaves parameters out): they must keep them. + defaults: Vec>, } impl GuardSnapshot { - fn new(entries: Vec<(String, Object)>) -> Self { + fn new(entries: Vec<(String, Object)>, defaults: Vec>) -> Self { Self { entries, last_ok: std::cell::Cell::new((0, 0, 0, 0, 0)), + defaults, } } } +/// Whether `f.__defaults__` no longer is the tuple `f` was defined with. +#[inline] +fn defaults_overridden(f: &PyFunction) -> bool { + f.defaults_maybe_overridden() && f.slot("__defaults__").is_some() +} + impl std::ops::Deref for GuardSnapshot { type Target = [(String, Object)]; fn deref(&self) -> &[(String, Object)] { @@ -1267,16 +1277,39 @@ impl JitState { // A callee already compiled as a guard-free scalar leaf is // entered directly by native code (its identity is guarded // with the callee table like any burned-in callee). + // A site that leaves trailing parameters out binds the + // function's scalar defaults there (guarded through + // `GuardSnapshot::defaults`). let mut direct = |token: u32| { let callees = callees.borrow(); - let (Object::Function(_), fcode) = callees.get(token as usize)? else { + let (Object::Function(f), fcode) = callees.get(token as usize)? else { return None; }; let k = Rc::as_ptr(fcode).cast::(); - match &cache_ref.get(&k)?.tier { - Tier::Compiled(a) => a.cf.direct_leaf(fcode.arg_count), - _ => None, + let leaf = match &cache_ref.get(&k)?.tier { + Tier::Compiled(a) => a.cf.direct_leaf(fcode.arg_count)?, + _ => return None, + }; + if f.defaults.is_empty() || defaults_overridden(f) { + return Some(leaf); } + let n = leaf.params().len(); + let first = n.saturating_sub(f.defaults.len()); + let defaults = (0..n) + .map(|i| { + let d = f.defaults.get((i + f.defaults.len()).checked_sub(n)?)?; + let lane = leaf.params()[i]; + // `bool` is not `int` here (the lanes are exact). + let exact = matches!( + (lane, d), + (JitType::Int, Object::Int(_)) + | (JitType::Float, Object::Float(_)) + | (JitType::Bool, Object::Bool(_)) + ); + (i >= first && exact).then(|| pack(d, lane)).flatten() + }) + .collect(); + Some(leaf.with_defaults(defaults)) }; ensure_obj_layout(); let r = engine.compile_frame_direct(code, &mut classify, &mut jit_probes, &mut direct); @@ -1374,10 +1407,23 @@ impl JitState { } let scalar_update = scalar_field_update_plan(code, &cf, &attr_guards).map(Box::new); + // Every callee whose defaults a direct leaf call could + // have bound (see `direct` above). + let burned_defaults = callees + .iter() + .filter_map(|(f, _)| match f { + Object::Function(f) + if !f.defaults.is_empty() && !defaults_overridden(f) => + { + Some(f.clone()) + } + _ => None, + }) + .collect(); let artifacts = StdRc::new(Artifacts { cf: StdRc::new(cf), scalar_update, - snap: StdRc::new(GuardSnapshot::new(snap)), + snap: StdRc::new(GuardSnapshot::new(snap, burned_defaults)), callees: StdRc::new(callees), obj_globals: StdRc::new(obj_globals), attr_guards: StdRc::new(attr_guards), @@ -3642,7 +3688,9 @@ fn guards_hold( { return false; } - (callees.is_empty() || callee_guards_hold(interp, callees)) && math.iter().all(MathGuard::holds) + (callees.is_empty() || callee_guards_hold(interp, callees)) + && math.iter().all(MathGuard::holds) + && !guard_snapshot.defaults.iter().any(|f| defaults_overridden(f)) } /// [`guards_hold`]'s globals, resolved name by name (and remembered as diff --git a/tests/regrtest/test_jit_leaf_defaults.py b/tests/regrtest/test_jit_leaf_defaults.py new file mode 100644 index 00000000..a64927a7 --- /dev/null +++ b/tests/regrtest/test_jit_leaf_defaults.py @@ -0,0 +1,135 @@ +"""Compiled code calls a compiled scalar function directly, defaults included. + +A call that leaves trailing or keyword-skipped parameters out binds the +callee's scalar defaults in compiled code. Each loop runs long enough for +the tier-2 compiler to take the callee and then the caller; then the +callee's defaults or code change, and every call must see them. +""" + +import unittest + + +def with_defaults(a, b=10, c=20): + return a + b + c + + +def scaled(x, k=1.5): + return x * k + + +def gated(a, flag=True): + if flag: + return a + return -a + + +def positional_loop(n): + t = 0 + for i in range(n): + t += with_defaults(i) + return t + + +def keyword_loop(n): + t = 0 + for i in range(n): + t += with_defaults(i, c=5) + return t + + +def middle_keyword_loop(n): + t = 0 + for i in range(n): + t += with_defaults(i, b=1) + return t + + +def float_loop(n): + t = 0.0 + for i in range(n): + t += scaled(float(i)) + return t + + +def bool_loop(n): + t = 0 + for i in range(n): + t += gated(i) + return t + + +def triangle(n): + # The sum of 0, 1, ..., n - 1. + return n * (n - 1) // 2 + + +class LeafDefaultTests(unittest.TestCase): + N = 3000 + + def setUp(self): + self.saved = ( + with_defaults.__defaults__, + with_defaults.__code__, + scaled.__defaults__, + gated.__defaults__, + ) + + def tearDown(self): + ( + with_defaults.__defaults__, + with_defaults.__code__, + scaled.__defaults__, + gated.__defaults__, + ) = self.saved + + def test_defaults_bind(self): + n = self.N + self.assertEqual(positional_loop(n), triangle(n) + 30 * n) + self.assertEqual(keyword_loop(n), triangle(n) + 15 * n) + self.assertEqual(middle_keyword_loop(n), triangle(n) + 21 * n) + self.assertEqual(float_loop(n), 1.5 * triangle(n)) + self.assertEqual(bool_loop(n), triangle(n)) + + def test_defaults_reassigned(self): + n = self.N + self.assertEqual(positional_loop(n), triangle(n) + 30 * n) + self.assertEqual(keyword_loop(n), triangle(n) + 15 * n) + with_defaults.__defaults__ = (100, 200) + self.assertEqual(positional_loop(n), triangle(n) + 300 * n) + self.assertEqual(keyword_loop(n), triangle(n) + 105 * n) + self.assertEqual(middle_keyword_loop(n), triangle(n) + 201 * n) + with_defaults.__defaults__ = (1.5, 2) + self.assertEqual(positional_loop(4), 6 + 4 * 3.5) + with_defaults.__defaults__ = (2,) + with self.assertRaises(TypeError): + positional_loop(1) + self.assertEqual(middle_keyword_loop(3), 3 + 3 * 3) + with self.assertRaises(TypeError): + keyword_loop(1) + with_defaults.__defaults__ = None + with self.assertRaises(TypeError): + keyword_loop(1) + + def test_lanes_change(self): + n = self.N + self.assertEqual(float_loop(n), 1.5 * triangle(n)) + scaled.__defaults__ = (2,) + self.assertEqual(float_loop(4), 12.0) + self.assertEqual(bool_loop(n), triangle(n)) + gated.__defaults__ = (False,) + self.assertEqual(bool_loop(4), -6) + gated.__defaults__ = (0,) + self.assertEqual(bool_loop(4), -6) + + def test_code_reassigned(self): + def other(a, b=1, c=2): + return a * b * c + + n = self.N + self.assertEqual(positional_loop(n), triangle(n) + 30 * n) + with_defaults.__code__ = other.__code__ + self.assertEqual(positional_loop(4), 10 * 20 * triangle(4)) + + +if __name__ == "__main__": + unittest.main() From 5e00d127b3d119482c3d06468c59626f09938f38 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:44:11 -0700 Subject: [PATCH 26/96] perf: plan scalar field updates over __slots__ members A method that adds to one of its own `__slots__` members and returns it now gets the same callback-free update plan as one over an instance-dict field: the member's slot descriptor can't acquire hooks the class version doesn't track, so the store guard certifies it like a plain class value. wpjit_call_method then runs such a method through native_scalar_field_update instead of a framed native call. attr_access's per-unit instructions drop from 2,713 to 1,799. --- crates/weavepy-vm/src/tier2.rs | 24 ++++++- tests/regrtest/test_jit_inline_methods.py | 77 +++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 14b81119..7b53c37c 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -2755,9 +2755,12 @@ fn attr_site_guard( return None; } let stable_descriptor = site.store - && matches!(storage, AttrStorage::Indexed(_)) && scalar_field_update_shape(&frame.code) - && scalar_update_class_value_stable(&recv, &site.name, ver); + && match storage { + AttrStorage::Indexed(_) => scalar_update_class_value_stable(&recv, &site.name, ver), + AttrStorage::Slot(_) => scalar_update_slot_stable(&recv, &site.name, ver), + AttrStorage::NewKey => false, + }; let split_idx = match &recv { Object::Instance(inst) => split_index(&inst.cls(), storage, &site.name), _ => u32::MAX, @@ -4153,6 +4156,23 @@ fn scalar_update_class_value_stable(receiver: &Object, name: &str, ver: u64) -> && cls.attr_version.get() == ver } +/// [`scalar_update_class_value_stable`] for a `__slots__` member: the +/// class value is the member's own slot descriptor, which has no hooks a +/// class mutation wouldn't version. +fn scalar_update_slot_stable(receiver: &Object, name: &str, ver: u64) -> bool { + if crate::object::exotic_str_keys_possible() { + return false; + } + let Object::Instance(inst) = receiver else { + return false; + }; + let cls = inst.cls(); + cls.native_kind.get() == 0 + && super::Interpreter::default_getattribute(&cls) + && matches!(cls.lookup(name), Some(Object::SlotDescriptor(sd)) if sd.name == name) + && cls.attr_version.get() == ver +} + fn scalar_field_update_plan( code: &CodeObject, cf: &CompiledFrame, diff --git a/tests/regrtest/test_jit_inline_methods.py b/tests/regrtest/test_jit_inline_methods.py index 98554baf..11d5c2be 100644 --- a/tests/regrtest/test_jit_inline_methods.py +++ b/tests/regrtest/test_jit_inline_methods.py @@ -178,5 +178,82 @@ def profile(frame, event, arg): self.assertEqual(c.n, self.N + 50) +class Slotted: + __slots__ = ("n", "m") + + def __init__(self): + self.n = 0 + self.m = 0 + + def tick(self): + self.n += 1 + return self.n + + def bump(self, by): + self.m += by + return self.m + + +class SlottedUpdateTests(unittest.TestCase): + N = 3000 + + def test_updates(self): + s = Slotted() + self.assertEqual(tick_loop(s, self.N), triangle(1, self.N)) + self.assertEqual(bump_loop(s, self.N), triangle(2, self.N)) + self.assertEqual((s.n, s.m), (self.N, 2 * self.N)) + + def test_deleted_slot(self): + s = Slotted() + self.assertEqual(tick_loop(s, self.N), triangle(1, self.N)) + del s.n + with self.assertRaises(AttributeError): + tick_loop(s, 1) + s.n = 5 + self.assertEqual(tick_loop(s, 2), 6 + 7) + + def test_slot_value_types(self): + s = Slotted() + self.assertEqual(bump_loop(s, self.N), triangle(2, self.N)) + s.m = 0.25 + self.assertEqual(bump_loop(s, 2), 2.25 + 4.25) + s.m = sys.maxsize + self.assertEqual(bump_loop(s, 1), sys.maxsize + 2) + s.m = "x" + with self.assertRaises(TypeError): + bump_loop(s, 1) + + def test_class_replaces_the_slot(self): + class S: + __slots__ = ("n",) + + def __init__(self): + self.n = 0 + + def tick(self): + self.n += 1 + return self.n + + s = S() + self.assertEqual(tick_loop(s, self.N), triangle(1, self.N)) + slot = S.n + seen = [] + S.n = property(lambda self: 100, lambda self, v: seen.append(v)) + self.assertEqual(tick_loop(s, 2), 200) + self.assertEqual(seen, [101, 101]) + S.n = slot + self.assertEqual(tick_loop(s, 1), self.N + 1) + + def test_subclass_with_dict(self): + class D(Slotted): + pass + + d = D() + self.assertEqual(tick_loop(d, self.N), triangle(1, self.N)) + d.tick = lambda: -1 + self.assertEqual(tick_loop(d, 3), -3) + self.assertEqual(tick_all([Slotted(), D(), Slotted()], 5), [5, 5, 5]) + + if __name__ == "__main__": unittest.main() From 7cf72a21442025f1c88ca73aa30d4b2dee8def0c Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:56:50 -0700 Subject: [PATCH 27/96] perf: lighten native calls, constructor guards and exit write-backs try_native_call keeps its common path small: the under-arity default check, the scalar-leaf entry, a fresh child context and everything after a deopt or an object result move out of line, and a clean scalar return puts the child back without the general result translation. Framed activations reuse the last finished activation's pin table allocation, so a loop that leaves at the pin limit doesn't regrow it from empty. guards_hold remembers, per burned-in class constructor, the class version at which the full construction probe last held; an unchanged class then rechecks only its metaclass and its `__init__` code instead of rebuilding the probe after every call that ran Python. Compiled code no longer writes back, at every exit and call, the locals its body never assigns: the frame already holds them, and keeping each one live to every exit only cost registers and compile time. WEAVEPY_JIT_CLIF_STATS=1 prints each compiled function's size. Per-unit instructions: call_overhead 4,952 -> 4,671, fannkuch 1,562 -> 1,510, float_math 7,347 -> 7,291; compile times drop 3-17% (maximize 8.2 -> 6.8 ms). --- crates/weavepy-jit/src/engine.rs | 9 + crates/weavepy-jit/src/lower.rs | 57 ++- crates/weavepy-vm/src/tier2.rs | 457 +++++++++++++++++-------- tests/regrtest/test_jit_ctor_guards.py | 133 +++++++ 4 files changed, 508 insertions(+), 148 deletions(-) create mode 100644 tests/regrtest/test_jit_ctor_guards.py diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index 9aa2a191..77444bab 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -601,6 +601,15 @@ impl JitEngine { }) .collect(), ); + if std::env::var_os("WEAVEPY_JIT_CLIF_STATS").is_some() { + let f = &self.ctx.func; + let insts: usize = f.layout.blocks().map(|b| f.layout.block_insts(b).count()).sum(); + eprintln!( + "jit clif {name}: {} blocks, {insts} insts, {} values", + f.layout.blocks().count(), + f.dfg.num_values() + ); + } self.module .define_function(id, &mut self.ctx) diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 845fa96d..2118143b 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -16,6 +16,7 @@ use cranelift_codegen::ir::{ types, AbiParam, Block, BlockArg, FuncRef, Function, InstBuilder, MemFlags, SigRef, Signature, StackSlot, StackSlotData, StackSlotKind, Type, Value, }; +use cranelift_codegen::entity::EntityRef; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; use crate::ir::{ArithKind, CmpKind, MathFunc, MethodRet, SliceOrigin, TFunc, TOp, TStmt, TTerm}; @@ -81,6 +82,7 @@ pub(crate) fn build_function( lc.self_func = self_func; lc.leaves = leaves; lc.build(); + lc.prune_writebacks(); builder.seal_all_blocks(); builder.finalize(); } @@ -151,6 +153,13 @@ struct Lowerer<'a, 'b> { self_slow_sig: Option, /// Direct scalar-leaf call targets by token. leaves: Vec, + /// Per variable (by index), whether the body ever assigns it after + /// the entry loads it from the frame. + assigned: Vec, + /// Every local write-back store, by variable: a variable the body + /// never assigns still holds the frame's own value, so its stores are + /// dropped once the body is built (see [`Self::def_local`]). + writebacks: Vec<(Variable, cranelift_codegen::ir::Inst)>, } impl<'a, 'b> Lowerer<'a, 'b> { @@ -186,9 +195,21 @@ impl<'a, 'b> Lowerer<'a, 'b> { self_sig: None, self_slow_sig: None, leaves: Vec::new(), + assigned: Vec::new(), + writebacks: Vec::new(), } } + /// Assign the managed local `var` (anything but its entry load). + fn def_local(&mut self, var: Variable, v: Value) { + let i = var.index(); + if self.assigned.len() <= i { + self.assigned.resize(i + 1, false); + } + self.assigned[i] = true; + self.b.def_var(var, v); + } + fn cl_ty(ty: JitType) -> Type { match ty { JitType::Float => types::F64, @@ -581,9 +602,9 @@ impl<'a, 'b> Lowerer<'a, 'b> { let eb = self.cl_blocks[exit]; self.b.ins().brif(cond, body_pre, &[], eb, &[]); self.b.switch_to_block(body_pre); - self.b.def_var(loop_var, cur); + self.def_local(loop_var, cur); let next = self.b.ins().iadd_imm(cur, 1); - self.b.def_var(cur_var, next); + self.def_local(cur_var, next); let bb = self.cl_blocks[body]; self.b.ins().jump(bb, &[]); } @@ -676,9 +697,9 @@ impl<'a, 'b> Lowerer<'a, 'b> { } self.b.switch_to_block(got_b); let res = self.b.block_params(got_b)[0]; - self.b.def_var(loop_var, res); + self.def_local(loop_var, res); let next = self.b.ins().iadd_imm(idx, 1); - self.b.def_var(idx_var, next); + self.def_local(idx_var, next); let bb = self.cl_blocks[body]; self.b.ins().jump(bb, &args); self.b.switch_to_block(rest_b); @@ -746,7 +767,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.b .ins() .load(Self::cl_ty(elem), trusted, self.frame_ptr, OFF_RET_BITS); - self.b.def_var(loop_var, res); + self.def_local(loop_var, res); let bb = self.cl_blocks[body]; self.b.ins().jump(bb, &args); self.b.switch_to_block(rest_b); @@ -834,12 +855,12 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.b .ins() .load(Self::cl_ty(elem1), trusted, self.frame_ptr, OFF_RET_BITS); - self.b.def_var(var1, e1); + self.def_local(var1, e1); let e2 = self .b .ins() .load(Self::cl_ty(elem2), trusted, self.call_args_base, 0); - self.b.def_var(var2, e2); + self.def_local(var2, e2); let bb = self.cl_blocks[body]; self.b.ins().jump(bb, &args); self.b.switch_to_block(rest_b); @@ -990,7 +1011,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { TOp::StoreLocal(slot) => { let (v, _) = self.pop(); let var = self.vars[slot as usize].expect("managed local"); - self.b.def_var(var, v); + self.def_local(var, v); } TOp::IntArith(kind) => self.emit_int_arith(kind, stmt.pc), TOp::FloatArith(kind) => self.emit_float_arith(kind, stmt.pc), @@ -1250,7 +1271,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); let iter_var = self.vars[iter_slot as usize].expect("managed iter slot"); - self.b.def_var(iter_var, res); + self.def_local(iter_var, res); } /// RFC 0074 WS2 — the opaque-call lane via `wpjit_call_dyn`: @@ -1748,7 +1769,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); let iter_var = self.vars[iter_slot as usize].expect("managed iter slot"); - self.b.def_var(iter_var, pin); + self.def_local(iter_var, pin); } /// RFC 0073 WS2 — the dict-loop capture behind an erased @@ -1774,7 +1795,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); let iter_var = self.vars[iter_slot as usize].expect("managed iter slot"); - self.b.def_var(iter_var, res); + self.def_local(iter_var, res); } /// RFC 0071 WS4 — `BUILD_LIST k`: the elements are staged through @@ -4519,7 +4540,19 @@ impl<'a, 'b> Lowerer<'a, 'b> { if let Some(var) = self.vars[slot] { let v = self.b.use_var(var); let off = (slot as i32) * 8; - self.b.ins().store(trusted, v, self.locals_base, off); + let st = self.b.ins().store(trusted, v, self.locals_base, off); + self.writebacks.push((var, st)); + } + } + } + + /// Drop the write-backs of locals the body never assigns: the frame + /// already holds their values, and keeping each one live to every + /// exit only costs registers. + fn prune_writebacks(&mut self) { + for (var, st) in std::mem::take(&mut self.writebacks) { + if !self.assigned.get(var.index()).copied().unwrap_or(false) { + self.b.func.layout.remove_inst(st); } } } diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 7b53c37c..a6a729fb 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -256,14 +256,20 @@ struct GuardSnapshot { /// Callees whose defaults the compiled code may have burned in (a /// direct leaf call that leaves parameters out): they must keep them. defaults: Vec>, + /// Per callee-table entry, the class version at which a burned-in + /// constructor's full probe last held (`u64::MAX` for none): the + /// construction plan is a function of it, so an unchanged class only + /// needs its `__init__` code and metaclass rechecked. + ctor_vers: Vec>, } impl GuardSnapshot { - fn new(entries: Vec<(String, Object)>, defaults: Vec>) -> Self { + fn new(entries: Vec<(String, Object)>, defaults: Vec>, callees: usize) -> Self { Self { entries, last_ok: std::cell::Cell::new((0, 0, 0, 0, 0)), defaults, + ctor_vers: (0..callees).map(|_| Cell::new(u64::MAX)).collect(), } } } @@ -1423,7 +1429,7 @@ impl JitState { let artifacts = StdRc::new(Artifacts { cf: StdRc::new(cf), scalar_update, - snap: StdRc::new(GuardSnapshot::new(snap, burned_defaults)), + snap: StdRc::new(GuardSnapshot::new(snap, burned_defaults, callees.len())), callees: StdRc::new(callees), obj_globals: StdRc::new(obj_globals), attr_guards: StdRc::new(attr_guards), @@ -3691,7 +3697,7 @@ fn guards_hold( { return false; } - (callees.is_empty() || callee_guards_hold(interp, callees)) + (callees.is_empty() || callee_guards_hold(interp, callees, &guard_snapshot.ctor_vers)) && math.iter().all(MathGuard::holds) && !guard_snapshot.defaults.iter().any(|f| defaults_overridden(f)) } @@ -3719,8 +3725,12 @@ fn global_guards_hold( /// [`guards_hold`]'s burned-in callees. #[inline(never)] -fn callee_guards_hold(interp: &super::Interpreter, callees: &CalleeTable) -> bool { - for (f, code_snap) in callees { +fn callee_guards_hold( + interp: &super::Interpreter, + callees: &CalleeTable, + ctor_vers: &[Cell], +) -> bool { + for (i, (f, code_snap)) in callees.iter().enumerate() { match f { Object::Function(pf) => { // SAFETY: GIL-serialized raw read of the code cell (see @@ -3740,6 +3750,11 @@ fn callee_guards_hold(interp: &super::Interpreter, callees: &CalleeTable) -> boo // is memoised on `attr_version`, so an unchanged class // revalidates with a version compare. Object::Type(t) => { + let ver = t.attr_version.get(); + let memo = ctor_vers.get(i); + if memo.is_some_and(|m| m.get() == ver) && ctor_still_inits_with(t, ver, code_snap) { + continue; + } let ok = matches!( probe_class_ctor_shape(interp, t), Some((init_code, ..)) if Rc::ptr_eq(&init_code, code_snap) @@ -3747,6 +3762,9 @@ fn callee_guards_hold(interp: &super::Interpreter, callees: &CalleeTable) -> boo if !ok { return false; } + if let Some(m) = memo { + m.set(ver); + } } _ => return false, } @@ -3754,6 +3772,30 @@ fn callee_guards_hold(interp: &super::Interpreter, callees: &CalleeTable) -> boo true } +/// Whether class `t`, unchanged since its constructor's probe held at +/// version `ver`, still constructs through type's own metaclass with the +/// plain `__init__` wearing `code`: the rest of the probe is a function of +/// the version (the memoised plan says so for the plan itself). +fn ctor_still_inits_with(t: &TypeObject, ver: u64, code: &Rc) -> bool { + if !t.metaclass_is_type() { + return false; + } + let Ok(plan) = t.instance_plan.try_borrow() else { + return false; + }; + match plan.as_ref() { + Some((v, plan)) if *v == ver => match plan.init_fn.as_ref() { + // SAFETY: GIL-serialized raw read of the code cell; only the + // pointer is compared. + Some(Object::Function(f)) => { + std::ptr::eq(unsafe { Rc::as_ptr(&*f.code.as_ptr()) }, Rc::as_ptr(code)) + } + _ => false, + }, + _ => false, + } +} + /// RFC 0067 WS1 — pooled exchange buffers for nested native entries /// (and the top-level `enter_compiled`), so a call-recursive program /// doesn't `malloc` five vectors per call. @@ -3831,6 +3873,34 @@ fn put_u32(v: Vec) { }); } +thread_local! { + /// The pin table of the last framed activation to finish, drained and + /// kept for the next one: a long loop that leaves at the pin limit and + /// re-enters would otherwise regrow its table from empty each time. + static SPARE_PINS: Cell = const { Cell::new(Vec::new()) }; +} + +/// The most pins [`SPARE_PINS`] keeps capacity for (the soft limit, plus +/// the entry pins a pressure exit's activation also carried). +const SPARE_PINS_CAP: usize = 2 * RUNTIME_PIN_SOFT_LIMIT; + +/// An empty pin table, with the spare's capacity when there is one. +fn take_pins() -> PinTable { + SPARE_PINS.with(Cell::take) +} + +/// Keep `pins`' allocation for the next activation (see [`SPARE_PINS`]). +fn put_pins(mut pins: PinTable) { + if pins.capacity() == 0 || pins.capacity() > SPARE_PINS_CAP { + return; + } + pins.clear(); + SPARE_PINS.with(|spare| { + let old = spare.take(); + spare.set(if old.capacity() >= pins.capacity() { old } else { pins }); + }); +} + #[cfg(test)] mod scratch_pool_tests { use super::*; @@ -4603,35 +4673,8 @@ unsafe fn try_native_call( if supplied > n_params { return None; } - let first_default = n_params - nc.func.defaults.len().min(n_params); - if supplied < n_params { - if nc.func.defaults_maybe_overridden() && nc.func.slot("__defaults__").is_some() { - return None; - } - if supplied < first_default { - // Unbindable — the interpreter path raises the faithful - // TypeError. - return None; - } - // Every spliced default must fit the callee's compiled - // parameter lane (checked before any buffer is taken). List - // lanes stay interpreted: a mutable default whose *identity* - // matters must go through the generic binder. - for k in supplied..n_params { - // Retained defaults can outnumber parameters after __code__ - // replacement. Match the parameter against the trailing suffix. - let d = &nc.func.defaults[nc.func.defaults.len() - (n_params - k)]; - match nc.cf.local_types.get(k).copied().flatten() { - None | Some(JitType::Obj) => {} - Some(ty @ (JitType::Int | JitType::Bool | JitType::Float)) => { - pack(d, ty)?; - } - Some(JitType::Str) if matches!(d, Object::Str(_)) => {} - Some(JitType::Bytes) if matches!(d, Object::Bytes(_)) => {} - Some(JitType::Dict) if matches!(d, Object::Dict(_)) => {} - Some(_) => return None, - } - } + if supplied < n_params && !native_defaults_fit(nc, supplied, n_params) { + return None; } // The receiver slot must be the object-pin lane the eligibility // check admitted (defensive — `native_method_callable` verified @@ -4697,36 +4740,15 @@ unsafe fn try_native_call( && nc.cf.n_locals as usize <= SCALAR_LEAF_SLOTS && (nc.cf.max_stack as usize) < SCALAR_LEAF_SLOTS { - // Grow before entering the separate function that owns the fixed - // arrays, preserving the ordinary native call's stack discipline. - native_stat(|s| s.scalar_leaf_calls.set(s.scalar_leaf_calls.get() + 1)); - // SAFETY: the checks above establish the helper's size, scalar + // SAFETY: the checks above establish the leaf's size, scalar // lane, default-binding, code-lifetime, and recursion invariants. - let enter = || unsafe { enter_scalar_leaf(&nc.cf, &nc.func, &nc.code, jf, argc_usize) }; - let result = if crate::stdlib::greenlet_native::on_greenlet_stack() { - enter() - } else { - stacker::maybe_grow(512 * 1024, 8 * 1024 * 1024, enter) - }; - return match result { - Some((bits, tag)) if tag == expect_tag => { - jf.ret_bits = bits; - jf.ret_tag = tag; - Some(CallStatus::Ok as i64) - } - Some((bits, tag)) => { - ctx.parked = Some(unpack(bits, tag)); - Some(CallStatus::Boxed as i64) - } - None => { - // Only pure numeric work has run. The ordinary call path - // can restart to compute a bignum or raise with a complete - // interpreter frame. This return releases the recursion - // guard before that path charges its own tick. - native_stat(|s| s.deopts.set(s.deopts.get() + 1)); - None - } - }; + let status = unsafe { scalar_leaf_call(jf, ctx, nc, argc_usize, expect_tag) }; + // Only pure numeric work has run when the leaf declines: the + // ordinary call path can restart it to compute a bignum or raise + // with a complete interpreter frame. This return releases the + // recursion guard before that path charges its own tick. + drop(recursion_guard); + return status; } // A framed native call of a *tiny* callee (a getter, a comparison) @@ -4862,68 +4884,7 @@ unsafe fn try_native_call( c.polls = 0; c } else { - // Self-recursion reuses this activation's own tables. An immutable - // compilation with no call tokens needs no resolution, even when - // another function compiles and advances the cache generation. - // Nonempty tables retain the generation-checked cache lookup. - let (native, method_native) = if callee_key == ctx.code_ptr { - (ctx.native.clone(), ctx.method_native.clone()) - } else { - ( - if nc.callees.is_empty() { - None - } else { - resolved_native_table(callee_key) - }, - if nc.methods.is_empty() { - None - } else { - resolved_method_native_table(callee_key) - }, - ) - }; - let table_gen = if callee_key == ctx.code_ptr { - ctx.table_gen - } else { - current_compile_gen() - }; - Box::new(CallCtx { - interp: ctx.interp, - callees: nc.callees.clone(), - cf: StdRc::as_ptr(&nc.cf), - guard_snapshot: nc.snap.clone(), - globals: nc.func.globals.clone(), - builtins: nc.func.builtins.clone(), - // Native callees are cell-free by `py_callee_ok`. - cells: crate::object::empty_cells(), - parked: None, - raised: None, - const_pins: Vec::new(), - pins: Vec::new(), - entry_pin_count: 0, - pin_pressure_exit: false, - obj_globals: nc.obj_globals.clone(), - obj_global_pins: Vec::new(), - attr_guards: nc.attr_guards.clone(), - methods: nc.methods.clone(), - math: nc.math.clone(), - dirty: false, - interp_calls: 0, - dyn_py_calls: 0, - native_calls: 0, - polls: 0, - child: None, - depth_cell: ctx.depth_cell, - code_ptr: callee_key, - native, - method_native, - table_gen, - // The native call lanes push no interpreter frame for the - // callee — keep it observable to callee-side stack walkers. - frameless_code: Some(nc.code.clone()), - dyn_callee: None, - pin_memo: [u32::MAX; PIN_MEMO], - }) + fresh_child(ctx, nc, callee_key) }; // The callee's pins: the receiver, then the arguments' objects (the // reused child's drained table keeps its capacity). @@ -5025,6 +4986,233 @@ unsafe fn try_native_call( }) }; + // The common return: a scalar in the expected lane from a callee that + // ran no Python. Nothing needs revalidating. + if status == JitStatus::Returned + && njf.ret_tag == expect_tag + && !child.dirty + && matches!( + SlotTag::from_raw(expect_tag), + SlotTag::Int | SlotTag::Float | SlotTag::Bool | SlotTag::None + ) + { + note_callee_exit(&nc.art, &nc.code, &child); + if !inline_bufs { + put_u64(u64_buf); + put_u32(u32_buf); + } + child.pins.clear(); + ctx.child = Some(child); + jf.ret_bits = njf.ret_bits; + jf.ret_tag = njf.ret_tag; + return Some(CallStatus::Ok as i64); + } + // SAFETY: as above; the buffers are the ones the callee ran on. + let status = unsafe { + finish_native_call( + jf, + ctx, + interp, + nc, + child, + NativeExit { + status, + njf: &njf, + locals: locals_buf, + spill, + tags, + expect_tag, + }, + recursion_guard, + ) + }; + if !inline_bufs { + put_u64(u64_buf); + put_u32(u32_buf); + } + Some(status) +} + +/// [`try_native_call`]'s under-arity check: every parameter from +/// `supplied` on must bind a default, and each must fit its compiled lane +/// (checked before any buffer is taken). +#[cold] +#[inline(never)] +fn native_defaults_fit(nc: &NativeCallee, supplied: usize, n_params: usize) -> bool { + if nc.func.defaults_maybe_overridden() && nc.func.slot("__defaults__").is_some() { + return false; + } + let first_default = n_params - nc.func.defaults.len().min(n_params); + if supplied < first_default { + // Unbindable — the interpreter path raises the faithful + // TypeError. + return false; + } + // List lanes stay interpreted: a mutable default whose *identity* + // matters must go through the generic binder. + (supplied..n_params).all(|k| { + // Retained defaults can outnumber parameters after __code__ + // replacement. Match the parameter against the trailing suffix. + let d = &nc.func.defaults[nc.func.defaults.len() - (n_params - k)]; + match nc.cf.local_types.get(k).copied().flatten() { + None | Some(JitType::Obj) => true, + Some(ty @ (JitType::Int | JitType::Bool | JitType::Float)) => pack(d, ty).is_some(), + Some(JitType::Str) => matches!(d, Object::Str(_)), + Some(JitType::Bytes) => matches!(d, Object::Bytes(_)), + Some(JitType::Dict) => matches!(d, Object::Dict(_)), + Some(_) => false, + } + }) +} + +/// [`try_native_call`] of a certified scalar leaf, on bounded stack +/// storage; `None` when the leaf declines (only pure numeric work ran). +/// +/// # Safety +/// +/// As [`enter_scalar_leaf`]; the caller holds the recursion tick. +#[inline(never)] +unsafe fn scalar_leaf_call( + jf: &mut JitFrame, + ctx: &mut CallCtx, + nc: &NativeCallee, + argc: usize, + expect_tag: u32, +) -> Option { + // Grow before entering the separate function that owns the fixed + // arrays, preserving the ordinary native call's stack discipline. + native_stat(|s| s.scalar_leaf_calls.set(s.scalar_leaf_calls.get() + 1)); + // SAFETY: the caller's contract. + let enter = || unsafe { enter_scalar_leaf(&nc.cf, &nc.func, &nc.code, jf, argc) }; + let result = if crate::stdlib::greenlet_native::on_greenlet_stack() { + enter() + } else { + stacker::maybe_grow(512 * 1024, 8 * 1024 * 1024, enter) + }; + match result { + Some((bits, tag)) if tag == expect_tag => { + jf.ret_bits = bits; + jf.ret_tag = tag; + Some(CallStatus::Ok as i64) + } + Some((bits, tag)) => { + ctx.parked = Some(unpack(bits, tag)); + Some(CallStatus::Boxed as i64) + } + None => { + native_stat(|s| s.deopts.set(s.deopts.get() + 1)); + None + } + } +} + +/// A new context for a native callee this activation hasn't cached (see +/// [`try_native_call`]): the callee's handles and resolved tables, and +/// empty per-activation state. +#[cold] +#[inline(never)] +fn fresh_child(ctx: &CallCtx, nc: &NativeCallee, callee_key: *const CodeObject) -> Box { + // Self-recursion reuses this activation's own tables. An immutable + // compilation with no call tokens needs no resolution, even when + // another function compiles and advances the cache generation. + // Nonempty tables retain the generation-checked cache lookup. + let (native, method_native) = if callee_key == ctx.code_ptr { + (ctx.native.clone(), ctx.method_native.clone()) + } else { + ( + if nc.callees.is_empty() { + None + } else { + resolved_native_table(callee_key) + }, + if nc.methods.is_empty() { + None + } else { + resolved_method_native_table(callee_key) + }, + ) + }; + let table_gen = if callee_key == ctx.code_ptr { + ctx.table_gen + } else { + current_compile_gen() + }; + Box::new(CallCtx { + interp: ctx.interp, + callees: nc.callees.clone(), + cf: StdRc::as_ptr(&nc.cf), + guard_snapshot: nc.snap.clone(), + globals: nc.func.globals.clone(), + builtins: nc.func.builtins.clone(), + // Native callees are cell-free by `py_callee_ok`. + cells: crate::object::empty_cells(), + parked: None, + raised: None, + const_pins: Vec::new(), + pins: Vec::new(), + entry_pin_count: 0, + pin_pressure_exit: false, + obj_globals: nc.obj_globals.clone(), + obj_global_pins: Vec::new(), + attr_guards: nc.attr_guards.clone(), + methods: nc.methods.clone(), + math: nc.math.clone(), + dirty: false, + interp_calls: 0, + dyn_py_calls: 0, + native_calls: 0, + polls: 0, + child: None, + depth_cell: ctx.depth_cell, + code_ptr: callee_key, + native, + method_native, + table_gen, + // The native call lanes push no interpreter frame for the + // callee — keep it observable to callee-side stack walkers. + frameless_code: Some(nc.code.clone()), + dyn_callee: None, + pin_memo: [u32::MAX; PIN_MEMO], + }) +} + +/// How a native callee's activation ended, and the buffers it ran on. +struct NativeExit<'a> { + status: JitStatus, + njf: &'a JitFrame, + locals: &'a [u64], + spill: &'a [u64], + tags: &'a [u32], + expect_tag: u32, +} + +/// [`try_native_call`] after the callee left native code any other way +/// than a clean scalar return: materialize a deopt or a raise, translate +/// an object result into this activation's lanes, revalidate this +/// activation's guards when Python ran, and put the callee's context back. +/// +/// # Safety +/// +/// Same contract as [`try_native_call`]; `child` is the context the callee +/// ran with and `exit` its frame and buffers. +#[inline(never)] +unsafe fn finish_native_call( + jf: &mut JitFrame, + ctx: &mut CallCtx, + interp: &mut super::Interpreter, + nc: &NativeCallee, + mut child: Box, + exit: NativeExit<'_>, + recursion_guard: crate::recursion::Guard, +) -> i64 { + let NativeExit { + status, + njf, + expect_tag, + .. + } = exit; + let nctx: &mut CallCtx = &mut child; + /// How the nested call concluded, before result-lane translation. enum Done { Scalar(u64, u32), @@ -5081,17 +5269,13 @@ unsafe fn try_native_call( } else { None }; - match finish_deopted_callee(interp, nc, nctx, locals_buf, spill, tags, &njf, pending) { + match finish_deopted_callee(interp, nc, nctx, exit.locals, exit.spill, exit.tags, njf, pending) { Ok(v) => Done::Obj(v), Err(e) => Done::Raised(e), } } }; note_callee_exit(&nc.art, &nc.code, nctx); - if !inline_bufs { - put_u64(u64_buf); - put_u32(u32_buf); - } // Reap the callee's pins (after every // pin-based rebuild above); a reap cascade runs Python, so it // dirties the call like any interpreter-path work. @@ -5106,7 +5290,7 @@ unsafe fn try_native_call( ctx.child = Some(child); ctx.dirty |= child_dirty; - Some(match done { + match done { Done::Raised(e) => { ctx.raised = Some(e); CallStatus::Raised as i64 @@ -5164,12 +5348,12 @@ unsafe fn try_native_call( // The procedure shape: nothing to write back. jf.ret_bits = 0; jf.ret_tag = expect_tag; - return Some(CallStatus::Ok as i64); + return CallStatus::Ok as i64; } // RFC 0071 WS1 — an object-lane return pins into the // *caller's* table (`None` rides as the nullable `-1`). if guards_ok && expect_tag == SlotTag::ObjPin as u32 { - return Some(match obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo) { + return match obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo) { Some(bits) => { jf.ret_bits = bits; jf.ret_tag = expect_tag; @@ -5179,7 +5363,7 @@ unsafe fn try_native_call( ctx.parked = Some(v); CallStatus::Boxed as i64 } - }); + }; } let expect = match SlotTag::from_raw(expect_tag) { SlotTag::Int => JitType::Int, @@ -5203,7 +5387,7 @@ unsafe fn try_native_call( } } } - }) + } } /// RFC 0071 WS2 — the native class-construction fast path: allocate @@ -11383,7 +11567,7 @@ fn enter_compiled( // slot carries an index into `pins`, and the table (not the slot) // keeps the object alive and reachable for the access helpers and // the deopt rebuild. - let mut pins: PinTable = Vec::new(); + let mut pins: PinTable = take_pins(); { let locals = frame.locals.borrow(); for (slot, dst) in locals_buf.iter_mut().enumerate() { @@ -11597,6 +11781,7 @@ fn enter_compiled( // ones dying with the activation (after every pin-based rebuild // above, so nothing is unpacked from a drained table). drain_activation_pins(interp, &mut ctx.pins); + put_pins(std::mem::take(&mut ctx.pins)); put_u64(locals_buf); put_u64(spill); put_u32(tags); diff --git a/tests/regrtest/test_jit_ctor_guards.py b/tests/regrtest/test_jit_ctor_guards.py new file mode 100644 index 00000000..f60c2400 --- /dev/null +++ b/tests/regrtest/test_jit_ctor_guards.py @@ -0,0 +1,133 @@ +"""Compiled code that constructs a class keeps checking how it constructs. + +A loop builds instances of a global class whose construction tier 2 burns +in, and an interpreted hook changes the class or its `__init__` partway +through: the constructions after the change must see it. +""" + +import unittest + + +class Plain: + def __init__(self, v): + self.v = v + + +class Rebound: + def __init__(self, v): + self.v = v + + +class Swapped: + def __init__(self, v): + self.v = v + + +class GainsNew: + def __init__(self, v): + self.v = v + + +def build_plain(n, hook): + t = 0 + for i in range(n): + t += Plain(i).v + hook(i) + return t + + +def build_rebound(n, hook): + t = 0 + for i in range(n): + t += Rebound(i).v + hook(i) + return t + + +def build_swapped(n, hook): + t = 0 + for i in range(n): + t += Swapped(i).v + hook(i) + return t + + +def build_gains_new(n, hook): + t = 0 + for i in range(n): + t += GainsNew(i).v + hook(i) + return t + + +def noop(i): + pass + + +def triangle(n): + # The sum of 0, 1, ..., n - 1. + return n * (n - 1) // 2 + + +class CtorGuardTests(unittest.TestCase): + N = 3000 + + def test_plain(self): + self.assertEqual(build_plain(self.N, noop), triangle(self.N)) + self.assertEqual(build_plain(self.N, noop), triangle(self.N)) + + def test_init_rebound_midway(self): + def init(self, v): + self.v = -1 + + half = self.N // 2 + + def hook(i): + if i == half: + Rebound.__init__ = init + + self.assertEqual(build_rebound(self.N, noop), triangle(self.N)) + expected = triangle(half + 1) - (self.N - half - 1) + saved = Rebound.__init__ + try: + self.assertEqual(build_rebound(self.N, hook), expected) + finally: + Rebound.__init__ = saved + + def test_init_code_swapped_midway(self): + def other(self, v): + self.v = 2 * v + + half = self.N // 2 + + def hook(i): + if i == half: + Swapped.__init__.__code__ = other.__code__ + + self.assertEqual(build_swapped(self.N, noop), triangle(self.N)) + expected = triangle(half + 1) + 2 * (triangle(self.N) - triangle(half + 1)) + saved = Swapped.__init__.__code__ + try: + self.assertEqual(build_swapped(self.N, hook), expected) + finally: + Swapped.__init__.__code__ = saved + + def test_new_added_midway(self): + half = self.N // 2 + + def new(cls, v): + o = object.__new__(cls) + o.extra = True + return o + + def hook(i): + if i == half: + GainsNew.__new__ = new + + self.assertEqual(build_gains_new(self.N, noop), triangle(self.N)) + self.assertEqual(build_gains_new(self.N, hook), triangle(self.N)) + self.assertTrue(GainsNew(1).extra) + + +if __name__ == "__main__": + unittest.main() From f6c34cec3d78b739984fe03b09c6fd3230ae677b Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:05:49 -0700 Subject: [PATCH 28/96] perf: evaluate keyword calls of pure leaves straight from tier 2 A dynamic keyword call of a warm pure-leaf function (or a bound method over one) now binds its marshaled values straight onto the leaf's argument list through the site's cached CallPyKwNames permutation, building any `**kwargs` dictionary in its cell, and delivers the result without revalidating the activation's guards (a pure leaf runs no Python). It used to stage the values in scratch vectors, fill a full locals vector and move the filled dictionary into its cell three times. A loop of `f(i, delta=2)` calls into `def f(a, **kw)` drops from 312 to 244 ns per iteration (CPython: 149); call_overhead's per-unit instructions drop from 4,671 to 3,953. WEAVEPY_JIT_CLIF_STATS=1 also prints Cranelift's pass timings. --- crates/weavepy-jit/src/engine.rs | 14 +++ crates/weavepy-vm/src/lib.rs | 98 ++++++++++++++++++ crates/weavepy-vm/src/tier2.rs | 88 ++++++++++++++++ tests/regrtest/test_jit_kw_leaf_calls.py | 125 +++++++++++++++++++++++ 4 files changed, 325 insertions(+) create mode 100644 tests/regrtest/test_jit_kw_leaf_calls.py diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index 77444bab..c8c7402d 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -611,13 +611,27 @@ impl JitEngine { ); } + let stats = std::env::var_os("WEAVEPY_JIT_CLIF_STATS").is_some(); + if stats { + drop(cranelift_codegen::timing::take_current()); + } + let t0 = stats.then(std::time::Instant::now); self.module .define_function(id, &mut self.ctx) .map_err(|_| JitVerdict::NotConverged)?; + let t1 = stats.then(std::time::Instant::now); self.module.clear_context(&mut self.ctx); self.module .finalize_definitions() .map_err(|_| JitVerdict::NotConverged)?; + if let (Some(t0), Some(t1)) = (t0, t1) { + eprintln!( + "jit define {name}: {:?}, finalize {:?}\n{}", + t1 - t0, + t1.elapsed(), + cranelift_codegen::timing::take_current() + ); + } let code_ptr = self.module.get_finalized_function(id); // SAFETY: `code_ptr` is a finalized function with exactly the diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 1a7927bc..aa9d1f62 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -14493,6 +14493,104 @@ impl Interpreter { self.pure_leaf_eval::(code_rc, f, &args[..arity]) } + /// A tier-2 keyword call (the `CALL_KW` at `pc` of `caller`) of the warm + /// pure leaf `f`, bound straight onto the evaluation's argument list: + /// `vals` holds the positional run (a bound receiver first) and then + /// the keyword values in call order, `names` the keywords. The site's + /// `CallPyKwNames` permutation places them, as [`Self::core_pure_kw_call`] + /// does for the interpreter. Its result, or `None` having done nothing + /// observable. + #[allow(clippy::too_many_arguments)] + pub(crate) fn kw_pure_leaf_call( + &self, + caller: &CodeObject, + pc: usize, + f: &Rc, + names: &[Object], + vals: &[Object], + eff_argc: usize, + depth_cell: *const std::cell::Cell, + ) -> Option { + use weavepy_compiler::InlineCache as IC; + let IC::CallPyKwNames { + func_id, + perm, + argc: ca, + kwc: ck, + } = caller.caches.get(pc as u32) + else { + return None; + }; + let kwc = names.len(); + if ca as usize != eff_argc + || ck as usize != kwc + || vals.len() != eff_argc + kwc + || !fn_is_pure_leaf(f) + || !vals.iter().all(Self::core_droppable) + { + return None; + } + // SAFETY: GIL-serialized raw read of the function's code cell; the + // caller keeps the function alive across the evaluation. + let code_rc: &Rc = unsafe { &*f.code.as_ptr() }; + if !code_is_pure_leaf(code_rc) || !pure_leaf_warm(code_rc) || !Self::leaf_code_ok(code_rc) { + return None; + } + let (_, covered) = + Self::kw_names_bind_cached(caller, pc, f, func_id, perm, names, eff_argc)?; + // The ordinary call's `RecursionError` check. + // SAFETY: this thread's own depth cell. + if unsafe { (*depth_cell).get() } >= crate::recursion::recursion_limit() { + return None; + } + // A pure leaf has no keyword-only parameters; its `**kwargs` + // dictionary, if any, follows the positional ones. + let total = code_rc.arg_count as usize; + let arity = leaf_arity(code_rc); + if arity > 8 { + return None; + } + let mut args: [*const Object; 8] = [std::ptr::null(); 8]; + for (k, o) in vals[..eff_argc].iter().enumerate() { + args[k] = o; + } + // The `**kwargs` dictionary is built where it lives: no move of a + // filled table into its cell. + let varkw = code_rc.has_varkeywords.then(|| { + Object::Dict(Rc::new(RefCell::new(DictData::with_capacity_and_hasher( + kwc, + crate::fasthash::FxBuildHasher, + )))) + }); + for (j, o) in vals[eff_argc..].iter().enumerate() { + let slot = ((perm >> (4 * j)) & 0xF) as usize; + if slot == specialize::KW_TO_VARKW as usize { + // The bind check proved the callee collects it. + let Some(Object::Dict(d)) = &varkw else { + return None; + }; + d.borrow_mut() + .insert(DictKey(names.get(j)?.clone()), o.clone()); + continue; + } + if slot >= total { + return None; + } + args[slot] = o; + } + for (slot, arg) in args.iter_mut().enumerate().take(total).skip(eff_argc) { + if covered & (1 << slot) == 0 { + *arg = f + .defaults + .get(f.defaults.len().checked_sub(total - slot)?)?; + } + } + if let Some(d) = &varkw { + args[total] = d; + } + self.pure_leaf_eval::(code_rc, f, &args[..arity]) + } + /// A keyword call's parameters, bound as `kw_names_fill_locals` leaves /// them in `locals` (a `**kwargs` dictionary included), evaluated /// frameless when `f` is a warm pure leaf: its result, or `None` diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index a6a729fb..cf63700a 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -9616,6 +9616,14 @@ unsafe fn call_dyn_impl( return status; } } + // A keyword call of a warm pure leaf evaluates frameless on its + // marshaled values (no Python runs, so nothing needs revalidating). + if kwc > 0 { + // SAFETY: per the function contract — same live buffers. + if let Some(v) = unsafe { dyn_kw_pure_leaf(jf, ctx, interp, &callee, argc, kwc, names) } { + return dyn_leaf_result(jf, ctx, v, int_result); + } + } // A keyword call of a plain function whose site carries the // interpreter's `CallPyKwNames` permutation binds through it, as the // `CALL_KW` handler's hit does: no name strings, no generic binder. @@ -9745,6 +9753,86 @@ unsafe fn dyn_call_result( } } +/// A dynamic keyword call of a warm pure-leaf function (or a bound method +/// over one), evaluated frameless on the marshaled operands through the +/// call site's `CallPyKwNames` permutation (see +/// [`super::Interpreter::kw_pure_leaf_call`]): its result, or `None` +/// having done nothing observable. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_py`]; the call site's pc is in +/// `jf.deopt_pc` (stored before every call helper runs). +unsafe fn dyn_kw_pure_leaf( + jf: &JitFrame, + ctx: &CallCtx, + interp: &super::Interpreter, + callee: &Object, + argc: u32, + kwc: u32, + names: u32, +) -> Option { + const MAX: usize = 9; + // SAFETY: per the function contract, the activation keeps its code + // object alive. + let code = unsafe { &*ctx.code_ptr }; + let Some(Object::Tuple(name_items)) = crate::code_const_objects(code).get(names as usize) + else { + return None; + }; + let (f, recv) = match callee { + Object::Function(f) => (f, None), + Object::BoundMethod(bm) if !bm.redispatch_descriptor => match &bm.function { + Object::Function(f) => (f, Some(&bm.receiver)), + _ => return None, + }, + _ => return None, + }; + let n = (argc + kwc) as usize; + let eff_argc = argc as usize + usize::from(recv.is_some()); + if n + usize::from(recv.is_some()) > MAX || name_items.len() != kwc as usize { + return None; + } + let mut vals: [Object; MAX] = std::array::from_fn(|_| Object::None); + let offset = usize::from(recv.is_some()); + if let Some(r) = recv { + vals[0] = r.clone(); + } + for j in 0..n { + // SAFETY: native code wrote `argc + kwc` entries, and the buffers + // are `max_call_args` wide. + let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; + vals[offset + j] = unpack_pins(bits, tag, &ctx.pins); + } + interp.kw_pure_leaf_call( + code, + jf.deopt_pc as usize, + f, + name_items, + &vals[..offset + n], + eff_argc, + ctx.depth_cell, + ) +} + +/// Deliver a pure leaf's result at a dynamic call site: it ran no Python, +/// so the activation's guards still hold. +fn dyn_leaf_result(jf: &mut JitFrame, ctx: &mut CallCtx, v: Object, int_result: bool) -> i64 { + if int_result { + if let Object::Int(value) = &v { + jf.ret_bits = *value as u64; + jf.ret_tag = SlotTag::Int as u32; + return CallStatus::Ok as i64; + } + } else if let Some(bits) = pin_any(v.clone(), &mut ctx.pins) { + jf.ret_bits = bits; + jf.ret_tag = SlotTag::ObjPin as u32; + return CallStatus::Ok as i64; + } + ctx.parked = Some(v); + CallStatus::Boxed as i64 +} + /// Bind a dynamic keyword call's marshaled operands for a plain function /// (or a bound method over one) through the call site's cached /// `CallPyKwNames` permutation, re-verified against the live function diff --git a/tests/regrtest/test_jit_kw_leaf_calls.py b/tests/regrtest/test_jit_kw_leaf_calls.py new file mode 100644 index 00000000..d9bfe2f4 --- /dev/null +++ b/tests/regrtest/test_jit_kw_leaf_calls.py @@ -0,0 +1,125 @@ +"""Compiled code calls keyword-taking pure functions without a frame. + +A keyword call of a small pure function binds its values onto the +function's parameters (a fresh `**kwargs` dictionary included) and +evaluates it frameless. Each loop runs long enough for the tier-2 compiler +to take it; then the callee changes, and every call must see the change. +""" + +import unittest + + +def with_kwargs(a, **kw): + return a + kw.get("delta", 0) + + +def mixed(a, b=1, **kw): + return a * b + kw.get("c", 0) + + +def give_back(a, **kw): + return kw + + +class Holder: + def scaled(self, a, **kw): + return a * kw.get("k", 1) + + +def kw_loop(n): + t = 0 + for i in range(n): + t += with_kwargs(i, delta=2) + return t + + +def mixed_loop(n): + t = 0 + for i in range(n): + t += mixed(i, c=3, b=2) + return t + + +def fresh_dicts(n): + seen = [] + for i in range(n): + seen.append(give_back(i, x=i)) + return seen + + +def bound_loop(n): + h = Holder() + scaled = h.scaled + t = 0 + for i in range(n): + t += scaled(i, k=3) + return t + + +def triangle(n): + # The sum of 0, 1, ..., n - 1. + return n * (n - 1) // 2 + + +class KwLeafCallTests(unittest.TestCase): + N = 3000 + + def setUp(self): + self.saved = (with_kwargs.__code__, mixed.__defaults__) + + def tearDown(self): + with_kwargs.__code__, mixed.__defaults__ = self.saved + + def test_calls(self): + n = self.N + self.assertEqual(kw_loop(n), triangle(n) + 2 * n) + self.assertEqual(mixed_loop(n), 2 * triangle(n) + 3 * n) + self.assertEqual(bound_loop(n), 3 * triangle(n)) + + def test_each_call_gets_a_fresh_dict(self): + dicts = fresh_dicts(self.N) + self.assertEqual(len({id(d) for d in dicts}), self.N) + self.assertEqual(dicts[7], {"x": 7}) + dicts[7]["y"] = 1 + self.assertEqual(dicts[8], {"x": 8}) + + def test_code_reassigned(self): + def other(a, **kw): + return len(kw) * 100 + + n = self.N + self.assertEqual(kw_loop(n), triangle(n) + 2 * n) + with_kwargs.__code__ = other.__code__ + self.assertEqual(kw_loop(5), 500) + + def test_defaults_reassigned(self): + n = self.N + self.assertEqual(mixed_loop(n), 2 * triangle(n) + 3 * n) + mixed.__defaults__ = (10,) + self.assertEqual(mixed_loop(5), 2 * triangle(5) + 15) + + def call_without_b(n): + t = 0 + for i in range(n): + t += mixed(i, c=1) + return t + + self.assertEqual(call_without_b(n), 10 * triangle(n) + n) + mixed.__defaults__ = None + with self.assertRaises(TypeError): + call_without_b(1) + + def test_bad_keyword(self): + def call_bad(n): + t = 0 + for i in range(n): + t += mixed(i, b=2, c=1) + return t + + self.assertEqual(call_bad(self.N), 2 * triangle(self.N) + self.N) + with self.assertRaises(TypeError): + mixed(1, a=2) + + +if __name__ == "__main__": + unittest.main() From c0f8a653021f8e8ecbd8cdc4efcfad4b6ead1e2c Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:26:22 -0700 Subject: [PATCH 29/96] perf: append constructor fields in line and return objects without the slow exit A new-key attribute store (the constructor pattern, `self.x = ...` in `__init__`) of a scalar now appends the field in compiled code when the fresh instance's split values already hold exactly the class's names before it and have room: its guard carries the name's position among the class's shared names, and the class version, the block's names and length and capacity, the native body and the dict-watcher gate are checked in line before anything is written. The first store of an instance still allocates the values through the helper. A native callee that returns `None` on the object lane (a procedure, a constructor's `__init__`) or an instance where the site takes an object (a method returning `self`) now puts its context back without going through the general result translation. float_math's per-unit instructions drop from 7,291 to 6,834. --- crates/weavepy-jit/src/lower.rs | 103 ++++++++++++++++++- crates/weavepy-jit/src/runtime.rs | 8 +- crates/weavepy-vm/src/inst_dict.rs | 1 + crates/weavepy-vm/src/tier2.rs | 94 +++++++++++++---- tests/regrtest/test_jit_inline_new_fields.py | 97 +++++++++++++++++ 5 files changed, 278 insertions(+), 25 deletions(-) create mode 100644 tests/regrtest/test_jit_inline_new_fields.py diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 2118143b..0c1be715 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -3164,6 +3164,86 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.b.ins().iadd_imm(at, i64::from(l.split_values)) } + /// The address the new-key store at `site` appends its field to, in + /// the pinned instance `pin`'s split values (see + /// [`runtime::ObjLayout`]), having counted it in: an ordinary instance + /// pin whose class still has the guard's version, whose values are + /// split over its class's names, unborrowed and exclusive, hold exactly + /// the names before the guard's index and have room for one more, with + /// no dict watcher active. Anything else branches to `miss` before + /// anything is written. + fn split_append_addr(&mut self, l: &runtime::ObjLayout, pin: Value, site: u32, miss: Block) -> Value { + let t = MemFlags::trusted(); + let ptr = self.ptr_ty; + let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); + let guards = self.b.ins().load(ptr, t, ctx, l.ctx_guards); + let gbuf = self.b.ins().load(ptr, t, guards, l.guards_buf); + let g = self + .b + .ins() + .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); + let idx = self.b.ins().uload32(t, g, l.guard_split_idx); + let none = self.b.ins().icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); + self.miss_if(none, miss); + let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); + self.miss_if(out, miss); + let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); + let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); + let p = self.b.ins().iadd(buf, off); + let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); + let otag = self.b.ins().uload8(types::I32, t, p, l.pin_obj); + let not_obj = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + let bad = self.b.ins().bor(not_obj, not_inst); + self.miss_if(bad, miss); + let inst = self.b.ins().load(ptr, t, p, l.pin_obj + 8); + let ver = self.b.ins().load(types::I64, t, g, l.guard_ver); + let cls = self.b.ins().load(ptr, t, inst, l.inst_class); + let cver = self.b.ins().load(types::I64, t, cls, l.type_attr_version); + let lazy = self.b.ins().load(ptr, t, inst, l.inst_dict_lazy); + let body = self.b.ins().load(ptr, t, inst, l.inst_c_body); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, t, flag, 0); + let watchers = self.b.ins().iconst(ptr, l.dict_watchers as i64); + let watchers = self.b.ins().uload8(types::I64, t, watchers, 0); + let borrow = self.b.ins().sload32(t, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, t, inst, l.inst_split_block); + let stale = self.b.ins().icmp(IntCC::NotEqual, cver, ver); + let busy = self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, body); + let other = self.b.ins().bor(other, shared); + let other = self.b.ins().bor(other, watchers); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(stale, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, miss); + let keys = self.b.ins().load(ptr, t, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, t, cls, l.type_shared_keys); + let len = self.b.ins().uload32(t, block, l.split_len); + let cap = self.b.ins().uload32(t, block, l.split_cap); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let elsewhere = self.b.ins().icmp(IntCC::NotEqual, idx, len); + let full = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, len, cap); + let bad = self.b.ins().bor(foreign, elsewhere); + let bad = self.b.ins().bor(bad, full); + self.miss_if(bad, miss); + let len1 = self.b.ins().iadd_imm(len, 1); + let len1 = self.b.ins().ireduce(types::I32, len1); + self.b.ins().store(t, len1, block, l.split_len); + let off = self.b.ins().ishl_imm(idx, 4); + let at = self.b.ins().iadd(block, off); + self.b.ins().iadd_imm(at, i64::from(l.split_values)) + } + /// The pinned list `pin`'s items (buffer pointer and length; and its /// capacity, and where the length lives, for an append): a list pin /// whose element lane is one of `lanes`, unborrowed (`read`) or @@ -3396,8 +3476,29 @@ impl<'a, 'b> Lowerer<'a, 'b> { (Some(l), JitType::Int | JitType::Float | JitType::Bool) => Some(*l), _ => None, }; + let new_key = self + .tfunc + .attr_sites + .get(site as usize) + .is_some_and(|s| s.new_key); let mut done = None; - if let Some(l) = inline { + if let (Some(l), true) = (inline, new_key) { + // The constructor pattern: a fresh instance's next field. + let miss = self.b.create_block(); + let stored = self.b.create_block(); + let addr = self.split_append_addr(&l, pin, site, miss); + let (tag, off, v) = match lane { + JitType::Int => (l.tag_int, 8, val), + JitType::Float => (l.tag_float, 8, val), + _ => (l.tag_bool, 1, self.b.ins().ireduce(types::I8, val)), + }; + let tagv = self.b.ins().iconst(types::I8, i64::from(tag)); + self.b.ins().store(trusted, tagv, addr, 0); + self.b.ins().store(trusted, v, addr, off); + self.b.ins().jump(stored, &[]); + self.b.switch_to_block(miss); + done = Some(stored); + } else if let Some(l) = inline { let miss = self.b.create_block(); let stored = self.b.create_block(); let addr = self.split_field_addr(&l, pin, site, miss, false); diff --git a/crates/weavepy-jit/src/runtime.rs b/crates/weavepy-jit/src/runtime.rs index b50af8ee..955f3b52 100644 --- a/crates/weavepy-jit/src/runtime.rs +++ b/crates/weavepy-jit/src/runtime.rs @@ -194,11 +194,15 @@ pub struct ObjLayout { /// names' pointer. pub type_attr_version: i32, pub type_shared_keys: i32, - /// A split block → its names' pointer, its length (`u32`) and its - /// first value. + /// A split block → its names' pointer, its length and capacity + /// (`u32`s) and its first value. pub split_keys: i32, pub split_len: i32, + pub split_cap: i32, pub split_values: i32, + /// An instance value's payload pointer → its native body's pointer + /// (a word; zero for an ordinary instance). + pub inst_c_body: i32, /// The byte that is nonzero once cells are shared between threads. pub cells_unguarded: usize, /// A list pin: its discriminant byte's value, where the list pointer diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs index cd98d69f..f0945c5d 100644 --- a/crates/weavepy-vm/src/inst_dict.rs +++ b/crates/weavepy-vm/src/inst_dict.rs @@ -239,6 +239,7 @@ impl SplitValues { pub(crate) const BLOCK_OFFSET: usize = std::mem::offset_of!(Self, block); pub(crate) const KEYS_OFFSET: usize = std::mem::offset_of!(SplitHeader, keys); pub(crate) const LEN_OFFSET: usize = std::mem::offset_of!(SplitHeader, len); + pub(crate) const CAP_OFFSET: usize = std::mem::offset_of!(SplitHeader, cap); pub(crate) const VALUES_OFFSET: usize = std::mem::size_of::(); #[inline] diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index cf63700a..76d907ef 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -2785,9 +2785,23 @@ fn attr_site_guard( /// [`AttrGuard::split_idx`] for an indexed site of `cls`: the index, when /// the class's shared names hold `name` there (they never move). +/// +/// A new-key site's is the name's position among its class's shared names +/// (compiled code appends the field there, in line, while a fresh instance +/// sets its fields in the class's order). fn split_index(cls: &TypeObject, storage: AttrStorage, name: &str) -> u32 { - let AttrStorage::Indexed(i) = storage else { - return u32::MAX; + let i = match storage { + AttrStorage::Indexed(i) => i, + AttrStorage::NewKey if cls.native_kind.get() == 0 => { + let Some(keys) = cls.shared_keys.get() else { + return u32::MAX; + }; + match u32::try_from(keys.names_before(name, crate::object::py_str_hash(name))) { + Ok(i) => i, + Err(_) => return u32::MAX, + } + } + _ => return u32::MAX, }; match cls.shared_keys.get().and_then(|keys| keys.get(i as usize)) { Some(DictKey(Object::Str(s))) if &**s == name => i, @@ -2869,6 +2883,19 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option` alone in its wrapper, + // checked on a sample. + let inst_c_body = arc_data + std::mem::offset_of!(PyInstance, c_body); + { + let sample = crate::types::CBody::default(); + sample.set(0x5eed_c0de); + if std::mem::size_of::() != 8 + || word(std::ptr::from_ref(&sample).cast::(), 0) != 0x5eed_c0de + || word(arc as *const u8, inst_c_body) != inst.c_body.get() + { + return None; + } + } let type_attr_version = arc_data + std::mem::offset_of!(TypeObject, attr_version); let type_shared_keys = arc_data + std::mem::offset_of!(TypeObject, shared_keys) @@ -3001,6 +3028,8 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option Some(0), + SlotTag::ObjPin => Some(u64::MAX), + _ => None, + } + } else if njf.ret_tag == SlotTag::ObjPin as u32 && expect == SlotTag::ObjPin { + // An instance pins into this activation's table (a method + // returning `self` finds its existing pin). + match child.pins.get(njf.ret_bits as usize) { + Some(Pin::Obj(v @ Object::Instance(_))) => { + pin_reusing(v, &mut ctx.pins, &mut ctx.pin_memo) + } + _ => None, + } + } else { + None + }; + if let Some(bits) = ret { + note_callee_exit(&nc.art, &nc.code, &child); + if !inline_bufs { + put_u64(u64_buf); + put_u32(u32_buf); + } + child.pins.clear(); + ctx.child = Some(child); + jf.ret_bits = bits; + jf.ret_tag = expect_tag; + return Some(CallStatus::Ok as i64); + } } // SAFETY: as above; the buffers are the ones the callee ran on. let status = unsafe { diff --git a/tests/regrtest/test_jit_inline_new_fields.py b/tests/regrtest/test_jit_inline_new_fields.py new file mode 100644 index 00000000..ffc60ecd --- /dev/null +++ b/tests/regrtest/test_jit_inline_new_fields.py @@ -0,0 +1,97 @@ +"""Compiled constructors append a fresh instance's fields in line. + +An `__init__` that sets scalar fields in its class's usual order stores +each one past the previous without calling a helper. Each loop runs long +enough for the tier-2 compiler to take the constructor; then instances +arrive in other shapes, and each must end up with exactly its fields. +""" + +import unittest + + +class Point: + def __init__(self, i): + self.x = i + self.y = i * 0.5 + self.z = i > 3 + + +def build(cls, n): + out = [] + for i in range(n): + out.append(cls(i)) + return out + + +class InlineNewFieldTests(unittest.TestCase): + N = 3000 + + def check(self, p, i): + self.assertEqual(vars(p), {"x": i, "y": i * 0.5, "z": i > 3}) + + def test_fields(self): + points = build(Point, self.N) + for i in (0, 1, 4, self.N - 1): + self.check(points[i], i) + self.assertEqual(sum(p.x for p in points), self.N * (self.N - 1) // 2) + + def test_reinitialized_instance(self): + points = build(Point, self.N) + p = points[10] + Point.__init__(p, 99) + self.check(p, 99) + + def test_other_order_first(self): + class Q: + def __init__(self, i, flip=False): + if flip: + self.b = -i + self.a = i + else: + self.a = i + self.b = -i + + def make(n, flip): + return [Q(i, flip) for i in range(n)] + + qs = make(self.N, False) + self.assertEqual(vars(qs[5]), {"a": 5, "b": -5}) + flipped = make(10, True) + self.assertEqual(list(vars(flipped[5]).items()), [("b", -5), ("a", 5)]) + again = make(10, False) + self.assertEqual(list(vars(again[5]).items()), [("a", 5), ("b", -5)]) + + def test_subclass_and_extra_fields(self): + class Base: + def __init__(self, i): + self.u = i + self.v = 2 * i + + class Sub(Base): + pass + + bases = build(Base, self.N) + subs = build(Sub, 10) + self.assertEqual(vars(bases[7]), {"u": 7, "v": 14}) + self.assertEqual(vars(subs[7]), {"u": 7, "v": 14}) + b = Base(1) + b.w = 3 + self.assertEqual(vars(Base(2)), {"u": 2, "v": 4}) + self.assertEqual(vars(b), {"u": 1, "v": 2, "w": 3}) + + def test_class_attribute_becomes_a_property(self): + class R: + def __init__(self, i): + self.k = i + + rs = build(R, self.N) + self.assertEqual(rs[3].k, 3) + seen = [] + R.k = property(lambda self: 0, lambda self, v: seen.append(v)) + r = R(5) + self.assertEqual(seen, [5]) + self.assertEqual(r.k, 0) + + +if __name__ == "__main__": + unittest.main() From 2781ae806d5d3f3334f5c24ff07f8a408ddce673 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:26:30 -0700 Subject: [PATCH 30/96] perf: compile tier-2 code without Cranelift's mid-end optimizer The lowering already emits the code it wants (its guards' loads aren't pure, so the e-graph pass merges almost nothing), and opt_level "none" keeps the backtracking register allocator. Measured over the five call fixtures, per-unit instructions are unchanged or slightly lower and the compile-inclusive totals drop 3-6M instructions; wall time vs CPython improves by 1-4 points (float_math 0.981 -> 0.959, richards 0.494 -> 0.458). WEAVEPY_JIT_OPT=1 restores opt_level "speed". --- crates/weavepy-jit/src/engine.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index c8c7402d..c8917682 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -269,8 +269,16 @@ impl JitEngine { // in-process. flag_builder.set("use_colocated_libcalls", "false").ok()?; flag_builder.set("is_pic", "false").ok()?; - // Favour fast compiles over the last few percent of codegen. - flag_builder.set("opt_level", "speed").ok()?; + // No mid-end optimization: the lowering already emits the code it + // wants, and the e-graph pass found nothing in it worth its share + // of compile time (measured: the fixtures ran no slower without it + // and compiled 3-8% faster). WEAVEPY_JIT_OPT=1 restores it. + let opt = if std::env::var_os("WEAVEPY_JIT_OPT").is_some() { + "speed" + } else { + "none" + }; + flag_builder.set("opt_level", opt).ok()?; if std::env::var_os("WEAVEPY_JIT_QUICK").is_some() { flag_builder.set("opt_level", "none").ok()?; flag_builder.set("regalloc_algorithm", "single_pass").ok()?; From 002bf5ed6febb862f30cdd6275afeef115ca855f Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:36:28 -0700 Subject: [PATCH 31/96] test: count armed and slot field updates in the update coverage test Compiled code now runs an armed field-update method in line, so the helper only completes the calls before arming (and the ones the in-line path declines), and `__slots__` members get an update plan too. The unit test counts arming and the helper's slot updates, accepts an armed method with any helper count, and expects slot updates through the evaluator. --- crates/weavepy-vm/src/lib.rs | 23 ++++++++++++++++++++--- crates/weavepy-vm/src/tier2.rs | 4 ++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index aa9d1f62..7556f24b 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -56135,6 +56135,8 @@ thread_local! { static SCALAR_FIELD_UPDATE_BOXED_RETURNS: std::cell::Cell = const { std::cell::Cell::new(0) }; #[cfg(feature = "jit")] static SCALAR_FIELD_UPDATE_NATIVE_CALLS: std::cell::Cell = const { std::cell::Cell::new(0) }; + #[cfg(feature = "jit")] + static SCALAR_FIELD_UPDATE_ARMED: std::cell::Cell = const { std::cell::Cell::new(0) }; static PURE_LITERAL_ARGUMENT_CALLS: std::cell::Cell<[u64; 4]> = const { std::cell::Cell::new([0; 4]) }; static PURE_SLOT_FIELD_READS: std::cell::Cell<[u64; 3]> = const { std::cell::Cell::new([0; 3]) }; @@ -65977,7 +65979,9 @@ assert loop(2000) == 1999000 ); let native = SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(std::cell::Cell::get); let attempts = SCALAR_FIELD_UPDATE_ATTEMPTS.with(std::cell::Cell::get); + let armed = SCALAR_FIELD_UPDATE_ARMED.with(std::cell::Cell::get); run(&mut interp, &source); + let armed = SCALAR_FIELD_UPDATE_ARMED.with(std::cell::Cell::get) - armed; let native = SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(std::cell::Cell::get) - native; eprintln!("Completed scalar field update {kind} calls: native {native}"); @@ -65986,7 +65990,15 @@ assert loop(2000) == 1999000 if crate::tier2::jit_off_for_process() { assert_eq!(native, 0, "native entry requires JIT"); assert_eq!(attempts, 0, "interpreted calls use their ordinary path"); - } else if matches!(kind, "parameter" | "constant" | "default" | "alias") { + assert_eq!(armed, 0, "in-line updates require JIT"); + } else if matches!(kind, "parameter" | "constant") { + // The helper runs the update until it arms the + // method entry; compiled code runs the rest in line. + assert!( + native > 100 || (armed > 0 && native > 0), + "compiled {kind} native update coverage: {native} (armed {armed})" + ); + } else if matches!(kind, "default" | "slots" | "alias") { assert!( native > 100, "compiled {kind} native update coverage: {native}" @@ -66014,15 +66026,20 @@ assert loop(2000) == 1999000 ), ] { let native = SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(std::cell::Cell::get); + let armed = SCALAR_FIELD_UPDATE_ARMED.with(std::cell::Cell::get); run( &mut interp, &format!("__name__ = 'update_coverage'\n{fixture}\nbench({work})\n"), ); let native = SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(std::cell::Cell::get) - native; - eprintln!("Application {name} completed updates: native {native}"); + let armed = SCALAR_FIELD_UPDATE_ARMED.with(std::cell::Cell::get) - armed; + eprintln!("Application {name} completed updates: native {native}, armed {armed}"); if name == "richards" && !crate::tier2::jit_off_for_process() { - assert!(native > 100, "Richards native update coverage: {native}"); + assert!( + native > 100 || (armed > 0 && native > 0), + "Richards native update coverage: {native} (armed {armed})" + ); } if crate::tier2::jit_off_for_process() { assert_eq!(native, 0, "{name} ordinary interpreter dispatch"); diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 76d907ef..bec1818d 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -4473,6 +4473,8 @@ unsafe fn native_scalar_field_update( let (_, slot) = slots.get_index_mut(index as usize)?; // An exact integer owns no destructor or GC edge. *slot = Object::Int(value); + #[cfg(test)] + crate::SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(|hits| hits.set(hits.get() + 1)); return Some(value); } // SAFETY: no callback, allocation, or Python execution can overlap this @@ -4534,6 +4536,8 @@ fn arm_update(entry: &MethodEntry, nc: &NativeCallee, receiver: &Object, argc: u // SAFETY: `Rc` is one pointer, compared and never dereferenced. u.code.set(unsafe { std::mem::transmute_copy::, usize>(&entry.code) }); u.idx.set(guard.split_idx); + #[cfg(test)] + crate::SCALAR_FIELD_UPDATE_ARMED.with(|armed| armed.set(armed.get() + 1)); } /// Bind the slots a keyword call skipped (tagged [`SlotTag::Default`] From b38b6c500c9a52f32fc414111b75933f795d5b74 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:05:49 -0700 Subject: [PATCH 32/96] perf: plan local write-backs before lowering and merge the first guard branch The locals a compiled body assigns are now listed from its IR before lowering (stores, loop steps, iterator captures), and exits and calls write back only those: a never-assigned local is no longer even read there, so the SSA builder no longer threads it through every join. A lowering that assigns an unlisted local discards the build. The in-line field access tests a missing split index and an out-of-range pin with one branch. WEAVEPY_JIT_CLIF_DUMP=1 prints each compiled function. Per-unit instructions are unchanged within noise; maximize's CLIF drops from 1,074 to 1,036 values. --- crates/weavepy-jit/src/engine.rs | 9 ++- crates/weavepy-jit/src/lower.rs | 125 +++++++++++++++++++++++-------- 2 files changed, 100 insertions(+), 34 deletions(-) diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index c8917682..32b748c0 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -591,7 +591,7 @@ impl JitEngine { } } - build_function( + let sound = build_function( &mut self.ctx.func, &mut self.fbctx, tfunc, @@ -609,6 +609,13 @@ impl JitEngine { }) .collect(), ); + if !sound { + self.module.clear_context(&mut self.ctx); + return Err(JitVerdict::NotConverged); + } + if std::env::var_os("WEAVEPY_JIT_CLIF_DUMP").is_some() { + eprintln!("jit clif {name}:\n{}", self.ctx.func.display()); + } if std::env::var_os("WEAVEPY_JIT_CLIF_STATS").is_some() { let f = &self.ctx.func; let insts: usize = f.layout.blocks().map(|b| f.layout.block_insts(b).count()).sum(); diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 0c1be715..dcaff703 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -68,7 +68,9 @@ struct SiteCall { gaps: u32, } -/// Build the Cranelift function body for `tfunc` into `func`. +/// Build the Cranelift function body for `tfunc` into `func`; `false` when +/// the body assigned a local the write-back plan missed (the build must be +/// discarded). pub(crate) fn build_function( func: &mut Function, fbctx: &mut FunctionBuilderContext, @@ -76,15 +78,16 @@ pub(crate) fn build_function( ptr_ty: Type, self_func: Option, leaves: Vec, -) { +) -> bool { let mut builder = FunctionBuilder::new(func, fbctx); let mut lc = Lowerer::new(&mut builder, tfunc, ptr_ty); lc.self_func = self_func; lc.leaves = leaves; lc.build(); - lc.prune_writebacks(); + let sound = !lc.unlisted_assignment; builder.seal_all_blocks(); builder.finalize(); + sound } struct Lowerer<'a, 'b> { @@ -153,13 +156,64 @@ struct Lowerer<'a, 'b> { self_slow_sig: Option, /// Direct scalar-leaf call targets by token. leaves: Vec, - /// Per variable (by index), whether the body ever assigns it after - /// the entry loads it from the frame. - assigned: Vec, - /// Every local write-back store, by variable: a variable the body - /// never assigns still holds the frame's own value, so its stores are - /// dropped once the body is built (see [`Self::def_local`]). - writebacks: Vec<(Variable, cranelift_codegen::ir::Inst)>, + /// Per local slot, whether the body assigns it (see + /// [`assigned_slots`]): a local it never assigns still holds the + /// frame's own value, so exits and calls don't write it back. + assignable: Vec, + /// The slot each local's variable stands for, by variable index. + var_slots: Vec, + /// Set when the body assigned a local [`assigned_slots`] missed: the + /// build is then unsound and `build_function` reports it. + unlisted_assignment: bool, +} + +/// Which local slots `tfunc`'s body assigns after the entry loads them: +/// every operation and loop step that defines a local's variable (see +/// [`Lowerer::def_local`], which checks the list). +fn assigned_slots(tfunc: &TFunc) -> Vec { + let mut out = vec![false; tfunc.n_locals as usize]; + let mut mark = |slot: u32| { + if let Some(a) = out.get_mut(slot as usize) { + *a = true; + } + }; + for b in &tfunc.blocks { + for st in &b.stmts { + match st.op { + TOp::StoreLocal(slot) + | TOp::IterCapture { + iter_slot: slot, .. + } + | TOp::DictIterNew { iter_slot: slot } => mark(slot), + _ => {} + } + } + match b.term { + TTerm::ForRange { + cur_slot, var_slot, .. + } => { + mark(cur_slot); + mark(var_slot); + } + TTerm::ForList { + idx_slot, var_slot, .. + } => { + mark(idx_slot); + mark(var_slot); + } + TTerm::ForIter { var_slot, .. } => mark(var_slot), + TTerm::ForIterPair { + var1_slot, + var2_slot, + .. + } => { + mark(var1_slot); + mark(var2_slot); + } + _ => {} + } + } + out } impl<'a, 'b> Lowerer<'a, 'b> { @@ -195,18 +249,24 @@ impl<'a, 'b> Lowerer<'a, 'b> { self_sig: None, self_slow_sig: None, leaves: Vec::new(), - assigned: Vec::new(), - writebacks: Vec::new(), + assignable: Vec::new(), + var_slots: Vec::new(), + unlisted_assignment: false, } } /// Assign the managed local `var` (anything but its entry load). fn def_local(&mut self, var: Variable, v: Value) { - let i = var.index(); - if self.assigned.len() <= i { - self.assigned.resize(i + 1, false); + let listed = self + .var_slots + .get(var.index()) + .and_then(|&slot| self.assignable.get(slot as usize)) + .copied() + .unwrap_or(false); + if !listed { + debug_assert!(false, "an assignment `assigned_slots` doesn't list"); + self.unlisted_assignment = true; } - self.assigned[i] = true; self.b.def_var(var, v); } @@ -319,6 +379,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { // Declare + initialise a variable per managed local. self.vars = vec![None; self.tfunc.n_locals as usize]; + self.assignable = assigned_slots(self.tfunc); for slot in 0..self.tfunc.local_types.len() { if let Some(ty) = self.tfunc.local_types[slot] { let cl = Self::cl_ty(ty); @@ -327,6 +388,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let v = self.b.ins().load(cl, trusted, self.locals_base, off); self.b.def_var(var, v); self.vars[slot] = Some(var); + if self.var_slots.len() <= var.index() { + self.var_slots.resize(var.index() + 1, u32::MAX); + } + self.var_slots[var.index()] = slot as u32; } } @@ -3114,11 +3179,11 @@ impl<'a, 'b> Lowerer<'a, 'b> { .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); let idx = self.b.ins().uload32(t, g, l.guard_split_idx); let none = self.b.ins().icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); - self.miss_if(none, miss); // The pin. let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); - self.miss_if(out, miss); + let bad = self.b.ins().bor(none, out); + self.miss_if(bad, miss); let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); let p = self.b.ins().iadd(buf, off); @@ -3184,10 +3249,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); let idx = self.b.ins().uload32(t, g, l.guard_split_idx); let none = self.b.ins().icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); - self.miss_if(none, miss); let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); - self.miss_if(out, miss); + let bad = self.b.ins().bor(none, out); + self.miss_if(bad, miss); let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); let p = self.b.ins().iadd(buf, off); @@ -4635,25 +4700,19 @@ impl<'a, 'b> Lowerer<'a, 'b> { } /// Write back every managed local into the frame's locals buffer. + /// Store the locals the body assigns back to the frame (the others + /// still hold the values the entry loaded: reading one here would only + /// keep it live to every exit and call). fn writeback_locals(&mut self) { let trusted = MemFlags::trusted(); for slot in 0..self.vars.len() { if let Some(var) = self.vars[slot] { + if !self.assignable.get(slot).copied().unwrap_or(true) { + continue; + } let v = self.b.use_var(var); let off = (slot as i32) * 8; - let st = self.b.ins().store(trusted, v, self.locals_base, off); - self.writebacks.push((var, st)); - } - } - } - - /// Drop the write-backs of locals the body never assigns: the frame - /// already holds their values, and keeping each one live to every - /// exit only costs registers. - fn prune_writebacks(&mut self) { - for (var, st) in std::mem::take(&mut self.writebacks) { - if !self.assigned.get(var.index()).copied().unwrap_or(false) { - self.b.func.layout.remove_inst(st); + self.b.ins().store(trusted, v, self.locals_base, off); } } } From 6ce459c12a63b2f30eea37a0a906d85edc2a39eb Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:18:40 -0700 Subject: [PATCH 33/96] perf: let compiled loops keep pins whose objects live on elsewhere The temporary-pin soft limit bounds how many objects a long activation keeps alive past their last other reference. A poll at the limit now counts, once per pin, how many of the activation's temporary pins hold their object's last reference; while fewer than half a soft limit's worth do, the limit rises by another soft limit (up to half the hard cap) instead of leaving the native loop. A loop over a list's elements, or one that stores each object it builds, no longer leaves and re-enters every 4,096 objects. float_math leaves its native loops 10 times instead of 63 per 100,000 units; its 100,000-unit run retires 873M -> 847M instructions. --- crates/weavepy-vm/src/tier2.rs | 68 +++++++++++++++++++++++++++- tests/regrtest/test_jit_pin_limit.py | 64 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 tests/regrtest/test_jit_pin_limit.py diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index bec1818d..53e25689 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -2287,6 +2287,47 @@ impl Pin { /// One activation's pinned objects (RFC 0061/0065 WS5). type PinTable = Vec; +/// Whether a pin holds an object that something else keeps alive too (or +/// one that owns nothing): releasing it would free nothing and finalize +/// nothing, so holding it costs only the table's slot. +fn pin_is_shared(p: &Pin) -> bool { + match p { + Pin::List(l, _) => Rc::strong_count(l) > 1, + Pin::Obj(o) => match o { + Object::None | Object::Int(_) | Object::Float(_) | Object::Bool(_) => true, + Object::Instance(i) => Rc::strong_count(i) > 1, + Object::List(l) => Rc::strong_count(l) > 1, + _ => false, + }, + } +} + +/// At the soft pin limit, decide whether the activation may keep going: +/// the limit bounds how many objects a long activation keeps alive past +/// their last other reference (delaying their release and finalizers), so +/// pins whose objects are still referenced elsewhere (a loop over a list's +/// elements, results stored as they're made) don't count. When few of the +/// temporaries are last references, the limit rises by another soft +/// limit's worth (up to half the hard cap); otherwise the poll leaves. +/// Each pin is counted once, when the limit is first reached past it. +fn relax_pin_limit(ctx: &mut CallCtx) -> bool { + let from = ctx.pins_counted.max(ctx.entry_pin_count); + if let Some(fresh) = ctx.pins.get(from..) { + ctx.last_ref_pins += fresh.iter().filter(|p| !pin_is_shared(p)).count(); + } + ctx.pins_counted = ctx.pins.len(); + let temporaries = ctx.pins.len().saturating_sub(ctx.entry_pin_count); + // Half the hard cap stays free: a poll interval that pins heavily + // must still reach the next poll before the helpers refuse to pin. + if ctx.last_ref_pins < RUNTIME_PIN_SOFT_LIMIT / 2 + && ctx.pins.len() + RUNTIME_PIN_SOFT_LIMIT <= RUNTIME_PIN_CAP / 2 + { + ctx.pin_limit = temporaries + RUNTIME_PIN_SOFT_LIMIT; + return true; + } + false +} + /// RFC 0070 WS1 — hard cap on an activation's pin table. Object-lane /// attribute loads append pins for changing results (a list traversal /// appends one per node), so an unbounded loop needs a bound: at the cap the @@ -3527,6 +3568,14 @@ struct CallCtx { pins: PinTable, /// Entry roots do not count toward the temporary-pin soft limit. entry_pin_count: usize, + /// How many temporary pins this activation may hold before its next + /// poll leaves (see [`relax_pin_limit`]): the soft limit, raised while + /// most of them aren't the last reference to their object. + pin_limit: usize, + /// The temporary pins [`relax_pin_limit`] has counted so far, and how + /// many of those held their object's last reference when it did. + pins_counted: usize, + last_ref_pins: usize, /// Set only when a helper is committed to a reconstruction exit. /// Generated code exits before another native operation can run. pin_pressure_exit: bool, @@ -3674,7 +3723,7 @@ fn pin_reusing(v: &Object, pins: &mut PinTable, memo: &mut [u32; PIN_MEMO]) -> O impl CallCtx { fn temporary_pin_limit_reached(&self) -> bool { - self.pins.len().saturating_sub(self.entry_pin_count) >= RUNTIME_PIN_SOFT_LIMIT + self.pins.len().saturating_sub(self.entry_pin_count) >= self.pin_limit } /// RFC 0073 WS1 — re-resolve this activation's native-callee @@ -4087,7 +4136,7 @@ unsafe extern "C" fn wpjit_poll(frame: *mut JitFrame) -> i64 { ) { return 1; } - if ctx.temporary_pin_limit_reached() { + if ctx.temporary_pin_limit_reached() && !relax_pin_limit(ctx) { // The generated poll immediately spills live locals and the // boundary stack, then exits. No pin is freed while native // registers can still refer to its index. @@ -4906,6 +4955,9 @@ unsafe fn try_native_call( c.const_pins.clear(); } c.pin_pressure_exit = false; + c.pin_limit = RUNTIME_PIN_SOFT_LIMIT; + c.pins_counted = 0; + c.last_ref_pins = 0; if !c.obj_global_pins.is_empty() { c.obj_global_pins.clear(); } @@ -5206,6 +5258,9 @@ fn fresh_child(ctx: &CallCtx, nc: &NativeCallee, callee_key: *const CodeObject) pins: Vec::new(), entry_pin_count: 0, pin_pressure_exit: false, + pin_limit: RUNTIME_PIN_SOFT_LIMIT, + pins_counted: 0, + last_ref_pins: 0, obj_globals: nc.obj_globals.clone(), obj_global_pins: Vec::new(), attr_guards: nc.attr_guards.clone(), @@ -11008,6 +11063,9 @@ pub(crate) fn try_call_native_direct( pins, entry_pin_count, pin_pressure_exit: false, + pin_limit: RUNTIME_PIN_SOFT_LIMIT, + pins_counted: 0, + last_ref_pins: 0, obj_globals: entry.art.obj_globals.clone(), obj_global_pins: Vec::new(), attr_guards: entry.art.attr_guards.clone(), @@ -11817,6 +11875,9 @@ fn enter_compiled( pins, entry_pin_count, pin_pressure_exit: false, + pin_limit: RUNTIME_PIN_SOFT_LIMIT, + pins_counted: 0, + last_ref_pins: 0, obj_globals: entry.obj_globals.clone(), obj_global_pins: Vec::new(), attr_guards: entry.attr_guards.clone(), @@ -12703,6 +12764,9 @@ fn resume_parked(interp: &mut super::Interpreter, frame: &mut super::Frame) -> J pins, entry_pin_count: act.entry_pin_count, pin_pressure_exit: false, + pin_limit: RUNTIME_PIN_SOFT_LIMIT, + pins_counted: 0, + last_ref_pins: 0, obj_globals: entry.obj_globals.clone(), obj_global_pins: std::mem::take(&mut act.obj_global_pins), attr_guards: entry.attr_guards.clone(), diff --git a/tests/regrtest/test_jit_pin_limit.py b/tests/regrtest/test_jit_pin_limit.py new file mode 100644 index 00000000..14856e1f --- /dev/null +++ b/tests/regrtest/test_jit_pin_limit.py @@ -0,0 +1,64 @@ +"""A long compiled loop bounds how many dead objects its pins keep alive. + +Compiled code pins the objects it handles. A loop over objects something +else keeps alive (a list's elements) may hold many pins; a loop whose pins +are the last references must leave and release them every few thousand, +so finalizers run with a bounded delay. +""" + +import unittest + + +class Probe: + alive = 0 + + def __init__(self): + Probe.alive += 1 + + def __del__(self): + Probe.alive -= 1 + + def ping(self): + return 1 + + +def make(): + return Probe() + + +def churn(n, peaks): + t = 0 + for i in range(n): + p = make() + t += p.ping() + if i % 1000 == 0: + peaks.append(Probe.alive) + return t + + +def walk(items): + t = 0 + for p in items: + t += p.ping() + return t + + +class PinLimitTests(unittest.TestCase): + def test_dead_objects_are_released_as_the_loop_runs(self): + peaks = [] + self.assertEqual(churn(40000, peaks), 40000) + # The soft limit is a few thousand pins; never most of the loop. + self.assertLess(max(peaks), 12000) + self.assertEqual(Probe.alive, 0) + + def test_shared_objects_stay_pinned_correctly(self): + items = [Probe() for _ in range(40000)] + self.assertEqual(walk(items), 40000) + self.assertEqual(walk(items[::-1]), 40000) + self.assertEqual(Probe.alive, 40000) + del items + self.assertEqual(Probe.alive, 0) + + +if __name__ == "__main__": + unittest.main() From f42ad95f72a235b7de9c61cc5f02c2b30f5097e4 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:46:29 -0700 Subject: [PATCH 34/96] perf: size a native constructor's fields up front and keep big spare pin tables A class constructed through the native constructor path whose instances share attribute names now gets its fresh instance's split values sized for all of them, names adopted, before `__init__` runs: the compiled constructor then appends its first field in line too, instead of calling the helper to allocate the values. An `__init__` that sets other names, or none, takes the same paths as before. The spare pin table a finished activation leaves for the next one may now be as large as an activation grows before a pressure exit, so a long loop doesn't regrow its table after each one. float_math's per-unit instructions drop from 6,947 to 6,728 (its 100,000-unit run: 847M -> 819M). --- crates/weavepy-vm/src/inst_dict.rs | 28 ++++++++++++++++ crates/weavepy-vm/src/tier2.rs | 19 +++++++++-- tests/regrtest/test_jit_inline_new_fields.py | 35 ++++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs index f0945c5d..5485a91e 100644 --- a/crates/weavepy-vm/src/inst_dict.rs +++ b/crates/weavepy-vm/src/inst_dict.rs @@ -456,6 +456,34 @@ impl SplitValues { self.push(keys, want, value); } + /// Make room for `cap` values before the first one arrives, adopting + /// the class's names (`keys`): a constructor that sets its fields in the + /// class's order then appends each in place, the first included (see + /// [`Self::append_over`] and compiled code's new-key stores). Nothing + /// when a block exists (values, or a recycled one) or `cap` is zero. + pub(crate) fn reserve_for(&mut self, keys: impl FnOnce() -> Rc, cap: usize) { + if self.block.is_some() || cap == 0 { + return; + } + let layout = Self::layout(cap); + // SAFETY: a nonzero-size layout (the header alone has a size). + #[allow(clippy::cast_ptr_alignment)] + let h = unsafe { std::alloc::alloc(layout) }.cast::(); + let Some(h) = std::ptr::NonNull::new(h) else { + std::alloc::handle_alloc_error(layout); + }; + // SAFETY: `h` is a fresh block of `cap` slots; the header owns one + // strong count of the names. + unsafe { + h.as_ptr().write(SplitHeader { + keys: Rc::into_raw(keys()), + len: 0, + cap: cap as u32, + }); + } + self.block = Some(h); + } + /// Whether [`Self::append_over`] of position `i` of `keys` succeeds /// once the positions from the current length up to `i` have been /// appended first (the next of a run of in-order appends). diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 53e25689..e4ba8a1d 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -3958,9 +3958,10 @@ thread_local! { static SPARE_PINS: Cell = const { Cell::new(Vec::new()) }; } -/// The most pins [`SPARE_PINS`] keeps capacity for (the soft limit, plus -/// the entry pins a pressure exit's activation also carried). -const SPARE_PINS_CAP: usize = 2 * RUNTIME_PIN_SOFT_LIMIT; +/// The most pins [`SPARE_PINS`] keeps capacity for: as many as an +/// activation reaches before a pressure exit (see [`relax_pin_limit`]), +/// with room for its entry pins (under a megabyte). +const SPARE_PINS_CAP: usize = RUNTIME_PIN_CAP / 2 + 2 * RUNTIME_PIN_SOFT_LIMIT; /// An empty pin table, with the spare's capacity when there is one. fn take_pins() -> PinTable { @@ -5540,6 +5541,18 @@ unsafe fn try_native_ctor( // Threshold collection ran finalizers — arbitrary Python. ctx.dirty = true; } + // Room for every field the class's instances set, so the compiled + // `__init__` appends even its first one in line. + if let (Object::Instance(i), Some(keys)) = (&inst, cls.shared_keys.get()) { + let n = keys.len(); + if n > 0 && cls.native_kind.get() == 0 && i.dict.published().is_none() { + // SAFETY: the instance is fresh and unshared; nothing else + // borrows its values. + if let Some(split) = unsafe { i.dict.split_cell().peek_mut() } { + split.reserve_for(|| cls.shared_keys.share(), n); + } + } + } // `__init__` is a procedure: its `None` rides the procedure lane. // A `try_native_call` rejection discards the fresh (empty, never // `__init__`-ed) instance and re-allocates on the interpreter path. diff --git a/tests/regrtest/test_jit_inline_new_fields.py b/tests/regrtest/test_jit_inline_new_fields.py index ffc60ecd..a4157a2c 100644 --- a/tests/regrtest/test_jit_inline_new_fields.py +++ b/tests/regrtest/test_jit_inline_new_fields.py @@ -92,6 +92,41 @@ def __init__(self, i): self.assertEqual(seen, [5]) self.assertEqual(r.k, 0) + def test_instances_that_set_fewer_fields(self): + class S: + def __init__(self, i, full=True): + if full: + self.a = i + self.b = 2 * i + + def make(n, full): + return [S(i, full) for i in range(n)] + + full = make(self.N, True) + self.assertEqual(vars(full[3]), {"a": 3, "b": 6}) + empty = make(10, False) + self.assertEqual(vars(empty[3]), {}) + empty[3].b = 1 + empty[3].a = 2 + self.assertEqual(list(vars(empty[3]).items()), [("b", 1), ("a", 2)]) + self.assertEqual(vars(make(1, True)[0]), {"a": 0, "b": 0}) + + def test_constructor_raising_midway(self): + class E: + def __init__(self, i): + self.a = i + if i < 0: + raise ValueError(i) + self.b = i + + def make(n): + return [E(i) for i in range(n)] + + self.assertEqual(vars(make(self.N)[5]), {"a": 5, "b": 5}) + with self.assertRaises(ValueError): + E(-1) + self.assertEqual(vars(make(3)[2]), {"a": 2, "b": 2}) + if __name__ == "__main__": unittest.main() From ac52e57b40f237c96b4d891c87cf983777e27f1d Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:12:02 -0700 Subject: [PATCH 35/96] perf: compile generator bodies the fast steps run without waiting on tier 2 The frame compiler waits for tier 2 to settle a code object's loops before taking them, but a generator body that the fast steps run never reaches the core loop's back-edge consultation, so tier 2 never settles it and the body stayed interpreted (generators' `_squared`). What warmed a code object is now passed along (an activation, a back edge, or a fast step), and a fast step's body compiles once hot. generators: -5% instructions per unit. --- crates/weavepy-vm/src/frame_jit.rs | 40 ++++++++++++++++++++---------- crates/weavepy-vm/src/gen_fast.rs | 2 +- crates/weavepy-vm/src/lib.rs | 4 +-- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs index 62bb9b8b..76e9b3a8 100644 --- a/crates/weavepy-vm/src/frame_jit.rs +++ b/crates/weavepy-vm/src/frame_jit.rs @@ -186,12 +186,10 @@ impl Slot { } } - /// Count one activation or back edge of code without native code, - /// compiling it once it is hot. + /// Count one activation, back edge or generator step of code without + /// native code, compiling it once it is hot. #[inline(always)] - /// Count a call (`at` 0) or a back edge (`at` its pc) toward - /// compiling `code`. - pub(crate) fn warm(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize, at: usize) { + pub(crate) fn warm(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize, at: Heat) { if self.native.get().is_some() { return; } @@ -206,15 +204,17 @@ impl Slot { #[cold] #[inline(never)] - fn try_compile(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize, at: usize) { + fn try_compile(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize, at: Heat) { if !enabled() { let _ = self.native.set(None); return; } // Loops the tier-2 compiler may still take keep their back edges' - // consultation in the core loop. + // consultation in the core loop (a generator body the fast steps + // run never reaches it). let hint = &code.jit_hint; - if !(hint.is_not_jitable() + if !(matches!(at, Heat::Step) + || hint.is_not_jitable() || crate::tier2::jit_off_for_process() || hint.loop_free(code) || hint.is_backedge_quiet()) @@ -245,13 +245,24 @@ impl Slot { } } -/// Whether the loop of `code` whose back edge is at `at` (any loop, for -/// a call) runs mostly in native code: each +/// What warmed a code object. +#[derive(Clone, Copy)] +pub(crate) enum Heat { + /// An activation. + Call, + /// The back edge at this pc. + BackEdge(usize), + /// A generator's fast step (see `gen_fast`). + Step, +} + +/// Whether the loop of `code` that got hot (any loop, for a call or a +/// generator step) runs mostly in native code: each /// instruction the core loop runs costs a round trip out of the native /// code and back, so a loop of calls and global loads gains nothing from /// compiling (nor does a loop-free body, whose entry costs as much as the /// little it would save). -fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: usize) -> bool { +fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool { use weavepy_compiler::OpCode; let ins = &code.instructions; // A method call on a local runs through a helper without leaving, when @@ -268,9 +279,12 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: usize) -> bool } } } - let at_loop = ins.get(at).is_some_and(|i| i.op == OpCode::JumpBackward); + let at_loop = match at { + Heat::BackEdge(pc) => Some(pc).filter(|&pc| ins.get(pc).is_some_and(|i| i.op == OpCode::JumpBackward)), + Heat::Call | Heat::Step => None, + }; ins.iter().enumerate().any(|(pc, i)| { - if i.op != OpCode::JumpBackward || (at_loop && pc != at) { + if i.op != OpCode::JumpBackward || at_loop.is_some_and(|at| pc != at) { return false; } let top = (pc + 1).saturating_sub(i.arg as usize); diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs index a56c311d..dd9fa54b 100644 --- a/crates/weavepy-vm/src/gen_fast.rs +++ b/crates/weavepy-vm/src/gen_fast.rs @@ -378,7 +378,7 @@ impl Interpreter { let native = { let nlocals = locals.len(); if ext.frame_jit.get(nlocals).is_none() { - ext.frame_jit.warm(code, ext, nlocals, 0); + ext.frame_jit.warm(code, ext, nlocals, crate::frame_jit::Heat::Step); } ext.frame_jit.get(nlocals) }; diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 7556f24b..46635726 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -10211,7 +10211,7 @@ impl Interpreter { #[cfg(feature = "jit")] if pc == 0 && !NATIVE { if let Some(ext) = ext { - ext.frame_jit.warm(code, ext, nlocals, 0); + ext.frame_jit.warm(code, ext, nlocals, frame_jit::Heat::Call); } } #[cfg(feature = "jit")] @@ -11120,7 +11120,7 @@ impl Interpreter { #[cfg(feature = "jit")] if !NATIVE { if let Some(ext) = ext { - ext.frame_jit.warm(code, ext, nlocals, last); + ext.frame_jit.warm(code, ext, nlocals, frame_jit::Heat::BackEdge(last)); if ext.frame_jit.get(nlocals).is_some() { break Some(CoreExit::Switch); } From 06f2bb70c5f24bc681392687050a903bc7bc3745 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:17:37 -0700 Subject: [PATCH 36/96] perf: allocate the frame compiler's registers with the backtracking allocator The single-pass allocator spilled around every helper call, which cost more at run time than the backtracking one costs to compile: with it, generators -3% (default work) to -10% (8x), deque_ops -4%, dict_ops at 4x -2%; deltablue at the default work +2% (compile), the rest within noise. Recompiling hot code with Cranelift's mid-end on top gained nothing over this, so there is one tier. --- crates/weavepy-vm/src/frame_jit.rs | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs index 76e9b3a8..f382ae4f 100644 --- a/crates/weavepy-vm/src/frame_jit.rs +++ b/crates/weavepy-vm/src/frame_jit.rs @@ -144,18 +144,18 @@ unsafe impl Send for Native {} unsafe impl Sync for Native {} impl Native { - /// Run from `st.pc` until an instruction the core loop must run. - /// - /// # Safety - /// - /// `st` is the running activation of the code this was compiled - /// from, as the core loop holds it, with `nlocals` locals. /// Whether entering at `pc` gets anywhere. #[inline(always)] pub(crate) fn enters_at(&self, pc: usize) -> bool { self.entries.get(pc).copied().unwrap_or(false) } + /// Run from `st.pc` until an instruction the core loop must run. + /// + /// # Safety + /// + /// `st` is the running activation of the code this was compiled + /// from, as the core loop holds it, with `nlocals` locals. #[inline(always)] pub(crate) unsafe fn run(&self, st: &mut State) -> u32 { let from = st.pc; @@ -790,14 +790,10 @@ impl Engine { flags.set("use_colocated_libcalls", "false").ok()?; flags.set("is_pic", "false").ok()?; // The lowering already does the optimizing that pays here (operands - // kept unboxed and in registers); Cranelift's own passes cost more - // to run than they save. `WEAVEPY_FRAME_JIT_OPT` turns them on. - if std::env::var_os("WEAVEPY_FRAME_JIT_OPT").is_some() { - flags.set("opt_level", "speed").ok()?; - } else { - flags.set("opt_level", "none").ok()?; - flags.set("regalloc_algorithm", "single_pass").ok()?; - } + // kept unboxed and in registers): Cranelift's mid-end costs more to + // run than it saves. Its register allocator pays for itself (the + // single-pass one spills around every helper call). + flags.set("opt_level", "none").ok()?; if !cfg!(debug_assertions) && !verify() { flags.set("enable_verifier", "false").ok()?; } From eab0132b57b21ea3865dff74bea62ad64839c87f Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:30:43 -0700 Subject: [PATCH 37/96] perf: step nested generators without reference churn or a thread-local read A fast step of a generator the step iterates cloned its reference (two atomic counts) though the iterating frame's stack slot keeps it alive and the inner step runs no code that could reach that frame, and every step read the thread's queued-finalizer flag from thread-local storage. The flag is now read once per resume and handed down the nested steps, and the nested generator is borrowed from its slot. generators: -3% instructions per unit (now level with CPython 3.14). test_genexps' tuple-reuse doctest still fails, as on main. --- crates/weavepy-vm/src/gen_fast.rs | 31 ++++++++++++++++++------------- crates/weavepy-vm/src/lib.rs | 6 +++--- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs index dd9fa54b..0c6779f9 100644 --- a/crates/weavepy-vm/src/gen_fast.rs +++ b/crates/weavepy-vm/src/gen_fast.rs @@ -335,13 +335,15 @@ impl Interpreter { /// back when it ends. /// /// With `fold` (a draining consumer's sink, top level only), a yield - /// the sink takes resumes the body at once with `None` sent. + /// the sink takes resumes the body at once with `None` sent. `dead` is + /// the thread's queued-finalizer flag (`gc_trace::maybe_dead_flag`). pub(crate) fn gen_fast_step( &mut self, frame: &mut Frame, snap_gen: u64, depth: u8, fold: Option, + dead: *const std::cell::Cell, ) -> GenStep { // SAFETY: the frame's code is immutable and outlives the step (the // frame holds it, and nothing here replaces it). @@ -392,12 +394,7 @@ impl Interpreter { last: pc, countdown: std::ptr::addr_of_mut!(self.gil_countdown), snap_gen, - // (Only read when the native form runs.) - maybe_dead: if native.is_some() { - crate::gc_trace::maybe_dead_flag() - } else { - std::ptr::null() - }, + maybe_dead: dead, interp: std::ptr::from_ref(self), out: 0, depth_cell: std::ptr::null(), @@ -620,7 +617,7 @@ impl Interpreter { } } } - match self.gen_fast_for_iter(top, snap_gen, depth) { + match self.gen_fast_for_iter(top, snap_gen, depth, dead) { ForNext::Value(v) => { unsafe { base.add(len).write(v) }; len += 1; @@ -691,7 +688,13 @@ impl Interpreter { /// value, a range's end (when the loop alone holds it), or a fast /// generator's next yield. #[inline(never)] - fn gen_fast_for_iter(&mut self, top: &Object, snap_gen: u64, depth: u8) -> ForNext { + fn gen_fast_for_iter( + &mut self, + top: &Object, + snap_gen: u64, + depth: u8, + dead: *const std::cell::Cell, + ) -> ForNext { match top { Object::Iter(it) => { let unique = Rc::strong_count(it) == 1; @@ -742,9 +745,10 @@ impl Interpreter { _ => ForNext::Bail, } } + // (The stack slot keeps `g` alive: the inner step runs no code + // that could reach this frame.) Object::Generator(g) => { - let g = g.clone(); - match self.gen_fast_next(&g, snap_gen, depth + 1) { + match self.gen_fast_next(g, snap_gen, depth + 1, dead) { GenNext::Yielded(v) => ForNext::Value(v), GenNext::Declined | GenNext::Partial => ForNext::Bail, } @@ -766,7 +770,7 @@ impl Interpreter { if !Self::gen_fast_frame_ok(frame) { return None; } - match self.gen_fast_step(frame, snap_gen, 0, fold) { + match self.gen_fast_step(frame, snap_gen, 0, fold, crate::gc_trace::maybe_dead_flag()) { GenStep::Yielded(v) => Some(v), GenStep::Bail => None, } @@ -780,6 +784,7 @@ impl Interpreter { g: &Rc, snap_gen: u64, depth: u8, + dead: *const std::cell::Cell, ) -> GenNext { if depth > MAX_DEPTH || crate::recursion::current_depth() + usize::from(depth) + 2 @@ -818,7 +823,7 @@ impl Interpreter { frame.stack.push(Object::None); } debug_assert!(!frame.stack.is_empty()); - let out = match self.gen_fast_step(frame, snap_gen, depth, None) { + let out = match self.gen_fast_step(frame, snap_gen, depth, None, dead) { GenStep::Yielded(v) => GenNext::Yielded(v), GenStep::Bail if frame.pc == start => { // Nothing ran: the resume is undone. diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 46635726..8cb66f3e 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -11172,9 +11172,10 @@ impl Interpreter { Object::Generator(g) => { // A simple body runs to its next yield in // place, without switching (see `gen_fast`). - let g = g.clone(); + // (The stack slot keeps `g` alive: the step + // runs no code that could reach this frame.) if let gen_fast::GenNext::Yielded(v) = - self.gen_fast_next(&g, snap_gen, 0) + self.gen_fast_next(g, snap_gen, 0, pending_work) { // SAFETY: `len < cap` (checked above). unsafe { base.add(len).write(v) }; @@ -11183,7 +11184,6 @@ impl Interpreter { pc += 1; continue; } - drop(g); // SAFETY: `len <= cap`, every slot initialized. unsafe { frame.stack.set_len(len) }; frame.pc = pc as u32; From 8faf477a82df5ae4276e9f295d1924c660dcbab5 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 04:27:12 -0700 Subject: [PATCH 38/96] perf: evaluate pure-leaf functions in place when native code calls them A builtin calling back into Python (a `key=` function for `sorted`, `min` or `max`, and every other `Interpreter::call` of a plain function) built a full activation even when the callee was a pure leaf the core loop evaluates without one. `call_python` now tries the leaf evaluation first, under the same guards as the core loop's (no observers, inline calls allowed, below the recursion limit, leaf warm), and falls back untouched when it declines. `min(data) + max(data, key=lambda x: -x)` over 50 ints: -57% (1.15x CPython, from 2.7x). --- crates/weavepy-vm/src/lib.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 8cb66f3e..c0142fb5 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -46937,9 +46937,40 @@ impl Interpreter { args: &[Object], kwargs: &[(String, Object)], ) -> Result { + if kwargs.is_empty() { + if let Some(v) = self.call_pure_leaf(f, args) { + return Ok(v); + } + } self.call_python_owned(f, args.to_vec(), kwargs.to_vec()) } + /// A native caller's positional call of a pure leaf (a `key=` + /// function, a `map` callable, ...), evaluated in place as the core + /// loop evaluates one (see [`Self::pure_leaf_eval`]): `None` leaves the + /// call to the full machinery, nothing having happened. + #[inline] + fn call_pure_leaf(&self, f: &PyFunction, args: &[Object]) -> Option { + const MAX_ARGS: usize = 6; + // SAFETY: a function's code is only replaced under the GIL, and + // nothing here runs code that could replace it. + let code = unsafe { &*f.code.as_ptr() }; + if args.len() > MAX_ARGS + || !code_is_pure_leaf(code) + || !pure_leaf_warm(code) + || crate::trace::any_observers_active() + || !self.inline_calls_ok() + || crate::recursion::current_depth() >= crate::recursion::recursion_limit() + { + return None; + } + let mut ptrs = [std::ptr::null::(); MAX_ARGS]; + for (p, a) in ptrs.iter_mut().zip(args) { + *p = a; + } + self.pure_leaf_eval::(code, f, &ptrs[..args.len()]) + } + /// The full argument binder for [`Self::call_python_owned`]: CPython's /// `initialize_locals` order (positional move, keyword loop with its /// unexpected-keyword / multiple-values errors, too-many-positional, From 70dd3d22a41157f24b5f23c1d2ab83088141eaac Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:06:22 -0700 Subject: [PATCH 39/96] perf: evaluate subscripts in frameless leaf bodies A leaf body with a subscript (`key=lambda t: t[1]`, a getter returning `self.items[i]`) wasn't a pure leaf, so every call built an activation. Leaf plans now run `BINARY_SUBSCR` for a list's or tuple's item at an int index, a dict's value for a str or int key (through the core loop's probe, which runs no `__eq__`), and an ASCII string's character; any other shape, a miss included, declines to the full call, which raises or dispatches as before. The plan compiler calls a helper for it. `sorted(words, key=lambda w: w[1])` over 16 words: -68% (1.4x CPython, from 4.8x); the fixtures within noise. --- crates/weavepy-vm/src/leaf_plan.rs | 61 ++++++++++++++++++++++ crates/weavepy-vm/src/leaf_plan/jit.rs | 24 ++++++++- crates/weavepy-vm/src/lib.rs | 1 + tests/regrtest/test_leaf_subscripts.py | 70 ++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 tests/regrtest/test_leaf_subscripts.py diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index fecf6478..9f728632 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -173,6 +173,12 @@ enum Op { b: u8, kind: u8, }, + /// `regs[dst] = regs[a][regs[b]]` (see [`subscr`]). + Subscr { + dst: u8, + a: u8, + b: u8, + }, /// Exchange two registers. Swap { a: u8, @@ -503,6 +509,12 @@ impl Builder<'_> { kind: arg as u8, }); } + OpCode::BinarySubscr => { + let b = self.pop()?; + let a = self.pop()?; + let dst = self.push_new()?; + self.ops.push(Op::Subscr { dst, a, b }); + } OpCode::CopyTop => { let n = (arg as usize).max(1); let depth = self.stack.len(); @@ -1044,6 +1056,7 @@ impl Interpreter { Op::Truth { dst, src } => set!(dst, V::B(truth(get!(src))?)), Op::Unary { dst, src, kind } => set!(dst, unary(get!(src), kind)?), Op::Binary { dst, a, b, kind } => set!(dst, binary(get!(a), get!(b), kind)?), + Op::Subscr { dst, a, b } => set!(dst, subscr(&mut owned, get!(a), get!(b))?), Op::Swap { a, b } => { let (x, y) = (get!(a), get!(b)); set!(a, y); @@ -1352,6 +1365,54 @@ pub(crate) fn binary(a: V, b: V, kind: u8) -> Option { }) } +/// `a[b]` that can't run code or raise: a list's or tuple's item at an +/// int index (borrowed: a leaf mutates no container), a dict's value for +/// a `str` or `int` key (the core loop's probe, which compares no other +/// key type), or an ASCII string's character (held by `owned`). Anything +/// else, a miss included, declines. +#[inline] +fn subscr(owned: &mut Owned, a: V, b: V) -> Option { + let V::R(c) = a else { + return None; + }; + let index = |i: i64, n: usize| { + let n = n as i64; + let i = if i < 0 { i + n } else { i }; + (0..n).contains(&i).then_some(i as usize) + }; + // SAFETY: as `norm`; the borrows end before anything could change the + // containers (no Python runs during an evaluation). + match (unsafe { &*c }, b) { + (Object::List(l), V::I(i)) => { + let xs = unsafe { l.peek() }?; + Some(norm(&xs[index(i, xs.len())?])) + } + (Object::Tuple(t), V::I(i)) => Some(norm(&t[index(i, t.len())?])), + (Object::Str(s), V::I(i)) => { + if crate::object::str_char_len(s) != s.len() { + return None; + } + owned.own(Object::from_char(s.as_bytes()[index(i, s.len())?] as char)) + } + (Object::Dict(d), V::I(_) | V::R(_)) => { + let int; + let key = match b { + V::I(i) => { + int = Object::Int(i); + &int + } + // SAFETY: as above. + V::R(k) if matches!(unsafe { &*k }, Object::Str(_)) => unsafe { &*k }, + _ => return None, + }; + let probe = crate::object::LeafProbe::new(key)?; + let v = unsafe { d.peek() }?.get(&probe)?; + Some(norm(v)) + } + _ => None, + } +} + /// `BUILD_LIST 0` / `BUILD_MAP 0`: a new empty list or dict, tracked like /// the core loop's and held by the owned scratch. An allocation that would /// trigger a collection is left to the core loop. diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index 799de7b5..cb8333a4 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -16,8 +16,8 @@ //! natively too once the callee's own plan is compiled). use super::{ - binary, compare, is_same, new_container, norm, plan_store, truth, unary, LeafPlan, LeafRet, Op, - Owned, Pending, NEST, V, + binary, compare, is_same, new_container, norm, plan_store, subscr, truth, unary, LeafPlan, + LeafRet, Op, Owned, Pending, NEST, V, }; use crate::object::{Object, PyFunction}; use crate::sync::Rc; @@ -514,6 +514,19 @@ unsafe extern "C" fn h_new(ctx: *mut Ctx<'static>, dict: u32) -> u32 { } } +/// `a[b]` (see [`subscr`]). +unsafe extern "C" fn h_subscr(ctx: *mut Ctx<'static>, ta: u64, pa: u64, tb: u64, pb: u64) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + match subscr(&mut c.owned, value(ta, pa), value(tb, pb)) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + /// The evaluation's own `recv.name = val` (only the top frame stores). unsafe extern "C" fn h_store( ctx: *mut Ctx<'static>, @@ -1539,6 +1552,13 @@ impl Lower<'_> { self.check(st); self.set_out(fl, dst, 0); } + Op::Subscr { dst, a, b } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + let st = self.call(h_subscr as *const () as usize, &[self.ctx, ta, pa, tb, pb]); + self.check(st); + self.set_out(fl, dst, 0); + } Op::Decline => { self.b.ins().jump(self.decline, &[]); return Some(false); diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index c0142fb5..9ebaa95e 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -62135,6 +62135,7 @@ fn code_pure_leaf_decide(code: &CodeObject, ext: &CodeConstObjects) -> bool { | OpCode::PopJumpIfNotNone | OpCode::JumpForward | OpCode::BinaryOp + | OpCode::BinarySubscr | OpCode::CopyTop | OpCode::Swap | OpCode::PopTop diff --git a/tests/regrtest/test_leaf_subscripts.py b/tests/regrtest/test_leaf_subscripts.py new file mode 100644 index 00000000..fba1993a --- /dev/null +++ b/tests/regrtest/test_leaf_subscripts.py @@ -0,0 +1,70 @@ +"""Subscripts in frameless leaf evaluation. + +Small functions such as `key=lambda t: t[1]` are evaluated without an +activation when native code calls them. A subscript there must give the +same result as the full call, and anything it can't settle (a miss, an +unusual container) must fall back so the full call raises or dispatches +exactly as usual. +""" + +import unittest + + +class LeafSubscripts(unittest.TestCase): + def test_sort_keys(self): + pairs = [(3, "c"), (1, "a"), (2, "b")] * 50 + self.assertEqual(sorted(pairs, key=lambda t: t[1])[:3], [(1, "a")] * 3) + self.assertEqual(sorted(pairs, key=lambda t: t[-2])[-1], (3, "c")) + rows = [[i, -i] for i in range(100)] + self.assertEqual(max(rows, key=lambda r: r[1]), [0, 0]) + words = ["delta", "alpha", "charlie", "bravo"] * 30 + self.assertEqual(sorted(words, key=lambda w: w[1])[0], "delta") + self.assertEqual(min(words, key=lambda w: w[-1]), "delta") + + def test_non_ascii_strings(self): + words = ["żółw", "ąb", "éa", "zz"] * 30 + self.assertEqual(sorted(words, key=lambda w: w[1])[0], "éa") + + def test_dicts(self): + tables = [{"k": i, 7: -i} for i in range(60)] + self.assertEqual(max(tables, key=lambda d: d["k"])["k"], 59) + self.assertEqual(max(tables, key=lambda d: d[7])["k"], 0) + + def test_misses_raise(self): + items = [(1,), (2,)] * 40 + with self.assertRaises(IndexError): + sorted(items, key=lambda t: t[1]) + dicts = [{"a": 1}] * 40 + with self.assertRaises(KeyError): + sorted(dicts, key=lambda d: d["b"]) + strs = ["a", "bb"] * 40 + with self.assertRaises(IndexError): + sorted(strs, key=lambda s: s[1]) + + def test_subclasses_dispatch(self): + class D(dict): + def __missing__(self, key): + return 42 + + class L(list): + def __getitem__(self, i): + return -super().__getitem__(i) + + ds = [D(a=i) for i in range(40)] + self.assertEqual(sorted(ds, key=lambda d: d["zz"])[0]["a"], 0) + ls = [L([i]) for i in range(40)] + self.assertEqual(max(ls, key=lambda x: x[0]), [0]) + + def test_warm_loop(self): + def second(t): + return t[1] + + data = [(i, i * i) for i in range(30)] + total = 0 + for _ in range(3000): + total += max(map(second, data)) + self.assertEqual(total, 3000 * 29 * 29) + + +if __name__ == "__main__": + unittest.main() From fcfecbfbfee59258b6d030e4bfc4f3a76709cf0e Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:32:26 -0700 Subject: [PATCH 40/96] perf: build and step map and filter natively over native iterables `map(f, xs)` and `filter(f, xs)` construct the lazy `_seqtools` classes, whose `__new__` and per-item `__next__` ran Python: making a map cost about 74K instructions and each item about 20K. The common shape (no keywords, every iterable a list, tuple, str or range, or already an iterator) is now built directly, and an exact instance over one native iterator is stepped by the interpreter (`iter_next` and the full `FOR_ITER` arm), calling the function once per item as `__next__` would. Anything else, and every traced run, takes the class's methods. `list(map(str, data[:10]))`: -89% (275K to 30K instructions); `map(str, xs)` construction: -86%. --- crates/weavepy-vm/src/lib.rs | 151 ++++++++++++++++++++++- tests/regrtest/test_native_map_filter.py | 86 +++++++++++++ 2 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 tests/regrtest/test_native_map_filter.py diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 9ebaa95e..5f74ddda 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -928,6 +928,9 @@ pub struct Interpreter { /// (process-unique) attribute version of the class that resolved it /// (see [`Interpreter::core_leaf_next`]). core_next: ThreadCell>)>>, + /// `_seqtools`' lazy `map` and `filter` classes, once imported (see + /// [`Interpreter::seqtools_step`]). + seqtools: std::cell::OnceCell>, /// How the last class seen in a boolean context answers it (see /// [`Interpreter::leaf_instance_truth`]), by attribute version. core_truth: ThreadCell>, @@ -1160,6 +1163,7 @@ impl Default for Interpreter { leaf_fns: std::cell::OnceCell::new(), leaf_opaque: ThreadCell::new((0, leaf_builtins::LeafMap::default())), core_next: ThreadCell::new(None), + seqtools: std::cell::OnceCell::new(), core_truth: ThreadCell::new(None), dbg_sample: crate::hot_gates::env_flags::dbg_sample(), }; @@ -1287,6 +1291,7 @@ impl Interpreter { leaf_fns: std::cell::OnceCell::new(), leaf_opaque: ThreadCell::new((0, leaf_builtins::LeafMap::default())), core_next: ThreadCell::new(None), + seqtools: std::cell::OnceCell::new(), core_truth: ThreadCell::new(None), dbg_sample: crate::hot_gates::env_flags::dbg_sample(), } @@ -21409,7 +21414,11 @@ impl Interpreter { } Err(e) => return Err(e), }, - Object::Instance(_) => { + Object::Instance(_) => 'next: { + // `map` and `filter` step natively when they can. + if let Some(r) = self.seqtools_step(&it_obj, &frame.globals) { + break 'next r?; + } // Call __next__; treat StopIteration as exhaustion. let next = match instance_native_dunder(&it_obj, "__next__", None) { Some(r) => Some(r), @@ -29909,6 +29918,13 @@ impl Interpreter { kwargs: &[(String, Object)], globals: &Rc>, ) -> Result, RuntimeError> { + // The common shape is built here (what its `__new__` would build): + // a callable over native containers or iterators, no keywords. + if kwargs.is_empty() { + if let Some(it) = self.seqtools_new(class_name, args, globals) { + return Ok(Some(it)); + } + } let module = match self.do_import("_seqtools", &Object::None, 0, globals) { Ok(m) => m, Err(_) => return Ok(None), @@ -29927,6 +29943,130 @@ impl Interpreter { } } + /// The `_seqtools` classes, importing them on first use. + fn seqtools(&mut self, globals: &Rc>) -> Option<&SeqTools> { + if self.seqtools.get().is_none() { + let module = self.do_import("_seqtools", &Object::None, 0, globals).ok(); + let class = |name: &str| match &module { + Some(Object::Module(m)) => match m.dict.borrow().get(&DictKey(Object::from_str(name))) { + Some(Object::Type(t)) => Some(t.clone()), + _ => None, + }, + _ => None, + }; + let tools = class("_MapIter") + .zip(class("_FilterIter")) + .map(|(map, filter)| SeqTools { map, filter }); + let _ = self.seqtools.set(tools); + } + self.seqtools.get()?.as_ref() + } + + /// A `map(func, *iterables)` or `filter(func, iterable)` instance + /// built without running its class's `__new__`, when every iterable is + /// a list, tuple, string or range (iterated natively) or already an + /// iterator; `None` leaves the call to the class. + fn seqtools_new( + &mut self, + class_name: &str, + args: &[Object], + globals: &Rc>, + ) -> Option { + let filter = match class_name { + "_MapIter" => false, + "_FilterIter" => true, + _ => return None, + }; + let (func, sources) = args.split_first()?; + if sources.is_empty() || (filter && sources.len() != 1) { + return None; + } + let mut iters = Vec::with_capacity(sources.len()); + for s in sources { + iters.push(match s { + Object::Iter(_) | Object::Generator(_) => s.clone(), + Object::List(_) | Object::Tuple(_) | Object::Str(_) | Object::Range(..) => { + Object::Iter(Rc::new(RefCell::new(s.make_iter().ok()?))) + } + _ => return None, + }); + } + let tools = self.seqtools(globals)?; + let cls = if filter { &tools.filter } else { &tools.map }; + let inst = crate::types::PyInstance::new(cls.clone()); + inst.slot_set("_func", func.clone()); + if filter { + inst.slot_set("_it", iters.pop()?); + } else { + inst.slot_set("_iters", Object::new_tuple(iters)); + inst.slot_set("_strict", Object::Bool(false)); + } + let obj = Object::Instance(Rc::new(inst)); + gc_trace::track(&obj); + Some(obj) + } + + /// The next value of an exact `map` or `filter` over one native + /// iterator, stepped here rather than through its `__next__`: + /// `Some(Ok(None))` at exhaustion, `None` when the class's own + /// `__next__` must run. + fn seqtools_step( + &mut self, + it: &Object, + globals: &Rc>, + ) -> Option, RuntimeError>> { + let Object::Instance(inst) = it else { + return None; + }; + let tools = self.seqtools.get()?.as_ref()?; + let cls = inst.cls_raw(); + let filter = if std::ptr::eq(cls, &*tools.map) { + false + } else if std::ptr::eq(cls, &*tools.filter) { + true + } else { + return None; + }; + if crate::trace::any_observers_active() { + return None; + } + let func = inst.slot_get("_func")?; + let source = if filter { + inst.slot_get("_it")? + } else { + match inst.slot_get("_iters")? { + Object::Tuple(t) if t.len() == 1 => t[0].clone(), + _ => return None, + } + }; + let Object::Iter(cursor) = &source else { + return None; + }; + loop { + let item = match cursor.borrow_mut().next_value_checked() { + Ok(Some(v)) => v, + Ok(None) => return Some(Ok(None)), + Err(e) => return Some(Err(e)), + }; + if !filter { + return Some(self.call(&func, std::slice::from_ref(&item), &[], globals).map(Some)); + } + let keep = match &func { + Object::None => self.op_truth(&item), + Object::Type(t) if Rc::ptr_eq(t, &builtin_types().bool_) => self.op_truth(&item), + f => match self.call(f, std::slice::from_ref(&item), &[], globals) { + Ok(r) => self.op_truth(&r), + Err(e) => Err(e), + }, + }; + match keep { + Ok(true) => return Some(Ok(Some(item))), + Ok(false) => {} + Err(e) => return Some(Err(e)), + } + } + } + /// `map(func, *iterables)` — VM-aware (the plain builtin can't call /// back into the interpreter). /// @@ -32785,6 +32925,9 @@ impl Interpreter { Err(e) => Err(e), }, Object::Instance(inst) => { + if let Some(r) = self.seqtools_step(iter, globals) { + return r; + } // A registered native leaf `__next__` (the class cache the // core loop's `FOR_ITER` uses): called directly. if !crate::gil::free_threading_enabled() && !crate::trace::any_observers_active() { @@ -72277,3 +72420,9 @@ print("native pickle coverage: ok") assert_eq!(run(src), "small\nsmall\nlarge\n"); } } + +/// The `_seqtools` classes behind `map` and `filter`. +struct SeqTools { + map: Rc, + filter: Rc, +} diff --git a/tests/regrtest/test_native_map_filter.py b/tests/regrtest/test_native_map_filter.py new file mode 100644 index 00000000..e0e24c58 --- /dev/null +++ b/tests/regrtest/test_native_map_filter.py @@ -0,0 +1,86 @@ +"""`map` and `filter` over native containers, stepped natively. + +The common shapes are built and stepped without running the lazy +classes' Python methods. Laziness, error timing, type identity and +pickling must stay exactly as with the classes' own methods. +""" + +import pickle +import unittest + + +class NativeMapFilter(unittest.TestCase): + def test_results(self): + data = list(range(20)) + self.assertEqual(list(map(str, data[:5])), ["0", "1", "2", "3", "4"]) + self.assertEqual(sum(map(abs, [-1, -2, 3])), 6) + self.assertEqual(list(map(lambda x: x * 2, (1, 2, 3))), [2, 4, 6]) + self.assertEqual("".join(map(str.upper, "abc")), "ABC") + self.assertEqual(list(map(lambda a, b: a + b, [1, 2], [10, 20, 30])), [11, 22]) + self.assertEqual(list(filter(None, [0, 1, "", "x", None])), [1, "x"]) + self.assertEqual(list(filter(bool, [0, 2, 0.0, 3])), [2, 3]) + self.assertEqual(list(filter(lambda x: x % 3 == 0, range(10))), [0, 3, 6, 9]) + total = 0 + for v in map(lambda x: x + 1, data): + total += v + self.assertEqual(total, sum(data) + len(data)) + + def test_type_identity(self): + m = map(str, [1]) + f = filter(None, [1]) + # (WeavePy's `map` and `filter` names are functions building these + # classes, so the types are compared with each other.) + self.assertIs(type(m), type(map(abs, iter([])))) + self.assertIs(type(f), type(filter(abs, iter([])))) + self.assertIs(iter(m), m) + self.assertEqual(type(m).__name__, "map") + self.assertEqual(type(f).__name__, "filter") + + def test_lazy_errors(self): + def boom(x): + if x == 2: + raise ValueError("two") + return x + + it = map(boom, [1, 2, 3]) + self.assertEqual(next(it), 1) + with self.assertRaises(ValueError): + next(it) + self.assertEqual(next(it), 3) + with self.assertRaises(StopIteration): + next(it) + fi = filter(boom, [1, 2]) + self.assertEqual(next(fi), 1) + with self.assertRaises(ValueError): + next(fi) + + def test_shared_iterator(self): + src = iter([1, 2, 3, 4]) + m = map(lambda x: x * 10, src) + self.assertEqual(next(m), 10) + self.assertEqual(next(src), 2) + self.assertEqual(list(m), [30, 40]) + + def test_pickle_round_trip(self): + m = map(abs, [-1, -2, -3]) + next(m) + self.assertEqual(list(pickle.loads(pickle.dumps(m))), [2, 3]) + f = filter(None, [0, 5, 0, 6]) + self.assertEqual(list(pickle.loads(pickle.dumps(f))), [5, 6]) + + def test_side_effect_order(self): + seen = [] + + def note(x): + seen.append(x) + return x + + m = map(note, range(5)) + self.assertEqual(seen, []) + next(m) + next(m) + self.assertEqual(seen, [0, 1]) + + +if __name__ == "__main__": + unittest.main() From c8ef39f9be00412f991cc62377b159adaccc014e Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:10:18 -0700 Subject: [PATCH 41/96] perf: build sets of simple keys from lists and tuples in one sized pass `set(xs)` walked a native iterator, checked each item's hashability and inserted under a per-item error scope into a table grown by doubling. A list or tuple whose items are all ints, strs, floats, bools or None (whose hashing and comparison run no Python and can't fail) now fills a table sized once, straight from the storage. `len(set(data) & {1, 2, 3, 49})` over 50 ints: -31%. --- crates/weavepy-vm/src/builtins.rs | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 5cd9bd8f..6c986179 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -7139,6 +7139,8 @@ fn b_set(args: &[Object]) -> Result { } let out = if args.is_empty() { crate::object::SetData::default() + } else if let Some(out) = simple_key_set(&args[0]) { + out } else { // `set([[]])` raises `TypeError: unhashable type: 'list'` — check // each element as it is admitted, like CPython's `set_init`. A @@ -7158,6 +7160,35 @@ fn b_set(args: &[Object]) -> Result { Ok(obj) } +/// The set of a list's or tuple's items when every item is an `int`, +/// `str`, `float`, `bool` or `None` (whose hashing and comparison run no +/// Python and can't fail): sized once and filled without the per-insert +/// error scope. `None` leaves the general path to it. +fn simple_key_set(src: &Object) -> Option { + fn build(items: &[Object]) -> Option { + let simple = |v: &Object| { + matches!( + v, + Object::Int(_) | Object::Str(_) | Object::Float(_) | Object::Bool(_) | Object::None + ) + }; + if !items.iter().all(simple) { + return None; + } + let mut out = + crate::object::SetData::with_capacity_and_hasher(items.len(), Default::default()); + for v in items { + out.insert(DictKey(v.clone())); + } + Some(out) + } + match src { + Object::List(l) => build(&l.try_borrow().ok()?), + Object::Tuple(t) => build(t), + _ => None, + } +} + fn b_frozenset(args: &[Object]) -> Result { // `frozenset()` takes at most one positional argument (CPython // `frozenset_new`); `frozenset([], 2)` is a `TypeError`. From 78e40b2e7499c0828f0cd0bba4af02ea32df8d36 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:10:18 -0700 Subject: [PATCH 42/96] perf: run big-int arithmetic in the core loop An operation with a big int operand left the core loop for the full handler, which dispatched through four layers to the same `bignum_op` an exact int reaches. The core loop now calls it in place when an operand is a big int (exact ints run no Python), raising its errors as the full handler would. `(i + 10**30) * 12345678901234567890 // 7`: -49% (2.4x CPython, from 4.8x). --- crates/weavepy-vm/src/lib.rs | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 5f74ddda..93b6c1f5 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -10907,6 +10907,33 @@ impl Interpreter { } } } + // Arithmetic on a big int: the full handler's own + // bignum path (exact ints run no Python), out of + // line. + (Object::Long(_), Object::Int(_) | Object::Long(_)) + | (Object::Int(_), Object::Long(_)) => { + let r = core_bignum_op(a, b, kind); + // SAFETY: ints leave by plain decrements. + unsafe { + drop_hot(base.add(len - 1).read()); + drop_hot(base.add(len - 2).read()); + } + len -= 2; + match r { + Ok(v) => { + // SAFETY: the operands' slots are free. + unsafe { base.add(len).write(v) }; + len += 1; + last = pc; + pc += 1; + continue; + } + Err(e) => { + pc += 1; + break Some(CoreExit::Stop(LeafStop::Raised(e))); + } + } + } // String concatenation, repetition and `%` // formatting over scalars, out of line. (Object::Str(_), _) | (Object::Int(_), Object::Str(_)) => { @@ -64073,6 +64100,12 @@ fn bigint_true_div(x: &num_bigint::BigInt, y: &num_bigint::BigInt) -> f64 { /// Bignum-aware integer arithmetic for `int`-flavoured operands. /// Both inputs are guaranteed `int`/`long`/`bool` by the caller; the /// fast path stays in `i64` until an overflow forces promotion. +/// [`bignum_op`] for the core loop, kept out of its body. +#[inline(never)] +fn core_bignum_op(a: &Object, b: &Object, op: BinOpKind) -> Result { + bignum_op(a, b, op) +} + fn bignum_op(a: &Object, b: &Object, op: BinOpKind) -> Result { use BinOpKind as B; From cdb11018654ddec2005f3a6e89acdb728abb4e0d Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:43:09 -0700 Subject: [PATCH 43/96] perf: share functions' name objects and presize their slot table Every `MAKE_FUNCTION` allocated fresh `__name__` and `__qualname__` strings and grew its getset table from empty. The functions made from one code object now share one pair of name objects (as CPython's share the code's `co_name` and `co_qualname`), and the table starts with room for its usual four entries. Creating and calling a lambda: -11%; a closure factory: -4%. --- crates/weavepy-vm/src/lib.rs | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 93b6c1f5..fde8056f 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -22122,7 +22122,7 @@ impl Interpreter { // Function getset slots live *outside* `__dict__` // (`f.__dict__` starts empty in CPython; only genuine // user attributes land there). - let slots = RefCell::new(DictData::default()); + let slots = RefCell::new(DictData::with_capacity_and_hasher(4, Default::default())); // Stamp __module__ from globals['__name__'] (mirrors CPython's // function dispatch). Pickle relies on this to serialise the // function by qualified name. @@ -22146,10 +22146,21 @@ impl Interpreter { // Both come from the code object (see // `CodeConstObjects::fn_names`), so two functions made // from one `def` share them exactly as CPython's do. - let (name_obj, qualname_obj) = ( - Object::from_str(name.clone()), - Object::from_str(code.qualname.clone()), - ); + let (name_obj, qualname_obj) = match code_vm_ext(&code) { + Some(ext) => ext + .fn_names + .get_or_init(|| { + ( + Object::from_str(code.name.clone()), + Object::from_str(code.qualname.clone()), + ) + }) + .clone(), + None => ( + Object::from_str(name.clone()), + Object::from_str(code.qualname.clone()), + ), + }; { let mut sl = slots.borrow_mut(); sl.insert(DictKey(keys[1].clone()), name_obj); @@ -61177,6 +61188,10 @@ struct CodeConstObjects { /// The code's native form for the core loop (see [`frame_jit`]). #[cfg(feature = "jit")] frame_jit: frame_jit::Slot, + /// The `__name__` and `__qualname__` objects of the functions made + /// from this code, shared by all of them (as CPython's share the code + /// object's `co_name` and `co_qualname`). + fn_names: std::sync::OnceLock<(Object, Object)>, } /// A `CALL` site's inline-call shape (see `Interpreter::core_call`): the @@ -62627,6 +62642,7 @@ fn code_vm_ext_build( returns_none: std::sync::atomic::AtomicU8::new(0), #[cfg(feature = "jit")] frame_jit: frame_jit::Slot::default(), + fn_names: std::sync::OnceLock::new(), }) }) } From 8a5f4053d356b5992c8adb1cb787699c2eaa779a Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:22:43 -0700 Subject: [PATCH 44/96] perf: call native deque operations from compiled code The JIT's burned method lane now also resolves a native leaf method that opts in (`leaf_builtins::register_jit_method`), so `q.append(x)`, `q.popleft()` and `q.rotate(n)` on a deque compile instead of deopting at the generic attribute load. The guard runs at the method-form load (`GuardMethod`), or at the call when nothing ran in between, and the call uses the builtin bound there, as the interpreter's bound method would. A fast half that declines, or an active profiler, rejects the call so the interpreter performs it on a materialized frame. `q[i]` on a container whose class resolves `__getitem__` to a native leaf compiles to `ObjGetItem`; `len(q)`, truth tests and iterator steps reach the deque's natives from their helpers. Element results carry an integer hint: arithmetic on two of them guards both operands, and an immediately consuming integer operation fuses into the producer. The deque bodies share `fast_op`, which the helpers call directly, and the slot accessors compare interned keys by identity before the full comparison. A Python `__bool__` in a compiled truth test now rejects to the interpreter, which shows the callee the caller's locals. deque_ops, instructions retired per unit of WEAVEPY_BENCH_WORK: 6748 to 4386 (CPython 3.14: 4256). Its later loops still run in the interpreter: the compile at the first loop meets their deques unbound and takes a cold exit there. --- crates/weavepy-jit/src/analyze.rs | 240 ++++++- crates/weavepy-jit/src/engine.rs | 5 +- crates/weavepy-jit/src/ir.rs | 23 + crates/weavepy-jit/src/lib.rs | 11 +- crates/weavepy-jit/src/lower.rs | 74 +- crates/weavepy-jit/src/runtime.rs | 63 ++ crates/weavepy-vm/src/lib.rs | 134 +++- .../src/stdlib/collections_native.rs | 220 ++++-- crates/weavepy-vm/src/tier2.rs | 677 +++++++++++++++++- crates/weavepy-vm/src/types.rs | 59 ++ tests/regrtest/test_jit_deque_lanes.py | 257 +++++++ 11 files changed, 1599 insertions(+), 164 deletions(-) create mode 100644 tests/regrtest/test_jit_deque_lanes.py diff --git a/crates/weavepy-jit/src/analyze.rs b/crates/weavepy-jit/src/analyze.rs index 45ac5419..e4332943 100644 --- a/crates/weavepy-jit/src/analyze.rs +++ b/crates/weavepy-jit/src/analyze.rs @@ -49,6 +49,10 @@ pub struct MethodResolution { pub arg_count: u32, pub min_args: u32, pub ret: MethodRet, + /// The callee is a native accelerator's leaf method (`deque.pop`): + /// its object-lane result is a container element, so arithmetic on + /// it speculates the integer lane (see [`SE::num_hint`]). + pub native: bool, } /// Maximum receiver path retained by analysis. The final attribute read adds @@ -2781,6 +2785,8 @@ struct MethodMark { ret: MethodRet, /// The method-form `LOAD_ATTR` pc. load_pc: u32, + /// See [`MethodResolution::native`]. + native: bool, } /// RFC 0073 WS3 — a burned-in native `str` method riding on its @@ -2852,6 +2858,12 @@ struct SE { /// native method (the [`Self::method`] discipline, statically /// resolved). str_method: Option, + /// An object-lane element read from a native accelerator's container + /// (a `deque` subscript or end operation). Arithmetic combining two + /// object-lane operands speculates the integer lane when either + /// carries this hint: each operand is guarded exactly like the + /// integral-peer case, so a miss resumes at the operation itself. + num_hint: bool, } impl SE { @@ -2871,6 +2883,7 @@ impl SE { ctor: None, elem_of: None, str_method: None, + num_hint: false, } } @@ -3984,8 +3997,15 @@ fn step_abstract( stack.push(SE::known(a.ty)); return Ok(()); } + let hinted = a.num_hint || b.num_hint; let (a, b) = resolve_pair(a, b, local_types, changed); - let res = bin_result_type(kind, a.ty, b.ty)?; + // Two native container elements (`q[0] + q[-1]`): both + // operands are guarded as integers at emission. + let res = if hinted && a.ty == JitType::Obj && b.ty == JitType::Obj { + bin_result_type(kind, JitType::Int, JitType::Int)? + } else { + bin_result_type(kind, a.ty, b.ty)? + }; stack.push(SE::known(res)); } OpCode::CompareOp => { @@ -4160,9 +4180,11 @@ fn step_abstract( // RFC 0069 WS1 — a class-resolved method on a pinned // instance receiver: the receiver stays on the abstract // stack (its pin is the native value), re-marked with the - // resolution. `.append` keeps its dedicated list path. - // RFC 0071 WS3 admits attribute-chain receivers. - if name != "append" { + // resolution. `.append` keeps its dedicated list path (a + // list receiver never resolves here: the probe answers + // instances only). RFC 0071 WS3 admits attribute-chain + // receivers. + if name != "append" || !recv.ty.is_list() { if let Some((slot, path)) = obj_recv_ref(recv.ty, recv.src, recv.path, probes.paths) { let names = probes.paths.names(path); @@ -4188,6 +4210,7 @@ fn step_abstract( min_args: res.min_args, ret: res.ret, load_pc: i as u32, + native: res.native, }), ..recv }); @@ -4341,7 +4364,10 @@ fn step_abstract( poison: true, ..SE::known(JitType::Unknown) }), - MethodRet::Scalar(ty) => stack.push(SE::known(ty)), + MethodRet::Scalar(ty) => stack.push(SE { + num_hint: m.native && ty == JitType::Obj, + ..SE::known(ty) + }), } return Ok(()); } @@ -4602,6 +4628,21 @@ fn step_abstract( stack.push(SE::known(JitType::Int)); return Ok(()); } + // A native container element (`q[0]` on a `deque`): the + // container's class resolves `__getitem__` to a native leaf + // method (see `TOp::ObjGetItem`). + if native_getitem(&cont, probes).is_some() { + if cont.path.is_none() && cont.ty == JitType::Unknown { + if let Some(slot) = cont.src { + set_local(local_types, slot, JitType::Obj, changed)?; + } + } + stack.push(SE { + num_hint: true, + ..SE::known(JitType::Obj) + }); + return Ok(()); + } // RFC 0073 WS3 — `s[i]` reads a single-codepoint `str` // (ASCII fast path; the helper deopts on anything else). if cont.ty == JitType::Str { @@ -5508,8 +5549,84 @@ fn bin_result_type(kind: ArithKind, a: JitType, b: JitType) -> Result bool { + for k in (0..stmts.len()).rev() { + match stmts[k].op { + TOp::GuardMethod { .. } if stmts[k].pc == load_pc => { + stmts[k].op = TOp::GuardNotNone; + return true; + } + TOp::LoadLocal(_) + | TOp::PushConstInt(_) + | TOp::PushConstFloat(_) + | TOp::PushConstBool(_) + | TOp::PushNone + | TOp::IntArith(_) + | TOp::FloatArith(_) + | TOp::IntNeg + | TOp::FloatNeg + | TOp::IntInvert + | TOp::IntToFloatTos { .. } + | TOp::IntToFloatSecond { .. } + | TOp::UnboxInt { .. } => {} + _ => return false, + } + } + false +} + +/// The burned `__getitem__` resolution for a subscript container on the +/// object lane (or an untyped local): the root local and the embedder's +/// method probe, when the class resolves the dunder to a native leaf +/// method (`deque.__getitem__`). Python-level `__getitem__` methods stay +/// out: the helper calls the native body directly. +fn native_getitem(cont: &SE, probes: &mut Probes<'_>) -> Option<(u32, MethodResolution)> { + if !matches!(cont.ty, JitType::Obj | JitType::Unknown) { + return None; + } + let (slot, path) = obj_recv_ref(cont.ty, cont.src, cont.path, probes.paths)?; + let names = probes.paths.names(path); + let res = (probes.method)(slot, &names, "__getitem__")?; + res.native.then_some((slot, res)) +} + +/// Fuse an integer consumer at `pc` into the object-lane producer that +/// ran immediately before it (the instruction at `pc - 1`): the producer +/// then hands over an exact `int` unboxed, and anything else leaves +/// through the producer's own completed-result exit (a generic call or a +/// native method parks its result and resumes here; a pure native +/// subscript is simply re-executed by the interpreter). Returns whether +/// the producer took the consumer's guard. `native` admits a burned-in +/// method call, which only a native method's hinted result asks for. +fn fuse_int_result(stmts: &mut [TStmt], pc: u32, native: bool) -> bool { + let Some(previous) = stmts.last_mut() else { + return false; + }; + if previous.pc.checked_add(1) != Some(pc) { + return false; + } + match &mut previous.op { + TOp::CallDyn { int_result, .. } | TOp::ObjGetItem { int_result, .. } => { + *int_result = true; + true + } + TOp::CallMethod { ret, .. } if native && *ret == MethodRet::Scalar(JitType::Obj) => { + *ret = MethodRet::Scalar(JitType::Int); + true + } + _ => false, + } +} + /// An integral peer provides a useful speculative lane for an opaque -/// operand. Two opaque operands have no such evidence and stay generic. +/// operand. Two opaque operands have no such evidence and stay generic +/// unless one is a native container element (see [`SE::num_hint`]). /// The depth is measured from TOS, so only 0 and 1 can be returned. fn integer_operand_guard(a: JitType, b: JitType) -> Option { if a.is_integral() && b == JitType::Obj { @@ -5671,6 +5788,8 @@ struct ESlot { /// RFC 0073 WS3 — a pinned `str` receiver carrying a burned-in /// native method (see [`SE::str_method`]). str_method: Option, + /// See [`SE::num_hint`]. + num_hint: bool, } impl ESlot { @@ -5689,6 +5808,7 @@ impl ESlot { slice: None, const_slice: false, str_method: None, + num_hint: false, } } @@ -6493,21 +6613,31 @@ fn emit_instr( } OpCode::BinaryOp => { let kind = bin_kind(ins.arg)?; + let b_hint = stack.last().is_some_and(|s| s.num_hint); + let hinted = b_hint || (stack.len() >= 2 && stack[stack.len() - 2].num_hint); let mut b = pop_val(stack)?; let mut a = pop_val(stack)?; + // Two native container elements (see `SE::num_hint`): the top + // operand is guarded (or its producer fused) like the + // integral-peer case below, then the lower one at depth 1. + if hinted && a == JitType::Obj && b == JitType::Obj { + if !fuse_int_result(stmts, pc, b_hint) { + stack.push(ESlot::val(a)); + stack.push(ESlot::val(b)); + push(TOp::UnboxInt { depth: 0 }, None, stack, stmts); + stack.pop(); + stack.pop(); + } + b = JitType::Int; + stack.push(ESlot::val(a)); + stack.push(ESlot::val(b)); + push(TOp::UnboxInt { depth: 1 }, None, stack, stmts); + stack.pop(); + stack.pop(); + a = JitType::Int; + } if let Some(depth) = integer_operand_guard(a, b) { - let fused = depth == 0 - && stmts.last_mut().is_some_and(|previous| { - if previous.pc.checked_add(1) != Some(pc) { - return false; - } - if let TOp::CallDyn { int_result, .. } = &mut previous.op { - *int_result = true; - true - } else { - false - } - }); + let fused = depth == 0 && fuse_int_result(stmts, pc, b_hint); // Re-model both original operands for the guard. A miss // restores the boxed value at this BINARY_OP, never its // producer, so calls and callbacks aren't repeated. @@ -6892,8 +7022,10 @@ fn emit_instr( let names = probes.paths.names(path); // RFC 0069 WS1 — a class-resolved method: re-mark the // receiver in place (its pin stays the native value) and - // record the site under the probe's token. - if name != "append" { + // record the site under the probe's token (an object-lane + // `.append`, such as `deque.append`, included: the list + // shape returned above). + if name != "append" || !top.ty.is_list() { // RFC 0073 WS1 (element residue) — mirror the // inference fallback: an unbound receiver local // resolves against an exemplar element of its source @@ -6927,7 +7059,17 @@ fn emit_instr( // fence out `None` here (deopt at this pc, receiver // still on the stack) so the interpreter re-executes // the load and raises the exact `AttributeError`. - push(TOp::GuardNotNone, None, stack, stmts); + // A native method binds at this load: its whole guard + // (class version, no instance shadowing, `None` + // fenced) runs here, and the call then uses the + // builtin bound here even when argument evaluation + // rebinds the class attribute, exactly as the + // interpreter's bound method would. + if res.native { + push(TOp::GuardMethod { token: res.token }, None, stack, stmts); + } else { + push(TOp::GuardNotNone, None, stack, stmts); + } let last = stack.len() - 1; stack[last].method = Some(MethodMark { token: res.token, @@ -6935,6 +7077,7 @@ fn emit_instr( min_args: res.min_args, ret: res.ret, load_pc: pc, + native: res.native, }); stack.push(ESlot { null: Some((pc, plan.hidden_at(i) + stack.len() as u32)), @@ -7220,6 +7363,15 @@ fn emit_instr( token: Some(m.token), }); *max_call_args = (*max_call_args).max(argc as u32); + // A native site whose arguments ran no code since the load + // checks its guard at the call (one helper call, not two). + let mut token = m.token; + if m.native { + token |= crate::runtime::METHOD_NATIVE; + if defer_method_guard(stmts, m.load_pc) { + token |= crate::runtime::METHOD_GUARD_AT_CALL; + } + } // Re-model the receiver + args for the statement // (`CallMethod` pops `argc` scalars and the pin). stack.push(ESlot::val(f.ty)); @@ -7228,7 +7380,7 @@ fn emit_instr( } push( TOp::CallMethod { - token: m.token, + token, argc: argc as u8, ret: m.ret, }, @@ -7252,7 +7404,10 @@ fn emit_instr( if !ty.is_representable() || (ty.is_pinned() && ty != JitType::Obj) { return Err(JitVerdict::TypeUnknown); } - stack.push(ESlot::val(ty)); + stack.push(ESlot { + num_hint: m.native && ty == JitType::Obj, + ..ESlot::val(ty) + }); } } *max_stack = (*max_stack).max(stack.len() as u32); @@ -8075,6 +8230,45 @@ fn emit_instr( push(TOp::BytesGetItem, Some(JitType::Int), stack, stmts); return Ok(()); } + // A native container element (see the inference arm): the + // site records the burned `__getitem__` resolution under its + // probe token, like a method site. + if cont == JitType::Obj { + let probe = SE { + src: cont_slot.src, + path: cont_slot.path, + ..SE::known(cont) + }; + if let Some((slot, res)) = native_getitem(&probe, probes) { + let idx = res.token as usize; + if method_sites.len() <= idx { + method_sites.resize(idx + 1, None); + } + method_sites[idx] = Some(MethodSiteMeta { + slot, + name: "__getitem__".to_owned(), + arg_count: res.arg_count, + min_args: res.min_args, + ret: res.ret, + }); + *max_call_args = (*max_call_args).max(1); + push( + TOp::ObjGetItem { + token: res.token, + int_result: false, + }, + None, + stack, + stmts, + ); + stack.push(ESlot { + num_hint: true, + ..ESlot::val(JitType::Obj) + }); + *max_stack = (*max_stack).max(stack.len() as u32); + return Ok(()); + } + } // RFC 0073 WS3 — `s[i]` through the registered helper // (ASCII-only O(1); anything else deopts there). if cont == JitType::Str { diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index 32b748c0..6ab70506 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -769,7 +769,10 @@ fn op_mix(tfunc: &TFunc) -> OpMix { TOp::CallDyn { .. } => mix.dyn_calls += 1, TOp::DynAttrGet { .. } | TOp::DynAttrSet { .. } => mix.dyn_attrs += 1, TOp::ContainsDyn { .. } => mix.dyn_other += 1, - TOp::CallPy { .. } | TOp::CallPyKw { .. } | TOp::CallMethod { .. } => { + TOp::CallPy { .. } + | TOp::CallPyKw { .. } + | TOp::CallMethod { .. } + | TOp::ObjGetItem { .. } => { mix.guarded_calls += 1; } _ => {} diff --git a/crates/weavepy-jit/src/ir.rs b/crates/weavepy-jit/src/ir.rs index eb1f3ead..eae027d0 100644 --- a/crates/weavepy-jit/src/ir.rs +++ b/crates/weavepy-jit/src/ir.rs @@ -312,6 +312,27 @@ pub enum TOp { argc: u8, ret: MethodRet, }, + /// `container[index]` (`BINARY_SUBSCR`) on an object-lane container + /// whose class resolved `__getitem__` to a native leaf method at + /// compile time (`deque.__getitem__`): pops the `int` index and the + /// container pin and calls the registered `wpjit_obj_getitem` helper + /// with the method-site `token` (the [`TOp::CallMethod`] protocol, + /// index staged as the one argument). A guard miss or a declined + /// fast half deopts *at* this pc with both operands spilled (nothing + /// ran; the interpreter re-executes the subscript). The result rides + /// the object lane, or the `int` lane when `int_result` fuses the + /// immediately following integer consumer (any other value parks and + /// resumes at that consumer). + ObjGetItem { token: u32, int_result: bool }, + /// The method-form `LOAD_ATTR` of a burned-in native method site + /// (see [`TOp::CallMethod`]): peeks the receiver pin and calls the + /// registered `wpjit_guard_method` helper with the site's `token`, + /// which checks the receiver's class version and that no instance + /// attribute shadows the name. A miss (or `None`) deopts *at* this + /// pc with the receiver spilled, so the interpreter performs the + /// load itself. The `CALL` then needs no guard: the method was bound + /// here. + GuardMethod { token: u32 }, /// RFC 0071 WS6 — `==`/`!=` on two pinned `str` values: pops both /// pins, calls the registered `wpjit_str_eq` helper (identical-pin /// and pointer equality answer before a content compare), and @@ -1486,6 +1507,8 @@ impl TOp { | TOp::CallPy { .. } | TOp::CallPyKw { .. } | TOp::CallMethod { .. } + | TOp::ObjGetItem { .. } + | TOp::GuardMethod { .. } | TOp::MathIntrinsic(_) | TOp::FloatArith(ArithKind::FloorDiv | ArithKind::Mod | ArithKind::Pow) | TOp::ListGet { .. } diff --git a/crates/weavepy-jit/src/lib.rs b/crates/weavepy-jit/src/lib.rs index 09ae43d8..7c5ebcfa 100644 --- a/crates/weavepy-jit/src/lib.rs +++ b/crates/weavepy-jit/src/lib.rs @@ -45,25 +45,28 @@ pub use runtime::{ set_obj_layout, ObjLayout, register_attr_get_chain_helper, register_attr_helpers, register_build_map_helper, register_build_set_helper, register_cached_attr_chain_helper, register_call_dyn_helper, - register_call_dyn_int_helper, register_call_method_helper, register_call_py_helper, + register_call_dyn_int_helper, register_call_method_helper, register_call_native_method_helper, + register_call_py_helper, register_cell_helpers, register_const_str_helper, register_contains_dyn_helper, register_dict_del_helper, register_dict_helpers, register_dict_iter_helper, register_dyn_attr_helpers, register_float_pow_helper, register_global_obj_helper, register_iter_helpers, register_iter_new_helper, register_iter_next_pair_helper, register_list_extra_helpers, register_list_from_range_helper, register_list_helpers, - register_list_next_helper, register_math_helpers, register_poll_helper, + register_guard_method_helper, register_list_next_helper, register_math_helpers, + register_obj_getitem_helper, register_poll_helper, register_self_call_helpers, register_str_format_helpers, register_str_helpers, register_str_method_helper, register_str_write_helpers, register_truth_helper, register_tuple_read_helpers, register_unbox_int_helper, AttrGetChainHelper, AttrGetHelper, AttrSetHelper, BuildListHelper, BuildTupleHelper, BytesGetHelper, CachedAttrChainHelper, CallDynHelper, CallMethodHelper, CallPyHelper, CallStatus, CellGetHelper, CellSetHelper, - DictAccessHelper, DynAttrHelper, GetIterHelper, IterNextHelper, IterNextPairHelper, JitFrame, + DictAccessHelper, DynAttrHelper, GetIterHelper, GuardMethodHelper, IterNextHelper, + IterNextPairHelper, JitFrame, JitStatus, ListAppendHelper, ListFromRangeHelper, ListGetHelper, ListLenHelper, ListNextHelper, ListRepeatHelper, ListSetHelper, ListSliceHelper, MathBinaryHelper, MathUnaryHelper, PollHelper, SelfEnterHelper, SelfExitHelper, SelfSlowHelper, SlotTag, StrEqHelper, StrLenHelper, StrModHelper, CALL_GAPS, DICT_KEY_INT, DICT_KEY_STR, DICT_VAL_FLOAT, DICT_VAL_INT, DICT_VAL_OBJ, ITER_ELEM_STR, JIT_POLL_STRIDE, MAX_ATTR_CHAIN_LEN, - MAX_CACHED_ATTR_CHAIN_LEN, + MAX_CACHED_ATTR_CHAIN_LEN, METHOD_GUARD_AT_CALL, METHOD_NATIVE, }; pub use value::JitType; diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index dcaff703..63934b82 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -1211,6 +1211,39 @@ impl<'a, 'b> Lowerer<'a, 'b> { TOp::CallMethod { token, argc, ret } => { self.emit_call_method(token, argc, ret, stmt.pc); } + TOp::GuardMethod { token } => { + let snapshot = self.vstack.clone(); + let &(pin, _) = self.vstack.last().expect("guard on empty stack"); + let sig = self.list_helper_sig(); + let helper = self + .b + .ins() + .iconst(self.ptr_ty, runtime::guard_method_helper_addr() as i64); + let tokenv = self.b.ins().iconst(types::I64, i64::from(token)); + let call = self + .b + .ins() + .call_indirect(sig, helper, &[self.frame_ptr, pin, tokenv]); + let status = self.b.inst_results(call)[0]; + let bad = self.b.ins().icmp_imm(IntCC::NotEqual, status, 0); + let cont = self.guard(bad, stmt.pc, &snapshot); + self.b.switch_to_block(cont); + } + TOp::ObjGetItem { token, int_result } => { + let ret = MethodRet::Scalar(if int_result { + JitType::Int + } else { + JitType::Obj + }); + self.emit_method_helper_call( + runtime::obj_getitem_helper_addr(), + token, + 1, + ret, + stmt.pc, + false, + ); + } TOp::CallStrMethod { site, argc, ret } => { self.emit_call_str_method(site, argc, ret, stmt.pc); } @@ -2620,6 +2653,29 @@ impl<'a, 'b> Lowerer<'a, 'b> { /// the bound method + `Unbound` pair), so the interpreter /// re-executes the call generically. fn emit_call_method(&mut self, token: u32, argc: u8, ret: MethodRet, pc: u32) { + let native = token & runtime::METHOD_NATIVE != 0; + let helper = if native { + runtime::call_native_method_helper_addr() + } else { + runtime::call_method_helper_addr() + }; + self.emit_method_helper_call(helper, token, argc, ret, pc, !native); + } + + /// [`Self::emit_call_method`]'s body over any helper sharing the + /// method helper's shape and status protocol (`wpjit_obj_getitem` + /// stages its index as the one argument). `writeback` stores the + /// locals first, for a helper that may run Python on this + /// activation's behalf. + fn emit_method_helper_call( + &mut self, + helper_addr: usize, + token: u32, + argc: u8, + ret: MethodRet, + pc: u32, + writeback: bool, + ) { let trusted = MemFlags::trusted(); let n = argc as usize; // Snapshot *including* receiver + args: the Reject exit re-runs @@ -2645,9 +2701,14 @@ impl<'a, 'b> Lowerer<'a, 'b> { let snapshot = self.vstack.clone(); // A method whose body is a field update, run in line once the - // helper has armed it; anything else takes the helper below. + // helper has armed it (a Python method's helper only); anything + // else takes the helper below. let inline = match (runtime::obj_layout(), ret, update_arg) { - (Some(l), MethodRet::Scalar(JitType::Int), Some(arg)) => Some((*l, arg)), + (Some(l), MethodRet::Scalar(JitType::Int), Some(arg)) + if helper_addr == runtime::call_method_helper_addr() => + { + Some((*l, arg)) + } _ => None, }; let mut join = None; @@ -2661,7 +2722,9 @@ impl<'a, 'b> Lowerer<'a, 'b> { join = Some(done); } - self.writeback_locals(); + if writeback { + self.writeback_locals(); + } self.store_call_site_pc(pc); let expect = match ret { @@ -2669,10 +2732,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { MethodRet::Scalar(t) => Self::tag(t), }; let sig = self.call_method_helper_sig(); - let helper = self - .b - .ins() - .iconst(self.ptr_ty, runtime::call_method_helper_addr() as i64); + let helper = self.b.ins().iconst(self.ptr_ty, helper_addr as i64); let tokenv = self.b.ins().iconst(types::I32, i64::from(token)); let argcv = self.b.ins().iconst(types::I32, i64::from(argc)); let expectv = self.b.ins().iconst(types::I32, expect); diff --git a/crates/weavepy-jit/src/runtime.rs b/crates/weavepy-jit/src/runtime.rs index 955f3b52..19fa9a4c 100644 --- a/crates/weavepy-jit/src/runtime.rs +++ b/crates/weavepy-jit/src/runtime.rs @@ -132,6 +132,18 @@ pub enum SlotTag { /// are tagged [`SlotTag::Default`]. pub const CALL_GAPS: u32 = 1 << 31; +/// Set in a [`crate::TOp::CallMethod`] token when a native method site's +/// guard runs at the call instead of at its load: nothing between the +/// two can run code (see `TOp::GuardMethod`), so the call helper checks +/// the receiver itself and resolves a miss generically. +pub const METHOD_GUARD_AT_CALL: u32 = 1 << 31; + +/// Set in a [`crate::TOp::CallMethod`] token when the site resolved a +/// native leaf method: its helper never runs Python on this activation's +/// behalf (any call that would leaves through a side exit, which writes +/// the locals back itself), so the call skips the locals write-back. +pub const METHOD_NATIVE: u32 = 1 << 30; + impl SlotTag { /// Decode a raw tag written by native code. #[inline] @@ -1147,6 +1159,57 @@ pub(crate) fn call_method_helper_addr() -> usize { CALL_METHOD_HELPER.load(std::sync::atomic::Ordering::Acquire) } +/// The native method-site load guard (see [`crate::TOp::GuardMethod`]): +/// `0` when the receiver pin still matches the site's burned +/// resolution, anything else to deopt at the load. +pub type GuardMethodHelper = + unsafe extern "C" fn(frame: *mut JitFrame, pin: i64, token: i64) -> i64; + +static GUARD_METHOD_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Register the process-wide native method-site load guard. Must precede +/// the first compile of a frame containing `GuardMethod` ops. +pub fn register_guard_method_helper(helper: GuardMethodHelper) { + GUARD_METHOD_HELPER.store(helper as usize, std::sync::atomic::Ordering::Release); +} + +#[must_use] +pub(crate) fn guard_method_helper_addr() -> usize { + GUARD_METHOD_HELPER.load(std::sync::atomic::Ordering::Acquire) +} + +static CALL_NATIVE_METHOD_HELPER: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + +/// Register the process-wide native method-call helper: the +/// [`CallMethodHelper`] for a [`crate::TOp::CallMethod`] whose token +/// carries [`METHOD_NATIVE`]. Must precede the first compile of a frame +/// containing such a call. +pub fn register_call_native_method_helper(helper: CallMethodHelper) { + CALL_NATIVE_METHOD_HELPER.store(helper as usize, std::sync::atomic::Ordering::Release); +} + +#[must_use] +pub(crate) fn call_native_method_helper_addr() -> usize { + CALL_NATIVE_METHOD_HELPER.load(std::sync::atomic::Ordering::Acquire) +} + +static OBJ_GETITEM_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Register the process-wide native-subscript helper (see +/// [`crate::TOp::ObjGetItem`]). Shares [`CallMethodHelper`]'s shape: the +/// `token` names the burned `__getitem__` resolution and the index rides +/// the marshal buffer as the one argument. Must precede the first +/// compile of a frame containing `ObjGetItem` ops. +pub fn register_obj_getitem_helper(helper: CallMethodHelper) { + OBJ_GETITEM_HELPER.store(helper as usize, std::sync::atomic::Ordering::Release); +} + +#[must_use] +pub(crate) fn obj_getitem_helper_addr() -> usize { + OBJ_GETITEM_HELPER.load(std::sync::atomic::Ordering::Acquire) +} + static STR_METHOD_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); /// Register the process-wide native `str`-method helper (RFC 0073 diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index fde8056f..3abba523 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -927,7 +927,7 @@ pub struct Interpreter { /// The core loop's last native iterator `__next__`, keyed by the /// (process-unique) attribute version of the class that resolved it /// (see [`Interpreter::core_leaf_next`]). - core_next: ThreadCell>)>>, + core_next: ThreadCell>, u8)>>, /// `_seqtools`' lazy `map` and `filter` classes, once imported (see /// [`Interpreter::seqtools_step`]). seqtools: std::cell::OnceCell>, @@ -11173,12 +11173,10 @@ impl Interpreter { // the full arm, which asks again (a leaf iterator // stays exhausted) and ends the loop. obj @ Object::Instance(inst) => { - let Some(b) = self.core_leaf_next_ptr(inst) else { + let Some(step) = self.leaf_next_step(obj, inst) else { break None; }; - // SAFETY: the cache and the class keep the - // builtin alive; its body runs no Python. - match unsafe { ((*b).call)(std::slice::from_ref(obj)) } { + match step { Ok(v) => { // SAFETY: `len < cap`. unsafe { base.add(len).write(v) }; @@ -17705,20 +17703,41 @@ impl Interpreter { return None; } let ver = cls.attr_version.get(); - let fresh = !matches!(&*self.core_truth.borrow(), Some((v, _)) if *v == ver); - if fresh { - let t = self.native_truth_of(v, cls); - *self.core_truth.borrow_mut() = Some((ver, t)); - } // The cache (and the class) keep the builtin alive through the // call, whose body runs no Python. - let (b, is_len): (*const crate::object::BuiltinFn, bool) = match &*self.core_truth.borrow() - { - Some((_, NativeTruth::AlwaysTrue)) => return Some(true), - Some((_, NativeTruth::Bool(b))) => (Rc::as_ptr(b), false), - Some((_, NativeTruth::Len(b))) => (Rc::as_ptr(b), true), - _ => return None, + let lookup = + |t: &NativeTruth| -> Result<(*const crate::object::BuiltinFn, bool, u8), bool> { + match t { + NativeTruth::AlwaysTrue => Err(true), + NativeTruth::Bool(b, op) => Ok((Rc::as_ptr(b), false, *op)), + NativeTruth::Len(b) => Ok((Rc::as_ptr(b), true, 0)), + NativeTruth::Decline => Err(false), + } + }; + let cached = match &*self.core_truth.borrow() { + Some((v, t)) if *v == ver => Some(lookup(t)), + _ => None, + }; + let found = match cached { + Some(found) => found, + None => { + let t = self.native_truth_of(v, cls); + let found = lookup(&t); + *self.core_truth.borrow_mut() = Some((ver, t)); + found + } }; + let (b, is_len, op) = match found { + Ok(found) => found, + Err(true) => return Some(true), + Err(false) => return None, + }; + // A deque's `__bool__`: its direct operation, without the call. + if op == crate::stdlib::collections_native::OP_BOOL { + if let Some(Object::Bool(b)) = crate::stdlib::collections_native::fast_op(op, v, None) { + return Some(b); + } + } // SAFETY: see above. let b = unsafe { &*b }; let r = match b.call_kw.as_ref() { @@ -17751,7 +17770,8 @@ impl Interpreter { return if is_len { NativeTruth::Len(b) } else { - NativeTruth::Bool(b) + let op = leaf_builtins::jit_method_op(&b).unwrap_or(0); + NativeTruth::Bool(b, op) }; } Some(_) => return NativeTruth::Decline, @@ -18136,14 +18156,43 @@ impl Interpreter { /// lookup. #[inline] fn core_leaf_next_ptr(&self, inst: &PyInstance) -> Option<*const crate::object::BuiltinFn> { + self.core_leaf_next_op(inst).map(|(b, _)| b) + } + + /// [`Self::core_leaf_next_ptr`] with the builtin's direct operation + /// (`collections_native::fast_op`, `0` for none). + #[inline] + fn core_leaf_next_op( + &self, + inst: &PyInstance, + ) -> Option<(*const crate::object::BuiltinFn, u8)> { let ver = inst.cls_raw().attr_version.get(); - if let Some((v, b)) = &*self.core_next.borrow() { + if let Some((v, b, op)) = &*self.core_next.borrow() { if *v == ver { - return b.as_ref().map(Rc::as_ptr); + return b.as_ref().map(|b| (Rc::as_ptr(b), *op)); } } self.core_leaf_next_resolve(inst, ver) - .map(|b| Rc::as_ptr(&b)) + } + + /// A native iterator step through [`Self::core_leaf_next_op`]: the + /// direct operation when it applies, else the builtin. `None` when the + /// class's `__next__` is not a registered leaf. + #[inline] + fn leaf_next_step( + &self, + it: &Object, + inst: &PyInstance, + ) -> Option> { + let (b, op) = self.core_leaf_next_op(inst)?; + if op != 0 { + if let Some(v) = crate::stdlib::collections_native::fast_op(op, it, None) { + return Some(Ok(v)); + } + } + // SAFETY: the cache and the class keep the builtin alive; its body + // runs no Python. + Some(unsafe { ((*b).call)(std::slice::from_ref(it)) }) } #[cold] @@ -18152,7 +18201,7 @@ impl Interpreter { &self, inst: &PyInstance, ver: u64, - ) -> Option> { + ) -> Option<(*const crate::object::BuiltinFn, u8)> { // A miss is remembered too: a Python-level `__next__` class asks // again on every step of a generic iteration. let found = match inst.cls().lookup("__next__") { @@ -18164,8 +18213,13 @@ impl Interpreter { } _ => None, }; - *self.core_next.borrow_mut() = Some((ver, found.clone())); - found + let op = found + .as_ref() + .and_then(leaf_builtins::jit_method_op) + .unwrap_or(0); + let out = found.as_ref().map(|b| (Rc::as_ptr(b), op)); + *self.core_next.borrow_mut() = Some((ver, found, op)); + out } /// Which leaf builtin `b` is, if any (pointer identity). @@ -56454,6 +56508,35 @@ pub(crate) mod leaf_builtins { GENERATION.load(Ordering::Acquire) } + /// Builtins opted into the JIT's native method lane (see + /// `tier2::MethodCallee::Native`), each with its direct operation. + static JIT_METHODS: parking_lot::Mutex, u8)>> = + parking_lot::Mutex::new(Vec::new()); + + /// Opt `b` (already registered as a leaf, binding its instance) into + /// the JIT's native method lane: a compiled caller may call its body + /// directly on a receiver whose class resolves to it, with no + /// interpreter frame. The body must also be safe to call with a + /// receiver that is a subclass instance or any other object (it + /// validates its receiver). A nonzero `op` names the operation's + /// direct form (`collections_native::fast_op`), which the caller + /// tries first. + pub(crate) fn register_jit_method(b: &Rc, op: u8) { + let mut reg = JIT_METHODS.lock(); + reg.retain(|(w, _)| w.strong_count() > 0); + reg.push((Rc::downgrade(b), op)); + } + + /// `b`'s direct operation when it opted into the JIT's native method + /// lane (by identity; probed at compile time only), `0` for none. + pub(crate) fn jit_method_op(b: &Rc) -> Option { + JIT_METHODS + .lock() + .iter() + .find(|(w, _)| std::ptr::eq(w.as_ptr(), Rc::as_ptr(b)) && w.strong_count() > 0) + .map(|&(_, op)| op) + } + /// The live registrations, keyed by address. pub(crate) fn snapshot() -> LeafMap { REGISTRY @@ -62769,8 +62852,9 @@ fn code_name_obj(code: &CodeObject, name_idx: u32) -> Option<&Object> { enum NativeTruth { /// Neither `__bool__` nor `__len__`: always true. AlwaysTrue, - /// A registered native `__bool__`. - Bool(Rc), + /// A registered native `__bool__`, with its direct operation + /// (`collections_native::fast_op`, `0` for none). + Bool(Rc, u8), /// A registered native `__len__`. Len(Rc), /// Anything else: the full path decides. diff --git a/crates/weavepy-vm/src/stdlib/collections_native.rs b/crates/weavepy-vm/src/stdlib/collections_native.rs index f207445f..2bea7e46 100644 --- a/crates/weavepy-vm/src/stdlib/collections_native.rs +++ b/crates/weavepy-vm/src/stdlib/collections_native.rs @@ -148,6 +148,7 @@ impl DequeState<'_> { // unguarded (`peek_mut` returns `None` if any guard is live) and neither // reference outlives the native call, which runs no Python. #[allow(clippy::mut_from_ref)] +#[inline(always)] fn fast_parts(args: &[Object]) -> Option> { let Object::Instance(inst) = args.first()? else { return None; @@ -156,7 +157,8 @@ fn fast_parts(args: &[Object]) -> Option> { // checks), and neither reference outlives the native call. let slots = unsafe { inst.slots.peek_mut() }?; let n = names(); - let [data, head, maxlen, state] = slots.leading_mut([&n.data, &n.head, &n.maxlen, &n.state])?; + let [data, head, maxlen, state] = + slots.leading_interned_mut([&n.data, &n.head, &n.maxlen, &n.state])?; let Object::List(data) = data else { return None; }; @@ -216,12 +218,14 @@ impl Fast<'_> { } } - /// [`popleft_locked`] over the unguarded parts. - fn popleft(&mut self) -> Result { + /// [`popleft_locked`] over the unguarded parts; `None` (nothing + /// touched) on an empty deque. + #[inline] + fn popleft(&mut self) -> Option { let mut h = self.head(); let d = &mut *self.d; if h >= d.len() { - return Err(index_error("pop from an empty deque")); + return None; } let x = std::mem::replace(&mut d[h], Object::None); h += 1; @@ -234,7 +238,101 @@ impl Fast<'_> { } self.bump_state(); self.set_head(h); - Ok(x) + Some(x) + } +} + +/// The deque operations a compiled caller may run directly (see +/// [`fast_op`]); `0` names none. The JIT's method registry carries one +/// per builtin (`leaf_builtins::register_jit_method`). +pub(crate) const OP_APPEND: u8 = 1; +pub(crate) const OP_APPENDLEFT: u8 = 2; +pub(crate) const OP_POP: u8 = 3; +pub(crate) const OP_POPLEFT: u8 = 4; +pub(crate) const OP_LEN: u8 = 5; +pub(crate) const OP_BOOL: u8 = 6; +pub(crate) const OP_GETITEM: u8 = 7; +/// A forward (`_deque_iterator`) or reverse iterator step; the receiver +/// is the iterator. +pub(crate) const OP_NEXT: u8 = 8; +pub(crate) const OP_RNEXT: u8 = 9; + +/// One deque operation's common case on `recv` (and its one argument), +/// over the unguarded views [`fast_parts`] takes: the whole operation, +/// or `None` with nothing touched when it needs the full body (a guarded +/// cell, a foreign receiver, an empty deque's `IndexError`, an index out +/// of range or not an exact `int`). The builtins' fast paths and the +/// JIT's direct calls share it; it runs no Python code. Inlined, so a +/// builtin's constant `op` folds the dispatch away. +#[inline(always)] +pub(crate) fn fast_op(op: u8, recv: &Object, arg: Option<&Object>) -> Option { + if op >= OP_NEXT { + let Object::Instance(iterator) = recv else { + return None; + }; + return deque_next_fast(iterator, op == OP_RNEXT); + } + let mut f = fast_parts(std::slice::from_ref(recv))?; + match op { + OP_APPEND => { + let x = arg?; + f.bump_state(); + f.d.push(x.clone()); + let trimmed = match f.maxlen() { + Some(m) if f.d.len() - f.head() > m => f.popleft(), + _ => None, + }; + drop(trimmed); + Some(Object::None) + } + OP_APPENDLEFT => { + let x = arg?; + f.bump_state(); + let mut h = f.head().min(f.d.len()); + if h == 0 { + h = std::cmp::max(8, f.d.len() / 2); + f.d.splice(0..0, std::iter::repeat_n(Object::None, h)); + } + h -= 1; + f.d[h] = x.clone(); + f.set_head(h); + let trimmed = match f.maxlen() { + Some(m) if f.d.len() - h > m => f.d.pop(), + _ => None, + }; + drop(trimmed); + Some(Object::None) + } + OP_POP => { + let h = f.head(); + if f.d.len() <= h { + return None; + } + f.bump_state(); + let x = f.d.pop().expect("len checked"); + // An emptied deque keeps a short free prefix for the next + // `appendleft` instead of splicing a new one. + if f.d.len() == h && h > 32 { + f.d.clear(); + f.set_head(0); + } + Some(x) + } + OP_POPLEFT => f.popleft(), + OP_LEN => Some(Object::Int(f.d.len().saturating_sub(f.head()) as i64)), + OP_BOOL => Some(Object::Bool(f.d.len() > f.head())), + OP_GETITEM => { + let Some(&Object::Int(i)) = arg else { + return None; + }; + let h = f.head(); + let n = f.d.len().saturating_sub(h) as i64; + let index = if i < 0 { i + n } else { i }; + (0..n) + .contains(&index) + .then(|| f.d[h + index as usize].clone()) + } + _ => None, } } @@ -277,16 +375,9 @@ fn popleft_locked(st: &mut DequeState<'_>, d: &mut Vec) -> Result Result { - if let [_, x] = args { - if let Some(mut f) = fast_parts(args) { - f.bump_state(); - f.d.push(x.clone()); - let trimmed = match f.maxlen() { - Some(m) if f.d.len() - f.head() > m => Some(f.popleft()?), - _ => None, - }; - drop(trimmed); - return Ok(Object::None); + if let [recv, x] = args { + if let Some(v) = fast_op(OP_APPEND, recv, Some(x)) { + return Ok(v); } } let mut st = receiver(args, "append")?; @@ -321,23 +412,9 @@ fn deque_append(args: &[Object]) -> Result { } fn deque_appendleft(args: &[Object]) -> Result { - if let [_, x] = args { - if let Some(mut f) = fast_parts(args) { - f.bump_state(); - let mut h = f.head().min(f.d.len()); - if h == 0 { - h = std::cmp::max(8, f.d.len() / 2); - f.d.splice(0..0, std::iter::repeat_n(Object::None, h)); - } - h -= 1; - f.d[h] = x.clone(); - f.set_head(h); - let trimmed = match f.maxlen() { - Some(m) if f.d.len() - h > m => f.d.pop(), - _ => None, - }; - drop(trimmed); - return Ok(Object::None); + if let [recv, x] = args { + if let Some(v) = fast_op(OP_APPENDLEFT, recv, Some(x)) { + return Ok(v); } } let mut st = receiver(args, "appendleft")?; @@ -374,20 +451,9 @@ fn deque_appendleft(args: &[Object]) -> Result { } fn deque_pop(args: &[Object]) -> Result { - if args.len() == 1 { - if let Some(mut f) = fast_parts(args) { - let h = f.head(); - if f.d.len() > h { - f.bump_state(); - let x = f.d.pop().expect("len checked"); - // An emptied deque keeps a short free prefix for the next - // `appendleft` instead of splicing a new one. - if f.d.len() == h && h > 32 { - f.d.clear(); - f.set_head(0); - } - return Ok(x); - } + if let [recv] = args { + if let Some(v) = fast_op(OP_POP, recv, None) { + return Ok(v); } } let mut st = receiver(args, "pop")?; @@ -413,11 +479,9 @@ fn deque_pop(args: &[Object]) -> Result { } fn deque_popleft(args: &[Object]) -> Result { - if args.len() == 1 { - if let Some(mut f) = fast_parts(args) { - if f.head() < f.d.len() { - return f.popleft(); - } + if let [recv] = args { + if let Some(v) = fast_op(OP_POPLEFT, recv, None) { + return Ok(v); } } let mut st = receiver(args, "popleft")?; @@ -433,9 +497,9 @@ fn deque_popleft(args: &[Object]) -> Result { } fn deque_len(args: &[Object]) -> Result { - if args.len() == 1 { - if let Some(f) = fast_parts(args) { - return Ok(Object::Int(f.d.len().saturating_sub(f.head()) as i64)); + if let [recv] = args { + if let Some(v) = fast_op(OP_LEN, recv, None) { + return Ok(v); } } let st = receiver(args, "__len__")?; @@ -447,9 +511,9 @@ fn deque_len(args: &[Object]) -> Result { } fn deque_bool(args: &[Object]) -> Result { - if args.len() == 1 { - if let Some(f) = fast_parts(args) { - return Ok(Object::Bool(f.d.len() > f.head())); + if let [recv] = args { + if let Some(v) = fast_op(OP_BOOL, recv, None) { + return Ok(v); } } let st = receiver(args, "__bool__")?; @@ -461,14 +525,9 @@ fn deque_bool(args: &[Object]) -> Result { } fn deque_getitem(args: &[Object]) -> Result { - if let [_, Object::Int(i)] = args { - if let Some(f) = fast_parts(args) { - let h = f.head(); - let n = f.d.len().saturating_sub(h) as i64; - let index = if *i < 0 { *i + n } else { *i }; - if (0..n).contains(&index) { - return Ok(f.d[h + index as usize].clone()); - } + if let [recv, index] = args { + if let Some(v) = fast_op(OP_GETITEM, recv, Some(index)) { + return Ok(v); } } let [_, index] = args else { @@ -573,13 +632,14 @@ const IT_STATE: usize = 2; /// [`fast_parts`]): a live iterator over an unmutated deque with an item /// left. `None` (nothing touched) leaves every other case to the full /// body. +#[inline(always)] fn deque_next_fast(iterator: &crate::types::PyInstance, reverse: bool) -> Option { // SAFETY: no guard is live on the cells (`peek`/`peek_mut` check), no // Python runs before the last use, and the iterator and its deque are // distinct objects. let its = unsafe { iterator.slots.peek_mut() }?; let n = names(); - let [deque, index, it_state] = its.leading_mut([&n.deq, &n.index, &n.deq_state])?; + let [deque, index, it_state] = its.leading_interned_mut([&n.deq, &n.index, &n.deq_state])?; let Object::Instance(deque) = deque else { return None; }; @@ -590,12 +650,14 @@ fn deque_next_fast(iterator: &crate::types::PyInstance, reverse: bool) -> Option }; // SAFETY: as above. let ds = unsafe { (*deque).slots.peek() }?; - let [data, head, _, state] = ds.leading([&n.data, &n.head, &n.maxlen, &n.state])?; + let [data, head, _, state] = ds.leading_interned([&n.data, &n.head, &n.maxlen, &n.state])?; let Object::List(data) = data else { return None; }; - if state.as_i64() != it_state.as_i64() { - return None; + // The counters are plain ints (any other value takes the full body). + match (state, &*it_state) { + (Object::Int(a), Object::Int(b)) if a == b => {} + _ => return None, } let h = match *head { Object::Int(h) if h >= 0 => h as usize, @@ -622,7 +684,7 @@ fn deque_next(args: &[Object], reverse: bool) -> Result { let [Object::Instance(iterator)] = args else { return Err(type_error("deque iterator __next__ requires one iterator")); }; - if let Some(v) = deque_next_fast(iterator, reverse) { + if let Some(v) = fast_op(if reverse { OP_RNEXT } else { OP_NEXT }, &args[0], None) { return Ok(v); } let (deque, index, it_state) = { @@ -768,6 +830,24 @@ pub fn build(_cache: &ModuleCache) -> Rc { crate::leaf_builtins::register_fast(b, fast); } } + // The deque's own methods ride the JIT's native method lane (the + // receiver checks above reject a foreign receiver exactly). + for (name, op) in [ + ("append", OP_APPEND), + ("appendleft", OP_APPENDLEFT), + ("pop", OP_POP), + ("popleft", OP_POPLEFT), + ("__len__", OP_LEN), + ("__bool__", OP_BOOL), + ("__getitem__", OP_GETITEM), + ("rotate", 0), + ("iterator_next", OP_NEXT), + ("reverse_iterator_next", OP_RNEXT), + ] { + if let Some(Object::Builtin(b)) = d.get(&DictKey(Object::from_static(name))) { + crate::leaf_builtins::register_jit_method(b, op); + } + } } Rc::new(PyModule { name: "_weave_collections".to_owned(), diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index e4ba8a1d..06a216d2 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -115,9 +115,12 @@ struct AttrGuard { /// instance-dict shadowing, and `__code__` rebinding introduced after /// compilation stay exact. struct MethodEntry { - func: Rc, - /// The function's `__code__` at compile time (rebindable). - code: Rc, + /// What the class resolves the name to. + callee: MethodCallee, + /// The class forbids an instance `__dict__` (its `__slots__` cover + /// every base), so no instance attribute can shadow the method: the + /// guard skips that probe. + no_dict: bool, /// The method name (for the shadow check and the span rebuild). name: String, /// `hash(name)`, so the per-call shadow probe hashes nothing. @@ -174,9 +177,37 @@ impl InlineUpdate { } } +/// The callee a burned-in method site resolved to. +enum MethodCallee { + /// A plain Python function and its `__code__` at compile time + /// (rebindable, so the guard compares it per call). + Py { + func: Rc, + code: Rc, + }, + /// A native accelerator's method: a builtin that binds its instance, + /// registered as a leaf and opted into this lane + /// (`leaf_builtins::register_jit_method`), such as `deque.append`. + /// Its body runs no Python code, so the helper calls it directly + /// with no interpreter frame; `fast` is its pure fast half when the + /// full body may reach Python (an argument's `__index__`). + Native { + builtin: Rc, + fast: Option, + /// The builtin's direct operation + /// (`collections_native::fast_op`), `0` for none. + op: u8, + }, +} + /// One slot per method token (parallel to `cf.method_sites`). type MethodTable = Vec; +/// The widest positional arity a native method site admits (receiver +/// included). The builtin validates its own arity, so this only bounds +/// the marshal buffer. +const NATIVE_METHOD_MAX_ARGS: u32 = 8; + /// One burned-in math intrinsic. The ordinary global snapshot guards the /// module's identity; this guard reads its current attribute by a checked /// dictionary index. Neither a value replacement nor a moved key is hidden. @@ -930,6 +961,11 @@ impl JitState { weavepy_jit::register_self_call_helpers(wpjit_self_enter, wpjit_self_exit, wpjit_self_slow); // RFC 0069 WS1 — the guarded method-call lane. weavepy_jit::register_call_method_helper(wpjit_call_method); + weavepy_jit::register_call_native_method_helper(wpjit_call_native_method); + // Native container subscripts (`deque.__getitem__`) and native + // method-site load guards. + weavepy_jit::register_obj_getitem_helper(wpjit_obj_getitem); + weavepy_jit::register_guard_method_helper(wpjit_guard_method); // RFC 0073 WS3 — the native `str`-method lane. weavepy_jit::register_str_method_helper(wpjit_str_method); // RFC 0069 WS2 — the libm sin/cos intrinsics and the Python- @@ -1199,6 +1235,7 @@ impl JitState { arg_count: e.arg_count, min_args: e.min_args, ret: e.ret, + native: matches!(e.callee, MethodCallee::Native { .. }), }); } let e = method(slot, path, name)?; @@ -1209,6 +1246,7 @@ impl JitState { arg_count: e.arg_count, min_args: e.min_args, ret: e.ret, + native: matches!(e.callee, MethodCallee::Native { .. }), }; methods.push(e); Some(res) @@ -1575,7 +1613,10 @@ impl JitState { }; let table: NativeTable = methods .iter() - .map(|m| self.resolve_native_func(&m.func, &m.code, true)) + .map(|m| match &m.callee { + MethodCallee::Py { func, code } => self.resolve_native_func(func, code, true), + MethodCallee::Native { .. } => None, + }) .collect(); let tbl = StdRc::new(table); if let Some(entry) = self.cache.get_mut(&key) { @@ -3138,8 +3179,40 @@ fn probe_method_entry( if inst.attr_get_str(name).is_some() { return None; } - let Some(Object::Function(f)) = cls.lookup(name) else { - return None; + let f = match cls.lookup(name) { + // The analyzer probes `__getitem__` for native subscripts only + // (`TOp::ObjGetItem`); a Python-level one must not take a token + // that no site would use. + Some(Object::Function(_)) if name == "__getitem__" => return None, + Some(Object::Function(f)) => f, + // A native accelerator's leaf method (`deque.append`): any + // positional arity (the body validates it and raises exactly), + // and an object-lane result (`None` rides the nullable `-1`, so a + // procedure's result costs no pin). + Some(Object::Builtin(b)) if b.binds_instance => { + let op = crate::leaf_builtins::jit_method_op(&b)?; + let fast = match interp.leaf_call_kind(&b)? { + super::LeafKind::Opaque => None, + super::LeafKind::Fast(fast) => Some(fast), + _ => return None, + }; + return Some(MethodEntry { + callee: MethodCallee::Native { + builtin: b, + fast, + op, + }, + no_dict: cls.forbids_dict, + name: name.to_owned(), + name_hash: crate::object::py_str_hash(name), + arg_count: NATIVE_METHOD_MAX_ARGS, + min_args: 1, + ret: MethodRet::Scalar(JitType::Obj), + ver: cls.attr_version.get(), + update: InlineUpdate::unarmed(), + }); + } + _ => return None, }; let fcode = f.code.borrow().clone(); if !py_callee_ok(&fcode) || fcode.arg_count == 0 { @@ -3163,8 +3236,11 @@ fn probe_method_entry( let ver = cls.attr_version.get(); let arg_count = fcode.arg_count; Some(MethodEntry { - func: f, - code: fcode, + callee: MethodCallee::Py { + func: f, + code: fcode, + }, + no_dict: cls.forbids_dict, name: name.to_owned(), name_hash: crate::object::py_str_hash(name), arg_count, @@ -4555,6 +4631,10 @@ fn arm_update(entry: &MethodEntry, nc: &NativeCallee, receiver: &Object, argc: u let Some(plan) = nc.scalar_update.as_deref() else { return; }; + // (A native method's body is no field update.) + let MethodCallee::Py { func, code } = &entry.callee else { + return; + }; let Some(guard) = nc.attr_guards.get(plan.store_token) else { return; }; @@ -4582,9 +4662,9 @@ fn arm_update(entry: &MethodEntry, nc: &NativeCallee, receiver: &Object, argc: u u.shadow.set(u32::try_from(shadow).unwrap_or(0)); u.from_arg.set(from_arg); u.inc.set(inc); - u.code_at.set(entry.func.code.as_ptr() as usize); + u.code_at.set(func.code.as_ptr() as usize); // SAFETY: `Rc` is one pointer, compared and never dereferenced. - u.code.set(unsafe { std::mem::transmute_copy::, usize>(&entry.code) }); + u.code.set(unsafe { std::mem::transmute_copy::, usize>(code) }); u.idx.set(guard.split_idx); #[cfg(test)] crate::SCALAR_FIELD_UPDATE_ARMED.with(|armed| armed.set(armed.get() + 1)); @@ -6459,6 +6539,9 @@ unsafe extern "C" fn wpjit_call_method( // SAFETY: the `&mut Interpreter` that entered native code is dormant // while the helper runs; this is the only live path to it. let interp = unsafe { &mut *ctx.interp }; + // A native site's deferred-guard miss (see `wpjit_call_native_method`) + // takes the surprise-receiver lane below. + let token = token & !(weavepy_jit::METHOD_GUARD_AT_CALL | weavepy_jit::METHOD_NATIVE); native_stat(|s| s.method_calls.set(s.method_calls.get() + 1)); let guard_miss = || { native_stat(|s| s.method_guard_misses.set(s.method_guard_misses.get() + 1)); @@ -6486,9 +6569,13 @@ unsafe extern "C" fn wpjit_call_method( // code (see `GilCell::peek`). attr_class_ok(inst, entry.ver) && unsafe { inst.attr_peek_has(probe.s, probe.hash) } == Some(false) - && match unsafe { entry.func.code.peek() } { - Some(code) => Rc::ptr_eq(code, &entry.code), - None => Rc::ptr_eq(&entry.func.code.borrow(), &entry.code), + && match &entry.callee { + MethodCallee::Py { func, code: burned } => match unsafe { func.code.peek() } { + Some(code) => Rc::ptr_eq(code, burned), + None => Rc::ptr_eq(&func.code.borrow(), burned), + }, + // Only a native site's deferred-guard miss gets here. + MethodCallee::Native { .. } => false, } } _ => false, @@ -6520,6 +6607,9 @@ unsafe extern "C" fn wpjit_call_method( }); return finish_interp_call(jf, ctx, interp, res, expect_tag); } + let MethodCallee::Py { func, code } = &entry.callee else { + unreachable!("native methods returned above"); + }; // A pure-leaf method (a getter, a predicate) evaluates frameless. // SAFETY: `argc` marshaled entries are live (the function contract), @@ -6529,8 +6619,8 @@ unsafe extern "C" fn wpjit_call_method( jf, ctx, interp, - &entry.func, - &entry.code, + func, + code, Some(&raw const recv), argc, expect_tag, @@ -6549,9 +6639,9 @@ unsafe extern "C" fn wpjit_call_method( .and_then(Option::as_ref) { if nc.scalar_update.is_some() - && entry.code.arg_count == argc + 1 - && Rc::ptr_eq(&nc.func, &entry.func) - && Rc::ptr_eq(&nc.code, &entry.code) + && code.arg_count == argc + 1 + && Rc::ptr_eq(&nc.func, func) + && Rc::ptr_eq(&nc.code, code) { // SAFETY: the method guard above pinned the binding; the update // checks its receiver, argument lane, and observers itself. @@ -6587,7 +6677,7 @@ unsafe extern "C" fn wpjit_call_method( .and_then(|t| t.get(token as usize)) .and_then(Option::as_ref) { - if Rc::ptr_eq(&nc.func, &entry.func) && Rc::ptr_eq(&nc.code, &entry.code) { + if Rc::ptr_eq(&nc.func, func) && Rc::ptr_eq(&nc.code, code) { // SAFETY: `jf`/`ctx` are this activation's live buffers // (see the function contract) and `nc` came from this // thread's tier cache via the activation's resolved table. @@ -6620,7 +6710,7 @@ unsafe extern "C" fn wpjit_call_method( .set(s.method_call_fallbacks.get() + 1); }); ctx.dirty = true; - let callee = Object::Function(entry.func.clone()); + let callee = Object::Function(func.clone()); let mut args: Vec = Vec::with_capacity(argc as usize + 1); args.push(recv.clone()); for j in 0..argc as usize { @@ -6638,6 +6728,446 @@ unsafe extern "C" fn wpjit_call_method( finish_interp_call(jf, ctx, interp, res, expect_tag) } +/// Deliver a native method's or subscript's completed result `v` to +/// the compiled caller on its `expect_tag` lane: `None` for the +/// procedure lane, an unboxed scalar, or an object-lane pin (`None` +/// rides the nullable `-1`, so a procedure's result costs no pin). A +/// value the lane can't carry, or pin pressure, parks `v` (`Boxed`: the +/// caller resumes after the operation, which never runs again). +#[inline(always)] +fn deliver_native_result(jf: &mut JitFrame, ctx: &mut CallCtx, v: Object, expect_tag: u32) -> i64 { + // The scalar arms own nothing, so they skip `v`'s drop glue. + match (SlotTag::from_raw(expect_tag), &v) { + (SlotTag::None, Object::None) => { + std::mem::forget(v); + return CallStatus::Ok as i64; + } + (SlotTag::Int, &Object::Int(value)) => { + std::mem::forget(v); + jf.ret_bits = value as u64; + jf.ret_tag = expect_tag; + return CallStatus::Ok as i64; + } + (SlotTag::ObjPin, Object::None) => { + std::mem::forget(v); + jf.ret_bits = u64::MAX; + jf.ret_tag = expect_tag; + return CallStatus::Ok as i64; + } + (SlotTag::ObjPin, _) => { + if ctx.temporary_pin_limit_reached() { + ctx.pin_pressure_exit = true; + } else if ctx.pins.len() < RUNTIME_PIN_CAP { + jf.ret_bits = ctx.pins.len() as u64; + jf.ret_tag = expect_tag; + ctx.pins.push(Pin::Obj(v)); + return CallStatus::Ok as i64; + } + } + _ => {} + } + ctx.parked = Some(v); + CallStatus::Boxed as i64 +} + +/// A bitwise view of one marshaled argument: scalars rebuild by value +/// and an object pin aliases its pinned object without taking a +/// reference (the pin owns it for the whole activation). The view must +/// never be dropped. `None` for a lane it can't alias (a list pin). +#[inline(always)] +fn arg_view(bits: u64, tag: u32, pins: &PinTable) -> Option { + Some(match SlotTag::from_raw(tag) { + SlotTag::Int => Object::Int(bits as i64), + SlotTag::Float => Object::Float(f64::from_bits(bits)), + SlotTag::Bool => Object::Bool(bits != 0), + SlotTag::ObjPin if bits == u64::MAX => Object::None, + SlotTag::ObjPin => match pins.get(bits as usize)? { + // SAFETY: a bitwise alias, never dropped (see above). + Pin::Obj(o) => unsafe { std::ptr::read(o) }, + Pin::List(..) => return None, + }, + _ => return None, + }) +} + +/// A native leaf body's direct call on bitwise views of the receiver pin +/// and the marshaled arguments (see [`arg_view`]): no references are +/// taken or released around the call, which runs no Python code. `None` +/// (nothing ran) when an argument has no view, the arity exceeds the +/// inline buffer, or the fast half declines. +/// +/// # Safety +/// +/// `argc` marshal entries are initialized (the helper contract). +#[inline(always)] +unsafe fn direct_native_call( + jf: &JitFrame, + ctx: &CallCtx, + builtin: &crate::object::BuiltinFn, + fast: Option, + recv_pin: i64, + argc: u32, +) -> Option> { + const N: usize = 4; + let n = argc as usize + 1; + if n > N { + return None; + } + let mut views = [const { std::mem::MaybeUninit::::uninit() }; N]; + match ctx.pins.get(usize::try_from(recv_pin).ok()?)? { + // SAFETY: a bitwise alias, never dropped (the pin owns it). + Pin::Obj(o) => views[0].write(unsafe { std::ptr::read(o) }), + Pin::List(..) => return None, + }; + for j in 0..argc as usize { + // SAFETY: native code wrote `argc` entries. + let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; + views[j + 1].write(arg_view(bits, tag, &ctx.pins)?); + } + // SAFETY: the first `n` entries were written; they are views, so the + // `MaybeUninit` array never drops them. + let args = unsafe { std::slice::from_raw_parts(views.as_ptr().cast::(), n) }; + match fast { + Some(fast) => fast(args), + None => Some(match builtin.call_kw.as_ref() { + Some(ckw) => ckw(args, &[]), + None => (builtin.call)(args), + }), + } +} + +/// A native method's direct operation (`collections_native::fast_op`) on +/// the receiver pin and at most one marshaled argument, viewed in place +/// (see [`arg_view`]). `None` (nothing ran) sends the call to the full +/// body. +/// +/// # Safety +/// +/// `argc` marshal entries are initialized (the helper contract). +#[inline(always)] +unsafe fn direct_native_op( + jf: &JitFrame, + ctx: &CallCtx, + op: u8, + recv_pin: i64, + argc: u32, +) -> Option { + let Pin::Obj(recv) = ctx.pins.get(usize::try_from(recv_pin).ok()?)? else { + return None; + }; + match argc { + 0 => crate::stdlib::collections_native::fast_op(op, recv, None), + 1 => { + // SAFETY: native code wrote one entry. + let (bits, tag) = unsafe { (*jf.call_args, *jf.call_tags) }; + let arg = std::mem::ManuallyDrop::new(arg_view(bits, tag, &ctx.pins)?); + crate::stdlib::collections_native::fast_op(op, recv, Some(&arg)) + } + _ => None, + } +} + +/// A burned-in native leaf method (see [`MethodCallee::Native`]), +/// called directly on the receiver bound at its load and the marshaled +/// arguments. No Python runs on the direct path, so the activation stays +/// clean and no round trip is charged. +/// +/// A fast half that declines (an argument whose conversion could run +/// Python), or an active profiler or tracer that must see the call, +/// needs the interpreter, and Python must see this activation's locals, +/// which live lane-packed in the native frame. While the receiver still +/// resolves the name to this builtin, the call rejects, and the +/// interpreter performs it (and the attribute load, which binds the same +/// builtin) on a materialized frame. A class rebound by the argument +/// evaluation since the load calls the builtin bound there through the +/// interpreter's profiled call path instead. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_method`]; `entry` lives in the +/// activation's method table. +unsafe fn call_native_method( + jf: &mut JitFrame, + ctx: &mut CallCtx, + interp: &mut super::Interpreter, + entry: &MethodEntry, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + let MethodCallee::Native { op, .. } = &entry.callee else { + return CallStatus::Reject as i64; + }; + if *op != 0 && !crate::trace::any_observers_active() { + // SAFETY: per the function contract. + if let Some(v) = unsafe { direct_native_op(jf, ctx, *op, recv_pin, argc) } { + return deliver_native_result(jf, ctx, v, expect_tag); + } + } + // SAFETY: per the function contract. + unsafe { call_native_body(jf, ctx, interp, entry, recv_pin, argc, expect_tag) } +} + +/// [`call_native_method`] past its direct operation: the builtin's body +/// (or fast half) on views of the operands, else the interpreter. +/// +/// # Safety +/// +/// Same contract as [`call_native_method`]. +#[cold] +#[inline(never)] +unsafe fn call_native_body( + jf: &mut JitFrame, + ctx: &mut CallCtx, + interp: &mut super::Interpreter, + entry: &MethodEntry, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + let MethodCallee::Native { builtin, fast, .. } = &entry.callee else { + return CallStatus::Reject as i64; + }; + if !crate::trace::any_observers_active() { + // SAFETY: per the function contract. + match unsafe { direct_native_call(jf, ctx, builtin, *fast, recv_pin, argc) } { + Some(Ok(v)) => return deliver_native_result(jf, ctx, v, expect_tag), + Some(Err(err)) => { + ctx.raised = Some(err); + return CallStatus::Raised as i64; + } + None => {} + } + } + let Some(recv) = ctx.pins.get(recv_pin as usize).map(Pin::to_object) else { + return CallStatus::Reject as i64; + }; + if matches!(&recv, Object::Instance(inst) if method_guard_ok(entry, inst)) { + return CallStatus::Reject as i64; + } + let mut args: Vec = Vec::with_capacity(argc as usize); + for j in 0..argc as usize { + // SAFETY: native code wrote `argc` entries, and the buffers are + // `max_call_args` wide. + let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; + args.push(unpack_pins(bits, tag, &ctx.pins)); + } + ctx.dirty = true; + let bound = Object::BoundMethod(Rc::new(crate::object::BoundMethod::new( + recv, + Object::Builtin(builtin.clone()), + ))); + let res = call_with_activation_shell(interp, ctx, jf, |i| { + i.call_c_profiled(&bound, &args, &[], &ctx.globals) + }); + finish_interp_call(jf, ctx, interp, res, expect_tag) +} + +/// The `wpjit_call_native_method` helper: [`wpjit_call_method`] for a +/// site that resolved a native leaf method (`METHOD_NATIVE` in the +/// token), kept apart so the common call pays none of the Python path's +/// setup. The method was guarded and bound at its load +/// (`TOp::GuardMethod`), or, when nothing ran in between, is guarded +/// here (`METHOD_GUARD_AT_CALL`): call the builtin bound there. A +/// deferred guard's miss is exactly the load the interpreter would +/// perform now, so it takes the generic helper's surprise-receiver lane. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_method`]. +unsafe extern "C" fn wpjit_call_native_method( + frame: *mut JitFrame, + token: u32, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &mut *jf.ctx.cast::() }; + let guard_at_call = token & weavepy_jit::METHOD_GUARD_AT_CALL != 0; + let index = + (token & !(weavepy_jit::METHOD_GUARD_AT_CALL | weavepy_jit::METHOD_NATIVE)) as usize; + // The common case in one pass: the entry's direct operation on the + // receiver pin (read once) and at most one argument view. + if let Some( + entry @ MethodEntry { + callee: MethodCallee::Native { op, .. }, + .. + }, + ) = ctx.methods.get(index) + { + if let Some(Pin::Obj(recv)) = ctx.pins.get(recv_pin as usize) { + let bound = !guard_at_call + || matches!(recv, Object::Instance(inst) if method_guard_ok(entry, inst)); + if bound && *op != 0 && argc <= 1 && !crate::trace::any_observers_active() { + let arg = if argc == 1 { + // SAFETY: native code wrote one entry. + let (bits, tag) = unsafe { (*jf.call_args, *jf.call_tags) }; + arg_view(bits, tag, &ctx.pins).map(std::mem::ManuallyDrop::new) + } else { + None + }; + if argc == 0 || arg.is_some() { + let arg = arg.as_deref(); + if let Some(v) = crate::stdlib::collections_native::fast_op(*op, recv, arg) { + return deliver_native_result(jf, ctx, v, expect_tag); + } + } + } + } + } + // SAFETY: per the function contract. + unsafe { call_native_method_slow(frame, token, recv_pin, argc, expect_tag) } +} + +/// [`wpjit_call_native_method`] past its direct operation (see there). +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_method`]. +#[cold] +#[inline(never)] +unsafe fn call_native_method_slow( + frame: *mut JitFrame, + token: u32, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &mut *jf.ctx.cast::() }; + let guard_at_call = token & weavepy_jit::METHOD_GUARD_AT_CALL != 0; + let index = + (token & !(weavepy_jit::METHOD_GUARD_AT_CALL | weavepy_jit::METHOD_NATIVE)) as usize; + if let Some( + entry @ MethodEntry { + callee: MethodCallee::Native { .. }, + .. + }, + ) = ctx.methods.get(index) + { + let bound = !guard_at_call + || matches!( + ctx.pins.get(recv_pin as usize), + Some(Pin::Obj(Object::Instance(inst))) if method_guard_ok(entry, inst) + ); + if bound { + let entry: *const MethodEntry = entry; + // SAFETY: the `&mut Interpreter` that entered native code is + // dormant while the helper runs; the activation's method table + // (and so the entry) lives until the activation ends. + return unsafe { + let interp = &mut *ctx.interp; + call_native_method(jf, ctx, interp, &*entry, recv_pin, argc, expect_tag) + }; + } + } + // SAFETY: per the function contract. + unsafe { wpjit_call_method(frame, token, recv_pin, argc, expect_tag) } +} + +/// The `wpjit_guard_method` helper (see `TOp::GuardMethod`): `0` when the +/// receiver pin is an instance of the class version the native method +/// site `token` burned, with no instance attribute shadowing the name; +/// `1` sends the load to the interpreter (nothing has run). +/// +/// # Safety +/// +/// Same live-activation contract as [`wpjit_call_py`]. +unsafe extern "C" fn wpjit_guard_method(frame: *mut JitFrame, pin: i64, token: i64) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &*jf.ctx.cast::() }; + let Some(entry) = usize::try_from(token).ok().and_then(|t| ctx.methods.get(t)) else { + return 1; + }; + let Some(Pin::Obj(Object::Instance(inst))) = + usize::try_from(pin).ok().and_then(|p| ctx.pins.get(p)) + else { + return 1; + }; + i64::from(!method_guard_ok(entry, inst)) +} + +/// A burned method entry still describes `inst`: the class version it +/// was resolved against (which pins the MRO hit), and no instance +/// attribute shadowing the name. +#[inline(always)] +fn method_guard_ok(entry: &MethodEntry, inst: &crate::types::PyInstance) -> bool { + attr_class_ok(inst, entry.ver) + && (entry.no_dict + // SAFETY: a read between two native ops; nothing here runs + // code (see `GilCell::peek`). + || unsafe { inst.attr_peek_has(&entry.name, entry.name_hash) } == Some(false)) +} + +/// The `wpjit_obj_getitem` helper: `container[index]` on an object-lane +/// container whose class resolved `__getitem__` to a native leaf method +/// (`deque.__getitem__`) at compile time. Shares the method helper's +/// shape: `token` names the burned resolution and the index rides +/// `call_args[0]`. Unlike a method call, a guard miss (another class, a +/// non-instance) or a fast half that declines rejects, so the +/// interpreter re-executes the `BINARY_SUBSCR` itself: nothing has run. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_method`]; `argc` is 1. +unsafe extern "C" fn wpjit_obj_getitem( + frame: *mut JitFrame, + token: u32, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &mut *jf.ctx.cast::() }; + if argc != 1 || crate::trace::any_observers_active() { + return CallStatus::Reject as i64; + } + let Some(MethodEntry { + callee: MethodCallee::Native { builtin, fast, op }, + ver, + .. + }) = ctx.methods.get(token as usize) + else { + return CallStatus::Reject as i64; + }; + // Subscription looks the dunder up on the type: no instance + // shadowing check. + let recv = match ctx.pins.get(recv_pin as usize) { + Some(Pin::Obj(recv @ Object::Instance(inst))) if attr_class_ok(inst, *ver) => recv, + _ => return CallStatus::Reject as i64, + }; + if *op != 0 { + // SAFETY: native code wrote one entry (the index). + let (bits, tag) = unsafe { (*jf.call_args, *jf.call_tags) }; + if let Some(index) = arg_view(bits, tag, &ctx.pins).map(std::mem::ManuallyDrop::new) { + if let Some(v) = crate::stdlib::collections_native::fast_op(*op, recv, Some(&index)) { + return deliver_native_result(jf, ctx, v, expect_tag); + } + } + } + let (builtin, fast) = (Rc::as_ptr(builtin), *fast); + // SAFETY: the activation's method table keeps the builtin alive; one + // marshaled argument (the index). + match unsafe { direct_native_call(jf, ctx, &*builtin, fast, recv_pin, 1) } { + Some(Ok(v)) => deliver_native_result(jf, ctx, v, expect_tag), + Some(Err(err)) => { + ctx.raised = Some(err); + CallStatus::Raised as i64 + } + // A declined fast half ran nothing: the interpreter re-executes. + None => CallStatus::Reject as i64, + } +} + /// RFC 0073 WS3 — the per-method resolved `str` builtin bodies, /// memoized process-wide (builtin type surfaces are immutable, so one /// resolution is good for the process lifetime). Indexed by the @@ -7576,10 +8106,25 @@ unsafe extern "C" fn wpjit_tuple_len(frame: *mut JitFrame, pin: i64) -> i64 { let jf = unsafe { &mut *frame }; #[allow(clippy::cast_ptr_alignment)] let ctx = unsafe { &mut *jf.ctx.cast::() }; - let Some(Pin::Obj(Object::Tuple(items))) = ctx.pins.get(pin as usize) else { - return -1; - }; - items.len() as i64 + match ctx.pins.get(pin as usize) { + Some(Pin::Obj(Object::Tuple(items))) => items.len() as i64, + // An instance whose class's `__len__` is a registered native leaf + // (`deque.__len__`): its body runs no Python and has no effects, + // so anything but a plain length deopts for the interpreter to + // call it again. + Some(Pin::Obj(obj @ Object::Instance(_))) if !crate::gil::free_threading_enabled() => { + // SAFETY: the `&mut Interpreter` is dormant while the helper runs. + let interp = unsafe { &*ctx.interp }; + match interp + .leaf_instance_dunder(obj, "__len__") + .map(|b| (b.call)(std::slice::from_ref(obj))) + { + Some(Ok(Object::Int(n))) if n >= 0 => n, + _ => -1, + } + } + _ => -1, + } } #[cfg(test)] @@ -7786,6 +8331,38 @@ unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i // SAFETY: the `&mut Interpreter` that entered native code is // dormant while the helper runs. let interp = unsafe { &mut *ctx.interp }; + // A native deque iterator's step runs no Python: no checkpoint (the + // loop header's countdown poll covers the back edge), and the element + // is read in place through the pin. + if let Some(Pin::Obj(it @ Object::Instance(inst))) = ctx.pins.get(pin as usize) { + if !crate::gil::free_threading_enabled() + && crate::hot_gates::load() == 0 + && !crate::trace::any_observers_active() + && super::Interpreter::default_getattribute(inst.cls_raw()) + && !crate::object::exotic_str_keys_possible() + { + if let Some((_, op)) = interp.core_leaf_next_op(inst) { + if op != 0 { + if let Some(v) = crate::stdlib::collections_native::fast_op(op, it, None) { + if let Some(bits) = + pack_iter_elem(&v, elem_tag, &mut ctx.pins, &mut ctx.pin_memo) + { + jf.ret_bits = bits; + return 0; + } + // Consumed but outside the lane: resume at the store. + jf.ret_bits = if matches!(v, Object::None) { + u64::MAX + } else { + ctx.pins.push(Pin::Obj(v)); + (ctx.pins.len() - 1) as u64 + }; + return 3; + } + } + } + } + } // The loop's poll point (the header's countdown poll covers the // native back edge; this covers the Python the step may run): // pending interpreter work and active observers route the loop @@ -7796,12 +8373,17 @@ unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i } let it = match ctx.pins.get(pin as usize) { Some(Pin::Obj(o @ (Object::Generator(_) | Object::Iter(_)))) => o.clone(), + // An instance iterator whose `__next__` is a registered native + // leaf (a `deque` iterator): stepped below without Python. + Some(Pin::Obj(o @ Object::Instance(_))) if !crate::gil::free_threading_enabled() => { + o.clone() + } _ => return 2, }; // A builtin-iterator step is pure native code; everything else // (generator resume) runs arbitrary Python on behalf of this // activation. - let runs_python = !matches!(it, Object::Iter(_)); + let runs_python = matches!(it, Object::Generator(_)); if runs_python { ctx.dirty = true; // A generator resume from native code rebuilds a whole interpreter @@ -7810,7 +8392,25 @@ unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i // drives a generator retires at the next poll (see `wpjit_poll`). ctx.dyn_py_calls = ctx.dyn_py_calls.saturating_add(1); } - match interp.iter_next(&it, &ctx.globals) { + let step = if let Object::Instance(inst) = &it { + // The core loop's cached `__next__` resolution (by class version). + // Any other `__next__` belongs to the interpreter (nothing was + // consumed). + if !super::Interpreter::default_getattribute(inst.cls_raw()) + || crate::object::exotic_str_keys_possible() + { + return 2; + } + match interp.leaf_next_step(&it, inst) { + None => return 2, + Some(Ok(v)) => Ok(Some(v)), + Some(Err(RuntimeError::PyException(e))) if e.type_name() == "StopIteration" => Ok(None), + Some(Err(e)) => Err(e), + } + } else { + interp.iter_next(&it, &ctx.globals) + }; + match step { Err(err) => { ctx.raised = Some(err); 4 @@ -10274,18 +10874,21 @@ unsafe extern "C" fn wpjit_truth(frame: *mut JitFrame, pin: i64, _reserved: i64) jf.ret_bits = 0; return 0; } + // A registered native `__bool__`/`__len__` (or neither): the + // interpreter's cached answer, which runs no Python (read through the + // pin: the table owns the object throughout). + if let Some(Pin::Obj(obj @ Object::Instance(_))) = ctx.pins.get(pin as usize) { + if !crate::gil::free_threading_enabled() { + if let Some(b) = interp.leaf_instance_truth(obj) { + jf.ret_bits = u64::from(b); + return 0; + } + } + } let v = match ctx.pins.get(pin as usize) { Some(p) => p.to_object(), None => return 3, }; - // A registered native `__bool__`/`__len__` (or neither): the - // interpreter's cached answer, which runs no Python. - if matches!(v, Object::Instance(_)) && !crate::gil::free_threading_enabled() { - if let Some(b) = interp.leaf_instance_truth(&v) { - jf.ret_bits = u64::from(b); - return 0; - } - } let pure = match &v { Object::Foreign(_) | Object::MappingProxyObj(_) => false, Object::Instance(_) => { @@ -10301,6 +10904,12 @@ unsafe extern "C" fn wpjit_truth(frame: *mut JitFrame, pin: i64, _reserved: i64) jf.ret_bits = u64::from(v.is_truthy()); return 0; } + // A Python `__bool__`/`__len__` may inspect its caller's frame, whose + // locals live lane-packed in the native activation: the interpreter + // re-executes the test with a materialized frame. + if matches!(v, Object::Instance(_)) { + return 3; + } ctx.dirty = true; match interp.obj_truthy(&v, &ctx.globals) { Err(err) => { diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index a1b7af1b..47ff3be6 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -2320,6 +2320,65 @@ impl SlotStorage { } } + /// Whether the first `N` keys of a small store are the given + /// interned names themselves (pointer identity, as attribute stores + /// intern them). + #[inline(always)] + fn leads_with_interned( + &self, + names: &[&crate::shared_value::SharedStr; N], + ) -> bool { + let SlotData::Small(entries) = &self.data else { + return false; + }; + if entries.len() < N { + return false; + } + let mut same = true; + for (i, name) in names.iter().enumerate() { + same &= matches!(&entries[i].0 .0, Object::Str(s) + if crate::shared_value::SharedStr::ptr_eq(s, name)); + } + same + } + + /// [`Self::leading`] for its common shape, inlined: a small store + /// whose first `N` keys are the given interned names themselves. Any + /// other shape takes the full comparison. + #[inline(always)] + pub fn leading_interned( + &self, + names: [&crate::shared_value::SharedStr; N], + ) -> Option<[&Object; N]> { + if !self.leads_with_interned(&names) { + return self.leading(names); + } + let SlotData::Small(entries) = &self.data else { + return None; + }; + let base = entries.as_ptr(); + // SAFETY: the store holds at least `N` entries (checked above). + Some(std::array::from_fn(|i| unsafe { &(*base.add(i)).1 })) + } + + /// [`Self::leading_interned`], mutably. + #[inline(always)] + pub fn leading_interned_mut( + &mut self, + names: [&crate::shared_value::SharedStr; N], + ) -> Option<[&mut Object; N]> { + if !self.leads_with_interned(&names) { + return self.leading_mut(names); + } + let SlotData::Small(entries) = &mut self.data else { + return None; + }; + let base = entries.as_mut_ptr(); + // SAFETY: the store holds at least `N` entries (checked above), + // and each index yields a distinct entry. + Some(std::array::from_fn(|i| unsafe { &mut (*base.add(i)).1 })) + } + /// [`Self::leading`], mutably. #[inline] pub fn leading_mut( diff --git a/tests/regrtest/test_jit_deque_lanes.py b/tests/regrtest/test_jit_deque_lanes.py new file mode 100644 index 00000000..293c3a94 --- /dev/null +++ b/tests/regrtest/test_jit_deque_lanes.py @@ -0,0 +1,257 @@ +"""Compiled loops over deques keep exact results, errors, and fallbacks. + +Hot loops call `deque` methods, subscripts, `len`, truth tests, and +iteration natively from compiled code. Each check below runs long enough +to compile and then changes the shape (a subclass override, a non-integer +element, a mutation during iteration) so the compiled code must fall back +without changing what Python observes. +""" + +from collections import deque +import gc +import pickle +import weakref + +N = 3000 + + +def fifo(n): + total = 0 + q = deque() + for i in range(n): + q.append(i) + q.append(i + 1) + total += q.popleft() + if i & 7 == 0: + total += q[0] + q[-1] + while q: + total += q.popleft() + return total + + +def fifo_list(n): + total = 0 + q = [] + for i in range(n): + q.append(i) + q.append(i + 1) + total += q.pop(0) + if i & 7 == 0: + total += q[0] + q[-1] + while q: + total += q.pop(0) + return total + + +assert fifo(N) == fifo_list(N) + + +def ring(n): + total = 0 + r = deque(maxlen=64) + for i in range(n): + r.append(i) + if i & 15 == 15: + r.rotate(3) + total += r[5] + return total + sum(r), list(r) + + +def ring_list(n): + total = 0 + r = [] + for i in range(n): + r.append(i) + if len(r) > 64: + del r[0] + if i & 15 == 15: + r[:] = r[-3:] + r[:-3] + total += r[5] + return total + sum(r), r + + +assert ring(N) == ring_list(N) + + +def stack(n): + total = 0 + s = deque() + for i in range(n): + s.appendleft(i) + if i & 3 == 3: + total += s.pop() + s.popleft() + total += len(s) + for x in s: + total += x + return total + + +def stack_list(n): + total = 0 + s = [] + for i in range(n): + s.insert(0, i) + if i & 3 == 3: + total += s.pop() + s.pop(0) + total += len(s) + for x in s: + total += x + return total + + +assert stack(N) == stack_list(N) + + +# Elements that are not integers: the integer speculation misses and the +# exact generic operation runs instead. +def concat(q, n): + out = None + for _ in range(n): + out = q[0] + q[-1] + return out + + +assert concat(deque([1, 2]), N) == 3 +assert concat(deque(["a", "b"]), N) == "ab" +assert concat(deque([1.5, 2]), 3) == 3.5 + + +def pops(q, n): + total = 0 + for i in range(n): + q.append(i) + total += q.pop() + return total + + +assert pops(deque(), N) == sum(range(N)) +# A non-integer element below the integers is never popped. +q = deque(["x"]) +assert pops(q, N) == sum(range(N)) +assert list(q) == ["x"] + + +# Errors raised by a native method inside a compiled loop. +def drain(q, n): + caught = 0 + for _ in range(n): + try: + q.popleft() + except IndexError as e: + assert str(e) == "pop from an empty deque", e + caught += 1 + return caught + + +assert drain(deque(range(10)), N) == N - 10 + + +def peek(q, i, n): + total = 0 + for _ in range(n): + total += q[i] + return total + + +assert peek(deque([5, 6, 7]), -1, N) == 7 * N +try: + peek(deque([5]), 3, 2) +except IndexError as e: + assert str(e) == "deque index out of range", e +else: + raise AssertionError("IndexError expected") + + +# Subclasses that override the operations keep their overrides. +class Doubling(deque): + def __getitem__(self, i): + return 2 * deque.__getitem__(self, i) + + def __len__(self): + return 100 + + def __iter__(self): + yield from (2 * x for x in deque.__iter__(self)) + + +class Scaling(deque): + def append(self, x): + deque.append(self, x * 10) + + +d = Doubling([1, 2, 3]) +assert peek(d, 0, N) == 2 * N + + +def lengths(q, n): + total = 0 + for _ in range(n): + total += len(q) + return total + + +assert lengths(deque([1, 2]), N) == 2 * N +assert lengths(d, 3) == 300 + + +def appends(q, n): + for i in range(n): + q.append(i) + return q + + +assert list(appends(deque(), 5)) == [0, 1, 2, 3, 4] +assert list(appends(Scaling(), 3)) == [0, 10, 20] + + +def iterate(q): + total = 0 + for x in q: + total += x + return total + + +assert iterate(deque(range(N))) == sum(range(N)) +assert iterate(Doubling([1, 2])) == 6 + +# Mutation during compiled iteration raises exactly. +def mutate(q): + seen = 0 + for x in q: + seen += 1 + if seen == 5: + q.append(x) + return seen + + +try: + mutate(deque(range(N))) +except RuntimeError as e: + assert str(e) == "deque mutated during iteration", e +else: + raise AssertionError("RuntimeError expected") + + +# Native iterators keep the iterator classes' surface. +q = deque([1, 2, 3]) +it = iter(q) +assert type(it).__name__ == "_deque_iterator", type(it) +assert next(it) == 1 +assert list(pickle.loads(pickle.dumps(it))) == [2, 3] +assert list(it) == [2, 3] +rit = reversed(q) +assert type(rit).__name__ == "_deque_reverse_iterator", type(rit) +assert list(rit) == [3, 2, 1] +assert it.__length_hint__() == 0 +assert iter(q).__length_hint__() == 3 + + +# A deque holding its own iterator is collectable. +q = deque() +q.append(iter(q)) +ref = weakref.ref(q) +del q +gc.collect() +assert ref() is None + +print("Compiled deque lanes: ok") From 22c6ee233659931575dcebab5b4c157361be85b4 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:38:15 -0700 Subject: [PATCH 45/96] perf: recompile from a later loop after a cold exit A frame compiled at its first loop meets the containers of later loops unbound, so their statements become cold exits and those loops run in the interpreter for the rest of the activation. After a cold exit, the next OSR request at a loop header the compiled frame can't enter now recompiles from there, with that loop's live values. Each code object gets at most three such recompiles. deque_ops compiles at each of its three loops, and its instructions retired per unit of WEAVEPY_BENCH_WORK fall from 4386 to 2926 (CPython 3.14: 4256). --- crates/weavepy-vm/src/tier2.rs | 42 ++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 06a216d2..1709d084 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -453,6 +453,15 @@ struct CacheEntry { /// (it would observe the same frame state); other entries still /// do, with their own live values. probe_misses: Vec, + /// A compiled frame took a cold exit (see `CompiledFrame::cold_exits`): + /// the code after it was unsupported *with the values live when it was + /// compiled* (typically a later loop's receiver, still unbound when an + /// earlier loop triggered the compile). The next OSR request at a loop + /// header the frame can't enter recompiles from there, with that + /// loop's live values, at most [`COLD_RECOMPILE_BUDGET`] times. + recompile_at_osr: bool, + /// Recompiles taken for [`Self::recompile_at_osr`]. + cold_recompiles: u8, /// Native side exits taken by this code's compiled frame. Healthy /// compiled code exits by *returning* (deopt is exceptional — a /// type-lane surprise or invalidated guard), so a frame that keeps @@ -1024,6 +1033,8 @@ impl JitState { native_entries: 0, generic_dyn_calls: 0, probe_misses: Vec::new(), + recompile_at_osr: false, + cold_recompiles: 0, native: None, method_native: None, direct: None, @@ -1034,7 +1045,22 @@ impl JitState { // compile even when an individual activation is short. entry.defer_osr = false; } + // A cold exit handed the activation to the interpreter, which + // now asks to enter a loop the frame can't: compile again from + // here (see `CacheEntry::recompile_at_osr`). + let recompile = entry.recompile_at_osr + && entry_pc != 0 + && matches!(&entry.tier, Tier::Compiled(a) + if !a.cf.osr_entries.iter().any(|e| e.pc == entry_pc)); + if recompile { + entry.recompile_at_osr = false; + entry.cold_recompiles = entry.cold_recompiles.saturating_add(1); + entry.osr_failures = 0; + entry.tier = Tier::Cold; + entry.counter = entry.counter.max(self.threshold); + } match &entry.tier { + Tier::Cold if recompile => {} Tier::Compiled(a) => { let out = CompiledEntry { cf: a.cf.clone(), @@ -1749,6 +1775,8 @@ impl JitState { native_entries: 0, generic_dyn_calls: 0, probe_misses: Vec::new(), + recompile_at_osr: false, + cold_recompiles: 0, native: None, method_native: None, direct: None, @@ -3445,6 +3473,8 @@ pub(crate) fn warm_compile(interp: &mut super::Interpreter, frame: &mut super::F native_entries: 0, generic_dyn_calls: 0, probe_misses: Vec::new(), + recompile_at_osr: false, + cold_recompiles: 0, native: None, method_native: None, direct: None, @@ -6448,6 +6478,12 @@ fn retire_native_driver(ctx: &CallCtx) { unsafe { (*ctx.code_ptr).jit_hint.mark_not_jitable() }; } +/// Recompiles one code object may take after cold exits (see +/// `CacheEntry::recompile_at_osr`): one per later loop is the common +/// shape (a function running several loops over containers it builds in +/// turn), and the budget bounds compile cost for anything else. +const COLD_RECOMPILE_BUDGET: u8 = 3; + /// Generic native-callee calls one activation may make between two /// loop polls before its code retires (see [`charge_native_roundtrip`]). const NATIVE_CALL_RETIRE_BUDGET: u32 = 4096; @@ -12630,6 +12666,12 @@ fn note_native_exit( st.stats.native_entries += 1; if cold_exit { st.stats.cold_exits += 1; + let key = Rc::as_ptr(&frame.code).cast::(); + if let Some(ce) = st.cache.get_mut(&key) { + if ce.cold_recompiles < COLD_RECOMPILE_BUDGET { + ce.recompile_at_osr = true; + } + } return; } if pin_pressure_exit { From 66fbc3c7f004fd5c8057f3fde22c931f63066090 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:39:19 -0700 Subject: [PATCH 46/96] perf: build deque iterators natively `deque.__iter__` and `__reversed__` are now native leaf builtins that build the `_deque_iterator` (or reverse iterator) instance directly, with the slots its `__init__` would assign, instead of running two Python frames. The iterator classes ride on the deque class as `_iter_types`, so each copy of `_collections` pairs with its own. A compiled `for x in d:` now captures the iterator without leaving native code: the JIT's iterator capture admits an instance whose `__iter__` is a native leaf, and each step runs the iterator's direct operation. deque_ops, instructions retired per unit of WEAVEPY_BENCH_WORK: 2926 to 2833 (CPython 3.14: 4256; 6748 before this series). --- .../src/stdlib/collections_native.rs | 55 ++++++++++++++++++- .../src/stdlib/python/_collections.py | 15 ++--- crates/weavepy-vm/src/tier2.rs | 24 ++++++-- 3 files changed, 80 insertions(+), 14 deletions(-) diff --git a/crates/weavepy-vm/src/stdlib/collections_native.rs b/crates/weavepy-vm/src/stdlib/collections_native.rs index 2bea7e46..4f3105c2 100644 --- a/crates/weavepy-vm/src/stdlib/collections_native.rs +++ b/crates/weavepy-vm/src/stdlib/collections_native.rs @@ -601,6 +601,50 @@ fn deque_rotate(args: &[Object]) -> Result { Ok(Object::None) } +/// `iter(d)` / `reversed(d)`: a fresh `_deque_iterator` (or reverse +/// iterator) over `d` at index 0, snapshotting the mutation counter, +/// exactly what the iterator classes' `__init__` builds, but with no +/// Python code running. The classes come from the deque class's +/// `_iter_types` pair (set by `_collections.py`), so a subclass and each +/// copy of the module use their own. +fn deque_make_iter(args: &[Object], reverse: bool) -> Result { + let method = if reverse { "__reversed__" } else { "__iter__" }; + let state = { + let st = receiver(args, method)?; + st.slots + .get_hinted(SLOT_STATE, &names().state) + .cloned() + .unwrap_or(Object::Int(0)) + }; + let [recv @ Object::Instance(inst)] = args else { + return Err(type_error(format!( + "deque.{method}() takes no arguments ({} given)", + args.len().saturating_sub(1) + ))); + }; + let it_cls = match inst.cls().lookup("_iter_types") { + Some(Object::Tuple(types)) => match types.get(usize::from(reverse)) { + Some(Object::Type(cls)) => cls.clone(), + _ => return Err(type_error("deque iterator types are not set")), + }, + _ => return Err(type_error("deque iterator types are not set")), + }; + let it = crate::types::PyInstance::new_deferred(it_cls); + // The iterator classes' slot order (see `deque_next_fast`). + it.slot_set("_deq", recv.clone()); + it.slot_set("_index", Object::Int(0)); + it.slot_set("_deq_state", state); + Ok(Object::Instance(it)) +} + +fn deque_iter(args: &[Object]) -> Result { + deque_make_iter(args, false) +} + +fn deque_reversed(args: &[Object]) -> Result { + deque_make_iter(args, true) +} + fn deque_iterator_index(args: &[Object]) -> Result { receiver(args, "__iter__")?; let [_, index] = args else { @@ -795,6 +839,8 @@ pub fn build(_cache: &ModuleCache) -> Rc { reg("__bool__", "__bool__", deque_bool); reg("__getitem__", "__getitem__", deque_getitem); reg("rotate", "rotate", deque_rotate); + reg("__iter__", "__iter__", deque_iter); + reg("__reversed__", "__reversed__", deque_reversed); reg("iterator_index", "iterator_index", deque_iterator_index); reg("iterator_next", "__next__", deque_iterator_next); reg( @@ -802,9 +848,10 @@ pub fn build(_cache: &ModuleCache) -> Rc { "__next__", deque_reverse_iterator_next, ); - // The end operations, length, truth, and iterator steps run no - // Python code. Indexing and rotation require an argument guard - // because index coercion can invoke Python's __index__ protocol. + // The end operations, length, truth, iterator construction, and + // iterator steps run no Python code. Indexing and rotation require + // an argument guard because index coercion can invoke Python's + // __index__ protocol. for name in [ "append", "appendleft", @@ -812,6 +859,8 @@ pub fn build(_cache: &ModuleCache) -> Rc { "popleft", "__len__", "__bool__", + "__iter__", + "__reversed__", "iterator_next", "reverse_iterator_next", ] { diff --git a/crates/weavepy-vm/src/stdlib/python/_collections.py b/crates/weavepy-vm/src/stdlib/python/_collections.py index 1d250216..a5fb91af 100644 --- a/crates/weavepy-vm/src/stdlib/python/_collections.py +++ b/crates/weavepy-vm/src/stdlib/python/_collections.py @@ -223,7 +223,7 @@ def _flat(self): # `builtin_function_or_method`, like the C accelerator's. from _weave_collections import ( append, appendleft, pop, popleft, rotate, - __len__, __bool__, __getitem__, + __len__, __bool__, __getitem__, __iter__, __reversed__, ) def extend(self, iterable): @@ -314,12 +314,6 @@ def reverse(self): self._state += 1 self._flat().reverse() - def __iter__(self): - return _deque_iterator(self) - - def __reversed__(self): - return _deque_reverse_iterator(self) - def __contains__(self, x): # CPython `deque_contains`: mutation during a comparison raises. data = self._flat() @@ -941,3 +935,10 @@ def __reduce__(self): if deq is None: return type(self), (deque(),) return type(self), (deq, self._index) + + +# The native `deque.__iter__` / `__reversed__` build these iterators +# directly (no Python `__init__` runs), so a compiled loop can capture a +# deque iterator without leaving native code. The classes ride on the +# deque class itself, so each copy of this module pairs with its own. +deque._iter_types = (_deque_iterator, _deque_reverse_iterator) diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 1709d084..6e3b7164 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -11110,13 +11110,29 @@ unsafe extern "C" fn wpjit_iter_new(frame: *mut JitFrame, pin: i64) -> i64 { Some(Pin::List(l, _)) => Object::List(l.clone()), None => return -1, }; + // An instance whose class's `__iter__` is a registered native leaf + // (`deque.__iter__`) builds its iterator without Python and without + // effects: a failure deopts for the interpreter to raise it. + if let Object::Instance(_) = &recv { + if crate::gil::free_threading_enabled() { + return -1; + } + let Some(b) = interp.leaf_instance_dunder(&recv, "__iter__") else { + return -1; + }; + return match (b.call)(std::slice::from_ref(&recv)) { + Ok(it) => { + let idx = ctx.pins.len() as i64; + ctx.pins.push(Pin::Obj(it)); + idx + } + Err(_) => -1, + }; + } // `make_iter` dispatches Python for exactly these receiver // shapes; a fresh compile would double `__iter__`'s side effects // on a post-hoc deopt, so they never enter the helper. - if matches!( - recv, - Object::Instance(_) | Object::Type(_) | Object::MappingProxyObj(_) - ) { + if matches!(recv, Object::Type(_) | Object::MappingProxyObj(_)) { return -1; } match interp.make_iter(&recv, &ctx.globals) { From 3dd9f16a7a7f9aff77bcd6e25a26b1af08537008 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 07:40:34 -0700 Subject: [PATCH 47/96] perf: load cached code's column spans on first use, and Fx-hash the intern pool Loading a module from the code cache decoded every instruction's PEP 657 column span (three varints each) although only tracebacks and `co_positions()` read them, and interned every name through a SipHash set. The spans are now stored as one length-prefixed block (cache format 3) that `ColTable` decodes when first read, and the intern pool hashes with Fx. Import-only instructions: typing -13%, json -11%, dataclasses -9%, asyncio -5%. --- crates/weavepy-compiler/src/flowgraph.rs | 2 +- crates/weavepy-compiler/src/lib.rs | 76 ++++++++++++++++++++- crates/weavepy-compiler/src/native_code.rs | 41 +++++++---- crates/weavepy-vm/src/builtins.rs | 4 +- crates/weavepy-vm/src/stdlib/marshal_mod.rs | 2 +- crates/weavepy-vm/src/stdlib/sys.rs | 6 +- 6 files changed, 107 insertions(+), 24 deletions(-) diff --git a/crates/weavepy-compiler/src/flowgraph.rs b/crates/weavepy-compiler/src/flowgraph.rs index a4be97f7..68740409 100644 --- a/crates/weavepy-compiler/src/flowgraph.rs +++ b/crates/weavepy-compiler/src/flowgraph.rs @@ -3092,7 +3092,7 @@ impl Cfg { } co.instructions = instructions; co.linetable = linetable; - co.coltable = coltable; + co.coltable = coltable.into(); co.exception_table = table; co.constants = self.consts; no_interrupt.sort_unstable(); diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index 64b87217..9d54e798 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -511,8 +511,9 @@ pub struct CodeObject { /// PEP-657 fine-grained column spans, one per instruction (same length /// as `instructions` once emission finishes). Drives the column fields /// of `co_positions()`. Empty when never populated (e.g. code objects - /// reconstructed from marshal, which doesn't carry columns). - pub coltable: Vec, + /// reconstructed from marshal, which doesn't carry columns). Decoded + /// from the code cache on first use (see [`ColTable`]). + pub coltable: ColTable, /// Number of positional + keyword arguments (excluding `*args`/`**kwargs`). pub arg_count: u32, /// Number of positional-only arguments. @@ -632,6 +633,74 @@ pub struct WireOverrides { /// A per-instruction source-column span (PEP-657). `col`/`end_col` are /// 0-based UTF-8 byte offsets within their respective source lines, and +/// A code object's column spans (see [`CodeObject::coltable`]): a +/// vector, or the code cache's encoding of one, decoded on first use. +/// Most code never reports a column (only tracebacks and +/// `co_positions()` read them), so a module loaded from the cache skips +/// the work until something does. +#[derive(Clone, Default)] +pub struct ColTable { + spans: std::sync::OnceLock>, + encoded: Option>, +} + +impl ColTable { + /// The spans `encoded` holds in the code cache's form (see + /// `native_code`), decoded when first read. + pub(crate) fn encoded(encoded: std::sync::Arc<[u8]>) -> Self { + ColTable { + spans: std::sync::OnceLock::new(), + encoded: Some(encoded), + } + } + + fn spans(&self) -> &Vec { + self.spans.get_or_init(|| { + self.encoded + .as_deref() + .and_then(native_code::decode_coltable) + .unwrap_or_default() + }) + } +} + +impl From> for ColTable { + fn from(spans: Vec) -> Self { + ColTable { + spans: std::sync::OnceLock::from(spans), + encoded: None, + } + } +} + +impl std::ops::Deref for ColTable { + type Target = Vec; + + fn deref(&self) -> &Vec { + self.spans() + } +} + +impl std::ops::DerefMut for ColTable { + fn deref_mut(&mut self) -> &mut Vec { + self.spans(); + self.encoded = None; + self.spans.get_mut().expect("decoded above") + } +} + +impl PartialEq for ColTable { + fn eq(&self, other: &Self) -> bool { + **self == **other + } +} + +impl std::fmt::Debug for ColTable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + (**self).fmt(f) + } +} + /// are `-1` when the column was not tracked. `end_lineno` is `0` when /// unknown (callers fall back to the instruction's start line). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -2630,7 +2699,8 @@ impl Compiler { } }; self.co.linetable.push(line); - self.co.coltable.push(self.resolve_colspan()); + let span = self.resolve_colspan(); + self.co.coltable.push(span); offset } diff --git a/crates/weavepy-compiler/src/native_code.rs b/crates/weavepy-compiler/src/native_code.rs index b430758a..f12aa104 100644 --- a/crates/weavepy-compiler/src/native_code.rs +++ b/crates/weavepy-compiler/src/native_code.rs @@ -16,7 +16,7 @@ use crate::bytecode::{CacheTable, Instruction, OpCode}; use crate::{CodeObject, ColSpan, Constant, ExcHandler}; /// The layout revision; bump it whenever the encoding changes. -pub const VERSION: u8 = 2; +pub const VERSION: u8 = 3; /// Encode `code` (and its nested code objects). `None` for a code object /// the format doesn't carry: one with raw CPython wire overrides, or with @@ -28,6 +28,21 @@ pub fn encode(code: &CodeObject) -> Option> { Some(w.0) } +/// Decode a column-span block [`encode`] wrote (see `ColTable`). +pub(crate) fn decode_coltable(bytes: &[u8]) -> Option> { + let mut r = Reader { bytes, pos: 0 }; + let end_lines = r.deltas()?; + let mut spans = Vec::with_capacity(end_lines.len()); + for end_lineno in end_lines { + spans.push(ColSpan { + end_lineno, + col: r.i32()?, + end_col: r.i32()?, + }); + } + (r.pos == bytes.len()).then_some(spans) +} + /// Decode what [`encode`] wrote, stamping `filename` on every code /// object. `None` for input it didn't write (or a different version's). pub fn decode(bytes: &[u8], filename: &str) -> Option { @@ -116,11 +131,15 @@ impl Writer { // Line numbers as deltas from the previous entry: mostly zero, so // one byte each whatever the line. self.deltas(c.linetable.iter().copied()); - self.deltas(c.coltable.iter().map(|s| s.end_lineno)); - for s in &c.coltable { - self.int(i64::from(s.col)); - self.int(i64::from(s.end_col)); + // The column spans as one length-prefixed block, which a reader + // keeps encoded until something reads a column (see `ColTable`). + let mut cols = Writer(Vec::new()); + cols.deltas(c.coltable.iter().map(|s| s.end_lineno)); + for s in c.coltable.iter() { + cols.int(i64::from(s.col)); + cols.int(i64::from(s.end_col)); } + self.bytes(&cols.0); self.uint(u64::from(c.arg_count)); self.uint(u64::from(c.posonly_count)); self.uint(u64::from(c.kwonly_count)); @@ -360,15 +379,7 @@ impl Reader<'_> { }); } let linetable = self.deltas()?; - let end_lines = self.deltas()?; - let mut coltable = Vec::with_capacity(end_lines.len()); - for end_lineno in end_lines { - coltable.push(ColSpan { - end_lineno, - col: self.i32()?, - end_col: self.i32()?, - }); - } + let coltable = crate::ColTable::encoded(Arc::from(self.raw()?)); let arg_count = self.u32()?; let posonly_count = self.u32()?; let kwonly_count = self.u32()?; @@ -472,7 +483,7 @@ mod tests { constants: vec![Constant::Int(-5)], varnames: vec!["x".to_owned()], linetable: vec![1, 2, 2], - coltable: vec![ColSpan::default(); 3], + coltable: vec![ColSpan::default(); 3].into(), arg_count: 1, is_generator: true, is_nested: true, diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 6c986179..48349b6e 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -4681,7 +4681,7 @@ fn install_wire_code(nc: &mut weavepy_compiler::CodeObject, bytes: Vec) { // instruction count; line info defaults to the first line. let first = nc.linetable.iter().copied().find(|l| *l > 0).unwrap_or(1); nc.linetable = vec![first; instructions.len()]; - nc.coltable = Vec::new(); + nc.coltable = Vec::new().into(); nc.caches = weavepy_compiler::CacheTable::with_len(instructions.len()); nc.instructions = instructions; nc.exception_table = Vec::new(); @@ -4869,7 +4869,7 @@ pub(crate) fn code_type_call( nc.caches = weavepy_compiler::CacheTable::with_len(decoded.instructions.len()); nc.instructions = decoded.instructions; nc.linetable = decoded.linetable; - nc.coltable = decoded.coltable; + nc.coltable = decoded.coltable.into(); nc.exception_table = decoded.exception_table; nc.no_interrupt_jumps = decoded.no_interrupt_jumps; } diff --git a/crates/weavepy-vm/src/stdlib/marshal_mod.rs b/crates/weavepy-vm/src/stdlib/marshal_mod.rs index e42c43dc..495f4e2d 100644 --- a/crates/weavepy-vm/src/stdlib/marshal_mod.rs +++ b/crates/weavepy-vm/src/stdlib/marshal_mod.rs @@ -1750,7 +1750,7 @@ impl<'a> MarshalReader<'a> { // PEP-657 columns recovered from long-form location entries // (RFC 0056 WS4): traceback caret underlines survive the // `.pyc` round-trip. - coltable: decoded.coltable, + coltable: decoded.coltable.into(), arg_count, posonly_count, kwonly_count, diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index 7259fe52..86df607a 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -1417,8 +1417,10 @@ fn sys_setrecursionlimit(args: &[Object]) -> Result { // (pickle's `load_build` inserts `sys.intern(k)` keys — // test_pickle test_attribute_name_interning). thread_local! { - static INTERN_POOL: RefCell> = - RefCell::new(std::collections::HashSet::new()); + // (Fx-hashed: interning runs for every name of every code object a + // module loads, and SipHash showed in import profiles.) + static INTERN_POOL: RefCell> = + RefCell::new(std::collections::HashSet::with_hasher(Default::default())); } /// Canonicalize `name` through the interpreter's intern pool, seeding it From c6122b375f3e3c560997fff9c2bc98a1a4dc6f4b Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:00:46 -0700 Subject: [PATCH 48/96] perf: check isinstance against tuples of classes in the core loop The core loop settled `isinstance(x, cls)` in place for one plain class but sent a tuple of classes (`isinstance(x, (str, int))`) to the full call. A tuple of plain classes (no `__instancecheck__`) is now checked in place too; anything else, and an instance whose class doesn't match (its `__class__` may still claim it), takes the full call as before. `isinstance(i, (str, int))` in a small function: -48%. --- crates/weavepy-vm/src/lib.rs | 40 ++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 3abba523..60dedc9f 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -13652,21 +13652,39 @@ impl Interpreter { /// shapes, see `leaf_builtin_call`); `None` declines untouched. #[inline(never)] fn core_isinstance(ops: &[Object]) -> Option> { - let [obj, Object::Type(cls)] = ops else { + let [obj, spec] = ops else { return None; }; - if !cls.metaclass_is_type() { - return None; - } - let r = match obj { - Object::Instance(inst) => { - if !inst.cls_raw().is_subclass_of(cls) { - return None; + // One class with no `__instancecheck__` of its own; `None` when + // the full call must decide (an instance whose class doesn't + // match may still claim the class through `__class__`). + let check = |cls: &Rc| -> Option { + if !cls.metaclass_is_type() { + return None; + } + match obj { + Object::Instance(inst) => inst.cls_raw().is_subclass_of(cls).then_some(true), + Object::File(_) => None, + obj => Some(builtins::class_of(obj).is_subclass_of(cls)), + } + }; + let r = match spec { + Object::Type(cls) => check(cls)?, + // A tuple of plain classes, matched in order. + Object::Tuple(items) => { + let mut found = false; + for item in items.iter() { + let Object::Type(cls) = item else { + return None; + }; + if check(cls)? { + found = true; + break; + } } - true + found } - Object::File(_) => return None, - obj => builtins::class_of(obj).is_subclass_of(cls), + _ => return None, }; Some(Ok(Object::Bool(r))) } From 9421987fc89271af65aca6afda1b7dd753145f5c Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:28:26 -0700 Subject: [PATCH 49/96] perf: store natively built datetime values packed, with a per-class pool Exact `timedelta`, `date` and `datetime` instances the natives build now hold their fields as two machine words (`SlotData::Packed`) instead of a vector of ten field objects, as CPython's C structs do. A dead one goes back to its class set's pool (CPython's per-type freelists), so a value's life allocates nothing. Python code that reads a slot gets field objects built on first use, and a slot write turns the storage back into a field vector, so `_pydatetime`'s methods, pickling and the C API see the same slots as before. The exact classes' constructors now run natively for `timedelta` and for keyword arguments (in the core loop's `CALL`/`CALL_KW` too), `__hash__` and `date.replace(day=...)` have native bodies, `datetime.fromisoformat` binds without the descriptor protocol, a shift within the day skips the calendar, and isoformat/strftime format without `core::fmt`. datetime_ops, instructions retired (python3.14 9.2K/unit, 674M default): per unit 10.1K -> 5.7K; default work 750M -> 487M. --- crates/weavepy-vm/src/lib.rs | 88 +- .../weavepy-vm/src/stdlib/datetime_native.rs | 1480 ++++++++++++++--- crates/weavepy-vm/src/sync.rs | 2 +- crates/weavepy-vm/src/types.rs | 117 +- crates/weavepy-vm/src/vm_singletons.rs | 14 + tests/regrtest/test_datetime_packed.py | 207 +++ 6 files changed, 1664 insertions(+), 244 deletions(-) create mode 100644 tests/regrtest/test_datetime_packed.py diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 60dedc9f..e76ee2c1 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -2435,6 +2435,15 @@ impl Interpreter { // A dead plain tuple's allocation is recycled (CPython's tuple // freelist; see `maybe_donate_tuple`). dropped @ Object::Tuple(_) => self.maybe_donate_tuple(dropped), + // A natively served `datetime` value goes back to its pool + // (see `stdlib::datetime_native::recycle`). + Object::Instance(i) + if Rc::strong_count(&i) == 1 && crate::stdlib::datetime_native::pooled_kind(&i) => + { + if let Err(i) = crate::stdlib::datetime_native::recycle(i) { + drop(i); + } + } // Likewise a dead instance's (see `PyInstance::try_recycle`). Object::Instance(i) if Rc::strong_count(&i) == 1 && i.dies_by_plain_drop() => { PyInstance::try_recycle(i); @@ -11699,8 +11708,15 @@ impl Interpreter { // SAFETY: `start + kwc + argc + 3 == len`. let ops = unsafe { std::slice::from_raw_parts(base.add(start), len - start) }; - let Some(r) = self.core_pure_kw_call(code, pc, ops, argc, sw.depth_cell) - else { + let r = match &ops[0] { + // A natively served class's constructor (see + // `stdlib::datetime_native`). + Object::Type(ty) if ty.native_kind.get() != 0 => { + Self::core_native_ctor_kw(ops, argc, true) + } + _ => self.core_pure_kw_call(code, pc, ops, argc, sw.depth_cell), + }; + let Some(r) = r else { // A plain Python callee switches in place, as // `CALL`'s does (see `core_call_kw`). if !matches!(&ops[0], Object::Function(_)) { @@ -11789,6 +11805,29 @@ impl Interpreter { } } } + // A natively served class's constructor (see + // `stdlib::datetime_native`), in place. + if python == 2 { + let start = len - argc - 2; + // SAFETY: `len >= argc + 2`: the call's operands. + let ops = + unsafe { std::slice::from_raw_parts(base.add(start), argc + 2) }; + if let Some(r) = Self::core_native_ctor_kw(ops, argc, false) { + // SAFETY: every operand was checked to leave by a + // plain decrement; the result takes the callee's + // slot. + unsafe { + for k in start..len { + drop_hot(base.add(k).read()); + } + base.add(start).write(r); + } + len = start + 1; + last = pc; + pc += 1; + continue; + } + } if python != 0 { // SAFETY: as above. unsafe { frame.stack.set_len(len) }; @@ -15504,6 +15543,38 @@ impl Interpreter { crate::stdlib::datetime_native::leaf_compare(kind, a, b) } + /// `cls(*args, **kwargs)` for a natively served class (see + /// [`crate::stdlib::datetime_native::construct_names`]): `ops` is the + /// core loop's `CALL` operands (callee, self slot, `argc` positional), + /// or with `kw` its `CALL_KW` operands (the keyword values and the + /// names tuple follow). The new instance, or `None` (nothing touched) + /// for the full handler, which also raises every error. + #[inline(never)] + fn core_native_ctor_kw(ops: &[Object], argc: usize, kw: bool) -> Option { + let (Object::Type(cls), Object::Unbound) = (ops.first()?, ops.get(1)?) else { + return None; + }; + if cls.native_kind.get() == 0 { + return None; + } + let args = ops.get(2..2 + argc)?; + let (names, values): (&[Object], &[Object]) = if kw { + let Object::Tuple(names) = ops.last()? else { + return None; + }; + (names, ops.get(2 + argc..ops.len() - 1)?) + } else { + (&[], &[]) + }; + if !ops + .iter() + .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o)) + { + return None; + } + crate::stdlib::datetime_native::construct_names(cls, args, names, values)?.ok() + } + /// A natively served instance's public field (`dt.hour`; see /// [`crate::stdlib::datetime_native::leaf_field`]), or `None`. #[inline(never)] @@ -19138,6 +19209,19 @@ impl Interpreter { if (cls.c_ext_ptr.get() != 0 && !cls.flags.is_builtin) || !Self::plain_metaclass(cls) { return None; } + // A natively served class method (`datetime.fromisoformat`; see + // `stdlib::datetime_native`), bound as `classmethod.__get__` + // binds it. + if cls.native_kind.get() != 0 { + if let Some(Object::Str(name)) = code_name_obj(code, name_idx) { + if let Some(f) = crate::stdlib::datetime_native::type_classmethod(cls, name) { + return Some(Object::BoundMethod(Rc::new(BoundMethod::py_method( + Object::Type(cls.clone()), + f, + )))); + } + } + } match Self::leaf_class_attr(code, cls, name_idx)? { LeafAttr::Method(f) => { if let Some(s) = slot { diff --git a/crates/weavepy-vm/src/stdlib/datetime_native.rs b/crates/weavepy-vm/src/stdlib/datetime_native.rs index df69bda3..d60b2688 100644 --- a/crates/weavepy-vm/src/stdlib/datetime_native.rs +++ b/crates/weavepy-vm/src/stdlib/datetime_native.rs @@ -22,13 +22,22 @@ //! ([`leaf_binop`], [`leaf_compare`], [`leaf_field`]) apply only while a //! class is in the state [`install`] left it in. //! +//! The natives build exact `timedelta`, `date` and `datetime` instances +//! with packed storage ([`PackedSlots`]): the fields as two machine words +//! instead of a vector of objects, as CPython's C structs hold them. The +//! exact classes' constructors (keywords included), arithmetic, and +//! `fromisoformat` produce such values, and a dead one returns to its +//! class set's [`Pool`], so a value's life costs no allocation. Python +//! code reading a slot (`self._year`) sees field objects built on first +//! use; writing one turns the storage back into a plain field vector. +//! //! Exact-class instances are never cycle-collector tracked: CPython's C //! datetime types are not GC types either (a cycle through `tzinfo` is //! uncollectable there too), and their fields are ints, `None` and the //! `tzinfo` reference. use std::collections::HashMap; -use std::fmt::Write as _; +use std::sync::atomic::{AtomicUsize, Ordering}; use crate::error::{overflow_error, type_error, RuntimeError}; use crate::object::{BuiltinFn, DictKey, Object}; @@ -63,15 +72,374 @@ const DT_TZINFO: usize = 7; const DT_FOLD: usize = 9; const TZ_OFFSET: usize = 0; +// --------------------------------------------------------------------- +// Packed storage. + +/// A natively built `timedelta`, `date` or `datetime`, packed into two +/// words: the instance's slot storage (see `SlotData::Packed`) instead +/// of a vector of field objects. CPython's C types store these as C +/// struct members; packing gets WeavePy the same allocation-free +/// representation, and the natives read a field with a shift. +/// +/// - `datetime` and `date`: `word` holds the fields in chronological +/// significance (see [`pack_dt`]), so two naive values compare as +/// `word >> 1`; `aux` owns the `tzinfo` (an `Rc` turned +/// into a raw pointer), or is `0` for `None`. +/// - `timedelta`: `word` is the days (as `i64`), `aux` the seconds and +/// microseconds (`seconds << 20 | microseconds`). +/// +/// Code that reads a slot by reference (a slot descriptor serving +/// `self._year` in a Python method, `copy`, the collector) gets a copy +/// of the field objects built on first use and kept until the storage +/// dies or is written; every write first converts the storage to a +/// field vector (`SlotStorage::unpack`). +pub(crate) struct PackedSlots { + word: u64, + aux: usize, + /// The kind (low three bits), and the lazily built field objects (a + /// thin pointer to `layout().len()` objects; `0` until built). + meta: AtomicUsize, +} + +const META_KIND: usize = 7; + +// Bit positions in a `datetime`/`date` word. +const W_FOLD: u32 = 0; +const W_US: u32 = 1; +const W_SS: u32 = 21; +const W_MM: u32 = 27; +const W_HH: u32 = 33; +const W_DAY: u32 = 38; +const W_MONTH: u32 = 43; +const W_YEAR: u32 = 47; + +/// A validated `datetime`'s fields as a packed word. +#[inline] +fn pack_dt(f: &Dt) -> u64 { + ((f.y as u64) << W_YEAR) + | ((f.m as u64) << W_MONTH) + | ((f.d as u64) << W_DAY) + | ((f.hh as u64) << W_HH) + | ((f.mm as u64) << W_MM) + | ((f.ss as u64) << W_SS) + | ((f.us as u64) << W_US) + | ((f.fold as u64) << W_FOLD) +} + +#[inline] +fn bits(word: u64, shift: u32, width: u32) -> i64 { + ((word >> shift) & ((1 << width) - 1)) as i64 +} + +/// `(year, month, day)` of a packed `date`/`datetime` word. +#[inline] +fn word_ymd(w: u64) -> (i64, i64, i64) { + (bits(w, W_YEAR, 14), bits(w, W_MONTH, 4), bits(w, W_DAY, 5)) +} + +/// `(hour, minute, second, microsecond, fold)` of a packed word. +#[inline] +fn word_time(w: u64) -> (i64, i64, i64, i64, i64) { + ( + bits(w, W_HH, 5), + bits(w, W_MM, 6), + bits(w, W_SS, 6), + bits(w, W_US, 20), + bits(w, W_FOLD, 1), + ) +} + +/// The slot layouts packed values describe themselves with, shared by +/// every interpreter: `[timedelta, date, datetime]`, in the order the +/// classes' `__new__` assigns the slots. +fn layouts() -> &'static [SharedSlice; 3] { + static LAYOUTS: std::sync::OnceLock<[SharedSlice; 3]> = std::sync::OnceLock::new(); + LAYOUTS.get_or_init(|| { + let layout = |names: &[&str]| -> SharedSlice { + names + .iter() + .map(|n| DictKey(Object::Str(interned(n)))) + .collect::>() + .into() + }; + [ + layout(&[ + "_days", + "_seconds", + "_microseconds", + "_hashcode", + "days", + "seconds", + "microseconds", + ]), + layout(&["_year", "_month", "_day", "_hashcode"]), + layout(&[ + "_year", + "_month", + "_day", + "_hour", + "_minute", + "_second", + "_microsecond", + "_tzinfo", + "_hashcode", + "_fold", + ]), + ] + }) +} + +#[inline] +fn layout_of(kind: u8) -> &'static SharedSlice { + let l = layouts(); + match kind { + KIND_TIMEDELTA => &l[0], + KIND_DATE => &l[1], + _ => &l[2], + } +} + +impl PackedSlots { + #[inline] + fn new(kind: u8, word: u64, aux: usize) -> Self { + Self { + word, + aux, + meta: AtomicUsize::new(kind as usize), + } + } + + #[inline] + pub(crate) fn kind(&self) -> u8 { + (self.meta.load(Ordering::Relaxed) & META_KIND) as u8 + } + + #[inline] + fn has_tz(&self) -> bool { + self.kind() == KIND_DATETIME + } + + /// The owned `tzinfo` pointer (`null` for `None` and other kinds). + #[inline] + fn tz_ptr(&self) -> *const PyInstance { + if self.has_tz() { + self.aux as *const PyInstance + } else { + std::ptr::null() + } + } + + /// A new reference to the `tzinfo`. + #[inline] + fn tz(&self) -> Object { + let p = self.tz_ptr(); + if p.is_null() { + return Object::None; + } + // SAFETY: `aux` owns one strong reference; this adds another. + unsafe { + Rc::increment_strong_count(p); + Object::Instance(Rc::from_raw(p)) + } + } + + pub(crate) fn layout(&self) -> &'static SharedSlice { + layout_of(self.kind()) + } + + /// The field objects (built on first use; see the type docs). + pub(crate) fn values(&self) -> &[Object] { + let len = self.layout().len(); + let m = self.meta.load(Ordering::Acquire); + let ptr = if m & !META_KIND != 0 { + (m & !META_KIND) as *const Object + } else { + self.materialize() + }; + // SAFETY: a built copy holds exactly `len` objects and lives until + // the storage is dropped, cleared or unpacked (all `&mut self`). + unsafe { std::slice::from_raw_parts(ptr, len) } + } + + #[cold] + #[inline(never)] + fn materialize(&self) -> *const Object { + let vals = self.build_values().into_boxed_slice(); + let len = vals.len(); + let ptr = Box::into_raw(vals).cast::(); + let kind = self.kind() as usize; + match self.meta.compare_exchange( + kind, + ptr as usize | kind, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => ptr, + Err(cur) => { + // Another reader built a copy first. + // SAFETY: `ptr` came from `Box::into_raw` just above. + drop(unsafe { Box::from_raw(std::ptr::slice_from_raw_parts_mut(ptr, len)) }); + (cur & !META_KIND) as *const Object + } + } + } + + fn build_values(&self) -> Vec { + let w = self.word; + match self.kind() { + KIND_TIMEDELTA => { + let (d, s, us) = self.td(); + vec![ + Object::Int(d), + Object::Int(s), + Object::Int(us), + Object::Int(-1), + Object::Int(d), + Object::Int(s), + Object::Int(us), + ] + } + KIND_DATE => { + let (y, m, d) = word_ymd(w); + vec![ + Object::Int(y), + Object::Int(m), + Object::Int(d), + Object::Int(-1), + ] + } + _ => { + let (y, m, d) = word_ymd(w); + let (hh, mm, ss, us, fold) = word_time(w); + vec![ + Object::Int(y), + Object::Int(m), + Object::Int(d), + Object::Int(hh), + Object::Int(mm), + Object::Int(ss), + Object::Int(us), + self.tz(), + Object::Int(-1), + Object::Int(fold), + ] + } + } + } + + /// `(days, seconds, microseconds)` of a packed `timedelta`. + #[inline] + fn td(&self) -> (i64, i64, i64) { + ( + self.word as i64, + (self.aux >> 20) as i64, + (self.aux & 0xF_FFFF) as i64, + ) + } + + /// Take the field objects, releasing the packed form. + pub(crate) fn into_values(self) -> Box<[Object]> { + let this = std::mem::ManuallyDrop::new(self); + let len = this.layout().len(); + let m = this.meta.load(Ordering::Acquire); + let vals = if m & !META_KIND != 0 { + // SAFETY: a built copy of `len` objects (see `materialize`), + // owned by the storage being consumed. + unsafe { + Box::from_raw(std::ptr::slice_from_raw_parts_mut( + (m & !META_KIND) as *mut Object, + len, + )) + } + } else { + this.build_values().into_boxed_slice() + }; + this.release_aux(); + vals + } + + /// Release the owned `tzinfo` reference (the caller forgets `aux`). + #[inline] + fn release_aux(&self) { + let p = self.tz_ptr(); + if !p.is_null() { + // SAFETY: `aux` owns one strong reference. + drop(unsafe { Rc::from_raw(p) }); + } + } + + /// Release the built copy, if any. + #[inline] + fn free_values(&mut self) { + let m = *self.meta.get_mut(); + if m & !META_KIND != 0 { + let len = self.layout().len(); + *self.meta.get_mut() = m & META_KIND; + // SAFETY: see `into_values`. + drop(unsafe { + Box::from_raw(std::ptr::slice_from_raw_parts_mut( + (m & !META_KIND) as *mut Object, + len, + )) + }); + } + } + + /// Release everything the value owns, keeping the kind (a pooled + /// instance's storage; see [`recycle`]). + fn clear(&mut self) { + self.release_aux(); + self.aux = 0; + self.free_values(); + } +} + +impl Drop for PackedSlots { + fn drop(&mut self) { + self.release_aux(); + self.free_values(); + } +} + +impl Clone for PackedSlots { + fn clone(&self) -> Self { + let p = self.tz_ptr(); + if !p.is_null() { + // SAFETY: `aux` keeps the allocation alive; the clone owns the + // added reference. + unsafe { Rc::increment_strong_count(p) }; + } + Self::new(self.kind(), self.word, self.aux) + } +} + +impl std::fmt::Debug for PackedSlots { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PackedSlots") + .field("kind", &self.kind()) + .field("word", &self.word) + .field("aux", &self.aux) + .finish_non_exhaustive() + } +} + +/// The instance's slot storage for a read (no Python code may run while +/// `f` reads). +#[inline(always)] +fn with_slots(i: &PyInstance, f: impl FnOnce(&SlotStorage) -> Option) -> Option { + // SAFETY: the view is dropped when `f` returns, and `f` runs no code. + if let Some(s) = unsafe { i.slots.peek() } { + return f(s); + } + let s = i.slots.try_borrow().ok()?; + f(&s) +} + /// Interned slot and field names. struct Names { days: SharedStr, seconds: SharedStr, microseconds: SharedStr, - hashcode: SharedStr, - pub_days: SharedStr, - pub_seconds: SharedStr, - pub_microseconds: SharedStr, year: SharedStr, month: SharedStr, day: SharedStr, @@ -93,6 +461,10 @@ struct Names { f_microsecond: SharedStr, f_tzinfo: SharedStr, f_fold: SharedStr, + f_days: SharedStr, + f_seconds: SharedStr, + f_microseconds: SharedStr, + f_fromisoformat: SharedStr, } fn interned(s: &str) -> SharedStr { @@ -108,10 +480,6 @@ impl Names { days: interned("_days"), seconds: interned("_seconds"), microseconds: interned("_microseconds"), - hashcode: interned("_hashcode"), - pub_days: interned("days"), - pub_seconds: interned("seconds"), - pub_microseconds: interned("microseconds"), year: interned("_year"), month: interned("_month"), day: interned("_day"), @@ -132,6 +500,10 @@ impl Names { f_microsecond: interned("microsecond"), f_tzinfo: interned("tzinfo"), f_fold: interned("fold"), + f_days: interned("days"), + f_seconds: interned("seconds"), + f_microseconds: interned("microseconds"), + f_fromisoformat: interned("fromisoformat"), } } } @@ -160,6 +532,11 @@ struct State { /// `(kind, name)` → what the class resolved the name to after /// [`install`]: the natives, and the field properties. expect: std::sync::OnceLock>, + /// Dead instances kept for reuse (see [`Pool`]). + pool: Pool, + /// The native `datetime.fromisoformat` (the function its + /// `classmethod` wraps). + fromiso: std::sync::OnceLock, } /// The dunders and fields the dispatch-loop shortcuts serve without a @@ -174,6 +551,11 @@ const SHORTCUT_NAMES: &[(u8, &str)] = &[ (KIND_TIMEDELTA, "__le__"), (KIND_TIMEDELTA, "__gt__"), (KIND_TIMEDELTA, "__ge__"), + (KIND_TIMEDELTA, "__new__"), + (KIND_TIMEDELTA, "__init__"), + (KIND_TIMEDELTA, "days"), + (KIND_TIMEDELTA, "seconds"), + (KIND_TIMEDELTA, "microseconds"), (KIND_DATE, "__add__"), (KIND_DATE, "__sub__"), (KIND_DATE, "__eq__"), @@ -206,6 +588,7 @@ const SHORTCUT_NAMES: &[(u8, &str)] = &[ (KIND_DATETIME, "microsecond"), (KIND_DATETIME, "tzinfo"), (KIND_DATETIME, "fold"), + (KIND_DATETIME, "fromisoformat"), ]; /// Whether `cls` (of `kind`) still resolves every shortcut name to what @@ -281,9 +664,20 @@ fn as_int(o: &Object) -> Option { // from the shared layout; any other storage is read by name. #[allow(clippy::index_refutable_slice)] +#[inline] fn td_fields(i: &PyInstance, st: &State) -> Option<(i64, i64, i64)> { + with_slots(i, |s| { + if let Some(p) = s.as_packed() { + return (p.kind() == KIND_TIMEDELTA).then(|| p.td()); + } + td_fields_slow(s, st) + }) +} + +#[allow(clippy::index_refutable_slice)] +#[inline(never)] +fn td_fields_slow(s: &SlotStorage, st: &State) -> Option<(i64, i64, i64)> { let n = &st.names; - let s = i.slots.try_borrow().ok()?; if let Some(v) = s.values_for_layout(&st.td_layout) { return Some(( as_int(&v[TD_DAYS])?, @@ -302,10 +696,20 @@ fn td_us(f: (i64, i64, i64)) -> i128 { (i128::from(f.0) * 86_400 + i128::from(f.1)) * 1_000_000 + i128::from(f.2) } -#[allow(clippy::index_refutable_slice)] +#[inline] fn date_fields(i: &PyInstance, st: &State) -> Option<(i64, i64, i64)> { + with_slots(i, |s| { + if let Some(p) = s.as_packed() { + return (p.kind() != KIND_TIMEDELTA).then(|| word_ymd(p.word)); + } + date_fields_slow(s, st) + }) +} + +#[allow(clippy::index_refutable_slice)] +#[inline(never)] +fn date_fields_slow(s: &SlotStorage, st: &State) -> Option<(i64, i64, i64)> { let n = &st.names; - let s = i.slots.try_borrow().ok()?; if let Some(v) = s .values_for_layout(&st.date_layout) .or_else(|| s.values_for_layout(&st.dt_layout)) @@ -335,9 +739,49 @@ struct Dt { fold: i64, } +impl Dt { + /// The fields of a packed `datetime` word, with `tz`. + #[inline] + fn from_word(w: u64, tz: Object) -> Self { + let (y, m, d) = word_ymd(w); + let (hh, mm, ss, us, fold) = word_time(w); + Dt { + y, + m, + d, + hh, + mm, + ss, + us, + tz, + fold, + } + } +} + +#[inline] fn dt_fields(i: &PyInstance, st: &State) -> Option
{ + with_slots(i, |s| { + if let Some(p) = s.as_packed() { + return (p.kind() == KIND_DATETIME).then(|| Dt::from_word(p.word, p.tz())); + } + dt_fields_slow(s, st) + }) +} + +/// A packed `datetime`'s word and borrowed `tzinfo` pointer (`null` for +/// `None`): the comparison and difference paths, which keep no field. +#[inline] +fn dt_packed(i: &PyInstance) -> Option<(u64, *const PyInstance)> { + with_slots(i, |s| { + let p = s.as_packed()?; + (p.kind() == KIND_DATETIME).then(|| (p.word, p.tz_ptr())) + }) +} + +#[inline(never)] +fn dt_fields_slow(s: &SlotStorage, st: &State) -> Option
{ let n = &st.names; - let s = i.slots.try_borrow().ok()?; if let Some(v) = s.values_for_layout(&st.dt_layout) { return Some(Dt { y: as_int(&v[D_YEAR])?, @@ -371,10 +815,25 @@ enum Tz { Fixed(i128), } +/// [`tz_of`] for a packed `tzinfo` pointer (`null` for `None`). +#[inline] +fn tz_of_ptr(p: *const PyInstance, st: &State) -> Option { + if p.is_null() { + return Some(Tz::Naive); + } + if std::ptr::eq(p, st.utc.as_ptr()) { + return Some(Tz::Fixed(0)); + } + // SAFETY: a borrowed view of the owner's reference, never dropped. + let o = std::mem::ManuallyDrop::new(Object::Instance(unsafe { Rc::from_raw(p) })); + tz_of(&o, st) +} + fn tz_of(tz: &Object, st: &State) -> Option { let n = &st.names; match tz { Object::None => Some(Tz::Naive), + Object::Instance(i) if std::ptr::eq(Rc::as_ptr(i), st.utc.as_ptr()) => Some(Tz::Fixed(0)), o => { let i = inst(o, KIND_TIMEZONE)?; let off = { @@ -466,72 +925,154 @@ fn instance_fixed( Object::Instance(Rc::new(i)) } -/// `date(y, m, d)` and `datetime(y, m, d[, hh[, mm[, ss[, us[, tz]]]]])` -/// (`tzinfo=` by keyword too) of the exact classes, with in-range `int` -/// fields and a naive or fixed-offset zone: the instance the replaced -/// `__new__` builds, built natively. `None` for every other shape, which -/// the Python constructor serves (and diagnoses). +/// `timedelta(...)`, `date(...)` and `datetime(...)` of the exact classes +/// with plain `int` fields (by position or keyword) and, for `datetime`, a +/// naive or fixed-offset zone: the instance the replaced `__new__` builds, +/// built natively. `None` for every other shape, which the Python +/// constructor serves (and diagnoses). pub(crate) fn construct( cls: &Rc, args: &[Object], kwargs: &[(String, Object)], +) -> Option> { + construct_with(cls, args, kwargs.len(), |i| { + (kwargs[i].0.as_str(), &kwargs[i].1) + }) +} + +/// [`construct`] with the keyword names as the call's names tuple +/// (`names[i]` names `values[i]`). +pub(crate) fn construct_names( + cls: &TypeObject, + args: &[Object], + names: &[Object], + values: &[Object], +) -> Option> { + if names.len() != values.len() { + return None; + } + construct_with(cls, args, names.len(), |i| match &names[i] { + Object::Str(s) => (s.as_ref(), &values[i]), + _ => ("", &values[i]), + }) +} + +/// The constructors' parameter names, by kind, in positional order. +const TD_PARAMS: [&str; 7] = [ + "days", + "seconds", + "microseconds", + "milliseconds", + "minutes", + "hours", + "weeks", +]; +const DATE_PARAMS: [&str; 3] = ["year", "month", "day"]; +const DT_PARAMS: [&str; 9] = [ + "year", + "month", + "day", + "hour", + "minute", + "second", + "microsecond", + "tzinfo", + "fold", +]; + +fn construct_with<'a>( + cls: &TypeObject, + args: &'a [Object], + nkw: usize, + kw: impl Fn(usize) -> (&'a str, &'a Object), ) -> Option> { let st = state_of_cls(cls)?; let kind = cls.native_kind.get(); - let exact = match kind { - KIND_DATE => st.date.upgrade(), - KIND_DATETIME => st.datetime.upgrade(), - _ => None, + let (exact, params): (*const TypeObject, &[&str]) = match kind { + KIND_TIMEDELTA => (st.timedelta.as_ptr(), &TD_PARAMS), + KIND_DATE => (st.date.as_ptr(), &DATE_PARAMS), + // `fold` is keyword-only. + KIND_DATETIME => (st.datetime.as_ptr(), &DT_PARAMS), + _ => return None, }; - if !exact.is_some_and(|c| Rc::ptr_eq(&c, cls)) || !verified(st, cls, kind) { + if !std::ptr::eq(cls, exact) || !verified(st, cls, kind) { return None; } - let int = |o: &Object| match o { - Object::Int(v) => Some(*v), - _ => None, + let npos = if kind == KIND_DATETIME { + 8 + } else { + params.len() }; - if kind == KIND_DATE { - let ([y, m, d], true) = (args, kwargs.is_empty()) else { - return None; - }; - let (y, m, d) = (int(y)?, int(m)?, int(d)?); - return valid_date(y, m, d) - .then(|| new_date(st, y, m, d)) - .flatten() - .map(Ok); - } - if !(3..=8).contains(&args.len()) { + if args.len() > npos { return None; } - let mut tz = args.get(7).cloned().unwrap_or(Object::None); - for (name, v) in kwargs { - if name != "tzinfo" || args.len() == 8 { + // Each parameter's argument, by position then keyword. + let mut slots: [Option<&'a Object>; 9] = [None; 9]; + for (slot, a) in slots.iter_mut().zip(args) { + *slot = Some(a); + } + for i in 0..nkw { + let (name, v) = kw(i); + let ix = params.iter().position(|p| *p == name)?; + if slots[ix].is_some() { + // A duplicate: the Python constructor raises. return None; } - tz = v.clone(); - } - let field = |ix: usize| args.get(ix).map_or(Some(0), int); - let f = Dt { - y: int(&args[0])?, - m: int(&args[1])?, - d: int(&args[2])?, - hh: field(3)?, - mm: field(4)?, - ss: field(5)?, - us: field(6)?, - tz, - fold: 0, + slots[ix] = Some(v); + } + let int = |ix: usize, default: Option| -> Option { + match slots[ix] { + Some(Object::Int(v)) => Some(*v), + None => default, + _ => None, + } }; - let ok = valid_date(f.y, f.m, f.d) - && (0..24).contains(&f.hh) - && (0..60).contains(&f.mm) - && (0..60).contains(&f.ss) - && (0..1_000_000).contains(&f.us); - if !ok { - return None; + match kind { + KIND_TIMEDELTA => { + let mut v = [0i128; 7]; + for (ix, slot) in v.iter_mut().enumerate() { + *slot = i128::from(int(ix, Some(0))?); + } + let [days, seconds, us, ms, minutes, hours, weeks] = v; + new_td( + st, + days + weeks * 7, + seconds + minutes * 60 + hours * 3600, + us + ms * 1000, + ) + } + KIND_DATE => { + let (y, m, d) = (int(0, None)?, int(1, None)?, int(2, None)?); + valid_date(y, m, d) + .then(|| new_date(st, y, m, d)) + .flatten() + .map(Ok) + } + _ => { + let f = Dt { + y: int(0, None)?, + m: int(1, None)?, + d: int(2, None)?, + hh: int(3, Some(0))?, + mm: int(4, Some(0))?, + ss: int(5, Some(0))?, + us: int(6, Some(0))?, + tz: slots[7].cloned().unwrap_or(Object::None), + fold: int(8, Some(0))?, + }; + let ok = valid_date(f.y, f.m, f.d) + && (0..24).contains(&f.hh) + && (0..60).contains(&f.mm) + && (0..60).contains(&f.ss) + && (0..1_000_000).contains(&f.us) + && (0..=1).contains(&f.fold); + if !ok { + return None; + } + tz_of(&f.tz, st)?; + new_dt(st, f).map(Ok) + } } - tz_of(&f.tz, st)?; - new_dt(st, &f).map(Ok) } /// A normalized exact `timedelta` from unnormalized components. @@ -558,38 +1099,57 @@ fn new_td(st: &State, d: i128, s: i128, us: i128) -> Option Option { - let cls = st.date.upgrade()?; - Some(instance_fixed( - cls, - &st.date_layout, - vec![ - Object::Int(y), - Object::Int(m), - Object::Int(d), - Object::Int(-1), - ], - )) + if !valid_date(y, m, d) { + let cls = st.date.upgrade()?; + return Some(instance_fixed( + cls, + &st.date_layout, + vec![ + Object::Int(y), + Object::Int(m), + Object::Int(d), + Object::Int(-1), + ], + )); + } + let word = ((y as u64) << W_YEAR) | ((m as u64) << W_MONTH) | ((d as u64) << W_DAY); + new_packed(st, &st.date, KIND_DATE, word, 0) } -#[allow(clippy::too_many_arguments)] -fn new_dt(st: &State, f: &Dt) -> Option { +fn new_dt(st: &State, f: Dt) -> Option { + let ok = valid_date(f.y, f.m, f.d) + && (0..24).contains(&f.hh) + && (0..60).contains(&f.mm) + && (0..60).contains(&f.ss) + && (0..1_000_000).contains(&f.us) + && (0..=1).contains(&f.fold); + if ok { + let word = pack_dt(&f); + match f.tz { + Object::None => return new_packed(st, &st.datetime, KIND_DATETIME, word, 0), + Object::Instance(tz) => { + let aux = Rc::into_raw(tz) as usize; + let r = new_packed(st, &st.datetime, KIND_DATETIME, word, aux); + if r.is_none() { + // SAFETY: not adopted; release the reference taken above. + drop(unsafe { Rc::from_raw(aux as *const PyInstance) }); + } + return r; + } + _ => {} + } + } let cls = st.datetime.upgrade()?; Some(instance_fixed( cls, @@ -602,16 +1162,171 @@ fn new_dt(st: &State, f: &Dt) -> Option { Object::Int(f.mm), Object::Int(f.ss), Object::Int(f.us), - f.tz.clone(), + f.tz, Object::Int(-1), Object::Int(f.fold), ], )) } +// --------------------------------------------------------------------- +// The instance pool: CPython's C types keep per-type freelists; a dead +// natively served instance keeps its allocation, class and packed +// storage here for the next value of its kind. + +const POOL_CAP: usize = 64; + +/// One class set's pooled instances, per kind (`timedelta`, `date`, +/// `datetime`). A pooled instance keeps its class, so the pool (in the +/// classes' shared [`State`]) and the classes keep each other alive, as +/// CPython's static types live for the whole process. +#[derive(Default)] +struct Pool(std::cell::UnsafeCell<[Vec>; 3]>); + +// SAFETY: the pool is touched only while `pool_ok()`: then every thread +// that reaches VM objects holds the GIL, which serializes the accesses +// (each is a push or a pop that runs no other code). +unsafe impl Sync for Pool {} +unsafe impl Send for Pool {} + +/// Whether the pools may be used: under the GIL with no thread reaching +/// objects outside it (see `sync::cells_unguarded`). Debug builds (the +/// unit-test binary runs interpreters on concurrent threads with no GIL +/// between them) never pool. +#[inline] +fn pool_ok() -> bool { + #[cfg(debug_assertions)] + { + false + } + #[cfg(not(debug_assertions))] + { + !crate::sync::cells_unguarded() + } +} + +impl Pool { + #[inline] + fn with(&self, kind: u8, f: impl FnOnce(&mut Vec>) -> R) -> Option { + if !pool_ok() { + return None; + } + let ix = match kind { + KIND_TIMEDELTA => 0, + KIND_DATE => 1, + KIND_DATETIME => 2, + _ => return None, + }; + // SAFETY: see the `Sync` impl; `f` only pushes or pops. + Some(f(unsafe { &mut (*self.0.get())[ix] })) + } +} + +/// An instance of `cls` (the exact class of `kind`) holding the packed +/// value `(word, aux)`; `aux` is adopted on success. +#[inline] +fn new_packed( + st: &State, + cls: &Weak, + kind: u8, + word: u64, + aux: usize, +) -> Option { + if let Some(inst) = st.pool.with(kind, Vec::pop).flatten() { + // SAFETY: pooled instances are unique (see `recycle`): nothing + // else can observe the class or storage while they change. + let m = unsafe { &mut *Rc::as_ptr(&inst).cast_mut() }; + if std::ptr::eq(Rc::as_ptr(m.class.get_mut()), cls.as_ptr()) { + if let Some(p) = m.slots.get_mut().as_packed_mut() { + p.word = word; + p.aux = aux; + return Some(Object::Instance(inst)); + } + } + // Not this class's (a `__class__` assignment cannot reach a pooled + // instance, so this does not happen): free it outright. + drop(Rc::into_arc(inst)); + } + let cls = cls.upgrade()?; + let mut i = PyInstance::new(cls); + *i.slots.get_mut() = SlotStorage::from_packed(PackedSlots::new(kind, word, aux)); + Some(Object::Instance(Rc::new(i))) +} + +/// Whether `i`'s class keeps a pool (see [`recycle`]). +#[inline] +pub(crate) fn pooled_kind(i: &PyInstance) -> bool { + // SAFETY: a read between two instructions (see `GilCell::peek`). + unsafe { i.class.peek() }.is_some_and(|c| { + matches!( + c.native_kind.get(), + KIND_TIMEDELTA | KIND_DATE | KIND_DATETIME + ) + }) +} + +/// Retire a dying natively served instance into its class set's pool +/// (see above). `Err` hands back an instance that must be freed normally +/// (dropped as an `Arc`, not through the finalizing `Rc` drop that may +/// have called here). +pub(crate) fn recycle(mut inst: Rc) -> Result<(), Rc> { + if !pool_ok() { + return Err(inst); + } + // While the refcount bias holds, this thread owns every count. + let unique = if crate::rc::refcounts_biased() { + Rc::strong_count(&inst) == 1 && Rc::weak_count(&inst) == 0 + } else { + Rc::get_mut(&mut inst).is_some() + }; + if !unique { + return Err(inst); + } + // SAFETY: `inst` is the only reference (checked above). + let m = unsafe { &mut *Rc::as_ptr(&inst).cast_mut() }; + let cls = m.class.get_mut(); + let kind = cls.native_kind.get(); + if !matches!(kind, KIND_TIMEDELTA | KIND_DATE | KIND_DATETIME) || cls.instances_need_finalize() + { + return Err(inst); + } + let Some(st) = state_of_cls(cls) else { + return Err(inst); + }; + if m.dict.published().is_some() + || !m.dict.split_mut().is_empty() + || m.native.get().is_some() + || m.c_body.get() != 0 + || m.finalize_ran.get() + { + return Err(inst); + } + match m.slots.get_mut().as_packed_mut() { + Some(p) if p.kind() == kind => { + // Releasing the `tzinfo` runs no code (a dying one is queued + // for finalization, not finalized here). + p.clear(); + } + _ => return Err(inst), + } + m.hash_cache = crate::sync::CachedHash::new(None); + // SAFETY: `st` lives in the class `inst` holds. + let st: &State = unsafe { &*std::ptr::from_ref(st) }; + let mut slot = Some(inst); + st.pool.with(kind, |v| { + if v.len() < POOL_CAP { + v.extend(slot.take()); + } + }); + match slot { + None => Ok(()), + Some(inst) => Err(inst), + } +} + /// `datetime` fields shifted by `delta_us` (fold cleared, `tzinfo` /// kept), or `None` past the representable range. -fn dt_shift(f: &Dt, delta_us: i128) -> Option
{ +fn dt_shift(f: Dt, delta_us: i128) -> Option
{ // Every representable datetime is under 2^59 microseconds, so a // delta that fits no `i64` lands out of range (the Python path // raises); the rest is 64-bit arithmetic over in-range fields. @@ -643,11 +1358,19 @@ fn dt_shift(f: &Dt, delta_us: i128) -> Option
{ mm: secs % 3600 / 60, ss: secs % 60, us: (rest % 1_000_000) as i64, - tz: f.tz.clone(), + tz: f.tz, fold: 0, }) } +/// Microseconds since 0001-01-01T00:00 of a packed (so valid) word. +#[inline] +fn word_us(w: u64) -> i64 { + let (y, m, d) = word_ymd(w); + let (hh, mm, ss, us, _) = word_time(w); + (ymd2ord(y, m, d) * 86_400 + hh * 3600 + mm * 60 + ss) * 1_000_000 + us +} + /// Microseconds since 0001-01-01T00:00, naive. fn dt_us(f: &Dt) -> i128 { (i128::from(ymd2ord(f.y, f.m, f.d)) * 86_400 + i128::from(f.hh * 3600 + f.mm * 60 + f.ss)) @@ -714,6 +1437,73 @@ fn td_bool(a: &[Object]) -> Option> { Some(Ok(Object::Bool(p != (0, 0, 0)))) } +/// `hash((days, seconds, microseconds))`: the replaced `__hash__`'s +/// value for a `timedelta` (`hash(self._getstate())`). +fn td_tuple_hash(d: i64, s: i64, us: i64) -> Option { + let h = |v: i64| crate::object::numeric_hash(&Object::Int(v)); + Some(crate::object::combine_tuple_hash(&[h(d)?, h(s)?, h(us)?])) +} + +fn td_hash(a: &[Object]) -> Option> { + let [x] = a else { return None }; + let st = state_of(x)?; + let (d, s, us) = td_fields(inst(x, KIND_TIMEDELTA)?, st)?; + Some(Ok(Object::Int(td_tuple_hash(d, s, us)?))) +} + +fn date_hash(a: &[Object]) -> Option> { + let [x] = a else { return None }; + let st = state_of(x)?; + let (y, m, d) = date_fields(inst(x, KIND_DATE)?, st)?; + if !valid_date(y, m, d) { + return None; + } + // `hash(self._getstate())`: a one-tuple of the state bytes. + let state = [(y / 256) as u8, (y % 256) as u8, m as u8, d as u8]; + let h = crate::object::py_bytes_hash(&state); + Some(Ok(Object::Int(crate::object::combine_tuple_hash(&[h])))) +} + +/// The replaced `datetime.__hash__` for a naive or fixed-offset value: +/// the state bytes' hash when naive, else the hash of the UTC value as a +/// `timedelta` since 0001-01-01 (`fold` never changes a fixed offset). +fn dt_hash(a: &[Object]) -> Option> { + let [x] = a else { return None }; + let st = state_of(x)?; + let f = dt_fields(inst(x, KIND_DATETIME)?, st)?; + if !valid_date(f.y, f.m, f.d) { + return None; + } + let h = match tz_of(&f.tz, st)? { + Tz::Naive => { + let state = [ + (f.y / 256) as u8, + (f.y % 256) as u8, + f.m as u8, + f.d as u8, + f.hh as u8, + f.mm as u8, + f.ss as u8, + (f.us >> 16) as u8, + (f.us >> 8 & 0xff) as u8, + (f.us & 0xff) as u8, + ]; + crate::object::py_bytes_hash(&state) + } + Tz::Fixed(off) => { + let total = dt_us(&f) - off; + let days = total.div_euclid(US_PER_DAY); + let rest = total.rem_euclid(US_PER_DAY); + td_tuple_hash( + i64::try_from(days).ok()?, + (rest / 1_000_000) as i64, + (rest % 1_000_000) as i64, + )? + } + }; + Some(Ok(Object::Int(h))) +} + fn td_cmp(a: &[Object]) -> Option { let [x, y] = a else { return None }; let st = state_of(x)?; @@ -833,7 +1623,75 @@ fn date_isoformat(a: &[Object]) -> Option> { if !valid_date(y, m, d) { return None; } - Some(Ok(Object::from_str(format!("{y:04}-{m:02}-{d:02}")))) + let mut out = Vec::with_capacity(10); + push_ymd(&mut out, y, m, d); + Some(Ok(text_object(&out))) +} + +/// Append `v` in decimal, zero-padded to `width` digits: +/// `format!("{v:0width$}")` without the formatting machinery. +#[inline] +fn push_num(out: &mut Vec, v: i64, width: usize) { + const LIMIT: [i64; 7] = [1, 10, 100, 1_000, 10_000, 100_000, 1_000_000]; + if width < LIMIT.len() && (0..LIMIT[width]).contains(&v) { + // Exactly `width` digits. + let start = out.len(); + out.resize(start + width, b'0'); + let mut n = v as u32; + for b in out[start..].iter_mut().rev() { + *b = b'0' + (n % 10) as u8; + n /= 10; + } + return; + } + let mut buf = [b'0'; 20]; + let mut n = v.unsigned_abs(); + let mut i = buf.len(); + loop { + i -= 1; + buf[i] = b'0' + (n % 10) as u8; + n /= 10; + if n == 0 { + break; + } + } + let start = i.min(buf.len() - width.min(buf.len())); + if v < 0 { + out.push(b'-'); + } + out.extend_from_slice(&buf[start..]); +} + +/// `YYYY-MM-DD`. +#[inline] +fn push_ymd(out: &mut Vec, y: i64, m: i64, d: i64) { + push_num(out, y, 4); + out.push(b'-'); + push_num(out, m, 2); + out.push(b'-'); + push_num(out, d, 2); +} + +/// `HH:MM:SS`. +#[inline] +fn push_hms(out: &mut Vec, hh: i64, mm: i64, ss: i64) { + push_num(out, hh, 2); + out.push(b':'); + push_num(out, mm, 2); + out.push(b':'); + push_num(out, ss, 2); +} + +/// A `str` of `text` (UTF-8: ASCII fields around the caller's text). +#[inline] +fn text_object(text: &[u8]) -> Object { + // SAFETY: the formatters write ASCII and copy whole UTF-8 sequences. + let text = unsafe { std::str::from_utf8_unchecked(text) }; + if text.is_ascii() { + Object::Str(SharedStr::from_ascii(text)) + } else { + Object::Str(SharedStr::from(text)) + } } /// `date.replace(year=None, month=None, day=None)`, positional form. @@ -861,16 +1719,52 @@ fn date_replace(a: &[Object]) -> Option> { Some(Ok(new_date(st, y, m, d)?)) } +/// A packed `datetime` shifted by `delta_us` within its day: the same +/// date, new time fields, fold cleared, the `tzinfo` shared. `None` when +/// `xi` is not packed or the result leaves the day. +#[inline] +fn dt_shift_in_day(st: &State, xi: &PyInstance, delta_us: i128) -> Option { + let (w, tz) = dt_packed(xi)?; + if delta_us.unsigned_abs() >= US_PER_DAY as u128 { + return None; + } + let (hh, mm, ss, us, _) = word_time(w); + let n = ((hh * 3600 + mm * 60 + ss) * 1_000_000 + us) + delta_us as i64; + if !(0..US_PER_DAY as i64).contains(&n) { + return None; + } + let (secs, us) = (n / 1_000_000, n % 1_000_000); + let word = (w & !((1u64 << W_DAY) - 1)) + | (((secs / 3600) as u64) << W_HH) + | (((secs % 3600 / 60) as u64) << W_MM) + | (((secs % 60) as u64) << W_SS) + | ((us as u64) << W_US); + if !tz.is_null() { + // SAFETY: `xi` holds a reference; the new value owns this one. + unsafe { Rc::increment_strong_count(tz) }; + } + let r = new_packed(st, &st.datetime, KIND_DATETIME, word, tz as usize); + if r.is_none() && !tz.is_null() { + // SAFETY: not adopted; release the reference taken above. + drop(unsafe { Rc::from_raw(tz) }); + } + r +} + fn dt_add(a: &[Object]) -> Option> { let [x, y] = a else { return None }; let st = state_of(x)?; - let f = dt_fields(inst(x, KIND_DATETIME)?, st)?; + let xi = inst(x, KIND_DATETIME)?; let delta = td_us(td_fields(inst(y, KIND_TIMEDELTA)?, st)?); + if let Some(r) = dt_shift_in_day(st, xi, delta) { + return Some(Ok(r)); + } + let f = dt_fields(xi, st)?; if !valid_date(f.y, f.m, f.d) { return None; } - match dt_shift(&f, delta) { - Some(g) => Some(Ok(new_dt(st, &g)?)), + match dt_shift(f, delta) { + Some(g) => Some(Ok(new_dt(st, g)?)), None => Some(Err(overflow_error("date value out of range"))), } } @@ -878,20 +1772,40 @@ fn dt_add(a: &[Object]) -> Option> { fn dt_sub(a: &[Object]) -> Option> { let [x, y] = a else { return None }; let st = state_of(x)?; - let f = dt_fields(inst(x, KIND_DATETIME)?, st)?; - if !valid_date(f.y, f.m, f.d) { - return None; - } + let xi = inst(x, KIND_DATETIME)?; match kind_of(y) { KIND_TIMEDELTA => { let delta = td_us(td_fields(inst(y, KIND_TIMEDELTA)?, st)?); - match dt_shift(&f, -delta) { - Some(g) => Some(Ok(new_dt(st, &g)?)), + if let Some(r) = dt_shift_in_day(st, xi, -delta) { + return Some(Ok(r)); + } + let f = dt_fields(xi, st)?; + if !valid_date(f.y, f.m, f.d) { + return None; + } + match dt_shift(f, -delta) { + Some(g) => Some(Ok(new_dt(st, g)?)), None => Some(Err(overflow_error("date value out of range"))), } } KIND_DATETIME => { - let g = dt_fields(inst(y, KIND_DATETIME)?, st)?; + let j = inst(y, KIND_DATETIME)?; + if let (Some((wf, tf)), Some((wg, tg))) = (dt_packed(xi), dt_packed(j)) { + let mut diff = i128::from(word_us(wf) - word_us(wg)); + if !std::ptr::eq(tf, tg) { + match (tz_of_ptr(tf, st)?, tz_of_ptr(tg, st)?) { + (Tz::Naive, Tz::Naive) => {} + (Tz::Fixed(a), Tz::Fixed(b)) => diff += b - a, + _ => return None, + } + } + return new_td(st, 0, 0, diff); + } + let f = dt_fields(xi, st)?; + if !valid_date(f.y, f.m, f.d) { + return None; + } + let g = dt_fields(j, st)?; if !valid_date(g.y, g.m, g.d) { return None; } @@ -915,8 +1829,23 @@ fn dt_sub(a: &[Object]) -> Option> { fn dt_cmp_raw(a: &[Object]) -> Option<(std::cmp::Ordering, bool)> { let [x, y] = a else { return None }; let st = state_of(x)?; - let f = dt_fields(inst(x, KIND_DATETIME)?, st)?; - let g = dt_fields(inst(y, KIND_DATETIME)?, st)?; + let (xi, yi) = (inst(x, KIND_DATETIME)?, inst(y, KIND_DATETIME)?); + if let (Some((wf, tf)), Some((wg, tg))) = (dt_packed(xi), dt_packed(yi)) { + // The fields sit in significance order above the fold bit. + if std::ptr::eq(tf, tg) { + return Some(((wf >> 1).cmp(&(wg >> 1)), true)); + } + return match (tz_of_ptr(tf, st)?, tz_of_ptr(tg, st)?) { + (Tz::Naive, Tz::Naive) => Some(((wf >> 1).cmp(&(wg >> 1)), true)), + (Tz::Fixed(a), Tz::Fixed(b)) => Some(( + (i128::from(word_us(wf)) - a).cmp(&(i128::from(word_us(wg)) - b)), + true, + )), + _ => Some((std::cmp::Ordering::Equal, false)), + }; + } + let f = dt_fields(xi, st)?; + let g = dt_fields(yi, st)?; if !valid_date(f.y, f.m, f.d) || !valid_date(g.y, g.m, g.d) { return None; } @@ -965,17 +1894,22 @@ fn dt_date(a: &[Object]) -> Option> { } /// `_format_offset(off, sep)` for a fixed offset in microseconds. -fn format_offset(out: &mut String, off: i128, sep: &str) { - let (sign, mut v) = if off < 0 { ('-', -off) } else { ('+', off) }; - let hh = v / 3_600_000_000; +fn format_offset(out: &mut Vec, off: i128, sep: &[u8]) { + let (sign, mut v) = if off < 0 { (b'-', -off) } else { (b'+', off) }; + let hh = (v / 3_600_000_000) as i64; v %= 3_600_000_000; - let mm = v / 60_000_000; + let mm = (v / 60_000_000) as i64; v %= 60_000_000; - let _ = write!(out, "{sign}{hh:02}{sep}{mm:02}"); + out.push(sign); + push_num(out, hh, 2); + out.extend_from_slice(sep); + push_num(out, mm, 2); if v != 0 { - let _ = write!(out, "{sep}{:02}", v / 1_000_000); + out.extend_from_slice(sep); + push_num(out, (v / 1_000_000) as i64, 2); if v % 1_000_000 != 0 { - let _ = write!(out, ".{:06}", v % 1_000_000); + out.push(b'.'); + push_num(out, (v % 1_000_000) as i64, 6); } } } @@ -1017,47 +1951,44 @@ fn dt_isoformat(a: &[Object]) -> Option> { return None; } let tz = tz_of(&f.tz, st)?; - let mut out = String::with_capacity(32); - let _ = write!(out, "{:04}-{:02}-{:02}{sep}", f.y, f.m, f.d); + let mut out = Vec::with_capacity(40); + push_ymd(&mut out, f.y, f.m, f.d); + out.extend_from_slice(sep.encode_utf8(&mut [0; 4]).as_bytes()); match spec { - "hours" => { - let _ = write!(out, "{:02}", f.hh); - } + "hours" => push_num(&mut out, f.hh, 2), "minutes" => { - let _ = write!(out, "{:02}:{:02}", f.hh, f.mm); + push_num(&mut out, f.hh, 2); + out.push(b':'); + push_num(&mut out, f.mm, 2); } "milliseconds" => { - let _ = write!( - out, - "{:02}:{:02}:{:02}.{:03}", - f.hh, - f.mm, - f.ss, - f.us / 1000 - ); + push_hms(&mut out, f.hh, f.mm, f.ss); + out.push(b'.'); + push_num(&mut out, f.us / 1000, 3); } "microseconds" => { - let _ = write!(out, "{:02}:{:02}:{:02}.{:06}", f.hh, f.mm, f.ss, f.us); - } - "seconds" => { - let _ = write!(out, "{:02}:{:02}:{:02}", f.hh, f.mm, f.ss); + push_hms(&mut out, f.hh, f.mm, f.ss); + out.push(b'.'); + push_num(&mut out, f.us, 6); } + "seconds" => push_hms(&mut out, f.hh, f.mm, f.ss), _ => { + push_hms(&mut out, f.hh, f.mm, f.ss); if f.us != 0 { - let _ = write!(out, "{:02}:{:02}:{:02}.{:06}", f.hh, f.mm, f.ss, f.us); - } else { - let _ = write!(out, "{:02}:{:02}:{:02}", f.hh, f.mm, f.ss); + out.push(b'.'); + push_num(&mut out, f.us, 6); } } } if let Tz::Fixed(off) = tz { - format_offset(&mut out, off, ":"); + format_offset(&mut out, off, b":"); } - Some(Ok(Object::from_str(out))) + Some(Ok(text_object(&out))) } /// `strftime` for the numeric directives (no locale involvement); any -/// other directive declines. +/// other directive declines. The result is UTF-8: literal text is copied +/// byte for byte, and a directive is ASCII. fn strftime_numeric( y: i64, m: i64, @@ -1065,69 +1996,55 @@ fn strftime_numeric( time: Option<(i64, i64, i64, i64)>, tz: Option<&Tz>, fmt: &str, -) -> Option { +) -> Option> { let (hh, mm, ss, us) = time.unwrap_or((0, 0, 0, 0)); - let mut out = String::with_capacity(fmt.len() + 16); - let mut chars = fmt.chars(); - while let Some(c) = chars.next() { - if c != '%' { + let fmt = fmt.as_bytes(); + let mut out = Vec::with_capacity(fmt.len() + 16); + let mut i = 0; + while i < fmt.len() { + let c = fmt[i]; + i += 1; + if c != b'%' { out.push(c); continue; } - match chars.next()? { - 'Y' => { - let _ = write!(out, "{y:04}"); - } - 'm' => { - let _ = write!(out, "{m:02}"); - } - 'd' => { - let _ = write!(out, "{d:02}"); - } - 'H' => { - let _ = write!(out, "{hh:02}"); - } - 'M' => { - let _ = write!(out, "{mm:02}"); - } - 'S' => { - let _ = write!(out, "{ss:02}"); - } - 'f' => { - let _ = write!(out, "{us:06}"); - } - 'y' => { - let _ = write!(out, "{:02}", y % 100); - } - 'j' => { + let k = *fmt.get(i)?; + i += 1; + match k { + b'Y' => push_num(&mut out, y, 4), + b'm' => push_num(&mut out, m, 2), + b'd' => push_num(&mut out, d, 2), + b'H' => push_num(&mut out, hh, 2), + b'M' => push_num(&mut out, mm, 2), + b'S' => push_num(&mut out, ss, 2), + b'f' => push_num(&mut out, us, 6), + b'y' => push_num(&mut out, y % 100, 2), + b'j' => { let doy = DAYS_BEFORE_MONTH[(m - 1) as usize] + i64::from(m > 2 && is_leap(y)) + d; - let _ = write!(out, "{doy:03}"); + push_num(&mut out, doy, 3); } - 'I' => { + b'I' => { let h12 = match hh % 12 { 0 => 12, h => h, }; - let _ = write!(out, "{h12:02}"); - } - 'F' => { - let _ = write!(out, "{y:04}-{m:02}-{d:02}"); - } - 'T' => { - let _ = write!(out, "{hh:02}:{mm:02}:{ss:02}"); + push_num(&mut out, h12, 2); } - '%' => out.push('%'), - 'z' => match tz? { + b'F' => push_ymd(&mut out, y, m, d), + b'T' => push_hms(&mut out, hh, mm, ss), + b'%' => out.push(b'%'), + b'z' => match tz? { Tz::Naive => {} - Tz::Fixed(off) => format_offset(&mut out, *off, ""), + Tz::Fixed(off) => format_offset(&mut out, *off, b""), }, - ':' => { - if chars.next()? != 'z' { + b':' => { + if *fmt.get(i)? != b'z' { return None; } + i += 1; match tz? { Tz::Naive => {} - Tz::Fixed(off) => format_offset(&mut out, *off, ":"), + Tz::Fixed(off) => format_offset(&mut out, *off, b":"), } } _ => return None, @@ -1154,7 +2071,7 @@ fn dt_strftime(a: &[Object]) -> Option> { Some(&tz), fmt, )?; - Some(Ok(Object::from_str(s))) + Some(Ok(text_object(&s))) } fn date_strftime(a: &[Object]) -> Option> { @@ -1168,7 +2085,7 @@ fn date_strftime(a: &[Object]) -> Option> { } // A date has no time or offset: `%z` formats as empty, like CPython. let s = strftime_numeric(y, m, d, None, Some(&Tz::Naive), fmt)?; - Some(Ok(Object::from_str(s))) + Some(Ok(text_object(&s))) } fn digits(b: &[u8]) -> Option { @@ -1254,7 +2171,7 @@ fn dt_fromisoformat(a: &[Object]) -> Option> { }; } } - Some(Ok(new_dt(st, &f)?)) + Some(Ok(new_dt(st, f)?)) } /// `HH[:MM[:SS[.f{1,6}]]]` (colon-separated only). @@ -1315,21 +2232,22 @@ fn new_timezone(st: &State, us: i128) -> Option { /// keeps the careful path, which frees that object promptly. pub(crate) fn plain_drop_ok(i: &PyInstance) -> bool { match i.cls_raw().native_kind.get() { - KIND_DATETIME | KIND_TIME => { - let Ok(s) = i.slots.try_borrow() else { - return false; - }; - let idx = if i.cls_raw().native_kind.get() == KIND_DATETIME { - DT_TZINFO - } else { - 4 - }; - match s.get_hinted(idx, "_tzinfo") { + kind @ (KIND_DATETIME | KIND_TIME) => with_slots(i, |s| { + if let Some(p) = s.as_packed() { + let tz = p.tz_ptr(); + // SAFETY: `tz` is owned by the packed value being read. + return Some( + tz.is_null() || unsafe { &*tz }.cls_raw().native_kind.get() == KIND_TIMEZONE, + ); + } + let idx = if kind == KIND_DATETIME { DT_TZINFO } else { 4 }; + Some(match s.get_hinted(idx, "_tzinfo") { Some(Object::None) => true, Some(o @ Object::Instance(_)) => kind_of(o) == KIND_TIMEZONE, _ => false, - } - } + }) + }) + .unwrap_or(false), KIND_TIMEDELTA | KIND_DATE | KIND_TIMEZONE => true, _ => false, } @@ -1364,15 +2282,16 @@ pub(crate) fn leaf_binop( return None; } } - let args = [a.clone(), b.clone()]; + // SAFETY: a shallow copy the fast halves only read; never dropped. + let args = std::mem::ManuallyDrop::new(unsafe { [std::ptr::read(a), std::ptr::read(b)] }); match (kind, op) { - (KIND_TIMEDELTA, B::Add) => td_add(&args), - (KIND_TIMEDELTA, B::Sub) => td_sub(&args), - (KIND_TIMEDELTA, B::Mult) => td_mul(&args), - (KIND_DATE, B::Add) => date_add(&args), - (KIND_DATE, B::Sub) => date_sub(&args), - (KIND_DATETIME, B::Add) => dt_add(&args), - (KIND_DATETIME, B::Sub) => dt_sub(&args), + (KIND_TIMEDELTA, B::Add) => td_add(&args[..]), + (KIND_TIMEDELTA, B::Sub) => td_sub(&args[..]), + (KIND_TIMEDELTA, B::Mult) => td_mul(&args[..]), + (KIND_DATE, B::Add) => date_add(&args[..]), + (KIND_DATE, B::Sub) => date_sub(&args[..]), + (KIND_DATETIME, B::Add) => dt_add(&args[..]), + (KIND_DATETIME, B::Sub) => dt_sub(&args[..]), _ => None, } } @@ -1396,11 +2315,12 @@ pub(crate) fn leaf_compare( if !verified(st, ci, kind) || !verified(st, cj, kind) { return None; } - let args = [a.clone(), b.clone()]; + // SAFETY: a shallow copy the fast halves only read; never dropped. + let args = std::mem::ManuallyDrop::new(unsafe { [std::ptr::read(a), std::ptr::read(b)] }); let r = match kind { - KIND_TIMEDELTA => td_cmp(&args).map(|o| (o, true)), - KIND_DATE => date_cmp(&args).map(|o| (o, true)), - KIND_DATETIME => dt_cmp_raw(&args), + KIND_TIMEDELTA => td_cmp(&args[..]).map(|o| (o, true)), + KIND_DATE => date_cmp(&args[..]).map(|o| (o, true)), + KIND_DATETIME => dt_cmp_raw(&args[..]), _ => None, }?; let (o, comparable) = r; @@ -1415,12 +2335,30 @@ pub(crate) fn leaf_compare( }))) } +/// The function under the native class method `name` of the exact class +/// `cls` while the class still has it (`datetime.fromisoformat`), for +/// the dispatch loop to bind to `cls` without the descriptor protocol. +pub(crate) fn type_classmethod(cls: &TypeObject, name: &SharedStr) -> Option { + let kind = cls.native_kind.get(); + if kind != KIND_DATETIME { + return None; + } + let st = state_of_cls(cls)?; + if !SharedStr::ptr_eq(name, &st.names.f_fromisoformat) + || !std::ptr::eq(cls, st.datetime.as_ptr()) + || !verified(st, cls, kind) + { + return None; + } + st.fromiso.get().cloned() +} + /// A public field (`dt.hour`, `d.year`, …) of a natively served instance /// whose class still has its original property. pub(crate) fn leaf_field(i: &PyInstance, name: &SharedStr) -> Option { let cls = i.cls_raw(); let kind = cls.native_kind.get(); - if kind != KIND_DATE && kind != KIND_DATETIME { + if kind != KIND_DATE && kind != KIND_DATETIME && kind != KIND_TIMEDELTA { return None; } let st = state_of_cls(cls)?; @@ -1428,6 +2366,19 @@ pub(crate) fn leaf_field(i: &PyInstance, name: &SharedStr) -> Option { return None; } let n = &st.names; + if kind == KIND_TIMEDELTA { + let ix = if SharedStr::ptr_eq(name, &n.f_days) { + 0 + } else if SharedStr::ptr_eq(name, &n.f_seconds) { + 1 + } else if SharedStr::ptr_eq(name, &n.f_microseconds) { + 2 + } else { + return None; + }; + let (d, s, us) = td_fields(i, st)?; + return Some(Object::Int([d, s, us][ix])); + } let (idx, slot) = if SharedStr::ptr_eq(name, &n.f_year) { (D_YEAR, &n.year) } else if SharedStr::ptr_eq(name, &n.f_month) { @@ -1451,8 +2402,23 @@ pub(crate) fn leaf_field(i: &PyInstance, name: &SharedStr) -> Option { } else { return None; }; - let s = i.slots.try_borrow().ok()?; - Some(s.get_hinted(idx, slot)?.clone()) + with_slots(i, |s| { + if let Some(p) = s.as_packed() { + let w = p.word; + return Some(match idx { + D_YEAR => Object::Int(bits(w, W_YEAR, 14)), + D_MONTH => Object::Int(bits(w, W_MONTH, 4)), + D_DAY => Object::Int(bits(w, W_DAY, 5)), + DT_HOUR => Object::Int(bits(w, W_HH, 5)), + DT_MINUTE => Object::Int(bits(w, W_MM, 6)), + DT_SECOND => Object::Int(bits(w, W_SS, 6)), + DT_US => Object::Int(bits(w, W_US, 20)), + DT_FOLD => Object::Int(bits(w, W_FOLD, 1)), + _ => p.tz(), + }); + } + Some(s.get_hinted(idx, slot)?.clone()) + }) } // --------------------------------------------------------------------- @@ -1469,6 +2435,42 @@ fn with_interp( /// One native method: its name, the class kind it lives on, and the /// pure fast half. +/// A keyword call of a native method whose fast half takes its optional +/// parameters by position, `None` standing for an omitted one: the +/// positional argument list (receiver first), or `None` for the replaced +/// Python method (unknown or repeated names, other methods). +fn kw_positional( + key: (u8, &'static str), + a: &[Object], + kw: &[(String, Object)], +) -> Option> { + let params: &[&str] = match key { + (KIND_DATE, "replace") => &["year", "month", "day"], + _ => return None, + }; + let (recv, pos) = a.split_first()?; + if pos.len() > params.len() { + return None; + } + let mut out = Vec::with_capacity(params.len() + 1); + out.push(recv.clone()); + out.extend(pos.iter().cloned()); + out.resize(params.len() + 1, Object::Unbound); + for (name, v) in kw { + let ix = params.iter().position(|p| p == name)? + 1; + if !matches!(out[ix], Object::Unbound) { + return None; + } + out[ix] = v.clone(); + } + for o in &mut out { + if matches!(o, Object::Unbound) { + *o = Object::None; + } + } + Some(out) +} + struct Spec { kind: u8, name: &'static str, @@ -1520,6 +2522,24 @@ const SPECS: &[Spec] = &[ fast: td_bool, classmethod: false, }, + Spec { + kind: KIND_TIMEDELTA, + name: "__hash__", + fast: td_hash, + classmethod: false, + }, + Spec { + kind: KIND_DATE, + name: "__hash__", + fast: date_hash, + classmethod: false, + }, + Spec { + kind: KIND_DATETIME, + name: "__hash__", + fast: dt_hash, + classmethod: false, + }, Spec { kind: KIND_TIMEDELTA, name: "__eq__", @@ -1761,34 +2781,7 @@ pub(crate) fn install(args: &[Object]) -> Result { orig.insert((spec.kind, spec.name), v); } let names = Names::new(); - let layout = |keys: &[&SharedStr]| -> SharedSlice { - keys.iter() - .map(|k| DictKey(Object::Str((*k).clone()))) - .collect::>() - .into() - }; - let td_layout = layout(&[ - &names.days, - &names.seconds, - &names.microseconds, - &names.hashcode, - &names.pub_days, - &names.pub_seconds, - &names.pub_microseconds, - ]); - let date_layout = layout(&[&names.year, &names.month, &names.day, &names.hashcode]); - let dt_layout = layout(&[ - &names.year, - &names.month, - &names.day, - &names.hour, - &names.minute, - &names.second, - &names.microsecond, - &names.tzinfo, - &names.hashcode, - &names.fold, - ]); + let [td_layout, date_layout, dt_layout] = layouts().clone(); let state = Rc::new(State { timedelta: Rc::downgrade(td), date: Rc::downgrade(date), @@ -1802,6 +2795,8 @@ pub(crate) fn install(args: &[Object]) -> Result { orig, sealed: Default::default(), expect: std::sync::OnceLock::new(), + pool: Pool::default(), + fromiso: std::sync::OnceLock::new(), }); // Each class carries the state (a class in `native_ext` resolves to // it from any of its instances). @@ -1836,6 +2831,8 @@ pub(crate) fn install(args: &[Object]) -> Result { if let Some(r) = fast(a) { return r; } + } else if let Some(r) = kw_positional(key, a, kw).and_then(|a| fast(&a)) { + return r; } let f = st_kw.orig.get(&key).cloned().unwrap_or(Object::None); with_interp(|i| i.call_object(f, a, kw)) @@ -1843,6 +2840,9 @@ pub(crate) fn install(args: &[Object]) -> Result { }); crate::leaf_builtins::register_fast(&b, fast); let value = if spec.classmethod { + if spec.name == "fromisoformat" { + let _ = state.fromiso.set(Object::Builtin(b.clone())); + } Object::ClassMethod(crate::object::MethodWrapper::new(Object::Builtin(b))) } else { Object::Builtin(b) diff --git a/crates/weavepy-vm/src/sync.rs b/crates/weavepy-vm/src/sync.rs index 165d0fbc..e9a6a243 100644 --- a/crates/weavepy-vm/src/sync.rs +++ b/crates/weavepy-vm/src/sync.rs @@ -342,7 +342,7 @@ fn cells_shared() -> bool { static CELLS_UNGUARDED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); #[inline] -fn cells_unguarded() -> bool { +pub(crate) fn cells_unguarded() -> bool { CELLS_UNGUARDED.load(Ordering::Relaxed) } diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index 47ff3be6..4a97dc74 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -2004,8 +2004,16 @@ enum SlotData { layout: SharedSlice, values: Box<[Object]>, }, + /// A natively built `datetime` value's fields, packed into words (see + /// [`crate::stdlib::datetime_native::PackedSlots`]). Readers that + /// need `&Object`s get a lazily built copy; any write first converts + /// to [`Self::Fixed`]. + Packed(crate::stdlib::datetime_native::PackedSlots), } +#[cfg(target_pointer_width = "64")] +const _: () = assert!(std::mem::size_of::() == 32); + /// Is `key` the slot name `name`? (Interned names usually share the /// probe's storage, settled without reading either length.) #[inline(always)] @@ -2109,6 +2117,7 @@ impl SlotStorage { SlotData::Fixed { layout, values } if SharedSlice::ptr_eq(layout, expected) => { Some(values) } + SlotData::Packed(p) if SharedSlice::ptr_eq(p.layout(), expected) => Some(p.values()), _ => None, } } @@ -2117,6 +2126,7 @@ impl SlotStorage { &mut self, expected: &SharedSlice, ) -> Option<&mut [Object]> { + self.unpack(); match &mut self.data { SlotData::Fixed { layout, values } if SharedSlice::ptr_eq(layout, expected) => { Some(values) @@ -2125,9 +2135,58 @@ impl SlotStorage { } } + /// Storage holding a natively packed value (see [`SlotData::Packed`]). + pub(crate) fn from_packed(packed: crate::stdlib::datetime_native::PackedSlots) -> Self { + Self { + data: SlotData::Packed(packed), + } + } + + /// The packed value, while the storage still holds one. + #[inline] + pub(crate) fn as_packed(&self) -> Option<&crate::stdlib::datetime_native::PackedSlots> { + match &self.data { + SlotData::Packed(p) => Some(p), + _ => None, + } + } + + /// [`Self::as_packed`], mutably. + #[inline] + pub(crate) fn as_packed_mut( + &mut self, + ) -> Option<&mut crate::stdlib::datetime_native::PackedSlots> { + match &mut self.data { + SlotData::Packed(p) => Some(p), + _ => None, + } + } + + /// Turn a [`SlotData::Packed`] storage into [`SlotData::Fixed`] + /// (before a slot is written). + #[inline] + fn unpack(&mut self) { + if let SlotData::Packed(_) = &self.data { + self.unpack_slow(); + } + } + + #[cold] + #[inline(never)] + fn unpack_slow(&mut self) { + if let SlotData::Packed(p) = std::mem::take(self).data { + let layout = p.layout().clone(); + self.data = SlotData::Fixed { + layout, + values: p.into_values(), + }; + } + } + /// Turn a [`SlotData::Fixed`] storage into the per-key form (before /// a key is added or removed). fn unfix(&mut self) { + self.unpack(); if let SlotData::Fixed { layout, values } = &mut self.data { let values = std::mem::take(values); let entries: Vec<(DictKey, Object)> = @@ -2154,6 +2213,24 @@ impl SlotStorage { pub fn from_layout(layout: SharedSlice, values: Vec) -> Self { Self::from_entries(layout.iter().cloned().zip(values).collect()) } + + /// Storage holding a packed value's slots (stored per key here). + pub(crate) fn from_packed(packed: crate::stdlib::datetime_native::PackedSlots) -> Self { + let layout = packed.layout().clone(); + Self::from_layout(layout, packed.into_values().into_vec()) + } + + #[inline] + pub(crate) fn as_packed(&self) -> Option<&crate::stdlib::datetime_native::PackedSlots> { + None + } + + #[inline] + pub(crate) fn as_packed_mut( + &mut self, + ) -> Option<&mut crate::stdlib::datetime_native::PackedSlots> { + None + } } #[cfg(not(target_pointer_width = "64"))] @@ -2201,6 +2278,13 @@ impl SlotStorage { |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)), ) .map(|index| index as u32), + SlotData::Packed(p) => p + .layout() + .iter() + .position( + |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)), + ) + .map(|index| index as u32), } } @@ -2213,12 +2297,14 @@ impl SlotStorage { SlotData::Small(entries) => entries.get(index).map(|(key, value)| (key, value)), SlotData::Many(table) => table.get_index(index), SlotData::Fixed { layout, values } => layout.get(index).zip(values.get(index)), + SlotData::Packed(p) => p.layout().get(index).zip(p.values().get(index)), _ => None, } } #[inline] pub fn get_index_mut(&mut self, index: usize) -> Option<(&DictKey, &mut Object)> { + self.unpack(); match &mut self.data { SlotData::Single { key, value } if index == 0 => key.as_ref().map(|key| (key, value)), SlotData::Small(entries) => entries.get_mut(index).map(|(key, value)| (&*key, value)), @@ -2244,6 +2330,13 @@ impl SlotStorage { |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)), ) .and_then(|i| values.get(i)), + SlotData::Packed(p) => p + .layout() + .iter() + .position( + |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)), + ) + .and_then(|i| p.values().get(i)), _ => None, } } @@ -2316,6 +2409,17 @@ impl SlotStorage { } Some(std::array::from_fn(|i| &values[i])) } + SlotData::Packed(p) => { + let (keys, values) = (p.layout().get(..N)?, p.values().get(..N)?); + if !keys + .iter() + .zip(names) + .all(|(key, name)| key_named(key, name)) + { + return None; + } + Some(std::array::from_fn(|i| &values[i])) + } _ => None, } } @@ -2385,6 +2489,7 @@ impl SlotStorage { &mut self, names: [&crate::shared_value::SharedStr; N], ) -> Option<[&mut Object; N]> { + self.unpack(); let values: &mut [Object] = match &mut self.data { SlotData::Small(entries) => { let entries = entries.get_mut(..N)?; @@ -2417,6 +2522,7 @@ impl SlotStorage { /// Mutable access to a populated slot's value, by name. pub fn get_mut(&mut self, name: &str) -> Option<&mut Object> { + self.unpack(); match &mut self.data { SlotData::Single { key: Some(DictKey(Object::Str(stored))), @@ -2456,6 +2562,7 @@ impl SlotStorage { value: Object, make_key: impl FnOnce() -> DictKey, ) -> Option { + self.unpack(); if let SlotData::Fixed { .. } = &self.data { if let Some(slot) = self.get_mut(name) { return Some(std::mem::replace(slot, value)); @@ -2542,7 +2649,8 @@ impl SlotStorage { } SlotData::Small(entries) => (None, Some(entries), None, None), SlotData::Many(table) => (None, None, Some(table), None), - SlotData::Fixed { layout, values } => (None, None, None, Some((layout, values))), + SlotData::Fixed { layout, values } => (None, None, None, Some((layout, &values[..]))), + SlotData::Packed(p) => (None, None, None, Some((p.layout(), p.values()))), }; single .into_iter() @@ -2823,6 +2931,13 @@ impl PyInstance { /// The caller established that `obj` is its only reference, the /// instance is still deferred and owns no C body. pub fn try_recycle(mut inst: Rc) { + // SAFETY: the caller holds the only reference. + if unsafe { &*inst.class.as_ptr() }.native_kind.get() != 0 { + if let Err(inst) = crate::stdlib::datetime_native::recycle(inst) { + drop(Rc::into_arc(inst)); + } + return; + } let Some(m) = Rc::get_mut(&mut inst) else { return; }; diff --git a/crates/weavepy-vm/src/vm_singletons.rs b/crates/weavepy-vm/src/vm_singletons.rs index 01873b63..a6f984b2 100644 --- a/crates/weavepy-vm/src/vm_singletons.rs +++ b/crates/weavepy-vm/src/vm_singletons.rs @@ -121,6 +121,20 @@ pub(crate) fn finalize_on_last_release(arc: std::sync::Arc< if t == TypeId::of::() { // SAFETY: `T` is `PyInstance` (the cast drops no metadata). let inst = unsafe { Arc::from_raw(Arc::into_raw(arc).cast::()) }; + // A natively served `datetime` value: back to its pool (see + // `stdlib::datetime_native::recycle`). With no weak reference the + // last owner is the only one who can reach the class. + let inst = if Arc::weak_count(&inst) == 0 + // SAFETY: as above. + && unsafe { &*inst.class.as_ptr() }.native_kind.get() != 0 + { + match crate::stdlib::datetime_native::recycle(Rc::from_arc(inst)) { + Ok(()) => return, + Err(inst) => Rc::into_arc(inst), + } + } else { + inst + }; let owes = !inst.finalize_ran.get() && inst .class diff --git a/tests/regrtest/test_datetime_packed.py b/tests/regrtest/test_datetime_packed.py new file mode 100644 index 00000000..b91f10a9 --- /dev/null +++ b/tests/regrtest/test_datetime_packed.py @@ -0,0 +1,207 @@ +"""Natively built datetime values (packed storage) behave like Python-built ones.""" +import copy +import gc +import pickle +from datetime import date, datetime, time, timedelta, timezone, tzinfo + + +class SubDate(date): + pass + + +class SubDateTime(datetime): + pass + + +class SubDelta(timedelta): + pass + + +utc = timezone.utc +east = timezone(timedelta(hours=5, minutes=30)) +odd = timezone(timedelta(hours=-3, seconds=7, microseconds=11)) + +# Slot reads from Python methods see the packed fields. +d = datetime(2024, 2, 29, 23, 59, 58, 999999, tzinfo=east) +# The pure-Python classes' private slots (CPython's C types have none). +PURE = hasattr(d, "_year") +if PURE: + assert (d._year, d._month, d._day, d._hour, d._minute) == (2024, 2, 29, 23, 59) + assert (d._second, d._microsecond, d._tzinfo, d._fold) == (58, 999999, east, 0) +assert repr(d) == ("datetime.datetime(2024, 2, 29, 23, 59, 58, 999999, " + "tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)))") +assert str(d) == "2024-02-29 23:59:58.999999+05:30" +assert d.utcoffset() == timedelta(hours=5, minutes=30) +assert d.timetuple()[:6] == (2024, 2, 29, 23, 59, 58) +assert d.astimezone(utc) == d and d.astimezone(utc).hour == 18 +t = timedelta(days=-3, seconds=5, microseconds=7) +assert (t.days, t.seconds, t.microseconds) == (-3, 5, 7) +if PURE: + assert (t._days, t._seconds, t._microseconds) == (-3, 5, 7) +assert repr(t) == "datetime.timedelta(days=-3, seconds=5, microseconds=7)" +assert t.total_seconds() == -259194.999993 + +# Pickling and copying round-trip, through every protocol. +for value in [d, t, date(1, 1, 1), datetime(9999, 12, 31, 23, 59, 59, 999999), + datetime(2000, 1, 1, fold=1), datetime(2000, 1, 1, tzinfo=odd)]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + back = pickle.loads(pickle.dumps(value, proto)) + assert back == value and type(back) is type(value) + assert copy.copy(value) == value and copy.deepcopy(value) == value +assert pickle.loads(pickle.dumps(datetime(2000, 1, 1, fold=1), 4)).fold == 1 + +# A written slot keeps the other fields. +w = datetime(2021, 3, 4, 5, 6, 7) +if PURE: + w._hashcode = 12345 + assert w._hashcode == 12345 and w == datetime(2021, 3, 4, 5, 6, 7) +assert (w.year, w.month, w.day, w.hour, w.minute, w.second) == (2021, 3, 4, 5, 6, 7) + +# Native hashes equal the Python implementation's (which subclasses run). +for a, b in [(date(2024, 2, 29), SubDate(2024, 2, 29)), + (datetime(2024, 2, 29, 1, 2, 3, 456789), + SubDateTime(2024, 2, 29, 1, 2, 3, 456789)), + (datetime(2024, 2, 29, 1, 2, 3, tzinfo=odd), + SubDateTime(2024, 2, 29, 1, 2, 3, tzinfo=odd)), + (datetime(1, 1, 1, tzinfo=east), SubDateTime(1, 1, 1, tzinfo=east)), + (datetime(2000, 1, 1, fold=1), SubDateTime(2000, 1, 1, fold=1)), + (timedelta(1, 2, 3), SubDelta(1, 2, 3)), + (timedelta(-999999999), SubDelta(-999999999))]: + assert a == b and hash(a) == hash(b), (a, b) +assert hash(datetime(2020, 1, 1, 12, tzinfo=utc)) == hash( + datetime(2020, 1, 1, 17, 30, tzinfo=east)) + +# The collector sees the tzinfo (CPython's C type is not a GC type). +assert not PURE or east in gc.get_referents(d) + +# Arithmetic within and across days, both directions. +base = datetime(2024, 1, 15, 8, 30, tzinfo=utc) +step = timedelta(minutes=37, seconds=11) +cur = base +for i in range(200): + cur = cur + step +assert cur == datetime(2024, 1, 20, 12, 26, 40, tzinfo=utc) +assert cur - base == step * 200 and base - cur == -(step * 200) +assert (cur - step * 200) == base +assert datetime(2024, 1, 1) - timedelta(microseconds=1) == datetime(2023, 12, 31, 23, 59, 59, 999999) +assert datetime(2000, 3, 1, fold=1) + timedelta(0) == datetime(2000, 3, 1) +assert (datetime(2000, 3, 1, fold=1) + timedelta(0)).fold == 0 +assert datetime(2024, 1, 1, 12, tzinfo=utc) - datetime(2024, 1, 1, 12, tzinfo=east) == timedelta(hours=5, minutes=30) +for big in [timedelta(days=4000000), timedelta(days=-800000)]: + try: + datetime(2000, 1, 1) + big + except OverflowError: + pass + else: + raise AssertionError("out-of-range datetime accepted") +try: + timedelta(days=999999999) + timedelta(days=1) +except OverflowError: + pass +else: + raise AssertionError("out-of-range timedelta accepted") + +# Comparisons: fixed offsets, naive and aware, mixed kinds. +assert datetime(2024, 1, 1, 12, tzinfo=utc) == datetime(2024, 1, 1, 17, 30, tzinfo=east) +assert datetime(2024, 1, 1, 12, tzinfo=utc) < datetime(2024, 1, 1, 18, tzinfo=east) +assert datetime(2024, 1, 1) != datetime(2024, 1, 1, tzinfo=utc) +try: + datetime(2024, 1, 1) < datetime(2024, 1, 1, tzinfo=utc) +except TypeError: + pass +else: + raise AssertionError("naive/aware ordering accepted") +assert datetime(2000, 1, 1, fold=1) == datetime(2000, 1, 1) +assert timedelta(-1) < timedelta(0) < timedelta(microseconds=1) < timedelta(seconds=1) +assert date(2024, 1, 1) != datetime(2024, 1, 1) + +# Constructors: keywords, defaults, fallbacks and errors. +assert timedelta(weeks=1, days=-1, hours=1, minutes=2, seconds=3, + milliseconds=4, microseconds=5) == timedelta(6, 3723, 4005) +assert timedelta(hours=1.5) == timedelta(seconds=5400) +assert timedelta(True) == timedelta(1) +for bad in [dict(days=10**9), dict(days=2**70)]: + try: + timedelta(**bad) + except OverflowError: + pass + else: + raise AssertionError("out-of-range timedelta accepted") +assert date(year=2020, day=2, month=1) == date(2020, 1, 2) +assert datetime(2020, 5, 3, hour=4, tzinfo=utc) == datetime(2020, 5, 3, 4, 0, 0, 0, utc) +assert datetime(2020, 1, 1, fold=1).fold == 1 +for args, kwargs, exc in [((2020, 2, 30), {}, ValueError), ((2020, 1, 1, 24), {}, ValueError), + ((2020, 1, 1), {"fold": 2}, ValueError), + ((2020, 1, 1), {"year": 2020}, TypeError), + ((2020, 1, 1), {"bogus": 1}, TypeError), + ((2020, 1, 1, 0, 0, 0, 0, None, 0), {}, TypeError)]: + try: + datetime(*args, **kwargs) + except exc: + pass + else: + raise AssertionError((args, kwargs)) +assert type(SubDateTime(2020, 1, 1)) is SubDateTime +assert type(SubDelta(hours=1)) is SubDelta + +# replace(), positional and keyword. +x = date(2024, 2, 29) +assert x.replace(day=1) == date(2024, 2, 1) +assert x.replace(2023, 3) == date(2023, 3, 29) +assert x.replace(year=2023, month=3, day=31) == date(2023, 3, 31) +try: + x.replace(year=2023) +except ValueError: + pass +else: + raise AssertionError("2023-02-29 accepted") +assert type(SubDate(2024, 1, 1).replace(day=2)) is SubDate + +# fromisoformat: bound to the class it is read from. +m = datetime.fromisoformat +assert m.__self__ is datetime and m == datetime.fromisoformat +assert datetime.fromisoformat("2024-01-15T08:30:00+00:00") == base +assert datetime.fromisoformat("2024-01-15T08:30:00+00:00").tzinfo is utc +assert datetime.fromisoformat("2024-02-29T23:59:58.999999+05:30") == d +assert type(SubDateTime.fromisoformat("2024-01-15T08:30:00")) is SubDateTime +assert type(base.fromisoformat("2024-01-15")) is datetime + +# Formatting. +assert d.isoformat() == "2024-02-29T23:59:58.999999+05:30" +assert d.isoformat(" ", "minutes") == "2024-02-29 23:59+05:30" +assert d.isoformat("é", "milliseconds") == "2024-02-29é23:59:58.999+05:30" +assert datetime(2020, 1, 1, tzinfo=odd).isoformat() == "2020-01-01T00:00:00-02:59:52.999989" +assert datetime(5, 1, 2, 3, 4, 5).isoformat() == "0005-01-02T03:04:05" +assert d.strftime("%Y-%m-%d %H:%M:%S.%f %z|%:z %j %y %I%%") == \ + "2024-02-29 23:59:58.999999 +0530|+05:30 060 24 11%" +assert date(2024, 2, 29).isoformat() == "2024-02-29" +assert date(2024, 2, 29).strftime("%F é %T") == "2024-02-29 é 00:00:00" + +# A tzinfo held only by a native result is released (and finalized) with it. +deaths = [] + + +class Zone(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=1) + + def dst(self, dt): + return timedelta(0) + + def __del__(self): + deaths.append(1) + + +v = datetime(2020, 1, 1, tzinfo=Zone()) + timedelta(hours=1) +assert v.utcoffset() == timedelta(hours=1) +del v +gc.collect() +assert deaths == [1], deaths + +# Values outlive their pooled neighbours. +keep = [datetime(2000, 1, 1) + timedelta(days=i) for i in range(300)] +for i in range(1000): + datetime(2000, 1, 1) + timedelta(seconds=i) +assert keep[-1] == datetime(2000, 10, 26) and len(set(keep)) == 300 + +print("datetime packed values: ok") From 964383baa017f09863856b18e1f21aa9105e4920 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:20:08 -0700 Subject: [PATCH 50/96] perf: call a bound datetime.fromisoformat in the core loop `datetime.fromisoformat(s)` binds the native class method without the descriptor protocol, but the call of the bound method still took the full `CALL` handler. The core loop now runs a natively served class method bound to its exact class through its fast half, on bitwise views of the operands, as it does for a native instance method. A `timedelta`'s public `days`/`seconds`/`microseconds` reads also go through the dispatch loop's field shortcut, reading the public slots of a Python-built value as before. datetime_ops, instructions retired: per unit 5.70K -> 5.61K; `datetime.fromisoformat(s)` alone 4.2K -> 2.8K per call. --- crates/weavepy-vm/src/lib.rs | 51 +++++++++++++++++-- .../weavepy-vm/src/stdlib/datetime_native.rs | 28 +++++++++- 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index e76ee2c1..c07aaa81 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -11805,14 +11805,20 @@ impl Interpreter { } } } - // A natively served class's constructor (see - // `stdlib::datetime_native`), in place. - if python == 2 { + // A natively served class's constructor or bound + // class method (see `stdlib::datetime_native`), in + // place. + if python != 1 { let start = len - argc - 2; // SAFETY: `len >= argc + 2`: the call's operands. let ops = unsafe { std::slice::from_raw_parts(base.add(start), argc + 2) }; - if let Some(r) = Self::core_native_ctor_kw(ops, argc, false) { + let r = match &ops[0] { + Object::Type(_) => Self::core_native_ctor_kw(ops, argc, false), + Object::BoundMethod(_) => Self::core_native_bound_call(ops), + _ => None, + }; + if let Some(r) = r { // SAFETY: every operand was checked to leave by a // plain decrement; the result takes the callee's // slot. @@ -15575,6 +15581,43 @@ impl Interpreter { crate::stdlib::datetime_native::construct_names(cls, args, names, values)?.ok() } + /// A call of a bound natively served class method + /// (`datetime.fromisoformat(s)`; see + /// [`crate::stdlib::datetime_native::bound_fast`]): `ops` is the core + /// loop's `CALL` operands (the bound method, an empty self slot and + /// the arguments). The result, or `None` (nothing touched) for the + /// full handler, which also raises every error. + #[inline(never)] + fn core_native_bound_call(ops: &[Object]) -> Option { + let (Object::BoundMethod(bm), Object::Unbound) = (ops.first()?, ops.get(1)?) else { + return None; + }; + let args = &ops[2..]; + if args.len() > 7 || bm.redispatch_descriptor { + return None; + } + let fast = crate::stdlib::datetime_native::bound_fast(&bm.function, &bm.receiver)?; + if !ops + .iter() + .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o)) + { + return None; + } + // Bitwise views of the receiver and the arguments, never dropped: + // the fast half reads them (and clones what it keeps). + let mut views = [const { std::mem::MaybeUninit::::uninit() }; 8]; + // SAFETY: the method and the core loop's stack keep them alive. + views[0].write(unsafe { std::ptr::read(&bm.receiver) }); + for (v, a) in views[1..].iter_mut().zip(args) { + // SAFETY: as above. + v.write(unsafe { std::ptr::read(a) }); + } + // SAFETY: the first `args.len() + 1` entries were written. + let views = + unsafe { std::slice::from_raw_parts(views.as_ptr().cast::(), args.len() + 1) }; + fast(views)?.ok() + } + /// A natively served instance's public field (`dt.hour`; see /// [`crate::stdlib::datetime_native::leaf_field`]), or `None`. #[inline(never)] diff --git a/crates/weavepy-vm/src/stdlib/datetime_native.rs b/crates/weavepy-vm/src/stdlib/datetime_native.rs index d60b2688..a058e0d2 100644 --- a/crates/weavepy-vm/src/stdlib/datetime_native.rs +++ b/crates/weavepy-vm/src/stdlib/datetime_native.rs @@ -2353,6 +2353,22 @@ pub(crate) fn type_classmethod(cls: &TypeObject, name: &SharedStr) -> Option Option { + let (Object::Builtin(f), Object::Type(cls)) = (function, receiver) else { + return None; + }; + if cls.native_kind.get() != KIND_DATETIME { + return None; + } + match state_of_cls(cls)?.fromiso.get()? { + Object::Builtin(g) if Rc::ptr_eq(f, g) => Some(dt_fromisoformat), + _ => None, + } +} + /// A public field (`dt.hour`, `d.year`, …) of a natively served instance /// whose class still has its original property. pub(crate) fn leaf_field(i: &PyInstance, name: &SharedStr) -> Option { @@ -2376,8 +2392,16 @@ pub(crate) fn leaf_field(i: &PyInstance, name: &SharedStr) -> Option { } else { return None; }; - let (d, s, us) = td_fields(i, st)?; - return Some(Object::Int([d, s, us][ix])); + // The public slots themselves (a Python-built value's are written + // apart from the private ones). + return with_slots(i, |s| match s.as_packed() { + Some(p) if p.kind() == KIND_TIMEDELTA => { + let (d, s, us) = p.td(); + Some(Object::Int([d, s, us][ix])) + } + Some(_) => None, + None => Some(s.get_hinted(4 + ix, name)?.clone()), + }); } let (idx, slot) = if SharedStr::ptr_eq(name, &n.f_year) { (D_YEAR, &n.year) From 73d7d23c6fd481d4fc10ab459d03d26f25d326da Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:52:33 -0700 Subject: [PATCH 51/96] perf: keep datetime operators and fields in the frame compiler's code The frame compiler lowers `BINARY_OP` and `COMPARE_OP` on locals to scalar arithmetic and hands any other operand to the core loop, and its `LOAD_ATTR` reads only split-dict fields, so a loop over datetimes left native code at every operator and field read and paid a round trip each time. The core loop now records the operator sites it ran on a natively served operand; the compiler sends those through helpers (two locals or constants are read in place, with no references taken), and a local's natively served field (`dt.hour`) is read through a helper before the core loop. A class's datetime state is reached by a plain cast instead of a `downcast_ref`. datetime_ops, instructions retired: per unit 5.61K -> 4.27K, default work 483M -> 402M (python3.14: 9.2K, 674M). attr_access, richards, nbody, float_math and nested_loops are within noise (+/-2%). --- crates/weavepy-vm/src/frame_jit.rs | 234 +++++++++++++++++- crates/weavepy-vm/src/lib.rs | 41 ++- .../weavepy-vm/src/stdlib/datetime_native.rs | 15 +- tests/regrtest/test_datetime_packed.py | 76 ++++++ 4 files changed, 355 insertions(+), 11 deletions(-) diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs index f382ae4f..dbec32b5 100644 --- a/crates/weavepy-vm/src/frame_jit.rs +++ b/crates/weavepy-vm/src/frame_jit.rs @@ -694,6 +694,114 @@ unsafe extern "C" fn h_binop(at: *mut Object, kind: u32) -> u32 { 0 } +/// `BINARY_OP` of `kind` over the values at `a` and `b` (a local's or a +/// constant's, read in place) for a site that has seen a natively served +/// operand (see `crate::native_site`): `0` the result written to the free +/// slot `dst`, anything else declined untouched (the core loop then runs +/// the instruction, and raises its errors). +unsafe extern "C" fn h_binop_ref( + dst: *mut Object, + a: *const Object, + b: *const Object, + kind: u32, +) -> u32 { + // SAFETY: the code passes a free stack slot, two initialized values + // that outlive the call (nothing it runs can reach them), and a kind + // the compiler emitted (`BinOpKind` is `repr(u8)`). + unsafe { + let (a, b) = (&*a, &*b); + let num = |o: &Object| match *o { + Object::Int(i) => Some(crate::leaf_plan::V::I(i)), + Object::Float(x) => Some(crate::leaf_plan::V::F(x)), + _ => None, + }; + let r = if let (Some(x), Some(y)) = (num(a), num(b)) { + match crate::leaf_plan::binary(x, y, kind as u8) { + Some(crate::leaf_plan::V::I(i)) => Object::Int(i), + Some(crate::leaf_plan::V::F(f)) => Object::Float(f), + _ => return 1, + } + } else { + let kind: BinOpKind = std::mem::transmute(kind as u8); + match a { + Object::Instance(i) if i.cls_raw().native_kind.get() != 0 => { + match Interpreter::core_native_binop(kind, a, b) { + Some(Ok(r)) => r, + _ => return 1, + } + } + _ => return 1, + } + }; + dst.write(r); + } + 0 +} + +/// The field `name` (a `str`) of the natively served instance at `recv` +/// (see `stdlib::datetime_native::leaf_field`): `0` the value written to +/// the free slot `dst`, anything else declined untouched. +unsafe extern "C" fn h_field(dst: *mut Object, recv: *const Object, name: *const Object) -> u32 { + // SAFETY: the code passes a free stack slot, an initialized local and + // one of its code's names; reading the field runs no code. + unsafe { + let (Object::Instance(i), Object::Str(name)) = (&*recv, &*name) else { + return 1; + }; + if i.cls_raw().native_kind.get() == 0 { + return 1; + } + match crate::stdlib::datetime_native::leaf_field(i, name) { + Some(v) => { + dst.write(v); + 0 + } + None => 1, + } + } +} + +/// `COMPARE_OP` with oparg `arg` over the values at `at` and above it, for +/// the shapes the core loop's arm runs (two ints, two floats, two natively +/// served instances): `0` the `bool` at `at` (the operands released), +/// anything else declined untouched. +unsafe extern "C" fn h_compare(at: *mut Object, arg: u32) -> u32 { + // SAFETY: the code passes two initialized stack slots, and the oparg + // the compiler emitted (as `compare_op_step` reads it). + unsafe { + let (a, b) = (&*at, &*at.add(1)); + let kind: CompareKind = std::mem::transmute((arg & !COMPARE_OP_TO_BOOL_FLAG) as u8); + let ord = match (a, b) { + (Object::Int(x), Object::Int(y)) => x.cmp(y), + (Object::Float(x), Object::Float(y)) => match x.partial_cmp(y) { + Some(o) => o, + None => return 1, + }, + (Object::Instance(i), Object::Instance(_)) if i.cls_raw().native_kind.get() != 0 => { + let r = match Interpreter::core_native_compare(kind, a, b) { + Some(Ok(r)) if droppable(a) && droppable(b) => r, + _ => return 1, + }; + crate::drop_hot(at.add(1).read()); + crate::drop_hot(at.read()); + at.write(r); + return 0; + } + _ => return 1, + }; + let r = match kind { + CompareKind::Lt => ord.is_lt(), + CompareKind::LtE => ord.is_le(), + CompareKind::Eq => ord.is_eq(), + CompareKind::NotEq => ord.is_ne(), + CompareKind::Gt => ord.is_gt(), + CompareKind::GtE => ord.is_ge(), + }; + at.write(Object::Bool(r)); + } + 0 +} + /// `LOAD_FAST x` (the `pc`th instruction of `code`) and the call /// `x.m()` after it, as the core loop's fused arm runs it /// (see `Interpreter::core_local_method`), on a stack `len` deep: `0` the @@ -2207,8 +2315,17 @@ impl<'a> Lower<'a> { let bi = self.pop(); let ai = self.pop(); // A value on the stack (a call's result, an element) is as likely - // a heap value as a number: the helper takes every shape. - if matches!(ai, Item::Mem(_)) || matches!(bi, Item::Mem(_)) { + // a heap value as a number: the helper takes every shape. So does + // a site the core loop has run on a natively served operand + // (`datetime`'s; see `crate::native_site`). + let native = crate::native_site(self.ext, pc); + if native + && matches!(ai, Item::Local(_) | Item::Const(_)) + && matches!(bi, Item::Local(_) | Item::Const(_)) + { + return self.binary_borrowed(pc, kind, ai, bi); + } + if matches!(ai, Item::Mem(_)) || matches!(bi, Item::Mem(_)) || native { return self.binary_helper(pc, kind, ai, bi); } let (Some(a), Some(b)) = (self.operand(ai), self.operand(bi)) else { @@ -2340,6 +2457,83 @@ impl<'a> Lower<'a> { true } + /// The address of a local's or a constant's value (`None` for any + /// other item). + fn item_addr(&mut self, item: Item) -> Option { + match item { + Item::Local(i) => Some(self.local_addr(i)), + Item::Const(k) => Some(self.const_addr(k)), + _ => None, + } + } + + /// `BINARY_OP` of two locals or constants through [`h_binop_ref`], + /// which reads them in place (no references taken or released): the + /// result in the next free slot. + fn binary_borrowed(&mut self, pc: usize, kind: u8, ai: Item, bi: Item) -> bool { + let (Some(a), Some(b)) = (self.item_addr(ai), self.item_addr(bi)) else { + return self.binary_helper(pc, kind, ai, bi); + }; + let slot = self.depth; + let dst = self.slot_addr(slot); + let k = self.b.ins().iconst(types::I32, i64::from(kind)); + let r = self + .call(h_binop_ref as *const () as usize, &[dst, a, b, k], true) + .expect("returns"); + let out = self.exit_with(pc, &[ai, bi], INTERP); + self.branch_out(r, out); + self.push(Item::Mem(slot)); + self.set_last(pc); + true + } + + /// `LOAD_ATTR` of a local where no split-layout field was seen: a + /// natively served instance's field through [`h_field`], anything + /// else in the core loop. + fn native_field(&mut self, pc: usize, arg: u32) -> bool { + let item = self.pop(); + let (Item::Local(i), Some(name @ Object::Str(_))) = + (item, self.ext.name_objs.get(arg as usize)) + else { + self.push(item); + self.exit(INTERP, pc); + return false; + }; + let at = self.local_addr(i); + let slot = self.depth; + let dst = self.slot_addr(slot); + let name = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(name) as i64); + let r = self + .call(h_field as *const () as usize, &[dst, at, name], true) + .expect("returns"); + let out = self.exit_with(pc, &[item], INTERP); + self.branch_out(r, out); + self.push(Item::Mem(slot)); + self.set_last(pc); + true + } + + /// `COMPARE_OP` through [`h_compare`]: the operands onto the stack, + /// and the result in the lower one's slot. + fn compare_helper(&mut self, pc: usize, arg: u32, ai: Item, bi: Item) -> bool { + let slot = self.depth; + self.materialize(ai, slot); + self.materialize(bi, slot + 1); + let at = self.slot_addr(slot); + let k = self.b.ins().iconst(types::I32, i64::from(arg)); + let r = self + .call(h_compare as *const () as usize, &[at, k], true) + .expect("returns"); + let out = self.exit_with(pc, &[Item::Mem(slot), Item::Mem(slot + 1)], INTERP); + self.branch_out(r, out); + self.push(Item::Mem(slot)); + self.set_last(pc); + true + } + /// A container instruction through [`h_container`] (the operands on /// the stack; the depth after is the static one). fn container(&mut self, pc: usize) -> bool { @@ -2501,6 +2695,11 @@ impl<'a> Lower<'a> { }; let bi = self.pop(); let ai = self.pop(); + // A site the core loop has run on natively served operands: through + // the helper (see `binary`). + if crate::native_site(self.ext, pc) { + return self.compare_helper(pc, arg, ai, bi); + } let (Some(a), Some(b)) = (self.operand(ai), self.operand(bi)) else { self.push(ai); self.push(bi); @@ -2713,10 +2912,9 @@ impl<'a> Lower<'a> { /// scalar copied, anything else with a reference taken); a stack /// receiver is released after. Anything else, and a miss, is the core /// loop's. - fn load_attr(&mut self, pc: usize, _arg: u32) -> bool { + fn load_attr(&mut self, pc: usize, arg: u32) -> bool { let (Some(l), Some(cache_slot)) = (self.layout, self.field_slots.get(pc)) else { - self.exit(INTERP, pc); - return false; + return self.native_field(pc, arg); }; let l = *l; let item = self.pop(); @@ -2729,7 +2927,16 @@ impl<'a> Lower<'a> { return false; } }; - let miss = self.exit_with(pc, &[item], INTERP); + let exit = self.exit_with(pc, &[item], INTERP); + // A local natively served instance's field (`dt.hour`; see + // `stdlib::datetime_native::leaf_field`) through [`h_field`] before + // the core loop. + let name = self.ext.name_objs.get(arg as usize); + let native = match (item, name) { + (Item::Local(_), Some(Object::Str(_))) => Some(self.b.create_block()), + _ => None, + }; + let miss = native.unwrap_or(exit); let otag = self.tag_at(at); let not_inst = self .b @@ -2778,6 +2985,21 @@ impl<'a> Lower<'a> { Item::Local(_) => { // The receiver is the local's: the value just takes a copy. self.copy_value(dst, v); + if let (Some(native), Some(name)) = (native, name) { + let done = self.b.create_block(); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(native); + let dst = self.slot_addr(slot); + let name = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(name) as i64); + let r = self + .call(h_field as *const () as usize, &[dst, at, name], true) + .expect("returns"); + self.b.ins().brif(r, exit, &[], done, &[]); + self.b.switch_to_block(done); + } } _ => { // The receiver is the stack's: the value goes above it diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index c07aaa81..b45210fa 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -10894,6 +10894,9 @@ impl Interpreter { } _ => break None, }; + if let Some(e) = ext { + native_site_note(e, ninstrs, pc); + } // SAFETY: both operands leave by plain // decrements (checked). unsafe { @@ -11059,6 +11062,9 @@ impl Interpreter { } _ => break None, }; + if let Some(e) = ext { + native_site_note(e, ninstrs, pc); + } // SAFETY: both operands leave by plain // decrements (checked); the result takes the // lower one's slot. @@ -15607,7 +15613,7 @@ impl Interpreter { // the fast half reads them (and clones what it keeps). let mut views = [const { std::mem::MaybeUninit::::uninit() }; 8]; // SAFETY: the method and the core loop's stack keep them alive. - views[0].write(unsafe { std::ptr::read(&bm.receiver) }); + views[0].write(unsafe { std::ptr::read(&raw const bm.receiver) }); for (v, a) in views[1..].iter_mut().zip(args) { // SAFETY: as above. v.write(unsafe { std::ptr::read(a) }); @@ -61404,6 +61410,11 @@ struct CodeConstObjects { /// Split-layout attribute shortcuts per `LOAD_ATTR` site (see /// [`FieldSlot`]); allocated on the first recorded one. field_slots: std::sync::OnceLock>, + /// The `BINARY_OP` and `COMPARE_OP` sites the core loop has run on a + /// natively served operand (see `stdlib::datetime_native`), for the + /// frame compiler to send through its helpers instead of the scalar + /// arms; allocated on the first recorded one. + native_sites: std::sync::OnceLock>, /// Verified frameless method calls per method-load site (see /// [`LeafSite`]); allocated on the first recorded one. leaf_sites: std::sync::OnceLock>, @@ -61814,6 +61825,33 @@ fn field_slot_note(ext: &CodeConstObjects, ninstrs: usize, pc: usize, inst: &PyI } } +/// Record that the operator at `pc` ran on a natively served operand (see +/// [`CodeConstObjects::native_sites`]). +#[inline] +fn native_site_note(ext: &CodeConstObjects, ninstrs: usize, pc: usize) { + use std::sync::atomic::{AtomicBool, Ordering}; + if let Some(site) = ext.native_sites.get().and_then(|s| s.get(pc)) { + if !site.load(Ordering::Relaxed) { + site.store(true, Ordering::Relaxed); + } + return; + } + let sites = ext + .native_sites + .get_or_init(|| (0..ninstrs).map(|_| AtomicBool::new(false)).collect()); + if let Some(site) = sites.get(pc) { + site.store(true, Ordering::Relaxed); + } +} + +/// Whether the operator at `pc` has run on a natively served operand. +pub(crate) fn native_site(ext: &CodeConstObjects, pc: usize) -> bool { + ext.native_sites + .get() + .and_then(|s| s.get(pc)) + .is_some_and(|s| s.load(std::sync::atomic::Ordering::Relaxed)) +} + /// `v.clone()` with the scalars copied and the common heap variants' /// increments in line (the enum's `Clone` is an out-of-line call and a /// variant switch per clone). @@ -62865,6 +62903,7 @@ fn code_vm_ext_build( fast_pairs: std::sync::OnceLock::new(), gen_fast: std::sync::atomic::AtomicU8::new(0), field_slots: std::sync::OnceLock::new(), + native_sites: std::sync::OnceLock::new(), leaf_sites: std::sync::OnceLock::new(), leaf_plan: std::sync::OnceLock::new(), returns_none: std::sync::atomic::AtomicU8::new(0), diff --git a/crates/weavepy-vm/src/stdlib/datetime_native.rs b/crates/weavepy-vm/src/stdlib/datetime_native.rs index a058e0d2..1337dd6b 100644 --- a/crates/weavepy-vm/src/stdlib/datetime_native.rs +++ b/crates/weavepy-vm/src/stdlib/datetime_native.rs @@ -621,10 +621,17 @@ fn verify_slow(st: &State, cls: &TypeObject, kind: u8, ver: u64) -> bool { ok } -// SAFETY-free: `State` is reached from an exact class through its -// `native_ext` slot (see `state_of`). +/// The [`State`] a class marked by [`install`] carries in its +/// `native_ext` slot. +#[inline] +#[allow(clippy::cast_ptr_alignment)] fn state_of_cls(cls: &TypeObject) -> Option<&State> { - cls.native_ext.get()?.downcast_ref::() + let ext = cls.native_ext.get()?; + debug_assert!(ext.downcast_ref::().is_some()); + // SAFETY: only `install` fills `native_ext`, always with a `State` + // (checked above in debug builds), so the payload pointer is a + // `State`'s, aligned for it; the cast drops the vtable. + Some(unsafe { &*Rc::as_ptr(ext).cast::() }) } #[inline] @@ -1206,7 +1213,7 @@ fn pool_ok() -> bool { } impl Pool { - #[inline] + #[inline(always)] fn with(&self, kind: u8, f: impl FnOnce(&mut Vec>) -> R) -> Option { if !pool_ok() { return None; diff --git a/tests/regrtest/test_datetime_packed.py b/tests/regrtest/test_datetime_packed.py index b91f10a9..9b165f6b 100644 --- a/tests/regrtest/test_datetime_packed.py +++ b/tests/regrtest/test_datetime_packed.py @@ -204,4 +204,80 @@ def __del__(self): datetime(2000, 1, 1) + timedelta(seconds=i) assert keep[-1] == datetime(2000, 10, 26) and len(set(keep)) == 300 + +# Hot loops (compiled to native code) over datetime operators and fields, +# and operator sites that see datetimes and other operands in turn. +def walk(n): + cur = base = datetime(2024, 1, 1, tzinfo=utc) + step = timedelta(minutes=37, seconds=11) + day = timedelta(days=1) + hours = days = resets = 0 + for i in range(n): + cur = cur + step + if cur - base > day * 30: + base = base + day + resets += 1 + if cur.hour == 12: + hours += 1 + days += cur.weekday() + return cur, base, hours, days, resets + + +assert walk(20000) == (datetime(2025, 5, 31, 10, 26, 40, tzinfo=utc), + datetime(2025, 5, 2, tzinfo=utc), 835, 59850, 487) + + +def add(a, b, n): + out = None + for i in range(n): + out = a + b + return out + + +def less(a, b, n): + out = None + for i in range(n): + out = a < b + return out + + +def hour(obj, n): + out = None + for i in range(n): + out = obj.hour + return out + + +class Clock: + def __init__(self): + self.hour = 7 + + +for _ in range(2): + assert add(1, 2, 3000) == 3 + assert add(base, step, 3000) == base + step + assert add(2**62, 2**62, 3000) == 2**63 + assert add("a", "b", 3000) == "ab" + assert add(1.5, 2, 3000) == 3.5 + assert add(step, step, 3000) == timedelta(minutes=74, seconds=22) + assert less(1, 2, 3000) is True + assert less(step, timedelta(0), 3000) is False + assert less(2.5, 1.0, 3000) is False + assert less(base, base + step, 3000) is True + try: + less(datetime(2024, 1, 1), base, 3000) + except TypeError: + pass + else: + raise AssertionError("naive/aware ordering accepted") + try: + add(datetime(9999, 12, 31), timedelta(days=1), 3000) + except OverflowError: + pass + else: + raise AssertionError("out-of-range datetime accepted") + assert hour(base, 3000) == 8 + assert hour(Clock(), 3000) == 7 + assert hour(datetime(2020, 1, 1, 23), 3000) == 23 + print("datetime packed values: ok") From 09ae555dddd7e31097d0edfe579c41bf4af85bdd Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:26:14 -0700 Subject: [PATCH 52/96] perf: read method, class-constant and global caches in line in compiled leaf plans A compiled leaf plan called a helper for every method load, every class attribute and every global it read, and deltablue's leaves are made of them: `self.output().value = self.input().value` loads two methods, and each in-line callee reads the global `BinaryConstraint` and its class constant `FORWARD`. Native code now checks each site's cache itself: - a method site remembers, per receiver kind, the class version it resolved under and the function; an instance hits when its class still has that version and it holds none of the class's shared names from the method's on (so no field can shadow the method), a class when its version matches; - a class attribute reads the site's existing class-attribute stamp (which the plan path now fills, as the interpreter's does); - a module global keeps its value under the globals' stamp and the global value epoch, read through the frame's compile-time function (the evaluation's own is checked to be it). Misses fall back to the helpers, which fill the caches. The object layout gains the class tag the checks need. Measured (retired instructions, min of 3): deltablue per unit 37.8M -> 36.3M; its plan-execution phase alone 11.2M -> 9.6M per 5,000 calls. The other five fixtures are unchanged within noise. --- crates/weavepy-jit/src/runtime.rs | 2 + crates/weavepy-vm/src/leaf_plan.rs | 9 +- crates/weavepy-vm/src/leaf_plan/jit.rs | 443 +++++++++++++++++++++---- crates/weavepy-vm/src/lib.rs | 1 + crates/weavepy-vm/src/object.rs | 8 + crates/weavepy-vm/src/tier2.rs | 6 + tests/regrtest/test_jit_plan_caches.py | 189 +++++++++++ 7 files changed, 588 insertions(+), 70 deletions(-) create mode 100644 tests/regrtest/test_jit_plan_caches.py diff --git a/crates/weavepy-jit/src/runtime.rs b/crates/weavepy-jit/src/runtime.rs index 19fa9a4c..17ef6939 100644 --- a/crates/weavepy-jit/src/runtime.rs +++ b/crates/weavepy-jit/src/runtime.rs @@ -195,6 +195,8 @@ pub struct ObjLayout { pub tag_float: u8, pub tag_bool: u8, pub tag_none: u8, + /// A class value's tag (its payload pointer is the class pointer). + pub tag_type: u8, /// An instance value's payload pointer → the instance's class pointer, /// its published dict pointer (null while split), its split values' /// borrow counter (`i32`) and block pointer (null while empty). diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index 9f728632..38aaecf7 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -1571,7 +1571,14 @@ impl Interpreter { .and_then(|s| crate::class_attr_hit(s, cls)) { Some(v) => owned.own(v), - None => owned.own(Self::leaf_load_type_attr(code, cls, pc, name)?), + None => { + let v = Self::leaf_load_type_attr(code, cls, pc, name)?; + // A scalar is a plain class value (see + // `leaf_load_type_attr`): remembered for the next + // read, which native code makes in line. + crate::class_attr_fill(code, cls, pc as usize, &v); + owned.own(v) + } } } Object::Module(module) => { diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index cb8333a4..a249ad46 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -3,8 +3,10 @@ //! A plan that keeps running is compiled, once per effect mode, into a //! function over the same registers: each register is a pair of SSA //! variables holding the [`V`] tag and payload words; moves, constants, -//! jumps, and scalar compares, truth tests and arithmetic run in line; and -//! every other operation calls a helper that runs the interpreter's own +//! jumps, and scalar compares, truth tests and arithmetic run in line; so +//! do the common hits of a site's caches (an instance field, a method, a +//! class constant, a module global), which their helpers fill; and every +//! other operation calls a helper that runs the interpreter's own //! implementation of it. Whatever the native code or a helper can't //! settle declines the evaluation exactly where the interpreter's runner //! would, so a native plan keeps the runner's contract: a result, or @@ -76,6 +78,8 @@ pub(super) struct Native { _fields: Vec>, #[allow(clippy::vec_box)] _globals: Vec>, + #[allow(clippy::vec_box)] + _methods: Vec>, } /// An attribute site's split-field positions by receiver class version: @@ -88,15 +92,58 @@ pub(super) struct FieldCache { next: std::cell::Cell, } -/// A global site's resolution: the globals and builtins stamps it was -/// found under (process-unique, so they name the dicts and their key -/// layouts), which of the two holds it, and at what index. +/// A module global site's value as register words, with the globals +/// stamp it was read under (process-unique, so it names the dict and its +/// key layout: the entry stays put) and the global value epoch (which an +/// in-place rebinding advances). Native code reads it in line. #[derive(Default)] +#[repr(C)] pub(super) struct GlobalCache { - stamps: std::cell::Cell<(u64, u64)>, - at: std::cell::Cell<(bool, u32)>, + stamp: std::cell::Cell, + epoch: std::cell::Cell, + tag: std::cell::Cell, + pay: std::cell::Cell, } +const GC_STAMP: i32 = std::mem::offset_of!(GlobalCache, stamp) as i32; +const GC_EPOCH: i32 = std::mem::offset_of!(GlobalCache, epoch) as i32; +const GC_TAG: i32 = std::mem::offset_of!(GlobalCache, tag) as i32; +const GC_PAY: i32 = std::mem::offset_of!(GlobalCache, pay) as i32; +/// Where native code finds the evaluation's own function. +const TOP_F_OFFSET: i32 = TOP_OFFSET + std::mem::offset_of!(Frame, f) as i32; + +/// A class-attribute stamp's scalar tags (see `class_attr_fill`) and the +/// register tags of the values they hold. +const CLASS_ATTR_SCALARS: [(u64, u64); 4] = [ + (crate::CLASS_ATTR_INT, T_I), + (crate::CLASS_ATTR_FLOAT, T_F), + (crate::CLASS_ATTR_BOOL, T_B), + (crate::CLASS_ATTR_NONE, T_N), +]; + +/// A method site's latest resolutions, which native code reads in line: +/// for an instance receiver, its class's version, the function, and how +/// many of the class's shared names (from the first) come before the +/// method's name, so an instance holding no more fields than that can't +/// shadow it; for a class receiver, its version and the function called +/// with an empty self slot. Versions are process-unique (one class in one +/// state), so an equal one means the same resolution. +#[derive(Default)] +#[repr(C)] +pub(super) struct MethodCache { + inst_ver: std::cell::Cell, + inst_func: std::cell::Cell, + inst_limit: std::cell::Cell, + type_ver: std::cell::Cell, + type_func: std::cell::Cell, +} + +const MC_INST_VER: i32 = std::mem::offset_of!(MethodCache, inst_ver) as i32; +const MC_INST_FUNC: i32 = std::mem::offset_of!(MethodCache, inst_func) as i32; +const MC_INST_LIMIT: i32 = std::mem::offset_of!(MethodCache, inst_limit) as i32; +const MC_TYPE_VER: i32 = std::mem::offset_of!(MethodCache, type_ver) as i32; +const MC_TYPE_FUNC: i32 = std::mem::offset_of!(MethodCache, type_func) as i32; + // SAFETY: the descriptors name immutable code objects and reserved // function allocations; native code is only entered with the GIL held. unsafe impl Send for Native {} @@ -204,10 +251,10 @@ const TOP_OFFSET: i32 = 32; /// cache's first entry's version and position. const PEND_N_OFFSET: i32 = (std::mem::offset_of!(Ctx<'static>, pend) + std::mem::offset_of!(Pending, n)) as i32; -const FIELD_VER_OFFSET: i32 = (std::mem::offset_of!(FieldCache, entries) - + std::mem::offset_of!((u64, u32), 0)) as i32; -const FIELD_IDX_OFFSET: i32 = (std::mem::offset_of!(FieldCache, entries) - + std::mem::offset_of!((u64, u32), 1)) as i32; +const FIELD_VER_OFFSET: i32 = + (std::mem::offset_of!(FieldCache, entries) + std::mem::offset_of!((u64, u32), 0)) as i32; +const FIELD_IDX_OFFSET: i32 = + (std::mem::offset_of!(FieldCache, entries) + std::mem::offset_of!((u64, u32), 1)) as i32; const _: () = assert!(std::mem::offset_of!(Ctx<'static>, top) == TOP_OFFSET as usize); /// Run `plan`'s native code for one evaluation, compiling it once it is @@ -235,8 +282,17 @@ pub(super) fn run( { return None; } - slot.get_or_init(|| compile(code, ext, plan, EFFECT)) - .as_ref()? + slot.get_or_init(|| { + // SAFETY: a function value is always an `Rc` allocation + // (`Object::Function`), alive for this evaluation: the + // count taken here is released with the handle. + let f = unsafe { + Rc::increment_strong_count(std::ptr::from_ref(f)); + Rc::from_raw(std::ptr::from_ref(f)) + }; + compile(code, ext, plan, &f, EFFECT) + }) + .as_ref()? } }; // In-line callees run where the interpreter would nest frames: near @@ -307,38 +363,23 @@ unsafe extern "C" fn h_global( // SAFETY: called by native code with its context, a live frame, the // site's stamp slot (its code keeps the table), and its cache. let (c, (code, _, f), slot, cache) = unsafe { (cx(ctx), (*fr).parts(), &*slot, &*cache) }; - // SAFETY (raw dict reads): nothing runs code here. - let (gs, bs) = unsafe { - ( - (*f.globals.as_ptr()).mutation_stamp(), - (*f.builtins.as_ptr()).mutation_stamp(), - ) - }; - if cache.stamps.get() == (gs, bs) { - let (builtin, idx) = cache.at.get(); - let dict = if builtin { &f.builtins } else { &f.globals }; - // SAFETY: as above. - if let Some((_, v)) = unsafe { (*dict.as_ptr()).get_index(idx as usize) } { - c.put(norm(v)); - return 0; - } - } let Some(v) = c.interp.plan_global_at(code, slot, f, pc as u16) else { return 1; }; - // Remember where the value lives (the site's inline cache says which - // namespace), under the stamps it was read with. - use weavepy_compiler::InlineCache as IC; - match code.caches.get(pc) { - IC::LoadGlobalModule { key_idx, .. } => { - cache.stamps.set((gs, bs)); - cache.at.set((false, key_idx)); - } - IC::LoadGlobalBuiltin { key_idx, .. } => { - cache.stamps.set((gs, bs)); - cache.at.set((true, key_idx)); - } - _ => {} + // A module global's value is remembered for the in-line read (a + // builtin's also depends on the globals not gaining the name, which + // the helper checks). + if matches!( + code.caches.get(pc), + weavepy_compiler::InlineCache::LoadGlobalModule { .. } + ) { + // SAFETY (raw dict read): nothing runs code here. + let gs = unsafe { (*f.globals.as_ptr()).mutation_stamp() }; + let [t, p] = words(v); + cache.stamp.set(gs); + cache.epoch.set(crate::object::global_value_epoch()); + cache.tag.set(t); + cache.pay.set(p); } c.put(v); 0 @@ -431,22 +472,42 @@ unsafe extern "C" fn h_method( ctx: *mut Ctx<'static>, fr: *const Frame, ms: *const crate::MethodSlot, + cache: *const MethodCache, t: u64, p: u64, name: u32, ) -> u32 { - // SAFETY: called by native code with its context, a live frame, and - // the site's method slot (its code keeps the table). - let (c, (code, _, _), ms) = unsafe { (cx(ctx), (*fr).parts(), &*ms) }; - match c.interp.plan_method_at(code, ms, value(t, p), name as u16) { - Some((func, recv)) => { - let [a, b] = words(func); - let [d, e] = words(recv); - c.out = [a, b, d, e]; - 0 + // SAFETY: called by native code with its context, a live frame, the + // site's method slot (its code keeps the table), and its cache. + let (c, (code, ext, _), ms, cache) = unsafe { (cx(ctx), (*fr).parts(), &*ms, &*cache) }; + let Some((func, recv)) = c.interp.plan_method_at(code, ms, value(t, p), name as u16) else { + return 1; + }; + let [a, b] = words(func); + let [d, e] = words(recv); + c.out = [a, b, d, e]; + // Remember the resolution for the in-line check. + // SAFETY: the method load succeeded on a heap receiver (as `norm`). + match (func, unsafe { &*(p as *const Object) }) { + (V::Fn(fp), Object::Instance(inst)) if inst.dict.published().is_none() => { + let cls = inst.cls_raw(); + let i = name as usize; + if let (Some(Object::Str(n)), Some(&hash)) = + (ext.name_objs.get(i), ext.name_hashes.get(i)) + { + let limit = cls.shared_keys.get().map_or(0, |k| k.names_before(n, hash)); + cache.inst_ver.set(cls.attr_version.get()); + cache.inst_func.set(fp as u64); + cache.inst_limit.set(limit as u64); + } } - None => 1, + (V::Fn(fp), Object::Type(cls)) if matches!(recv, V::Null) => { + cache.type_ver.set(cls.attr_version.get()); + cache.type_func.set(fp as u64); + } + _ => {} } + 0 } /// `0`/`1` for the comparison's result, `2` to decline. @@ -731,6 +792,7 @@ fn compile( code: &CodeObject, ext: &CodeConstObjects, plan: &LeafPlan, + f: &Rc, effect: bool, ) -> Option { crate::tier2::ensure_obj_layout(); @@ -739,7 +801,7 @@ fn compile( .unwrap_or_else(std::sync::PoisonError::into_inner); let engine = guard.get_or_insert_with(Engine::new).as_mut()?; let built = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - compile_with(engine, code, ext, plan, effect) + compile_with(engine, code, ext, plan, f, effect) })); match built { Ok(native) => native, @@ -755,6 +817,7 @@ fn compile_with( code: &CodeObject, ext: &CodeConstObjects, plan: &LeafPlan, + f: &Rc, effect: bool, ) -> Option { engine.module.clear_context(&mut engine.ctx); @@ -783,8 +846,9 @@ fn compile_with( keep: Vec::new(), fields: Vec::new(), globals: Vec::new(), + methods: Vec::new(), }; - let done = lower.lower(code, ext, plan); + let done = lower.lower(code, ext, plan, f); let Lower { b, depth, @@ -792,14 +856,15 @@ fn compile_with( keep, fields, globals, + methods, .. } = lower; done.map(|()| { b.finalize(); - (depth, frames, keep, fields, globals) + (depth, frames, keep, fields, globals, methods) }) }; - let Some((depth, frames, keep, fields, globals)) = lowered else { + let Some((depth, frames, keep, fields, globals, methods)) = lowered else { // An unfinished function leaves the builder's state behind. engine.fbctx = FunctionBuilderContext::new(); engine.module.clear_context(&mut engine.ctx); @@ -826,6 +891,7 @@ fn compile_with( _keep: keep, _fields: fields, _globals: globals, + _methods: methods, }) } @@ -853,6 +919,10 @@ struct FrameLower<'p> { /// Whether an attribute store may have run before an op (only the /// evaluation's own frame stores). may_store: Vec, + /// The function the frame runs: an in-line callee's (its guard proved + /// it), or the one the plan was compiled for (the evaluation's own + /// may be another with the same code). + func: *const PyFunction, } struct Lower<'b> { @@ -872,6 +942,8 @@ struct Lower<'b> { fields: Vec>, #[allow(clippy::vec_box)] globals: Vec>, + #[allow(clippy::vec_box)] + methods: Vec>, } impl Lower<'_> { @@ -947,7 +1019,10 @@ impl Lower<'_> { let not_ref = self.b.ins().icmp_imm(IntCC::NotEqual, t, T_R as i64); self.miss_if(not_ref, miss); let otag = self.b.ins().uload8(types::I32, f, p, 0); - let mut bad = self.b.ins().icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + let mut bad = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); if effect { let n = self.b.ins().load(types::I64, f, self.ctx, PEND_N_OFFSET); let pending = self.b.ins().icmp_imm(IntCC::NotEqual, n, 0); @@ -982,7 +1057,10 @@ impl Lower<'_> { let ckeys = self.b.ins().load(ptr, f, cls, l.type_shared_keys); let len = self.b.ins().uload32(f, block, l.split_len); let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); - let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); let bad = self.b.ins().bor(foreign, absent); self.miss_if(bad, miss); let off = self.b.ins().ishl_imm(idx, 4); @@ -1009,6 +1087,186 @@ impl Lower<'_> { (rt, rp) } + /// [`h_method`]'s cache in line, for the receiver register `(t, p)`: an + /// instance whose class has the cached version and that holds none of + /// the class's names from the method's on (in its split values over + /// the class's names, with no dictionary of its own), or a class with + /// the cached version. Jumps to `done` with the function's and the + /// self slot's words, or to `slow`. + fn inline_method( + &mut self, + l: &weavepy_jit::ObjLayout, + t: Value, + p: Value, + cache: i64, + done: Block, + slow: Block, + ) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + let not_ref = self.b.ins().icmp_imm(IntCC::NotEqual, t, T_R as i64); + self.miss_if(not_ref, slow); + let otag = self.b.ins().uload8(types::I32, f, p, 0); + let at = self.b.ins().iconst(ptr, cache); + let is_inst = self + .b + .ins() + .icmp_imm(IntCC::Equal, otag, i64::from(l.tag_instance)); + let inst_b = self.b.create_block(); + let other = self.b.create_block(); + self.b.ins().brif(is_inst, inst_b, &[], other, &[]); + // A class: its version. + self.b.switch_to_block(other); + let is_type = self + .b + .ins() + .icmp_imm(IntCC::Equal, otag, i64::from(l.tag_type)); + let type_b = self.b.create_block(); + self.b.ins().brif(is_type, type_b, &[], slow, &[]); + self.b.switch_to_block(type_b); + let cls = self.b.ins().load(ptr, f, p, 8); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let want = self.b.ins().load(types::I64, f, at, MC_TYPE_VER); + let func = self.b.ins().load(types::I64, f, at, MC_TYPE_FUNC); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let bad = self.b.ins().bor(unset, stale); + self.miss_if(bad, slow); + let tf = self.b.ins().iconst(types::I64, T_FN as i64); + let tn = self.b.ins().iconst(types::I64, T_NULL as i64); + let zero = self.b.ins().iconst(types::I64, 0); + self.b + .ins() + .jump(done, &[tf.into(), func.into(), tn.into(), zero.into()]); + // An instance: its class's version, and its fields. + self.b.switch_to_block(inst_b); + let inst = self.b.ins().load(ptr, f, p, 8); + let cls = self.b.ins().load(ptr, f, inst, l.inst_class); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let want = self.b.ins().load(types::I64, f, at, MC_INST_VER); + let func = self.b.ins().load(types::I64, f, at, MC_INST_FUNC); + let limit = self.b.ins().load(types::I64, f, at, MC_INST_LIMIT); + let lazy = self.b.ins().load(ptr, f, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, f, flag, 0); + let borrow = self.b.ins().sload32(f, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, f, inst, l.inst_split_block); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, busy); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, slow); + let tf = self.b.ins().iconst(types::I64, T_FN as i64); + let tr = self.b.ins().iconst(types::I64, T_R as i64); + let hit_args = [tf.into(), func.into(), tr.into(), p.into()]; + // No fields at all, or none from the method's name on. + let fields = self.b.create_block(); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + self.b.ins().brif(empty, done, &hit_args, fields, &[]); + self.b.switch_to_block(fields); + let keys = self.b.ins().load(ptr, f, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, f, cls, l.type_shared_keys); + let len = self.b.ins().uload32(f, block, l.split_len); + let len = self.b.ins().uextend(types::I64, len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let shadows = self.b.ins().icmp(IntCC::UnsignedGreaterThan, len, limit); + let bad = self.b.ins().bor(foreign, shadows); + self.miss_if(bad, slow); + self.b.ins().jump(done, &hit_args); + } + + /// [`h_global`]'s cache in line: the frame runs its compile-time + /// function, whose globals still have the cached stamp, under the + /// cached value epoch. Jumps to `done` with the value's words, or to + /// `slow`. + fn inline_global(&mut self, fl: &FrameLower<'_>, cache: i64, done: Block, slow: Block) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + if fl.depth == 0 { + // The evaluation's own function may be another with this code. + let top = self.b.ins().load(ptr, f, self.ctx, TOP_F_OFFSET); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, top, fl.func as i64); + self.miss_if(other, slow); + } + // SAFETY: the frame's function is alive while its code runs (see + // `FrameLower::func`), and so its globals. + let stamp_at = + unsafe { (*fl.func).globals.as_ptr() } as usize + crate::object::DictData::STAMP_OFFSET; + let stamp_at = self.b.ins().iconst(ptr, stamp_at as i64); + let epoch_at = self + .b + .ins() + .iconst(ptr, crate::object::global_value_epoch_ptr() as i64); + let at = self.b.ins().iconst(ptr, cache); + let stamp = self.b.ins().load(types::I64, f, stamp_at, 0); + let epoch = self.b.ins().load(types::I64, f, epoch_at, 0); + let want = self.b.ins().load(types::I64, f, at, GC_STAMP); + let want_epoch = self.b.ins().load(types::I64, f, at, GC_EPOCH); + let t = self.b.ins().load(types::I64, f, at, GC_TAG); + let p = self.b.ins().load(types::I64, f, at, GC_PAY); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, stamp, want); + let moved = self.b.ins().icmp(IntCC::NotEqual, epoch, want_epoch); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, moved); + self.miss_if(bad, slow); + self.b.ins().jump(done, &[t.into(), p.into()]); + } + + /// A class attribute the site's stamp remembers, in line: the receiver + /// register `(t, p)` is a class at the stamp's version, which resolved + /// the name to a scalar (see `class_attr_fill`). Jumps to `done` with + /// its words, or to `slow`. + fn inline_class_attr( + &mut self, + l: &weavepy_jit::ObjLayout, + t: Value, + p: Value, + slot: i64, + done: Block, + slow: Block, + ) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + let not_ref = self.b.ins().icmp_imm(IntCC::NotEqual, t, T_R as i64); + self.miss_if(not_ref, slow); + let otag = self.b.ins().uload8(types::I32, f, p, 0); + let not_type = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_type)); + self.miss_if(not_type, slow); + let cls = self.b.ins().load(ptr, f, p, 8); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let at = self.b.ins().iconst(ptr, slot); + let want = self.b.ins().load(types::I64, f, at, 0); + let tag = self.b.ins().load(types::I64, f, at, 8); + let bits = self.b.ins().load(types::I64, f, at, 16); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let bad = self.b.ins().bor(unset, stale); + self.miss_if(bad, slow); + for (stamp, vtag) in CLASS_ATTR_SCALARS { + let next = self.b.create_block(); + let hit = self.b.ins().icmp_imm(IntCC::Equal, tag, stamp as i64); + let tv = self.b.ins().iconst(types::I64, vtag as i64); + let pay = if vtag == T_N { + self.b.ins().iconst(types::I64, 0) + } else { + bits + }; + self.b + .ins() + .brif(hit, done, &[tv.into(), pay.into()], next, &[]); + self.b.switch_to_block(next); + } + self.b.ins().jump(slow, &[]); + } + /// Load the result words a helper left at `ctx.out[k..k + 2]` into `r`. fn set_out(&mut self, fl: &FrameLower<'_>, r: u8, k: i32) { let flags = MemFlags::trusted(); @@ -1077,7 +1335,13 @@ impl Lower<'_> { self.b.block_params(merge)[0] } - fn lower(&mut self, code: &CodeObject, ext: &CodeConstObjects, plan: &LeafPlan) -> Option<()> { + fn lower( + &mut self, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + f: &Rc, + ) -> Option<()> { let entry = self.b.create_block(); self.b.append_block_params_for_function_params(entry); self.b.switch_to_block(entry); @@ -1085,7 +1349,8 @@ impl Lower<'_> { let argw = self.b.block_params(entry)[1]; self.decline = self.b.create_block(); let fr = self.b.ins().iadd_imm(self.ctx, i64::from(TOP_OFFSET)); - let fl = self.frame(plan, code, ext, fr, 0, None); + let fl = self.frame(plan, code, ext, fr, 0, None, Rc::as_ptr(f)); + self.keep.push((Rc::downgrade(f), f.code())); let flags = MemFlags::trusted(); for k in 0..usize::from(plan.nargs) { let t = self.b.ins().load(types::I64, flags, argw, (16 * k) as i32); @@ -1114,6 +1379,7 @@ impl Lower<'_> { fr: Value, depth: u8, ret: Option<(Variable, Variable, Block)>, + func: *const PyFunction, ) -> FrameLower<'p> { let nregs = usize::from(plan.nregs); let mut tags = Vec::with_capacity(nregs); @@ -1150,6 +1416,7 @@ impl Lower<'_> { blocks, ret, may_store, + func, } } @@ -1242,18 +1509,28 @@ impl Lower<'_> { .ins() .iconst(self.ptr, std::ptr::from_ref(slot) as i64); let cache = Box::::default(); - let cache_v = self - .b - .ins() - .iconst(self.ptr, std::ptr::from_ref(&*cache) as i64); + let cache_at = std::ptr::from_ref(&*cache) as i64; self.globals.push(cache); + let done = self.b.create_block(); + self.b.append_block_param(done, types::I64); + self.b.append_block_param(done, types::I64); + let slow = self.b.create_block(); + self.inline_global(fl, cache_at, done, slow); + self.b.switch_to_block(slow); + let cache_v = self.b.ins().iconst(self.ptr, cache_at); let pc = self.u32c(u32::from(pc)); let st = self.call( h_global as *const () as usize, &[self.ctx, fl.fr, slot, cache_v, pc], ); self.check(st); - self.set_out(fl, dst, 0); + let flags = MemFlags::trusted(); + let t = self.b.ins().load(types::I64, flags, self.ctx, 0); + let p = self.b.ins().load(types::I64, flags, self.ctx, 8); + self.b.ins().jump(done, &[t.into(), p.into()]); + self.b.switch_to_block(done); + let (t, p) = (self.b.block_params(done)[0], self.b.block_params(done)[1]); + self.set(fl, dst, t, p); } Op::Attr { dst, src, pc, name } => { let (t, p) = self.get(fl, src); @@ -1271,6 +1548,12 @@ impl Lower<'_> { let (vt, vp) = self.inline_field(l, t, p, cache_at, effect, slow); self.b.ins().jump(done, &[vt.into(), vp.into()]); self.b.switch_to_block(slow); + if let Some(slot) = crate::code_stamp_slot(fl.code, u32::from(pc)) { + let slow = self.b.create_block(); + let slot = std::ptr::from_ref(slot) as i64; + self.inline_class_attr(l, t, p, slot, done, slow); + self.b.switch_to_block(slow); + } } let addr = if effect { h_field:: as *const () as usize @@ -1293,15 +1576,37 @@ impl Lower<'_> { return Some(false); }; let (t, p) = self.get(fl, src); + let cache = Box::::default(); + let cache_at = std::ptr::from_ref(&*cache) as i64; + self.methods.push(cache); + let done = self.b.create_block(); + for _ in 0..4 { + self.b.append_block_param(done, types::I64); + } + let slow = self.b.create_block(); + if let Some(l) = crate::tier2::published_obj_layout() { + self.inline_method(l, t, p, cache_at, done, slow); + } else { + self.b.ins().jump(slow, &[]); + } + self.b.switch_to_block(slow); let ms = self.b.ins().iconst(self.ptr, std::ptr::from_ref(ms) as i64); + let cache_v = self.b.ins().iconst(self.ptr, cache_at); let nm = self.u32c(u32::from(name)); let st = self.call( h_method as *const () as usize, - &[self.ctx, fl.fr, ms, t, p, nm], + &[self.ctx, fl.fr, ms, cache_v, t, p, nm], ); self.check(st); - self.set_out(fl, dst, 0); - self.set_out(fl, dst + 1, 2); + let flags = MemFlags::trusted(); + let w: Vec<_> = (0..4) + .map(|k| self.b.ins().load(types::I64, flags, self.ctx, 8 * k).into()) + .collect(); + self.b.ins().jump(done, &w); + self.b.switch_to_block(done); + let ps = self.b.block_params(done).to_vec(); + self.set(fl, dst, ps[0], ps[1]); + self.set(fl, dst + 1, ps[2], ps[3]); } Op::Compare { dst, a, b, kind } => { let (ta, pa) = self.get(fl, a); @@ -1629,7 +1934,7 @@ impl Lower<'_> { let depth = fl.depth + 1; self.depth = self.depth.max(depth); let ret = Some((fl.tags[usize::from(at)], fl.pays[usize::from(at)], cont)); - let cfl = self.frame(cplan, &ccode, cext, fr, depth, ret); + let cfl = self.frame(cplan, &ccode, cext, fr, depth, ret, Rc::as_ptr(&f)); // The receiver and the arguments become the callee's first // registers. for k in 0..=usize::from(argc) { diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index b45210fa..7b568f0c 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -61636,6 +61636,7 @@ fn code_attr_poly(code: &CodeObject, cache_pc: u32) -> Option<&AttrPoly> { /// and (for a builtin hit) the builtins dict's stamp. While all three /// match, the dicts are exactly as they were when the site last ran its /// full guards, so the cached index can be read straight away. +#[repr(transparent)] struct StampSlot(std::cell::UnsafeCell<[u64; 3]>); // SAFETY: as `MethodSlot`: read and written only from the dispatch loop diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 4acf6af2..b72951b8 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -4058,6 +4058,11 @@ pub fn global_value_epoch() -> u64 { GLOBAL_VALUE_EPOCH.load(std::sync::atomic::Ordering::Relaxed) } +/// Where [`GLOBAL_VALUE_EPOCH`] lives (native code reads it in line). +pub(crate) fn global_value_epoch_ptr() -> *const u64 { + GLOBAL_VALUE_EPOCH.as_ptr() +} + /// Advance [`GLOBAL_VALUE_EPOCH`]. #[inline] pub fn bump_global_value_epoch() { @@ -4190,6 +4195,9 @@ impl DictData { crate::gc_trace::track_deferred_owner(owner); } + /// Where the stamp sits in a dict (native code reads it in line). + pub(crate) const STAMP_OFFSET: usize = std::mem::offset_of!(DictData, stamp); + /// The stamp of the dict's current state (see the type docs). #[inline] pub fn mutation_stamp(&self) -> u64 { diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 6e3b7164..aaad80a0 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -2993,6 +2993,11 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option(), 8) != cls_arc { + return None; + } // The native body's word: a `Cell` alone in its wrapper, // checked on a sample. let inst_c_body = arc_data + std::mem::offset_of!(PyInstance, c_body); @@ -3130,6 +3135,7 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option b.value + + @staticmethod + def weakest(a, b): + return a if Strength.weaker(a, b) else b + + +class StaticC: + def __init__(self, s, t): + self.s = s + self.t = t + + def get(self): + return Strength.weakest(self.s, self.t).value + + +def drive(o, n): + if n < 0: + return locals() + t = 0 + for _ in range(n): + t += o.get() + return t + + +def set_scale(v): + global SCALE + SCALE = v + + +class PlanCacheTests(unittest.TestCase): + def test_instance_attribute_shadows_method(self): + class P: + def __init__(self, x): + self.x = x + + def pick(self): + return self.x + + def get(self): + return self.pick().v + + p = P(Val(2)) + self.assertEqual(drive(p, WARM), 2 * WARM) + p.pick = lambda: Val(10) + self.assertEqual(drive(p, 10), 100) + self.assertEqual(drive(P(Val(3)), 10), 30) + + def test_method_replaced_on_class(self): + class P: + def __init__(self, x): + self.x = x + + def pick(self): + return self.x + + def get(self): + return self.pick().v + + p = P(Val(2)) + self.assertEqual(drive(p, WARM), 2 * WARM) + P.pick = lambda self: Val(5) + self.assertEqual(drive(p, 10), 50) + + def test_subclass_receiver(self): + class P: + def __init__(self, x): + self.x = x + + def pick(self): + return self.x + + def get(self): + return self.pick().v + + class Q(P): + def pick(self): + return Val(7) + + self.assertEqual(drive(P(Val(2)), WARM), 2 * WARM) + self.assertEqual(drive(Q(Val(2)), 10), 70) + self.assertEqual(drive(P(Val(4)), 10), 40) + + def test_class_constant_rebound(self): + q = ConstQ(Val(1), Val(2)) + self.assertEqual(drive(q, WARM), 2 * WARM) + ConstQ.FWD = 0 + self.assertEqual(drive(q, 10), 20) + ConstP.FWD = 0 + self.assertEqual(drive(q, 10), 10) + ConstP.FWD = 1 + self.assertEqual(drive(q, 10), 20) + + def test_global_rebound(self): + class P: + def __init__(self, x): + self.x = x + + def get(self): + return self.x.v * SCALE + + p = P(Val(3)) + try: + self.assertEqual(drive(p, WARM), 6 * WARM) + set_scale(5) + self.assertEqual(drive(p, 10), 150) + globals()["SCALE"] = 7 + self.assertEqual(drive(p, 10), 210) + del globals()["SCALE"] + with self.assertRaises(NameError): + p.get() + globals()["SCALE"] = 11 + self.assertEqual(drive(p, 10), 330) + finally: + set_scale(2) + + def test_shared_code_other_globals(self): + class P: + def __init__(self, x): + self.x = x + + def get(self): + return self.x.v * SCALE + + p = P(Val(3)) + self.assertEqual(drive(p, WARM), 6 * WARM) + ns = dict(globals()) + ns["SCALE"] = 100 + + class Q(P): + get = types.FunctionType(P.get.__code__, ns) + + self.assertEqual(drive(Q(Val(3)), 10), 3000) + self.assertEqual(drive(p, 10), 60) + + def test_static_method_through_class(self): + c = StaticC(Strength(1), Strength(2)) + self.assertEqual(drive(c, WARM), 2 * WARM) + Strength.weaker = staticmethod(lambda x, y: x.value < y.value) + self.assertEqual(drive(c, 10), 10) + +if __name__ == "__main__": + unittest.main() From 8b04438e41f6dc4e0d0678d4e0c8e6bcd09f6c80 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 03:26:35 -0700 Subject: [PATCH 53/96] perf: compile leaf plans without the mid-end optimizer, and long frames later Two compile costs deltablue paid without earning them back: - Leaf plans compiled at Cranelift's `speed` level, about ten million instructions a plan. Without the e-graph pass (tier 2 dropped it for the same reason) a plan compiles several times faster, and the code it runs is no slower: the lowering already emits what it wants. `WEAVEPY_JIT_OPT=1` restores the pass, as for tier 2. - The frame JIT compiled a code object once 1,000 activations and back edges had run, whatever its length. Compiling costs about the same for each instruction, while the saving comes from the loop that got hot, so a long driver around a short loop (deltablue's `projection_test`, 177 instructions for a few hundred iterations a unit) paid some thirty million instructions for nothing. The threshold now scales with the code's length in 16-instruction units; short loops compile as before. Measured (retired instructions, min of 3), deltablue: fixed cost 255M -> 230M, per unit 36.3M -> 36.0M; at the default work of 10 units, 618M -> 589M. The other five fixtures are unchanged within noise. --- crates/weavepy-vm/src/frame_jit.rs | 14 +++++++++++++- crates/weavepy-vm/src/leaf_plan/jit.rs | 10 +++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs index dbec32b5..ae4ec55c 100644 --- a/crates/weavepy-vm/src/frame_jit.rs +++ b/crates/weavepy-vm/src/frame_jit.rs @@ -71,6 +71,18 @@ fn hot() -> u32 { }) } +/// How many times [`hot`] a code object's heat must reach: compiling +/// costs about the same for every instruction of the code, while what it +/// saves comes from the loop that got hot alone, so a long body (a +/// driver's setup around a short loop) waits longer to earn its compile. +#[inline(always)] +fn size_factor(code: &CodeObject) -> u32 { + (code.instructions.len() / SIZE_UNIT).max(1) as u32 +} + +/// The code length [`hot`] alone covers. +const SIZE_UNIT: usize = 16; + /// The longest code compiled. const MAX_INSTRUCTIONS: usize = 4096; @@ -196,7 +208,7 @@ impl Slot { // (A racing thread losing a count is harmless.) let h = self.heat.load(Ordering::Relaxed) + 1; self.heat.store(h, Ordering::Relaxed); - if h >= hot() { + if h >= hot().saturating_mul(size_factor(code)) { self.heat.store(0, Ordering::Relaxed); self.try_compile(code, ext, nlocals, at); } diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index a249ad46..5f49efe9 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -750,7 +750,15 @@ impl Engine { let mut flags = settings::builder(); flags.set("use_colocated_libcalls", "false").ok()?; flags.set("is_pic", "false").ok()?; - flags.set("opt_level", "speed").ok()?; + // No mid-end optimization (as for tier 2): the lowering emits the + // code it wants, and the pass cost more to run than it saved. + // `WEAVEPY_JIT_OPT=1` restores it. + let opt = if std::env::var_os("WEAVEPY_JIT_OPT").is_some() { + "speed" + } else { + "none" + }; + flags.set("opt_level", opt).ok()?; if std::env::var_os("WEAVEPY_JIT_QUICK").is_some() { flags.set("opt_level", "none").ok()?; flags.set("regalloc_algorithm", "single_pass").ok()?; From 5d3a905e953373aabd7d1cbf4251e02d5a49ff6b Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:58:04 -0700 Subject: [PATCH 54/96] perf: read module-global lists on their list lanes in tier 2 A module global holding a list rode the generic object lane: a loop or comprehension over it (`[x * 2 for x in data]`) captured a real list iterator and stepped it through the generic iterator helper, about 270 instructions an element. A non-empty global list with a uniform element lane (ints, floats, instances, float lists) now grades to its pinned-list lane, as a list local does: its loads pin it as a list, loops index it natively, and every element access re-validates the lane, so later changes to the list's contents deopt. The obj-global helper learns the element lane from the high word of its argument. The generic iterator helper also steps a list, tuple or range iterator in place, without cloning it or polling for Python it can't run (the loop header polls). Measured (retired instructions per call, the coordinator's micro-suite): listcomp 32.2K -> 15.2K (CPython 12.5K). Deltablue and the five fixtures are unchanged within noise. --- crates/weavepy-jit/src/lib.rs | 2 +- crates/weavepy-jit/src/lower.rs | 5 +- crates/weavepy-jit/src/runtime.rs | 30 +++++ crates/weavepy-vm/src/tier2.rs | 148 ++++++++++++++++-------- tests/regrtest/test_jit_global_lists.py | 127 ++++++++++++++++++++ 5 files changed, 264 insertions(+), 48 deletions(-) create mode 100644 tests/regrtest/test_jit_global_lists.py diff --git a/crates/weavepy-jit/src/lib.rs b/crates/weavepy-jit/src/lib.rs index 7c5ebcfa..f3704b99 100644 --- a/crates/weavepy-jit/src/lib.rs +++ b/crates/weavepy-jit/src/lib.rs @@ -42,7 +42,7 @@ pub use ir::{ TOp, TStmt, TTerm, }; pub use runtime::{ - set_obj_layout, ObjLayout, + global_obj_list_elem, set_obj_layout, ObjLayout, register_attr_get_chain_helper, register_attr_helpers, register_build_map_helper, register_build_set_helper, register_cached_attr_chain_helper, register_call_dyn_helper, register_call_dyn_int_helper, register_call_method_helper, register_call_native_method_helper, diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 63934b82..5edb07e2 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -1334,7 +1334,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { .b .ins() .iconst(self.ptr_ty, runtime::global_obj_helper_addr() as i64); - let tokenv = self.b.ins().iconst(types::I64, i64::from(token)); + let tokenv = self + .b + .ins() + .iconst(types::I64, runtime::global_obj_list_code(token, lane)); let call = self .b .ins() diff --git a/crates/weavepy-jit/src/runtime.rs b/crates/weavepy-jit/src/runtime.rs index 17ef6939..d9f56410 100644 --- a/crates/weavepy-jit/src/runtime.rs +++ b/crates/weavepy-jit/src/runtime.rs @@ -1473,6 +1473,36 @@ pub(crate) fn global_obj_helper_addr() -> usize { GLOBAL_OBJ_HELPER.load(std::sync::atomic::Ordering::Acquire) } +/// The obj-global helper's argument for `token` on `lane`: a pinned-list +/// lane's element lane rides the high word (`1` int, `2` float, `3` +/// object, `4` float list), so the helper pins the list as a list. +#[must_use] +pub(crate) fn global_obj_list_code(token: u32, lane: crate::JitType) -> i64 { + use crate::JitType; + let elem = match lane.elem_lane() { + Some(JitType::Int) => 1, + Some(JitType::Float) => 2, + Some(JitType::Obj) => 3, + Some(JitType::ListFloat) => 4, + _ => 0, + }; + i64::from(token) | (elem << 32) +} + +/// Inverse of [`global_obj_list_code`]: the element lane a list-lane +/// global pins with, `None` for an object pin. +#[must_use] +pub fn global_obj_list_elem(code: i64) -> Option { + use crate::JitType; + match code >> 32 { + 1 => Some(JitType::Int), + 2 => Some(JitType::Float), + 3 => Some(JitType::Obj), + 4 => Some(JitType::ListFloat), + _ => None, + } +} + static ITER_NEW_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); /// Register the process-wide *generic* iterator-capture helper (RFC diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index aaad80a0..0b601e4c 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -2557,8 +2557,14 @@ fn probe_list_lane(frame: &super::Frame, slot: u32) -> Option { if items.is_empty() { return Some(JitType::Unknown); } + list_elem_lane(&items) +} + +/// The uniform element lane of a non-empty list's `items` (see +/// [`probe_list_lane`]), `None` for a mixed or unsupported list. +fn list_elem_lane(items: &[Object]) -> Option { let mut lane: Option = None; - for it in items.iter() { + for it in items { let t = match it { Object::Int(_) => JitType::Int, Object::Float(_) => JitType::Float, @@ -3596,9 +3602,23 @@ fn resolve_plain_dicts( /// ops (`CallDyn`, `DynAttrGet`/`Set`, iterator capture) apply and /// every other access helper deopts on the lane surprise. The identity /// guard makes the grade stable for the compilation's whole life. +/// +/// A non-empty list with a uniform element lane rides its pinned-list +/// lane, as a list local does (see [`probe_list_lane`]): the identity +/// guard keeps the list object, and every element access re-validates +/// the element lane, so later changes to the list's contents deopt. fn grade_obj_global(obj: &Object) -> JitType { match obj { Object::Str(_) => JitType::Str, + Object::List(l) => { + let items = l.borrow(); + if items.is_empty() { + return JitType::Obj; + } + list_elem_lane(&items) + .and_then(JitType::list_of) + .unwrap_or(JitType::Obj) + } _ => JitType::Obj, } } @@ -8365,6 +8385,22 @@ unsafe extern "C" fn wpjit_get_iter(frame: *mut JitFrame, pin: i64) -> i64 { /// # Safety /// /// Same contract as [`wpjit_call_py`]. +/// One step of a list, tuple or range iterator, without the generic +/// dispatch: the next item, or `None` once exhausted (the iterator then +/// detached as [`PyIterator::next_value`] leaves it). `None` outside for +/// any other iterator, or one already borrowed. +#[inline] +fn builtin_seq_step(cell: &Rc>) -> Option> { + use crate::object::PyIterator; + let mut it = cell.try_borrow_mut().ok()?; + match &*it { + PyIterator::List { .. } | PyIterator::Tuple { .. } | PyIterator::Range { .. } => { + Some(it.next_value()) + } + _ => None, + } +} + unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i64) -> i64 { // SAFETY: see wpjit_call_py — same live-buffer contract. let jf = unsafe { &mut *frame }; @@ -8405,52 +8441,65 @@ unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i } } } - // The loop's poll point (the header's countdown poll covers the - // native back edge; this covers the Python the step may run): - // pending interpreter work and active observers route the loop - // through the interpreter. - crate::gil::yield_checkpoint(); - if crate::hot_gates::load() != 0 || crate::trace::any_observers_active() { - return 2; - } - let it = match ctx.pins.get(pin as usize) { - Some(Pin::Obj(o @ (Object::Generator(_) | Object::Iter(_)))) => o.clone(), - // An instance iterator whose `__next__` is a registered native - // leaf (a `deque` iterator): stepped below without Python. - Some(Pin::Obj(o @ Object::Instance(_))) if !crate::gil::free_threading_enabled() => { - o.clone() - } - _ => return 2, + // A list, tuple or range iterator steps in place: no Python runs, so + // the header's countdown poll covers the step, and nothing needs the + // iterator cloned. + let fast = match ctx.pins.get(pin as usize) { + Some(Pin::Obj(Object::Iter(cell))) => builtin_seq_step(cell), + _ => None, }; - // A builtin-iterator step is pure native code; everything else - // (generator resume) runs arbitrary Python on behalf of this - // activation. - let runs_python = matches!(it, Object::Generator(_)); - if runs_python { - ctx.dirty = true; - // A generator resume from native code rebuilds a whole interpreter - // activation, several times what the interpreter's own inline - // resume costs: charged like an interpreter call, so a loop that - // drives a generator retires at the next poll (see `wpjit_poll`). - ctx.dyn_py_calls = ctx.dyn_py_calls.saturating_add(1); - } - let step = if let Object::Instance(inst) = &it { - // The core loop's cached `__next__` resolution (by class version). - // Any other `__next__` belongs to the interpreter (nothing was - // consumed). - if !super::Interpreter::default_getattribute(inst.cls_raw()) - || crate::object::exotic_str_keys_possible() - { - return 2; - } - match interp.leaf_next_step(&it, inst) { - None => return 2, - Some(Ok(v)) => Ok(Some(v)), - Some(Err(RuntimeError::PyException(e))) if e.type_name() == "StopIteration" => Ok(None), - Some(Err(e)) => Err(e), + let (step, runs_python) = match fast { + Some(step) => (Ok(step), false), + None => { + // The loop's poll point (the header's countdown poll covers the + // native back edge; this covers the Python the step may run): + // pending interpreter work and active observers route the loop + // through the interpreter. + crate::gil::yield_checkpoint(); + if crate::hot_gates::load() != 0 || crate::trace::any_observers_active() { + return 2; + } + let it = match ctx.pins.get(pin as usize) { + Some(Pin::Obj(o @ (Object::Generator(_) | Object::Iter(_)))) => o.clone(), + // An instance iterator whose `__next__` is a registered native + // leaf (a `deque` iterator): stepped below without Python. + Some(Pin::Obj(o @ Object::Instance(_))) if !crate::gil::free_threading_enabled() => { + o.clone() + } + _ => return 2, + }; + // A builtin-iterator step is pure native code; everything else + // (generator resume) runs arbitrary Python on behalf of this + // activation. + let runs_python = matches!(it, Object::Generator(_)); + if runs_python { + ctx.dirty = true; + // A generator resume from native code rebuilds a whole interpreter + // activation, several times what the interpreter's own inline + // resume costs: charged like an interpreter call, so a loop that + // drives a generator retires at the next poll (see `wpjit_poll`). + ctx.dyn_py_calls = ctx.dyn_py_calls.saturating_add(1); + } + let step = if let Object::Instance(inst) = &it { + // The core loop's cached `__next__` resolution (by class version). + // Any other `__next__` belongs to the interpreter (nothing was + // consumed). + if !super::Interpreter::default_getattribute(inst.cls_raw()) + || crate::object::exotic_str_keys_possible() + { + return 2; + } + match interp.leaf_next_step(&it, inst) { + None => return 2, + Some(Ok(v)) => Ok(Some(v)), + Some(Err(RuntimeError::PyException(e))) if e.type_name() == "StopIteration" => Ok(None), + Some(Err(e)) => Err(e), + } + } else { + interp.iter_next(&it, &ctx.globals) + }; + (step, runs_python) } - } else { - interp.iter_next(&it, &ctx.globals) }; match step { Err(err) => { @@ -9957,6 +10006,9 @@ unsafe extern "C" fn wpjit_global_obj(frame: *mut JitFrame, token: i64) -> i64 { let jf = unsafe { &mut *frame }; #[allow(clippy::cast_ptr_alignment)] let ctx = unsafe { &mut *jf.ctx.cast::() }; + // A list-lane global's element lane rides the high word (see + // `weavepy_jit::global_obj_list_code`). + let elem = weavepy_jit::global_obj_list_elem(token); let token = token as u32; if let Some(&(_, pin)) = ctx.obj_global_pins.iter().find(|&&(t, _)| t == token) { return pin as i64; @@ -9971,7 +10023,11 @@ unsafe extern "C" fn wpjit_global_obj(frame: *mut JitFrame, token: i64) -> i64 { return -2; } let pin = ctx.pins.len() as u64; - ctx.pins.push(Pin::Obj(obj)); + match (elem, obj) { + (Some(elem), Object::List(l)) => ctx.pins.push(Pin::List(l, elem)), + (Some(_), _) => return -2, + (None, obj) => ctx.pins.push(Pin::Obj(obj)), + } ctx.obj_global_pins.push((token, pin)); pin as i64 } diff --git a/tests/regrtest/test_jit_global_lists.py b/tests/regrtest/test_jit_global_lists.py new file mode 100644 index 00000000..bae7c48e --- /dev/null +++ b/tests/regrtest/test_jit_global_lists.py @@ -0,0 +1,127 @@ +"""Compiled code reads a module-global list on its list lane. + +A global holding a uniform list (all ints, all floats, all instances) +compiles like a list local: loops, comprehensions and reads index it +natively. Each loop here runs long enough to compile; then the list's +contents or the global itself change, and every result must still match +what the interpreter computes. +""" + +import unittest + +WARM = 3000 + +INTS = list(range(50)) +FLOATS = [i * 0.5 for i in range(20)] + + +class Box: + def __init__(self, v): + self.v = v + + +BOXES = [Box(i) for i in range(10)] + + +def doubled(): + return [x * 2 for x in INTS] + + +def total(): + t = 0 + for x in INTS: + t += x + return t + + +def ftotal(): + t = 0.0 + for x in FLOATS: + t += x + return t + + +def btotal(): + t = 0 + for b in BOXES: + t += b.v + return t + + +def nth(i): + return INTS[i] + len(INTS) + + +def run(fn, n, *args): + r = None + for _ in range(n): + r = fn(*args) + return r + + +class GlobalListTests(unittest.TestCase): + def test_comprehension_then_mixed_elements(self): + global INTS + saved = INTS + try: + self.assertEqual(run(doubled, WARM), [x * 2 for x in range(50)]) + INTS.append("a") + self.assertEqual(doubled()[-1], "aa") + INTS = [1.5, 2] + self.assertEqual(doubled(), [3.0, 4]) + INTS = [] + self.assertEqual(doubled(), []) + finally: + INTS = saved + if INTS and INTS[-1] == "a": + INTS.pop() + + def test_loop_sees_mutations(self): + global INTS + saved = list(INTS) + try: + self.assertEqual(run(total, WARM), sum(range(50))) + INTS[0] = 1000 + self.assertEqual(total(), sum(range(50)) + 1000) + INTS[1] = 2**70 + self.assertEqual(total(), sum(range(2, 50)) + 1000 + 2**70) + del INTS[10:] + self.assertEqual(total(), 1000 + 2**70 + sum(range(2, 10))) + finally: + INTS[:] = saved + + def test_float_and_instance_lists(self): + global FLOATS, BOXES + self.assertEqual(run(ftotal, WARM), sum(i * 0.5 for i in range(20))) + FLOATS[3] = 7 + self.assertEqual(ftotal(), sum(i * 0.5 for i in range(20)) - 1.5 + 7) + self.assertEqual(run(btotal, WARM), sum(range(10))) + saved = BOXES + BOXES = BOXES + [None] + try: + with self.assertRaises(AttributeError): + btotal() + finally: + BOXES = saved + self.assertEqual(btotal(), sum(range(10))) + + def test_index_and_len(self): + global INTS + self.assertEqual(run(nth, WARM, 3), 3 + 50) + INTS.append(99) + try: + self.assertEqual(nth(-1), 99 + 51) + with self.assertRaises(IndexError): + nth(100) + finally: + INTS.pop() + saved = INTS + INTS = list(range(5)) + try: + self.assertEqual(nth(4), 4 + 5) + finally: + INTS = saved + + +if __name__ == "__main__": + unittest.main() From 35311741857c6afc7b42c1f521891179d9b0d62e Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:58:43 -0700 Subject: [PATCH 55/96] perf: compile int compares with object operands and identity tests in tier 2 Two common shapes kept whole functions out of tier 2: - A compare between an integral value and one the analyzer only knows as an object (`self.direction == BinaryConstraint.FORWARD`, a class constant read generically) was rejected as mixed arithmetic. The object operand is now guarded to be an exact machine `int`, as `BINARY_OP` already does (a miss restores both operands at the `COMPARE_OP`). - `a is b` was only compiled against the `None` constant. Two object operands now compare by identity: equal machine values answer in line, and a new helper compares the pinned objects otherwise. Deltablue's leaves (`output`, `input`, `is_satisfied`, `execute`) and its planner loops get further into tier 2. Measured (retired instructions, min of 3): deltablue per unit 35.4M -> 34.8M; its fixed cost rises by about 17M (four more compiles), so the default ten units are about 3% slower and forty units about 1% faster. The five fixtures are unchanged within noise. --- crates/weavepy-jit/src/analyze.rs | 76 +++++++++++++---- crates/weavepy-jit/src/engine.rs | 8 ++ crates/weavepy-jit/src/ir.rs | 7 ++ crates/weavepy-jit/src/lib.rs | 1 + crates/weavepy-jit/src/lower.rs | 35 ++++++++ crates/weavepy-jit/src/runtime.rs | 15 ++++ crates/weavepy-vm/src/tier2.rs | 37 ++++++++ tests/regrtest/test_jit_object_compares.py | 99 ++++++++++++++++++++++ 8 files changed, 262 insertions(+), 16 deletions(-) create mode 100644 tests/regrtest/test_jit_object_compares.py diff --git a/crates/weavepy-jit/src/analyze.rs b/crates/weavepy-jit/src/analyze.rs index e4332943..d99578f8 100644 --- a/crates/weavepy-jit/src/analyze.rs +++ b/crates/weavepy-jit/src/analyze.rs @@ -4029,24 +4029,30 @@ fn step_abstract( } let b = stack.pop().ok_or(JitVerdict::StackUnderflow)?; let a = stack.pop().ok_or(JitVerdict::StackUnderflow)?; - let val = match (a.none_const, b.none_const) { - (false, true) => a, - (true, false) => b, + // Two object operands (`c is not determining`): both on the + // object lane, compared by identity. + let vals = match (a.none_const, b.none_const) { + (false, true) => vec![a], + (true, false) => vec![b], + (false, false) => vec![a, b], _ => return Err(JitVerdict::UnsupportedOpcode("IS_OP shape")), }; - if !val.is_plain() { - return Err(escape_verdict(code, &[&val], "CALL (callee escapes)")); - } - match val.ty { - JitType::Obj => {} - JitType::Unknown => { - if let Some(slot) = val.src { - set_local(local_types, slot, JitType::Obj, changed)?; + for val in vals { + if !val.is_plain() { + return Err(escape_verdict(code, &[&val], "CALL (callee escapes)")); + } + match val.ty { + JitType::Obj => {} + JitType::Unknown => { + if let Some(slot) = val.src { + set_local(local_types, slot, JitType::Obj, changed)?; + } + // No provenance: transient — a later iteration + // may type it; emission bails if it never + // resolves. } - // No provenance: transient — a later iteration may - // type it; emission bails if it never resolves. + _ => return Err(JitVerdict::UnsupportedOpcode("IS_OP operand lane")), } - _ => return Err(JitVerdict::UnsupportedOpcode("IS_OP operand lane")), } stack.push(SE::known(JitType::Bool)); } @@ -5645,6 +5651,11 @@ fn cmp_check(kind: CmpKind, a: JitType, b: JitType) -> Result<(), JitVerdict> { if !a.is_representable() || !b.is_representable() { return Ok(()); } + // An object compared with an integral operand is guarded to be an + // exact machine-sized `int` (see `integer_operand_guard`). + if integer_operand_guard(a, b).is_some() { + return Ok(()); + } if (a.is_integral() || a == JitType::Float) && (b.is_integral() || b == JitType::Float) { return Ok(()); } @@ -6702,8 +6713,24 @@ fn emit_instr( } OpCode::CompareOp => { let kind = cmp_kind(ins.arg)?; - let b = pop_val(stack)?; - let a = pop_val(stack)?; + let mut b = pop_val(stack)?; + let mut a = pop_val(stack)?; + // An object compared with an integral operand (a constant on + // a class, a field read generically) must be an exact + // machine-sized `int`; a miss restores both operands at this + // `COMPARE_OP` and the interpreter compares them. + if let Some(depth) = integer_operand_guard(a, b) { + stack.push(ESlot::val(a)); + stack.push(ESlot::val(b)); + push(TOp::UnboxInt { depth }, None, stack, stmts); + stack.pop(); + stack.pop(); + if depth == 0 { + b = JitType::Int; + } else { + a = JitType::Int; + } + } if a.is_integral() && b.is_integral() { push(TOp::IntCmp(kind), Some(JitType::Bool), stack, stmts); } else if a == JitType::Float && b == JitType::Float { @@ -6751,6 +6778,23 @@ fn emit_instr( let val = match (a.none_const, b.none_const) { (false, true) => a, (true, false) => b, + (false, false) => { + // Two object operands: identity through the pins. + for v in [&a, &b] { + if !v.is_plain() || v.ty != JitType::Obj { + return Err(JitVerdict::UnsupportedOpcode("IS_OP operand lane")); + } + } + push( + TOp::IsObj { + negate: ins.arg == 1, + }, + Some(JitType::Bool), + stack, + stmts, + ); + return Ok(()); + } _ => return Err(JitVerdict::UnsupportedOpcode("IS_OP shape")), }; if !val.is_plain() || val.ty != JitType::Obj { diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index 6ab70506..7b0538fc 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -516,6 +516,14 @@ impl JitEngine { "frame coverage (no helper registered)", )); } + if runtime::is_obj_helper_addr() == 0 + && tfunc + .blocks + .iter() + .any(|b| b.stmts.iter().any(|s| matches!(s.op, TOp::IsObj { .. }))) + { + return Err(JitVerdict::UnsupportedOpcode("IS_OP (no helper registered)")); + } let needs_float_divmod = tfunc.blocks.iter().any(|b| { b.stmts .iter() diff --git a/crates/weavepy-jit/src/ir.rs b/crates/weavepy-jit/src/ir.rs index eae027d0..7cfa5f87 100644 --- a/crates/weavepy-jit/src/ir.rs +++ b/crates/weavepy-jit/src/ir.rs @@ -284,6 +284,12 @@ pub enum TOp { /// `-1` for `None`) and pushes the `bool` result. Purely native — /// no helper call, no deopt. IsNone { negate: bool }, + /// `a is b` / `a is not b` on two object lanes: pops both (pin + /// indices, or `-1` for `None`) and pushes the `bool` identity + /// result. Equal machine values answer in line; otherwise the + /// registered helper compares the pinned objects (a pin miss + /// deopts). + IsObj { negate: bool }, /// RFC 0070 WS1 — push the `None` singleton in the nullable object /// lane (machine value `-1`). Emitted where a `None` constant must /// occupy a *native* stack slot: a `StoreFast` into an `Obj` local, @@ -1522,6 +1528,7 @@ impl TOp { | TOp::AttrSet { .. } | TOp::GuardNotNone | TOp::StrEq { .. } + | TOp::IsObj { .. } | TOp::StrLen | TOp::BytesLen | TOp::BytesGetItem diff --git a/crates/weavepy-jit/src/lib.rs b/crates/weavepy-jit/src/lib.rs index f3704b99..b3eb78eb 100644 --- a/crates/weavepy-jit/src/lib.rs +++ b/crates/weavepy-jit/src/lib.rs @@ -50,6 +50,7 @@ pub use runtime::{ register_cell_helpers, register_const_str_helper, register_contains_dyn_helper, register_dict_del_helper, register_dict_helpers, register_dict_iter_helper, register_dyn_attr_helpers, register_float_pow_helper, register_global_obj_helper, + register_is_obj_helper, register_iter_helpers, register_iter_new_helper, register_iter_next_pair_helper, register_list_extra_helpers, register_list_from_range_helper, register_list_helpers, register_guard_method_helper, register_list_next_helper, register_math_helpers, diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 5edb07e2..d1fb8eca 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -1248,6 +1248,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.emit_call_str_method(site, argc, ret, stmt.pc); } TOp::StrEq { negate } => self.emit_str_eq(negate, stmt.pc), + TOp::IsObj { negate } => self.emit_is_obj(negate, stmt.pc), TOp::StrLen => self.emit_pin_len(runtime::str_len_helper_addr(), stmt.pc), TOp::BytesLen => self.emit_pin_len(runtime::bytes_len_helper_addr(), stmt.pc), TOp::BytesGetItem => self.emit_bytes_get(stmt.pc), @@ -2335,6 +2336,40 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.vstack.push((res, JitType::Bool)); } + /// `a is b` on two object lanes (see [`TOp::IsObj`]): equal machine + /// values (the same pin, or both `None`) are the same object; any + /// other pair asks the helper, whose status above `1` deopts. + fn emit_is_obj(&mut self, negate: bool, pc: u32) { + let snapshot = self.vstack.clone(); + let (b_pin, _) = self.pop(); + let (a_pin, _) = self.pop(); + let same = self.b.ins().icmp(IntCC::Equal, a_pin, b_pin); + let slow = self.b.create_block(); + let merge = self.b.create_block(); + self.b.append_block_param(merge, types::I64); + let one = self.b.ins().iconst(types::I64, 1); + self.b.ins().brif(same, merge, &[one.into()], slow, &[]); + self.b.switch_to_block(slow); + let sig = self.list_helper_sig(); + let helper = self + .b + .ins() + .iconst(self.ptr_ty, runtime::is_obj_helper_addr() as i64); + let call = self + .b + .ins() + .call_indirect(sig, helper, &[self.frame_ptr, a_pin, b_pin]); + let status = self.b.inst_results(call)[0]; + let bad = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThan, status, 1); + let cont = self.guard(bad, pc, &snapshot); + self.b.switch_to_block(cont); + self.b.ins().jump(merge, &[status.into()]); + self.b.switch_to_block(merge); + let r = self.b.block_params(merge)[0]; + let res = if negate { self.b.ins().bxor_imm(r, 1) } else { r }; + self.vstack.push((res, JitType::Bool)); + } + /// RFC 0071 WS6 — pinned-`str`/`bytes` length (helper selected by /// address; both share the [`Self::pin_helper_sig`] shape and the /// negative-status deopt of `emit_list_len`). diff --git a/crates/weavepy-jit/src/runtime.rs b/crates/weavepy-jit/src/runtime.rs index d9f56410..7e0f0295 100644 --- a/crates/weavepy-jit/src/runtime.rs +++ b/crates/weavepy-jit/src/runtime.rs @@ -590,6 +590,21 @@ pub type StrLenHelper = unsafe extern "C" fn(frame: *mut JitFrame, pin: i64) -> pub type BytesGetHelper = unsafe extern "C" fn(frame: *mut JitFrame, pin: i64, idx: i64) -> i64; static STR_EQ_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); +static IS_OBJ_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Register the process-wide object identity helper ([`crate::ir::TOp::IsObj`]): +/// `1` when the two object-lane values (pin indices, or `-1` for `None`) +/// name the same object, `0` when not, anything above `1` to deopt (a +/// pin miss). Never runs Python code. Shares the [`StrEqHelper`] shape. +/// Must precede the first compile of a frame containing `IsObj`. +pub fn register_is_obj_helper(f: StrEqHelper) { + IS_OBJ_HELPER.store(f as usize, std::sync::atomic::Ordering::Release); +} + +#[must_use] +pub(crate) fn is_obj_helper_addr() -> usize { + IS_OBJ_HELPER.load(std::sync::atomic::Ordering::Acquire) +} static STR_LEN_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); static BYTES_LEN_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); static BYTES_GET_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 0b601e4c..00d23632 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -958,6 +958,7 @@ impl JitState { weavepy_jit::register_const_str_helper(wpjit_const_str); weavepy_jit::register_tuple_read_helpers(wpjit_const_tuple, wpjit_tuple_len); weavepy_jit::register_unbox_int_helper(wpjit_unbox_int); + weavepy_jit::register_is_obj_helper(wpjit_is_obj); weavepy_jit::register_dict_iter_helper(wpjit_dict_iter_new); // RFC 0073 WS3 — the string write lanes. weavepy_jit::register_str_write_helpers( @@ -7781,6 +7782,42 @@ unsafe extern "C" fn wpjit_str_eq(frame: *mut JitFrame, a: i64, b: i64) -> i64 { i64::from(sa == sb) } +/// The `wpjit_is_obj` helper ([`weavepy_jit::TOp::IsObj`]): whether two +/// object-lane values (pin indices, `-1` for `None`) are the same object; +/// `2` deopts on a pin miss. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_py`]. +unsafe extern "C" fn wpjit_is_obj(frame: *mut JitFrame, a: i64, b: i64) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &*jf.ctx.cast::() }; + let none = Object::None; + let at = |p: i64| -> Option> { + if p == -1 { + return Some(Ok(&none)); + } + match ctx.pins.get(usize::try_from(p).ok()?)? { + Pin::Obj(o) => Some(Ok(o)), + Pin::List(l, _) => Some(Err(Rc::as_ptr(l).cast())), + } + }; + let (Some(x), Some(y)) = (at(a), at(b)) else { + return 2; + }; + let same = match (x, y) { + (Ok(x), Ok(y)) => x.is_same(y), + (Err(x), Err(y)) => x == y, + (Ok(Object::List(l)), Err(y)) | (Err(y), Ok(Object::List(l))) => { + Rc::as_ptr(l).cast::<()>() == y + } + _ => false, + }; + i64::from(same) +} + /// The `wpjit_str_len` helper (RFC 0071 WS6): `len` of a pinned `str` /// — the *character* count, matching `str.__len__`. Negative return /// deopts (pin miss). diff --git a/tests/regrtest/test_jit_object_compares.py b/tests/regrtest/test_jit_object_compares.py new file mode 100644 index 00000000..81af2fbf --- /dev/null +++ b/tests/regrtest/test_jit_object_compares.py @@ -0,0 +1,99 @@ +"""Compiled code compares objects with ints, and objects by identity. + +An integer compared with a value the compiler only knows as an object (a +class constant, an attribute read generically) is guarded to be an exact +machine `int`; `a is b` on two objects compares identities. Each loop +here runs long enough to compile; then the values change shape, and the +results must still match what the interpreter computes. +""" + +import unittest + +WARM = 3000 + + +class Dir: + NONE = 0 + FORWARD = 1 + + +class Edge: + def __init__(self, a, b, d): + self.a = a + self.b = b + self.d = d + + +def pick(e, n): + t = 0 + for _ in range(n): + if e.d == Dir.FORWARD: + t += 1 + else: + t -= 1 + return t + + +def ahead(e, n): + t = 0 + for i in range(n): + if Dir.FORWARD < i: + t += 1 + return t + + +def count_same(items, target): + t = 0 + for x in items: + if x is target: + t += 1 + if x is not target: + t -= 1 + return t + + +class ObjectCompareTests(unittest.TestCase): + def test_class_constant_compare(self): + e = Edge(1, 2, 1) + self.assertEqual(pick(e, WARM), WARM) + e.d = 0 + self.assertEqual(pick(e, 10), -10) + saved = Dir.FORWARD + try: + Dir.FORWARD = "fwd" + e.d = "fwd" + self.assertEqual(pick(e, 10), 10) + Dir.FORWARD = 2**70 + e.d = 2**70 + self.assertEqual(pick(e, 10), 10) + Dir.FORWARD = 1.0 + e.d = 1 + self.assertEqual(pick(e, 10), 10) + Dir.FORWARD = True + self.assertEqual(pick(e, 10), 10) + finally: + Dir.FORWARD = saved + self.assertEqual(ahead(e, WARM), WARM - 2) + Dir.FORWARD = 5 + try: + self.assertEqual(ahead(e, 10), 4) + Dir.FORWARD = None + with self.assertRaises(TypeError): + ahead(e, 10) + finally: + Dir.FORWARD = saved + + def test_identity(self): + a, b = Edge(1, 2, 0), Edge(1, 2, 0) + items = [a, b, a, None, a] + self.assertEqual(count_same(items * 600, a), 600) + self.assertEqual(count_same([b, b, None], b), 1) + self.assertEqual(count_same([None, None, a], None), 1) + self.assertEqual(count_same([[], [], a], a), -1) + lst = [] + self.assertEqual(count_same([lst, [], lst], lst), 1) + self.assertEqual(count_same(["x", "y", "x"], "x"), 1) + + +if __name__ == "__main__": + unittest.main() From 73f8efcc390d3a8669626aad4bbdfc2d7553baf5 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 06:28:54 -0700 Subject: [PATCH 56/96] perf: land a compiled effect leaf's last store at once An effect leaf buffers its attribute stores until it returns, so a declined evaluation leaves nothing behind: each store cloned its value (and a receiver that isn't an argument) into the evaluation's scratch, and the return committed the buffer and released the scratch. Most effect leaves end in the store (`self.output().value = ...`, `self.x = v`), and nothing after it can decline: a compiled plan now marks such a store when it compiles, and its helper, with no store buffered before it, stores straight into the instance and skips the buffering, the commit and both releases. A declined store still touches nothing. Measured (retired instructions, min of 3): deltablue per unit 34.8M -> 34.5M; its plan-execution phase alone 9.6M -> 8.4M per 5,000 calls. --- crates/weavepy-vm/src/leaf_plan.rs | 2 +- crates/weavepy-vm/src/leaf_plan/jit.rs | 41 +++++++++- tests/regrtest/test_jit_plan_stores.py | 107 +++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 tests/regrtest/test_jit_plan_stores.py diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index 38aaecf7..e91a69bd 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -913,7 +913,7 @@ impl LeafRet { /// An owned object for a leaf value (the return value, a buffered /// store's value); `None` for the markers that are never values. #[inline(always)] -fn to_object(v: V) -> Option { +pub(super) fn to_object(v: V) -> Option { Some(match v { // SAFETY: as `norm` (an owned value is cloned before its holder // drops). diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index 5f49efe9..8d6b2867 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -589,6 +589,10 @@ unsafe extern "C" fn h_subscr(ctx: *mut Ctx<'static>, ta: u64, pa: u64, tb: u64, } /// The evaluation's own `recv.name = val` (only the top frame stores). +/// A store nothing after it can decline (`last`, decided when the plan +/// compiled), with no store buffered before it, lands at once: the +/// evaluation can no longer abandon it. Any other store is buffered for +/// the return to commit. unsafe extern "C" fn h_store( ctx: *mut Ctx<'static>, tr: u64, @@ -596,6 +600,7 @@ unsafe extern "C" fn h_store( tv: u64, pv: u64, pc_name: u32, + last: u32, ) -> u32 { // SAFETY: called by native code with its context. let c = unsafe { cx(ctx) }; @@ -603,6 +608,29 @@ unsafe extern "C" fn h_store( return 1; }; let (pc, name) = (pc_name as u16, (pc_name >> 16) as u16); + if last != 0 && c.pend.n == 0 { + // SAFETY: as `norm`; the receiver is rooted by an argument or a + // field of one, and nothing has run since it was read. A handle + // of its own keeps it while the store runs (a field's slot may + // move when the store grows the instance holding it). + let inst = match unsafe { &*rp } { + Object::Instance(inst) => inst.clone(), + _ => return 1, + }; + let Some(v) = super::to_object(value(tv, pv)) else { + return 1; + }; + // SAFETY: the evaluation's own frame (its code is alive). + let (code, _, _) = unsafe { c.top.parts() }; + // On `true` the value moved into the instance; a declined store + // touched nothing, and the evaluation declines whole. + let v = std::mem::ManuallyDrop::new(v); + if Interpreter::core_store_attr(code, &inst, usize::from(pc), u32::from(name), &v) { + return 0; + } + drop(std::mem::ManuallyDrop::into_inner(v)); + return 1; + } match plan_store( c.args, &mut c.owned, @@ -1889,9 +1917,10 @@ impl Lower<'_> { let (tr, pr) = self.get(fl, recv); let (tv, pv) = self.get(fl, val); let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); + let last = self.u32c(u32::from(Self::infallible_after(fl.plan, i))); let st = self.call( h_store as *const () as usize, - &[self.ctx, tr, pr, tv, pv, pn], + &[self.ctx, tr, pr, tv, pv, pn, last], ); self.check(st); } @@ -1899,6 +1928,16 @@ impl Lower<'_> { Some(true) } + /// Whether nothing after op `i` can decline the evaluation: every + /// later op is a move, a constant, a return or a forward jump. + fn infallible_after(plan: &LeafPlan, i: usize) -> bool { + plan.ops.iter().enumerate().skip(i + 1).all(|(k, op)| match *op { + Op::Move { .. } | Op::Const { .. } | Op::Return { .. } => true, + Op::Jump { target } => usize::from(target) > k, + _ => false, + }) + } + /// A `CALL` (op `i`): the callee its site last resolved in line behind /// a guard, falling back to the helper. fn call_op(&mut self, fl: &FrameLower<'_>, i: usize, at: u8, argc: u8, pc: u16) -> Option<()> { diff --git a/tests/regrtest/test_jit_plan_stores.py b/tests/regrtest/test_jit_plan_stores.py new file mode 100644 index 00000000..ad8a8806 --- /dev/null +++ b/tests/regrtest/test_jit_plan_stores.py @@ -0,0 +1,107 @@ +"""A compiled leaf plan's last attribute store lands at once. + +An effect leaf (a method whose only side effects are attribute stores) +buffers its stores until it returns; a store nothing after it can decline +lands directly instead. Each loop here runs long enough for the plan to +compile; then the receiver changes shape, and every store must behave as +the interpreter's would. The drivers read `locals()` on a path they never +take, which keeps the tier-2 compiler (whose calls take other paths) off +them. +""" + +import unittest + +WARM = 33000 + + +class Cell: + def __init__(self): + self.v = 0 + + +class Node: + def __init__(self, x, peer): + self.x = x + self.peer = peer + + def put(self): + self.peer.v = self.x + + +def drive(o, n): + if n < 0: + return locals() + for _ in range(n): + o.put() + + +class Noisy: + deaths = 0 + + def __del__(self): + Noisy.deaths += 1 + + +class PlanStoreTests(unittest.TestCase): + def test_plain_store(self): + c = Cell() + n = Node(7, c) + drive(n, WARM) + self.assertEqual(c.v, 7) + n.x = "s" + drive(n, 3) + self.assertEqual(c.v, "s") + + def test_receiver_is_self(self): + n = Node(5, None) + n.peer = n + drive(n, WARM) + self.assertEqual(n.v, 5) + m = Node(6, None) + m.peer = m + drive(m, 1) + self.assertEqual(vars(m), {"x": 6, "peer": m, "v": 6}) + + def test_setter_and_finalizer(self): + c = Cell() + n = Node(1, c) + drive(n, WARM) + + class Watched: + def __init__(self): + self.seen = [] + + @property + def v(self): + return self.seen[-1] + + @v.setter + def v(self, value): + self.seen.append(value) + + w = Watched() + n.peer = w + drive(n, 2) + self.assertEqual(w.seen, [1, 1]) + n.peer = c + c.v = Noisy() + before = Noisy.deaths + drive(n, 1) + self.assertEqual(Noisy.deaths, before + 1) + self.assertEqual(c.v, 1) + + def test_slots_receiver(self): + class Slotted: + __slots__ = ("v",) + + c = Cell() + n = Node(3, c) + drive(n, WARM) + s = Slotted() + n.peer = s + drive(n, 1) + self.assertEqual(s.v, 3) + + +if __name__ == "__main__": + unittest.main() From b7413c9c69d95c247998b76f4d7edd0e0bd7ff5c Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 07:04:29 -0700 Subject: [PATCH 57/96] perf: step dict.items() loops in tier 2 without building pairs A tier-2 `for k, v in d.items()` loop stepped the dict's iterator through the generic helper, which built a `(key, value)` tuple for each entry only for the loop to unpack it and drop it, and polled for Python it couldn't run. The pair helper now asks a forward items iterator for the key and value directly (the same checked step, mutation guards and all, factored out of `next_value_checked`), packs them into the two lanes, and leaves polling to the loop header. A pair that doesn't fit the lanes still surrenders as the tuple the erased unpack consumes. Measured (retired instructions per call, the coordinator's micro-suite, 20 items): items 29.4K -> 22.0K (CPython 8.2K). --- crates/weavepy-vm/src/object.rs | 92 ++++++++++++++++++++++++++------- crates/weavepy-vm/src/tier2.rs | 36 +++++++++++++ 2 files changed, 108 insertions(+), 20 deletions(-) diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index b72951b8..6c70c2b9 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -8521,25 +8521,10 @@ impl PyIterator { } } - /// Like [`next_value`], but enforces CPython's "container changed - /// size during iteration" invariant before yielding. Used on the - /// *user-visible* `__next__` boundaries (`FOR_ITER`, the `next()` - /// builtin, `iter.__next__()`); the many internal consumers that - /// drain a freshly-built iterator without mutating its source can - /// keep calling the cheaper [`next_value`]. - pub fn next_value_checked(&mut self) -> Result, RuntimeError> { - if let PyIterator::Set { set, len, index } = self { - // `setiter_iternext`: a size change since creation is fatal, - // even on the step that would otherwise raise StopIteration. - if set.borrow().len() != *len { - // Sticky, like CPython's `si_used = -1`: every further - // `next()` keeps raising, and `__length_hint__` reports 0 - // (test_iterlen.test_immutable_during_iteration). - *len = usize::MAX; - *index = usize::MAX; - return Err(runtime_error("Set changed size during iteration")); - } - } + /// A dict iterator's mutation guards, before each checked step: + /// `Ok(true)` when the step must end silently; an error once the dict + /// changed under it. Nothing for any other iterator. + fn dict_iter_guard(&mut self) -> Result { if let PyIterator::DictKeys { dict: Some(d), len, @@ -8571,7 +8556,7 @@ impl PyIterator { // array leaves `di_pos` past `dk_nentries`, which // is a silent StopIteration, not an error. *index = 0; - return Ok(None); + return Ok(true); } return Err(runtime_error("dictionary keys changed during iteration")); } @@ -8579,6 +8564,73 @@ impl PyIterator { _ => {} } } + Ok(false) + } + + /// The checked step of a forward `dict.items()` iterator without its + /// `(key, value)` tuple, for a consumer that unpacks it at once (a + /// native tuple-target loop). `None` for any other iterator. + pub fn next_item_pair(&mut self) -> Option, RuntimeError>> { + if !matches!( + self, + PyIterator::DictKeys { + kind: DictViewKind::Items, + reverse: false, + .. + } + ) { + return None; + } + match self.dict_iter_guard() { + Err(e) => return Some(Err(e)), + Ok(true) => return Some(Ok(None)), + Ok(false) => {} + } + let PyIterator::DictKeys { + index, dict, owner, .. + } = self + else { + return None; + }; + let Some(d) = dict.as_ref() else { + return Some(Ok(None)); + }; + let entry = d + .borrow() + .get_index(*index) + .map(|(k, v)| (k.0.clone(), v.clone())); + if entry.is_some() { + *index += 1; + } else { + // Exhausted: detached, as `next_value` leaves it. + *dict = None; + *owner = None; + } + Some(Ok(entry)) + } + + /// Like [`next_value`], but enforces CPython's "container changed + /// size during iteration" invariant before yielding. Used on the + /// *user-visible* `__next__` boundaries (`FOR_ITER`, the `next()` + /// builtin, `iter.__next__()`); the many internal consumers that + /// drain a freshly-built iterator without mutating its source can + /// keep calling the cheaper [`next_value`]. + pub fn next_value_checked(&mut self) -> Result, RuntimeError> { + if let PyIterator::Set { set, len, index } = self { + // `setiter_iternext`: a size change since creation is fatal, + // even on the step that would otherwise raise StopIteration. + if set.borrow().len() != *len { + // Sticky, like CPython's `si_used = -1`: every further + // `next()` keeps raising, and `__length_hint__` reports 0 + // (test_iterlen.test_immutable_during_iteration). + *len = usize::MAX; + *index = usize::MAX; + return Err(runtime_error("Set changed size during iteration")); + } + } + if self.dict_iter_guard()? { + return Ok(None); + } if let PyIterator::File { file } = self { // Surface read errors (OSError, decode errors) at the // user-visible `__next__` boundary instead of swallowing them. diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 00d23632..3ef874ee 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -11442,6 +11442,42 @@ unsafe extern "C" fn wpjit_iter_next_pair( let ctx = unsafe { &mut *jf.ctx.cast::() }; // SAFETY: the `&mut Interpreter` is dormant while the helper runs. let interp = unsafe { &mut *ctx.interp }; + // A `dict.items()` step yields the key and value straight into the + // two lanes, without the tuple; no Python runs, so the header's poll + // covers it. + let items = match ctx.pins.get(pin as usize) { + Some(Pin::Obj(Object::Iter(cell))) => cell + .try_borrow_mut() + .ok() + .and_then(|mut it| it.next_item_pair()), + _ => None, + }; + if let Some(step) = items { + return match step { + Err(err) => { + ctx.raised = Some(err); + 4 + } + Ok(None) => 1, + Ok(Some((k, v))) => { + let b1 = pack_iter_elem(&k, tag1, &mut ctx.pins, &mut ctx.pin_memo); + let b2 = + b1.and_then(|_| pack_iter_elem(&v, tag2, &mut ctx.pins, &mut ctx.pin_memo)); + if let (Some(b1), Some(b2)) = (b1, b2) { + jf.ret_bits = b1; + // SAFETY: the marshal buffer is at least one slot + // wide (`max_call_args.max(1)`). + unsafe { *jf.call_args = b2 }; + return 0; + } + // Not in the lanes: the pair surrenders as the tuple the + // erased `UNPACK_SEQUENCE` consumes. + ctx.pins.push(Pin::Obj(Object::new_tuple_array([k, v]))); + jf.ret_bits = (ctx.pins.len() - 1) as u64; + 3 + } + }; + } // The loop's poll point — see `wpjit_iter_next`. crate::gil::yield_checkpoint(); if crate::hot_gates::load() != 0 || crate::trace::any_observers_active() { From 7ebc3fbc8d72267761098e2f762a42970e9a8bd8 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 07:04:46 -0700 Subject: [PATCH 58/96] perf: check isinstance against a tuple of plain classes in place The core loop's and the leaf plans' `isinstance` handled one class only; `isinstance(x, (str, int))` took the general call (and a leaf calling it declined to a full frame). A flat tuple of plain classes (metaclass `type`, at most eight) is now checked in place, each class in turn, with the single-class rules: a miss on an instance still defers to the full check, which also consults `__class__`, and anything else (a nested tuple, an ABC, a non-class) declines. Measured (retired instructions per call, the coordinator's micro-suite): isinstance 6.7K -> 3.3K (CPython 1.5K). --- crates/weavepy-vm/src/lib.rs | 53 ++++---- .../regrtest/test_jit_items_and_isinstance.py | 118 ++++++++++++++++++ 2 files changed, 142 insertions(+), 29 deletions(-) create mode 100644 tests/regrtest/test_jit_items_and_isinstance.py diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 7b568f0c..36c248f7 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -13706,38 +13706,33 @@ impl Interpreter { let [obj, spec] = ops else { return None; }; - // One class with no `__instancecheck__` of its own; `None` when - // the full call must decide (an instance whose class doesn't - // match may still claim the class through `__class__`). - let check = |cls: &Rc| -> Option { + // One class, or a flat tuple of them (`isinstance(x, (str, int))` + // asks each in turn). + let classes: &[Object] = match spec { + Object::Type(_) => std::slice::from_ref(spec), + Object::Tuple(items) if items.len() <= 8 => items, + _ => return None, + }; + let mut hit = false; + for cls in classes { + let Object::Type(cls) = cls else { + return None; + }; if !cls.metaclass_is_type() { return None; } - match obj { - Object::Instance(inst) => inst.cls_raw().is_subclass_of(cls).then_some(true), - Object::File(_) => None, - obj => Some(builtins::class_of(obj).is_subclass_of(cls)), - } - }; - let r = match spec { - Object::Type(cls) => check(cls)?, - // A tuple of plain classes, matched in order. - Object::Tuple(items) => { - let mut found = false; - for item in items.iter() { - let Object::Type(cls) = item else { - return None; - }; - if check(cls)? { - found = true; - break; - } - } - found - } - _ => return None, - }; - Some(Ok(Object::Bool(r))) + hit |= match obj { + // A miss on an instance is left to the full check, which + // also consults `__class__`. + Object::Instance(inst) => inst.cls_raw().is_subclass_of(cls), + Object::File(_) => return None, + obj => builtins::class_of(obj).is_subclass_of(cls), + }; + } + if !hit && matches!(obj, Object::Instance(_)) { + return None; + } + Some(Ok(Object::Bool(hit))) } /// The `(function, has_self, effective argc)` of a plain-function diff --git a/tests/regrtest/test_jit_items_and_isinstance.py b/tests/regrtest/test_jit_items_and_isinstance.py new file mode 100644 index 00000000..22bada75 --- /dev/null +++ b/tests/regrtest/test_jit_items_and_isinstance.py @@ -0,0 +1,118 @@ +"""Native `dict.items()` loops and `isinstance` against a tuple of classes. + +A tier-2 loop `for k, v in d.items()` takes each key and value straight +from the dict, and `isinstance(x, (A, B))` checks a flat tuple of plain +classes without the general call. Each loop here runs long enough to +compile; then the dict or the classes change, and every result must match +the interpreter's. +""" + +import unittest + +WARM = 3000 + +TABLE = {str(i): i for i in range(20)} + + +def total(): + t = 0 + for k, v in TABLE.items(): + t += v + return t + + +def keys_len(d): + n = 0 + for k, v in d.items(): + n += len(k) + return n + + +def mutate_during(d): + for k, v in d.items(): + d[k + "!"] = v + + +def churn_during(d): + for k, v in d.items(): + del d[k] + d[k + "?"] = v + + +def overwrite_during(d): + t = 0 + for k, v in d.items(): + d[k] = v + 1 + t += v + return t + + +class Base: + pass + + +class Sub(Base): + pass + + +class Masked: + @property + def __class__(self): + return Base + + +def kinds(xs, spec): + n = 0 + for x in xs: + if isinstance(x, spec): + n += 1 + return n + + +class ItemsAndIsinstanceTests(unittest.TestCase): + def test_items_loop(self): + for _ in range(WARM): + r = total() + self.assertEqual(r, sum(range(20))) + TABLE["x"] = 1.5 + try: + self.assertEqual(total(), sum(range(20)) + 1.5) + finally: + del TABLE["x"] + self.assertEqual(total(), sum(range(20))) + d = {"ab": 1, "c": 2} + for _ in range(WARM): + r = keys_len(d) + self.assertEqual(r, 3) + self.assertEqual(keys_len({}), 0) + + def test_items_mutation(self): + for _ in range(WARM): + overwrite_during({"a": 1, "b": 2}) + d = {"a": 1, "b": 2} + self.assertEqual(overwrite_during(d), 3) + self.assertEqual(d, {"a": 2, "b": 3}) + with self.assertRaises(RuntimeError): + mutate_during({"a": 1, "b": 2}) + with self.assertRaises(RuntimeError): + churn_during({"a": 1, "b": 2}) + + def test_isinstance_tuples(self): + xs = [1, "s", 2.0, Sub(), Base(), None, Masked()] + for _ in range(WARM): + r = kinds(xs, (int, str)) + self.assertEqual(r, 2) + self.assertEqual(kinds(xs, (Base,)), 3) + self.assertEqual(kinds(xs, (float, Sub)), 2) + self.assertEqual(kinds(xs, (bool, type(None))), 1) + self.assertEqual(kinds(xs, ()), 0) + self.assertEqual(kinds(xs, (int, (str, float))), 3) + import numbers + + self.assertEqual(kinds(xs, (numbers.Number, Base)), 5) + with self.assertRaises(TypeError): + kinds(xs, (int, 3)) + + +if __name__ == "__main__": + unittest.main() From e8475ba1754c3bf00409fab22ce3dabbd65fd8f9 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:01:15 -0700 Subject: [PATCH 59/96] perf: make functions in the leaf burst and key call shapes by code A `def` or `lambda` that runs in a loop (`(lambda x: x + 1)(i)`, `mk(i)(1)`) paid three costs per function: - `MAKE_FUNCTION` left the core loop for the full step, and each `SET_FUNCTION_ATTRIBUTE` after it (defaults, closure) did the same, untracking and retracking the fresh function each time. The burst's slow arms now build the function there, folding the defaults and closure attributes that follow into the construction. - Every function built its getset slots dict afresh, hashing three names and allocating two strings. The slots are now built once per code object (for the module name they were built for) and copied, so every function from one `def` also shares its name objects, as CPython's do. - A call site's cached shape matched the function by identity, so each fresh function missed it and took the general inline call. The shape depends only on the code (callers still check the function's own defaults), so it now matches by code. Measured (retired instructions per call, the coordinator's micro-suite): lambda 9.3K -> 6.5K (CPython 1.9K), closure 17.7K -> 15.9K. Deltablue and the five fixtures are unchanged within noise. --- crates/weavepy-vm/src/lib.rs | 247 +++++++++++++++-------- tests/regrtest/test_function_creation.py | 103 ++++++++++ 2 files changed, 267 insertions(+), 83 deletions(-) create mode 100644 tests/regrtest/test_function_creation.py diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 36c248f7..f2420b9d 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -16317,6 +16317,59 @@ impl Interpreter { last = pc; pc += 1; } + // A `def` or `lambda` (CPython 3.14's bare `MAKE_FUNCTION`): + // the function, with the defaults and closure the + // `SET_FUNCTION_ATTRIBUTE`s right after it attach built in + // (each takes the value next below the function). No Python + // runs, unless a function watcher would see the creation. + OpCode::MakeFunction if ins.arg == 0 => { + let n = stack.len(); + if crate::capi_watchers::funcs_active() + || !matches!(stack.last(), Some(Object::Code(_))) + { + break; + } + let mut defaults: Option> = None; + let mut closure: Option> = None; + let mut k = 0; + while let (Some(next), Some(value)) = + (instrs.get(pc + 1 + k), n.checked_sub(2 + k).map(|i| &stack[i])) + { + let (Object::Tuple(items), OpCode::SetFunctionAttribute) = (value, next.op) + else { + break; + }; + let field = match next.arg { + 0x01 => &mut defaults, + 0x08 => &mut closure, + _ => break, + }; + if field.is_some() { + break; + } + *field = Some(items.iter().cloned().collect()); + k += 1; + } + let Some(Object::Code(code)) = stack.pop() else { + unreachable!("checked above") + }; + for _ in 0..k { + let v = stack.pop().expect("checked above"); + self.release(v); + } + let obj = new_function( + code, + globals, + builtins, + defaults.unwrap_or_default(), + Vec::new(), + closure.unwrap_or_default(), + None, + ); + stack.push(obj); + last = pc + k; + pc += 1 + k; + } OpCode::LoadConst => { let Some(v) = consts.get(ins.arg as usize) else { break; @@ -22318,84 +22371,15 @@ impl Interpreter { defaults = items.iter().cloned().collect(); } } - let name = code.name.clone(); - // Function getset slots live *outside* `__dict__` - // (`f.__dict__` starts empty in CPython; only genuine - // user attributes land there). - let slots = RefCell::new(DictData::with_capacity_and_hasher(4, Default::default())); - // Stamp __module__ from globals['__name__'] (mirrors CPython's - // function dispatch). Pickle relies on this to serialise the - // function by qualified name. - let keys = fn_slot_keys(); - if let Some(name_obj) = frame - .globals - .borrow() - .get(&DictKey(keys[1].clone())) - .cloned() - { - slots - .borrow_mut() - .insert(DictKey(keys[0].clone()), name_obj); - } - // Pin __name__ and __qualname__ as stable objects so - // repeated `func.__name__` reads (and delegated reads - // through classmethod/staticmethod wrappers) return the - // *same* object — CPython exposes these as slots with - // stable identity, which `assertIs(wrapper.__name__, - // func.__name__)` in test_decorators relies on. - // Both come from the code object (see - // `CodeConstObjects::fn_names`), so two functions made - // from one `def` share them exactly as CPython's do. - let (name_obj, qualname_obj) = match code_vm_ext(&code) { - Some(ext) => ext - .fn_names - .get_or_init(|| { - ( - Object::from_str(code.name.clone()), - Object::from_str(code.qualname.clone()), - ) - }) - .clone(), - None => ( - Object::from_str(name.clone()), - Object::from_str(code.qualname.clone()), - ), - }; - { - let mut sl = slots.borrow_mut(); - sl.insert(DictKey(keys[1].clone()), name_obj); - // `__qualname__` is the code object's PEP 3155 dotted - // name (computed at compile time from lexical - // nesting), not the bare `__name__`. - sl.insert(DictKey(keys[2].clone()), qualname_obj); - if let Some(ann) = annotations_obj { - sl.insert(DictKey(keys[3].clone()), ann); - } - } - let f = PyFunction { - name, - code: RefCell::new(code), - globals: frame.globals.clone(), - // The defining frame's builtins, not a fresh - // resolution: same mapping CPython's - // `PyFunction_New` snapshots (framestate's - // f_builtins came from the same globals). - builtins: frame.builtins.clone(), + let obj = new_function( + code, + &frame.globals, + &frame.builtins, defaults, kw_defaults, closure, - attrs: RefCell::new(None), - slots, - closure_cells: std::sync::OnceLock::new(), - defaults_override: crate::object::OverrideFlag::new(false), - }; - // A function participates in cycles through its globals - // (`exec(src, d)` builds `d -> f -> d`), its `__dict__`, and - // any closure cell that closes over the function. Track it so - // `gc.collect()` can reclaim those cycles and `gc.is_tracked(f)` - // reports True, matching CPython (functions are GC objects). - let obj = Object::Function(Rc::new(f)); - gc_trace::track(&obj); + annotations_obj, + ); // PEP 590-era function watchers: PyFunction_EVENT_CREATE // fires from `PyFunction_New*` (test_capi.test_watchers). if crate::capi_watchers::funcs_active() { @@ -61387,6 +61371,10 @@ struct CodeConstObjects { /// Inline-call shapes per `CALL` site (see [`CallSlot`]); allocated /// on the first inline call in this code object. call_slots: std::sync::OnceLock>, + /// The getset slots of a function made from this code (see + /// [`function_slots`]): the module name they were built for, and the + /// entries, copied into each new function. + fn_slots: std::sync::OnceLock<(Option, crate::object::DictMap)>, /// Whether this code is a *pure leaf* (see [`code_is_pure_leaf`]): /// `0` not yet decided, `1` no, `2` general leaf, `3` argument return, /// `4` constant return, `5` small-int return, `6` attribute return, @@ -61422,10 +61410,6 @@ struct CodeConstObjects { /// The code's native form for the core loop (see [`frame_jit`]). #[cfg(feature = "jit")] frame_jit: frame_jit::Slot, - /// The `__name__` and `__qualname__` objects of the functions made - /// from this code, shared by all of them (as CPython's share the code - /// object's `co_name` and `co_qualname`). - fn_names: std::sync::OnceLock<(Object, Object)>, } /// A `CALL` site's inline-call shape (see `Interpreter::core_call`): the @@ -61469,14 +61453,18 @@ unsafe impl Send for CallSlot {} unsafe impl Sync for CallSlot {} impl CallShape { + /// The shape for a call of `code`: it depends only on the code and + /// the site (every caller still checks the function's own defaults), + /// so a fresh function made from the same `def` or `lambda` hits too. + /// (The weak handle keeps the code's allocation reserved, so an equal + /// address is the same code object.) #[inline] fn matches( &self, - func: *const crate::object::PyFunction, + _func: *const crate::object::PyFunction, code: *const CodeObject, ) -> Option<(u32, bool)> { - (std::ptr::eq(self.func.as_ptr(), func) && std::ptr::eq(self.code.as_ptr(), code)) - .then_some((self.missing, self.has_self)) + std::ptr::eq(self.code.as_ptr(), code).then_some((self.missing, self.has_self)) } } @@ -62895,6 +62883,7 @@ fn code_vm_ext_build( stamp_slots: std::sync::OnceLock::new(), attr_poly: std::sync::OnceLock::new(), call_slots: std::sync::OnceLock::new(), + fn_slots: std::sync::OnceLock::new(), pure_leaf: std::sync::atomic::AtomicU8::new(0), fast_pairs: std::sync::OnceLock::new(), gen_fast: std::sync::atomic::AtomicU8::new(0), @@ -62905,7 +62894,6 @@ fn code_vm_ext_build( returns_none: std::sync::atomic::AtomicU8::new(0), #[cfg(feature = "jit")] frame_jit: frame_jit::Slot::default(), - fn_names: std::sync::OnceLock::new(), }) }) } @@ -62963,6 +62951,99 @@ fn code_names_anywhere(code: &CodeObject, name: &str) -> bool { /// The function getset-slot keys, interned once: a fresh /// `Object::from_static` would allocate the text and recompute its hash /// at every `MAKE_FUNCTION`. +/// `MAKE_FUNCTION`'s function: `code` with the defining frame's globals +/// and builtins, and the operands the instruction's flags popped. +#[allow(clippy::too_many_arguments)] +fn new_function( + code: Rc, + globals: &Rc>, + builtins: &Rc>, + defaults: Vec, + kw_defaults: Vec<(String, Object)>, + closure: Vec, + annotations: Option, +) -> Object { + // Function getset slots live *outside* `__dict__` (`f.__dict__` + // starts empty in CPython; only genuine user attributes land there): + // `__module__` (pickle serialises a function by qualified name), + // and `__name__` and `__qualname__` as stable objects, so repeated + // `func.__name__` reads (and delegated reads through classmethod or + // staticmethod wrappers) return the *same* object, which + // `assertIs(wrapper.__name__, func.__name__)` in test_decorators + // relies on. + let mut slots = function_slots(&code, globals); + if let Some(ann) = annotations { + slots.insert(DictKey(fn_slot_keys()[3].clone()), ann); + } + let f = PyFunction { + name: code.name.clone(), + code: RefCell::new(code), + globals: globals.clone(), + // The defining frame's builtins, not a fresh resolution: the same + // mapping CPython's `PyFunction_New` snapshots (the frame's + // builtins came from the same globals). + builtins: builtins.clone(), + defaults, + kw_defaults, + closure, + attrs: RefCell::new(None), + slots: RefCell::new(slots), + closure_cells: std::sync::OnceLock::new(), + defaults_override: crate::object::OverrideFlag::new(false), + }; + // A function participates in cycles through its globals (`exec(src, + // d)` builds `d -> f -> d`), its `__dict__`, and any closure cell that + // closes over the function. Tracked so `gc.collect()` can reclaim + // those cycles and `gc.is_tracked(f)` reports True, matching CPython + // (functions are GC objects). + let obj = Object::Function(Rc::new(f)); + gc_trace::track(&obj); + obj +} + +/// The getset slots a new function made from `code` in `globals` starts +/// with: `__module__` (the globals' `__name__`, when set), `__name__` and +/// `__qualname__`. Built once per code object (for the module name it +/// first sees) and copied into each function after, so two functions made +/// from one `def` share their name objects, as CPython's do. +fn function_slots(code: &CodeObject, globals: &Rc>) -> DictData { + let keys = fn_slot_keys(); + let module = globals.borrow().get(&DictKey(keys[1].clone())).cloned(); + let build = |module: Option| { + let mut map = crate::object::DictMap::with_capacity_and_hasher( + 3, + crate::fasthash::FxBuildHasher, + ); + if let Some(m) = module { + map.insert(DictKey(keys[0].clone()), m); + } + map.insert( + DictKey(keys[1].clone()), + Object::from_str(code.name.clone()), + ); + // `__qualname__` is the code object's PEP 3155 dotted name + // (computed at compile time from lexical nesting), not the bare + // `__name__`. + map.insert( + DictKey(keys[2].clone()), + Object::from_str(code.qualname.clone()), + ); + map + }; + let same = |a: &Option, b: &Option| match (a, b) { + (Some(a), Some(b)) => a.is_same(b), + (None, None) => true, + _ => false, + }; + if let Some(ext) = code_vm_ext(code) { + let (built_for, map) = ext.fn_slots.get_or_init(|| (module.clone(), build(module.clone()))); + if same(built_for, &module) { + return DictData::from(map.clone()); + } + } + DictData::from(build(module)) +} + fn fn_slot_keys() -> &'static [Object; 4] { static KEYS: std::sync::OnceLock<[Object; 4]> = std::sync::OnceLock::new(); KEYS.get_or_init(|| { diff --git a/tests/regrtest/test_function_creation.py b/tests/regrtest/test_function_creation.py new file mode 100644 index 00000000..afdbc349 --- /dev/null +++ b/tests/regrtest/test_function_creation.py @@ -0,0 +1,103 @@ +"""Functions made in a hot loop keep their attributes and call shapes. + +A `def` or `lambda` that runs many times builds each function from a +template of its code's name slots, attaches its defaults and closure as it +is built, and is called through the call site's shape for its code. Every +function made here must still behave as CPython's do. +""" + +import types +import unittest + + +def make_adder(n, scale=2): + def add(x, y=10): + return (x + y + n) * scale + + return add + + +def make_lambda(k): + return lambda x, z=k: x * z + + +class FunctionCreationTests(unittest.TestCase): + def test_closures_and_defaults(self): + total = 0 + for i in range(5000): + total += make_adder(i)(1) + self.assertEqual(total, sum((1 + 10 + i) * 2 for i in range(5000))) + f = make_adder(3, scale=5) + self.assertEqual(f(1, 2), 30) + self.assertEqual(f.__defaults__, (10,)) + self.assertEqual(f.__closure__[0].cell_contents, 3) + self.assertEqual([make_lambda(k)(3) for k in range(5)], [0, 3, 6, 9, 12]) + g = make_lambda(4) + self.assertEqual(g(2, 5), 10) + self.assertEqual(g.__defaults__, (4,)) + + def test_names_and_identity(self): + fs = [make_adder(i) for i in range(100)] + self.assertEqual({f.__name__ for f in fs}, {"add"}) + self.assertEqual(fs[0].__qualname__, "make_adder..add") + self.assertEqual(fs[0].__module__, __name__) + self.assertIs(fs[0].__name__, fs[1].__name__) + fs[0].__name__ = "renamed" + fs[0].__qualname__ = "q" + self.assertEqual(fs[0].__name__, "renamed") + self.assertEqual(fs[1].__name__, "add") + self.assertEqual(fs[1].__qualname__, "make_adder..add") + self.assertEqual(fs[2].__dict__, {}) + fs[2].tag = 1 + self.assertEqual(fs[2].__dict__, {"tag": 1}) + self.assertEqual(fs[3].__dict__, {}) + + def test_module_name_follows_globals(self): + src = "def f():\n return __name__\n" + code = compile(src, "", "exec") + first, second = {"__name__": "first"}, {"__name__": "second"} + for _ in range(100): + exec(code, first) + exec(code, second) + self.assertEqual(first["f"].__module__, "first") + self.assertEqual(second["f"].__module__, "second") + self.assertEqual(second["f"](), "second") + anon = {} + exec(code, anon) + self.assertIsNone(anon["f"].__module__) + clone = types.FunctionType(first["f"].__code__, {"__name__": "third"}) + self.assertEqual(clone.__module__, "third") + + def test_keyword_defaults_and_annotations(self): + def build(i): + def h(a: int, *, b=i) -> int: + return a + b + + return h + + hs = [build(i) for i in range(2000)] + self.assertEqual(hs[7](1), 8) + self.assertEqual(hs[7].__kwdefaults__, {"b": 7}) + self.assertEqual(hs[7].__annotations__, {"a": int, "return": int}) + + def test_fresh_functions_share_call_shapes(self): + def caller(n): + t = 0 + for i in range(n): + t += (lambda x, y=2: x * y)(i) + return t + + self.assertEqual(caller(5000), 2 * sum(range(5000))) + + def defaults_change(n): + out = [] + for i in range(n): + f = lambda x, y=i: x + y + out.append(f(1)) + return out + + self.assertEqual(defaults_change(300)[-3:], [298, 299, 300]) + + +if __name__ == "__main__": + unittest.main() From 67006d7dded41b979d3e76c3af283555de1d4c3f Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 05:31:42 -0700 Subject: [PATCH 60/96] perf: build Rust-side exceptions without name lookups or eager messages `make_exception_with_class` walked the class's MRO comparing names to find the StopIteration/SyntaxError/ImportError families, and `make_exception_with_object` (every `KeyError(key)` from `dict[k]`) looked the class up by name and formatted `repr(key)` into a `message` pseudo-slot nobody reads in CPython. Exceptions now take the memoized family flags, populate their slots in one sized vector, and leave the message to `str()`/`exception_message`, which derive it from `args` (including KeyError's repr rendering) the way CPython does. Instructions retired per iteration (/usr/bin/time -l slopes): try: d[5] except KeyError 15,070 -> 13,238 (CPython 2,301) same, raised in a callee 20,110 -> 18,255 (CPython 3,593) --- crates/weavepy-vm/src/builtin_types.rs | 100 ++++++++++++++----------- crates/weavepy-vm/src/lib.rs | 7 +- crates/weavepy-vm/src/types.rs | 2 +- 3 files changed, 64 insertions(+), 45 deletions(-) diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index c51543c1..87855344 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -1557,15 +1557,17 @@ pub fn make_exception(class_name: &str, message: impl Into) -> Object { /// Build a built-in exception instance whose single `args[0]` element is the /// *object* `arg`, not a stringified message — `KeyError(key)` where /// `e.args[0] is key`. CPython's `KeyError.__str__` renders `repr(args[0])`, -/// which our `exc_str` already reproduces; we set `message` to that repr so -/// the Rust Display/traceback path matches too. +/// which our `exc_str` already reproduces, and so does +/// [`exception_message`] for the Rust Display/traceback path. pub fn make_exception_with_object(class_name: &str, arg: Object) -> Object { - let exc = make_exception(class_name, ""); - if let Object::Instance(inst) = &exc { - inst.slot_set("args", Object::new_tuple_array([arg.clone()])); - inst.slot_set("message", Object::from_str(arg.repr())); - } - exc + let bt = builtin_types(); + let class = match class_name { + "KeyError" => bt.key_error.clone(), + _ => bt + .by_name(class_name) + .unwrap_or_else(|| bt.exception.clone()), + }; + exception_from_parts(class, Some(arg)) } /// Build a faithful `UnicodeEncodeError` instance carrying the 5-tuple @@ -4858,47 +4860,53 @@ fn install_exception_str_repr(base_exception: &Rc) { } pub fn make_exception_with_class(class: Rc, message: impl Into) -> Object { - use crate::types::PyInstance; - let (mut is_syntax, mut is_stop_iteration, mut is_import) = (false, false, false); - for t in class.mro.borrow().iter() { - match t.name.as_str() { - "SyntaxError" => is_syntax = true, - "StopIteration" => is_stop_iteration = true, - "ImportError" => is_import = true, - _ => {} - } - } - let inst = PyInstance::new(class); - let msg = Object::from_str(message); + let message: String = message.into(); // A messageless raise (`StopIteration()`, `GeneratorExit()`, …) // has *empty* args in CPython, not `("",)`. - let args = if msg.to_str().is_empty() { - Object::new_tuple(Vec::new()) + let msg = if message.is_empty() { + None } else { - Object::new_tuple_array([msg.clone()]) + Some(Object::from_str(message)) + }; + exception_from_parts(class, msg) +} + +/// A built-in exception of `class` whose `args` is `(arg,)`, or `()` +/// for `None`: the shared tail of the Rust-side constructors. +/// +/// The pseudo-slots land in one vector sized for the traceback and +/// chaining links a raise adds next, and the class's families come +/// from its memoised flags rather than an MRO walk. CPython keeps no +/// `message` attribute: `str()` and [`exception_message`] derive the +/// text from `args` on demand, so nothing is formatted here. +pub(crate) fn exception_from_parts(class: Rc, arg: Option) -> Object { + use crate::types::{slot_key, PyInstance, SlotStorage}; + let families = crate::exc_family_flags(&class); + let inst = PyInstance::new(class); + let args = match &arg { + None => Object::new_tuple(Vec::new()), + Some(a) => Object::new_tuple_array([a.clone()]), }; + inst.note_slot_store(&args); + let mut entries: Vec<(crate::object::DictKey, Object)> = Vec::with_capacity(4); // PEP 380: `StopIteration.value` is always present (CPython sets it // in `StopIteration.__init__`, defaulting to None). A Rust-raised - // bare `StopIteration` must answer `.value` too — asyncio's + // bare `StopIteration` must answer `.value` too: asyncio's // `Task.__step` reads `exc.value` on every coroutine return, and a // missing attribute leaves the task wedged (gh: shutdown_asyncgens). - if is_stop_iteration { - let value = if msg.to_str().is_empty() { - Object::None - } else { - msg.clone() - }; - inst.slot_set("value", value); + if families & crate::EXC_FAM_STOP_ITERATION != 0 { + let value = arg.clone().unwrap_or(Object::None); + inst.note_slot_store(&value); + entries.push((slot_key("value"), value)); } - inst.slot_set("args", args); - inst.slot_set("message", msg.clone()); + entries.push((slot_key("args"), args)); // The BaseException pseudo-slots (`__context__`/`__cause__`/ // `__suppress_context__`/`__traceback__`) and OSError's named fields // (`errno`/`strerror`/…) are *not* seeded per-instance: their class // slot descriptors answer the CPython getset defaults while unset, // and a subclass's own class attribute (`class Err(OSError): errno = // EINVAL`, raised bare via `raise Err`) stays visible. - if is_import { + if families & (crate::EXC_FAM_IMPORT_ERROR | crate::EXC_FAM_SYNTAX_ERROR) != 0 { // ImportError/ModuleNotFoundError expose `msg` (the message // string). CPython always defines the slot; a Rust-raised // ImportError must answer `.msg` so consumers (e.g. numpy's @@ -4906,15 +4914,15 @@ pub fn make_exception_with_class(class: Rc, message: impl Into Option { return Some(s.to_string()); } if let Some(Object::Tuple(items)) = exc_attr(inst, "args") { + // `KeyError.__str__` renders a lone key by its repr. + if let [single] = &items[..] { + if inst.cls().is_subclass_of(&builtin_types().key_error) { + return Some(single.repr()); + } + } if let Some(first) = items.first() { return Some(first.to_str()); } diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index f2420b9d..c6e88538 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -63070,6 +63070,11 @@ const EXC_FAMILIES: [&str; 9] = [ "UnicodeTranslateError", ]; +/// [`EXC_FAMILIES`] bits that Rust-side raise sites test directly. +pub(crate) const EXC_FAM_STOP_ITERATION: u16 = 1 << 0; +pub(crate) const EXC_FAM_SYNTAX_ERROR: u16 = 1 << 4; +pub(crate) const EXC_FAM_IMPORT_ERROR: u16 = 1 << 5; + /// `name`'s bit in an [`exc_family_flags`] set (`0` for a name that is /// not a family — never asked in practice). #[inline] @@ -63082,7 +63087,7 @@ fn exc_family_bit(name: &str) -> u16 { /// Which exception families `cls` inherits from, memoised on the type /// against its attribute version (an MRO reshape bumps it). -fn exc_family_flags(cls: &crate::types::TypeObject) -> u16 { +pub(crate) fn exc_family_flags(cls: &crate::types::TypeObject) -> u16 { let ver = cls.attr_version.get(); let memo = cls.exc_families.get(); if memo & 1 != 0 && memo >> 11 == ver { diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index 4a97dc74..3d7f646a 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -2053,7 +2053,7 @@ fn slot_name_eq(stored: &str, name: &str) -> bool { /// The key for a newly populated slot `name`. The slots every raise /// populates (`args`, `__traceback__`, the chaining links) share one /// interned key each instead of allocating a string per exception. -fn slot_key(name: &str) -> DictKey { +pub(crate) fn slot_key(name: &str) -> DictKey { const COMMON: [&str; 6] = [ "args", "__traceback__", From d571fae7ace3e146f8e4656768152effcadf5b5c Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:51:28 -0700 Subject: [PATCH 61/96] perf: catch exceptions in the quiet loop and record tracebacks lazily An exception raised in the quiet loop used to stop it and finish the whole activation in the general loop (`run_activation`), and every traceback entry materialized the frame object and its `f_back` chain, which also pushed the frame's callers off their quiet paths. - `quiet_catch`: when nothing observes the frame (no trace, profile or monitoring callback) and the code's exception table has a handler for the raise, the quiet loop chains the exception, records the traceback entry and jumps to the handler itself, for raises from the core loop, `step`, lean calls and inline callees alike. With no handler, an inline activation unwinds through `quiet_unwind` (the general loop's traceback entry and exceptional-exit epilogue) without entering it. - Lazy traceback frames: an entry for an ordinary, never-materialized activation holds its spine shell (`PyTraceback::new_lazy`) and builds the frame object on the first `tb_frame` read. If the activation leaves the spine while such an entry lives, `pop_frame_shell` materializes it then, linking `f_back` to the caller's frame object if one exists or lazily to the caller's shell (`PyFrame::lazy_back`). `tb_frame` identity, `f_lineno`/`tb_lasti` (including the PEP 626 RERAISE restore), `f_locals`, `frame.clear()` refusal and `sys._current_exceptions()` frames are unchanged. - A dead frame object releases its locals top-down before `f_back`, as CPython's `frame_dealloc` does, so finalizers run innermost-first. - `leaf_burst_slow` runs PUSH_EXC_INFO, CHECK_EXC_MATCH (a class or a tuple of BaseException subclasses) and POP_EXCEPT, and the core loop raises a plain dict's KeyError for an exact str/int miss itself. - Exception attribute probes read the instance dict without publishing one. The JIT string-argument unit test ran its loop-free cases through the general call path only because its driver frame had been materialized by a caught exception; it now turns the quiet loops off explicitly. Instructions retired per iteration (/usr/bin/time -l slopes): try: d[5] except KeyError 13,238 -> 8,102 (CPython 2,301) same, raised in a callee 18,255 -> 12,578 (CPython 3,593) raise ValueError; except as e 18,145 -> 14,171 (CPython 2,687) --- crates/weavepy-vm/src/builtin_types.rs | 8 +- crates/weavepy-vm/src/lib.rs | 647 ++++++++++++++---- crates/weavepy-vm/src/object.rs | 220 +++++- .../weavepy-vm/src/stdlib/weave_frame_mod.rs | 1 + crates/weavepy-vm/src/tier2.rs | 5 +- 5 files changed, 728 insertions(+), 153 deletions(-) diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index 87855344..e0bb50ba 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -1536,12 +1536,8 @@ fn exc_slot_readonly(name: &str, class_name: &str, default: Object) -> Object { /// applied by an old pickle, or `self.args = …` in a shadowing /// `__init__` that ran before the class descriptor was reachable). pub(crate) fn exc_attr(inst: &crate::types::PyInstance, name: &str) -> Option { - inst.slot_get(name).or_else(|| { - inst.dict_cell() - .borrow() - .get(&crate::object::StrKey(name)) - .cloned() - }) + // Read without publishing a `__dict__` the instance never needed. + inst.slot_get(name).or_else(|| inst.attr_get_str(name)) } /// Construct an exception instance of `class_name` with `message` as diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index c6e88538..45179b82 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -486,7 +486,12 @@ fn py_frame_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) { /// test_ssl.test_handshake_timeout_handler_leak). fn py_traceback_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) { let Object::Traceback(tb) = obj else { return }; - visit(&Object::Frame(tb.frame.clone())); + // A lazy entry's activation is still running (its locals are roots + // the collector reaches through the spine); materializing here would + // only add a frame object for the traversal to find. + if let Some(frame) = tb.frame_if_ready() { + visit(&Object::Frame(frame)); + } if let Ok(next) = tb.next.try_borrow() { if let Some(n) = next.as_ref() { visit(&Object::Traceback(n.clone())); @@ -3314,7 +3319,7 @@ impl Interpreter { let mut cur = Some(tb); while let Some(node) = cur { entries.push(crate::error::TracebackEntry { - code: node.frame.code.clone(), + code: node.code(), lineno: node.lineno, }); cur = node.next.borrow().clone(); @@ -4489,6 +4494,10 @@ impl Interpreter { if let Some(py) = shell.materialized.borrow().as_ref() { py.lasti.set(at); } + } else if let Some(exit) = &quiet_exit { + // Likewise the shell a lazy traceback entry will + // materialize from (see `pop_frame_shell`). + Self::sync_lazy_tb_lasti(Some(&shell), &frame.code, exit); } match quiet_exit { Some(QuietExit::Yield) => continue, @@ -5035,35 +5044,7 @@ impl Interpreter { Ok(StepOutcome::StartGenerator) => break Ok(FrameOutcome::StartGenerator), Err(err) => { if let RuntimeError::PyException(mut exc) = err { - // CPython's `_PyErr_SetObject` chains *every* - // fresh exception — including ones raised from C - // (here: Rust opcodes and builtins) — to the - // currently handled exception. `RAISE_VARARGS` - // already did this at the raise site - // (`context_settled`); a fresh Rust-raised error - // has empty traceback and unset context/cause. - if !exc.context_settled - && exc.context.is_none() - && exc.cause.is_none() - && exc.traceback.is_empty() - && !exc.suppress_tb_once - && !instance_has_nonnull_attr(&exc.instance, "__context__") - { - self.attach_implicit_context(&mut exc); - if exc.context.is_some() { - Self::sync_exc_attrs(&exc); - } - } else if exc.traceback.is_empty() - && (exc.cause.is_some() || exc.context.is_some()) - { - // A fresh Rust-raised error that already carries an - // explicit `cause`/`context` (e.g. `_io` chaining a - // misbehaving raw `readinto`'s `TypeError` into an - // `OSError`, `test_bad_readinto_type`) must still - // mirror those onto the instance dict so Python's - // `except` sees `e.__cause__`. - Self::sync_exc_attrs(&exc); - } + self.chain_fresh_exception(&mut exc); // CPython's RERAISE fires the separate RERAISE // monitoring event, which `sys.settrace` does // *not* subscribe to — only fresh raises @@ -5435,6 +5416,7 @@ impl Interpreter { lasti: std::sync::atomic::AtomicU32::new(frame.pc), has_materialized: std::sync::atomic::AtomicBool::new(materialized.is_some()), materialized: RefCell::new(materialized), + tb_refs: std::sync::atomic::AtomicU32::new(0), }); self.frame_stack.borrow_mut().push(shell.clone()); shell @@ -5606,6 +5588,7 @@ impl Interpreter { on_stack: Cell::new(0), extra_locals: RefCell::new(None), cleared: Cell::new(false), + lazy_back: RefCell::new(None), }) } @@ -5961,7 +5944,80 @@ impl Interpreter { } } + /// A quiet loop leaves its shell's `lasti` behind the instruction it + /// stopped on. A lazy traceback entry materializes from the shell + /// once the activation leaves the spine (see `pop_frame_shell`), so + /// bring it current for the frame object's final `f_lineno`. + /// A `RERAISE` leaves the restored raise site there itself (PEP 626). + #[inline] + fn sync_lazy_tb_lasti( + shell: Option<&crate::object::FrameShell>, + code: &CodeObject, + exit: &QuietExit, + ) { + if let (Some(shell), QuietExit::Outcome { cur_pc, stepped }) = (shell, exit) { + if shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) != 0 + && !(stepped.is_err() + && code + .instructions + .get(*cur_pc) + .is_some_and(|i| i.op == OpCode::Reraise)) + { + shell + .lasti + .store(*cur_pc as u32, std::sync::atomic::Ordering::Relaxed); + } + } + } + + /// Materialize the departing top activation for the lazy references + /// that outlive it (see `pop_frame_shell`). Its `f_back` is its + /// caller's frame object when one exists, else a lazy link to the + /// caller's shell (CPython's `take_ownership` links `f_back` at exit + /// too, but its frame objects cost nothing to keep current; here a + /// materialized caller would leave its quiet loop). + #[cold] + #[inline(never)] + fn materialize_departing_top(&self) { + let (top, caller) = { + let stack = self.frame_stack.borrow(); + let n = stack.len(); + ( + stack[n - 1].clone(), + n.checked_sub(2).map(|i| stack[i].clone()), + ) + }; + let py = top.materialize_departing(None); + let Some(caller) = caller else { return }; + let existing = caller.materialized.borrow().clone(); + match existing { + Some(back) => { + // A new holder of the caller's frame object: its loop + // re-derives whether the object is observed. + caller.refresh_materialized(&back); + py.set_back(Some(back)); + crate::hot_gates::bump_loop_gen(); + } + None => { + *py.lazy_back.borrow_mut() = Some(crate::object::LazyShellRef::new(caller)); + } + } + } + fn pop_frame_shell(&self) -> Option> { + // A lazy reference to this activation (a traceback entry, see + // `PyTraceback::new_lazy`, or a callee frame's `f_back`) still + // lives: build the frame object CPython would have by now, while + // the shell is still whole. + let lazy_top = self.frame_stack.borrow().last().is_some_and(|top| { + top.tb_refs.load(std::sync::atomic::Ordering::Relaxed) != 0 + && !top + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) + }); + if lazy_top { + self.materialize_departing_top(); + } let popped = self.frame_stack.borrow_mut().pop(); if let Some(shell) = &popped { // Borrowed slots (see `FrameSlot`) read through the frame, @@ -5988,7 +6044,7 @@ impl Interpreter { // generator-family frames are re-entered and observed // while suspended. if shell.is_gen { - *py.back.borrow_mut() = None; + py.set_back(None); } } } @@ -6518,13 +6574,14 @@ impl Interpreter { _ => None, }; let tb_arg = if synthesize_head { - Object::Traceback(Rc::new(PyTraceback { - lineno: py_frame.last_line.get().unwrap_or(1), - lasti: py_frame.lasti.get(), - frame: py_frame.clone(), - raw_lasti: None, - next: RefCell::new(inherited_tb.clone()), - })) + let tb = PyTraceback::new( + py_frame.clone(), + py_frame.last_line.get().unwrap_or(1), + py_frame.lasti.get(), + None, + ); + *tb.next.borrow_mut() = inherited_tb.clone(); + Object::Traceback(Rc::new(tb)) } else { inherited_tb.clone().map_or(Object::None, Object::Traceback) }; @@ -7449,18 +7506,21 @@ impl Interpreter { } } } - QuietEntry::Raised { err, cur_pc } => { - // The raise path drops the staged operands wholesale - // (see the `CALL` arm of `step`). - self.flush_lean(frame, shell); - if *last != usize::MAX { - *prev_pc = Some(*last); + QuietEntry::Raised { err, cur_pc } => match self.quiet_catch(frame, shell, err) { + Ok(()) => *last = cur_pc, + Err(err) => { + // The raise path drops the staged operands wholesale + // (see the `CALL` arm of `step`). + self.flush_lean(frame, shell); + if *last != usize::MAX { + *prev_pc = Some(*last); + } + return FrameEv::Exit(QuietExit::Outcome { + stepped: Err(err), + cur_pc, + }); } - return FrameEv::Exit(QuietExit::Outcome { - stepped: Err(err), - cur_pc, - }); - } + }, } loop { let stop = match pending.take() { @@ -7477,9 +7537,16 @@ impl Interpreter { match stop { LeafStop::Breaker => break 'run QuietExit::Yield, LeafStop::Raised(err) => { + let cur_pc = frame.pc as usize - 1; + let err = match self.quiet_catch(frame, shell, err) { + Ok(()) => { + *last = cur_pc; + continue; + } + Err(err) => err, + }; // The `CALL` arm of `step` marks on the raise path. self.flush_lean(frame, shell); - let cur_pc = frame.pc as usize - 1; if *last != usize::MAX { *prev_pc = Some(*last); } @@ -7558,6 +7625,13 @@ impl Interpreter { }; if let Some(called) = called { if let Err(err) = called { + let err = match self.quiet_catch(frame, shell, err) { + Ok(()) => { + *last = cur_pc; + continue; + } + Err(err) => err, + }; // The raise path drops the staged operands // wholesale (see the `CALL` arm of `step`). self.flush_lean(frame, shell); @@ -7622,6 +7696,13 @@ impl Interpreter { }; if let Some(loaded) = loaded { if let Err(err) = loaded { + let err = match self.quiet_catch(frame, shell, err) { + Ok(()) => { + *last = cur_pc; + continue; + } + Err(err) => err, + }; self.flush_lean(frame, shell); if *last != usize::MAX { *prev_pc = Some(*last); @@ -7672,7 +7753,13 @@ impl Interpreter { live_shell .lasti .store(frame.pc, std::sync::atomic::Ordering::Relaxed); - let stepped = self.step_hot(frame); + let stepped = match self.step_hot(frame) { + Err(err) => match self.quiet_catch(frame, shell, err) { + Ok(()) => Ok(StepOutcome::Continue), + Err(err) => Err(err), + }, + stepped => stepped, + }; if !matches!(stepped, Ok(StepOutcome::Continue)) { return FrameEv::Exit(QuietExit::Outcome { stepped, cur_pc }); } @@ -8123,6 +8210,7 @@ impl Interpreter { ) -> Result { let exc_depth = done.exc_depth; let frame: &mut Frame = &mut done.frame; + Self::sync_lazy_tb_lasti(done.act.shell.as_deref(), &frame.code, &exit); let outcome = match (exit, done.act.shell.take()) { // The whole activation ran without ever needing its shell. ( @@ -8157,12 +8245,27 @@ impl Interpreter { ) if !shell .has_materialized .load(std::sync::atomic::Ordering::Relaxed) + && shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) == 0 && self.exc_info_len() <= exc_depth => { self.pop_frame_shell(); self.recycle_frame_shell(shell); Ok(FrameOutcome::Returned(v)) } + // An exception leaving an unobserved activation with no handler + // for it: the general loop's unwind, without entering it. + ( + QuietExit::Outcome { + stepped: Err(RuntimeError::PyException(exc)), + .. + }, + Some(shell), + ) if !crate::trace::any_observers_active() + && find_handler(&frame.code.exception_table, frame.pc.wrapping_sub(1)) + .is_none() => + { + Err(self.quiet_unwind(frame, shell, exc, exc_depth)) + } // Everything else finishes in the general loop, which takes // over exactly where the quiet loop stopped. (exit, shell) => { @@ -8376,6 +8479,7 @@ impl Interpreter { lasti: std::sync::atomic::AtomicU32::new(lasti), has_materialized: std::sync::atomic::AtomicBool::new(false), materialized: RefCell::new(None), + tb_refs: std::sync::atomic::AtomicU32::new(0), }); self.frame_stack.borrow_mut().push(shell.clone()); shell @@ -9725,6 +9829,7 @@ impl Interpreter { }; let mut prev_pc: Option = None; let exit = self.quiet_run(frame, QuietShell::Lazy(&mut act), snap_gen, &mut prev_pc); + Self::sync_lazy_tb_lasti(act.shell.as_deref(), &frame.code, &exit); let outcome = match (exit, act.shell) { // The whole activation ran without ever needing its shell. ( @@ -9744,6 +9849,7 @@ impl Interpreter { ) if !shell .has_materialized .load(std::sync::atomic::Ordering::Relaxed) + && shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) == 0 && self.exc_info_len() <= exc_depth_on_entry => { self.pop_frame_shell(); @@ -10993,6 +11099,17 @@ impl Interpreter { // An instance whose class's `__getitem__` is a // native fast subscript the site cached. None if ins.op == OpCode::BinarySubscr && len >= 2 => { + // A plain dict without the key raises its + // KeyError here, for the quiet loop's handler + // dispatch (the `try: d[k] except KeyError:` + // idiom). + // SAFETY: the `len >= 2` operand slots are + // initialized; on a raise both are consumed. + if let Some(e) = unsafe { Self::core_dict_miss(base, len) } { + len -= 2; + pc += 1; + break Some(CoreExit::Stop(LeafStop::Raised(e))); + } let Some(fast) = self.core_native_subscript( code, pc, @@ -13975,6 +14092,40 @@ impl Interpreter { } } + /// `dict[key]` on the two operand slots ending at `len` when a plain + /// dict misses a str or int key: the full handler's `KeyError(key)`, + /// with both operands released. `None` (nothing touched) otherwise. + /// + /// # Safety + /// + /// The `len >= 2` slots at `base` are initialized; on `Some` the top + /// two are moved out (the caller drops them from its count). + #[cold] + #[inline(never)] + unsafe fn core_dict_miss(base: *mut Object, len: usize) -> Option { + // SAFETY: the caller's contract. + let (c, k) = unsafe { (&*base.add(len - 2), &*base.add(len - 1)) }; + let (Object::Dict(d), Object::Str(_) | Object::Int(_)) = (c, k) else { + return None; + }; + if !Self::core_droppable(c) { + return None; + } + let probe = crate::object::LeafProbe::new(k)?; + // A stored key of another kind sharing the bucket might equal the + // probe through Python (`__eq__`, which may also raise): only a + // miss that met none is the full lookup's miss. + if d.try_borrow().ok()?.get(&probe).is_some() || !probe.miss_is_exact() { + return None; + } + // SAFETY: both slots are initialized and leave the stack here; the + // key is a str or int and the container a shared dict, so neither + // release runs code. + let (c, k) = unsafe { (base.add(len - 2).read(), base.add(len - 1).read()) }; + drop_hot(c); + Some(crate::error::key_error_object(k)) + } + /// The native fast `__getitem__` the `BINARY_SUBSCR` at `pc` cached for /// `ops[0]`'s class (see [`Self::leaf_instance_subscript`]), when both /// operands leave by plain decrements. @@ -16279,6 +16430,7 @@ impl Interpreter { globals, builtins, builtins_obj, + exc_handlers, .. } = frame; let code: &CodeObject = code_rc; @@ -17662,6 +17814,55 @@ impl Interpreter { last = pc; pc += 1; } + // An `except` clause's entry and exit (the full handlers' + // common shapes; see `step`): an exception instance on + // entry, class operands that are BaseException subclasses. + OpCode::PushExcInfo => { + if !matches!(stack.last(), Some(Object::Instance(_))) { + break; + } + let exc = stack.pop().expect("checked above"); + let prev = self + .exc_info_stack + .borrow() + .last() + .map_or(Object::None, |pe| pe.instance.clone()); + stack.push(prev); + stack.push(exc.clone()); + let pe = PyException::new(exc); + exc_handlers.push((ins.arg, pe.clone())); + self.exc_info_stack.borrow_mut().push(pe); + last = pc; + pc += 1; + } + OpCode::CheckExcMatch => { + let n = stack.len(); + if n < 2 { + break; + } + let Some(matched) = Self::leaf_exc_match(&stack[n - 2], &stack[n - 1]) else { + break; + }; + let ty = std::mem::replace(&mut stack[n - 1], Object::Bool(matched)); + last = pc; + pc += 1; + self.release(ty); + } + OpCode::PopExcept => { + let Some(prev) = stack.pop() else { break }; + let popped = exc_handlers.pop(); + let top = self.exc_info_stack.borrow_mut().pop(); + // See the full handler: the handled exception dies + // here unless something else holds it. + self.recheck_frame_observed = true; + last = pc; + pc += 1; + drop(top); + if let Some((_, pe)) = popped { + self.release(pe.instance); + } + self.release(prev); + } _ => break, } } @@ -17670,6 +17871,43 @@ impl Interpreter { stop } + /// `CHECK_EXC_MATCH` of the exception instance `exc` against `ty`, a + /// class or a tuple of classes, when every class derives from + /// BaseException (`None` leaves the full handler its TypeError). + #[inline] + fn leaf_exc_match(exc: &Object, ty: &Object) -> Option { + let Object::Instance(inst) = exc else { + return None; + }; + let cls = inst.cls_raw(); + let base = &builtin_types().base_exception; + match ty { + Object::Type(t) => { + if std::ptr::eq(cls, Rc::as_ptr(t)) { + return Some(true); + } + if !t.is_subclass_of(base) { + return None; + } + Some(cls.is_subclass_of(t)) + } + Object::Tuple(items) => { + let mut matched = false; + for t in items.iter() { + let Object::Type(t) = t else { + return None; + }; + if !t.is_subclass_of(base) { + return None; + } + matched = matched || cls.is_subclass_of(t); + } + Some(matched) + } + _ => None, + } + } + /// The truth of an instance when it is settled natively: a class with /// default attribute access whose `__bool__` (else `__len__`) is a /// registered leaf builtin, or neither is defined (always true). @@ -22756,13 +22994,7 @@ impl Interpreter { .get(raised_at as usize) .copied() .unwrap_or(0); - let tb = Rc::new(PyTraceback { - frame: py_frame, - lineno, - lasti: raised_at, - raw_lasti: None, - next: RefCell::new(None), - }); + let tb = Rc::new(PyTraceback::new(py_frame, lineno, raised_at, None)); inst.slot_set("__traceback__", Object::Traceback(tb)); } (wrapper, Object::None) @@ -23626,6 +23858,111 @@ impl Interpreter { // ---------- exception handling ---------- + /// The raise-site half of the general loop's exception path, for an + /// exception arriving at a frame: CPython's `_PyErr_SetObject` chains + /// *every* fresh exception, including ones raised from C (here: Rust + /// opcodes and builtins), to the currently handled exception. + /// `RAISE_VARARGS` already did this at the raise site + /// (`context_settled`); a fresh Rust-raised error has empty traceback + /// and unset context/cause. + fn chain_fresh_exception(&self, exc: &mut PyException) { + if !exc.context_settled + && exc.context.is_none() + && exc.cause.is_none() + && exc.traceback.is_empty() + && !exc.suppress_tb_once + && !instance_has_nonnull_attr(&exc.instance, "__context__") + { + self.attach_implicit_context(exc); + if exc.context.is_some() { + Self::sync_exc_attrs(exc); + } + } else if exc.traceback.is_empty() && (exc.cause.is_some() || exc.context.is_some()) { + // A fresh Rust-raised error that already carries an + // explicit `cause`/`context` (e.g. `_io` chaining a + // misbehaving raw `readinto`'s `TypeError` into an + // `OSError`, `test_bad_readinto_type`) must still + // mirror those onto the instance dict so Python's + // `except` sees `e.__cause__`. + Self::sync_exc_attrs(exc); + } + } + + /// The quiet loop's exception path: `err`, raised by the instruction + /// before `frame.pc` in the running activation, goes to this frame's + /// handler without leaving the loop, exactly as the general loop's + /// path does when nothing observes the frame (no trace or monitoring + /// callback to fire, so no `'exception'`/RERAISE/EXCEPTION_HANDLED + /// event either). Hands `err` back when the frame has no handler for + /// it: the activation then finishes, and the exception propagates, + /// through the general loop as before. + #[inline(never)] + fn quiet_catch( + &mut self, + frame: &mut Frame, + shell: &mut QuietShell<'_>, + err: RuntimeError, + ) -> Result<(), RuntimeError> { + let RuntimeError::PyException(mut exc) = err else { + return Err(err); + }; + let raise_pc = frame.pc.wrapping_sub(1); + if find_handler(&frame.code.exception_table, raise_pc).is_none() + || crate::trace::any_observers_active() + { + return Err(RuntimeError::PyException(exc)); + } + self.chain_fresh_exception(&mut exc); + // The traceback entry records this activation's shell, and the + // handler's instructions run through `step`, which needs it. + self.flush_lean(frame, shell) + .lasti + .store(raise_pc, std::sync::atomic::Ordering::Relaxed); + match self.handle_exception(frame, exc) { + Ok(_) => Ok(()), + Err(e) => Err(e), + } + } + + /// The general loop's unwind of an inline activation that `exc` + /// leaves, when nothing observes the frame and no handler in it covers + /// the raise (see `quiet_catch`): this frame's traceback entry, then + /// `run_activation`'s epilogue for an exceptional exit. Returns the + /// propagating error. + #[inline(never)] + fn quiet_unwind( + &mut self, + frame: &mut Frame, + shell: Rc, + mut exc: PyException, + exc_depth: usize, + ) -> RuntimeError { + self.chain_fresh_exception(&mut exc); + let err = match self.handle_exception(frame, exc) { + Err(e) => e, + Ok(_) => RuntimeError::Internal("quiet unwind found a handler".to_owned()), + }; + self.pop_frame_shell(); + if shell + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) + { + // An exceptional exit leaves a frame object that outlives the + // activation (its traceback) at the last instruction, with + // its `f_locals` brought up to date. + let py = shell.materialized.borrow().clone(); + if let Some(py) = py { + shell.refresh_materialized(&py); + py.invalidate_locals(); + } + } + if self.exc_info_len() > exc_depth { + self.exc_info_stack.borrow_mut().truncate(exc_depth); + } + self.recycle_frame_shell(shell); + err + } + /// Look up a handler for `exc` at the current pc. If found, /// truncate the stack and jump to the handler. Otherwise propagate. fn handle_exception( @@ -23722,51 +24059,68 @@ impl Interpreter { // out-of-range `tb_lasti` values that made `traceback`'s // `_get_code_position` StopIterate (RFC 0054: asyncio cancellation // tracebacks crashed `format_exc` with PEP 479 RuntimeError). - let py_frame = frame - .py_frame - .clone() - .or_else(|| { - // Materialise this activation's shell — the traceback - // must reference *this* frame's snapshot, and it must be - // the *same* object the live stack holds so - // `frame.clear()` can refuse to wipe an executing frame - // (its shared locals storage!). Activation identity is - // the shared locals storage, which is exact even under - // recursion (the code object is shared). Searching the - // whole spine (not just the top) matters when a trace - // callback raises: the exception unwinds while the stack - // still holds dispatch-machinery entries above the frame - // being annotated (test_sys_settrace jump-from-'call'). - let idx = self - .frame_stack - .borrow() - .iter() - .rposition(|shell| Rc::ptr_eq(&shell.locals, &frame.locals)); - idx.and_then(|idx| crate::object::materialize_stack_at(&self.frame_stack, idx)) - }) - .unwrap_or_else(|| { - // Fall back to a fresh snapshot when neither source - // matches (e.g. a not-yet-entered generator frame - // receiving a throw, or an empty stack) so the chain - // stays non-empty. Cache it on generator-family frames - // so `gi_frame` keeps a stable identity. - let py = self.build_py_frame(frame, None); - py.lasti.set(lasti); - if frame.code.is_generator - || frame.code.is_coroutine - || frame.code.is_async_generator - { - frame.py_frame = Some(py.clone()); + let new_tb = match frame.py_frame.clone() { + Some(py_frame) => PyTraceback::new(py_frame, lineno, lasti, None), + None => { + // This activation's shell: the traceback must reference + // *this* frame's snapshot, and it must be the *same* + // object the live stack holds so `frame.clear()` can + // refuse to wipe an executing frame (its shared locals + // storage!). Activation identity is the shared locals + // storage, which is exact even under recursion (the code + // object is shared). Searching the whole spine (not just + // the top) matters when a trace callback raises: the + // exception unwinds while the stack still holds + // dispatch-machinery entries above the frame being + // annotated (test_sys_settrace jump-from-'call'). + let found = { + let stack = self.frame_stack.borrow(); + stack + .iter() + .rposition(|shell| Rc::ptr_eq(&shell.locals, &frame.locals)) + .map(|idx| (idx, stack[idx].clone())) + }; + match found { + // An ordinary activation nobody has looked at yet + // records its shell: the frame object is built only + // if something reads `tb_frame` or the entry outlives + // the activation (see `PyTraceback::new_lazy`). + Some((_, shell)) + if !shell.is_gen + && !shell + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) => + { + PyTraceback::new_lazy(shell, lineno, lasti) + } + found => { + let py_frame = found + .and_then(|(idx, _)| { + crate::object::materialize_stack_at(&self.frame_stack, idx) + }) + .unwrap_or_else(|| { + // Fall back to a fresh snapshot when neither + // source matches (e.g. a not-yet-entered + // generator frame receiving a throw, or an + // empty stack) so the chain stays non-empty. + // Cache it on generator-family frames so + // `gi_frame` keeps a stable identity. + let py = self.build_py_frame(frame, None); + py.lasti.set(lasti); + if frame.code.is_generator + || frame.code.is_coroutine + || frame.code.is_async_generator + { + frame.py_frame = Some(py.clone()); + } + py + }); + PyTraceback::new(py_frame, lineno, lasti, None) + } } - py - }); - let new_tb = Rc::new(PyTraceback { - frame: py_frame, - lineno, - lasti, - raw_lasti: None, - next: RefCell::new(None), - }); + } + }; + let new_tb = Rc::new(new_tb); // CPython chains outward: the catching frame ends up at the // *head*; `tb_next` walks toward the raise site. Each // propagation prepends the current frame's `tb` to the @@ -23805,13 +24159,12 @@ impl Interpreter { let Some(py_frame) = self.materialize_top_py_frame() else { return; }; - let new_tb = Rc::new(PyTraceback { - lineno: py_frame.last_line.get().unwrap_or(1), - lasti: py_frame.lasti.get(), - frame: py_frame, - raw_lasti: None, - next: RefCell::new(None), - }); + let new_tb = Rc::new(PyTraceback::new( + py_frame.clone(), + py_frame.last_line.get().unwrap_or(1), + py_frame.lasti.get(), + None, + )); inst.slot_set("__traceback__", Object::Traceback(new_tb)); } @@ -25322,6 +25675,7 @@ impl Interpreter { // it suspends (see `pop_frame_shell`); a later resume // re-derives it here, from the live spine. if fr.back.borrow().is_none() + && fr.lazy_back.borrow().is_none() && fr.on_stack.get() > 0 && (fr.code.is_generator || fr.code.is_coroutine @@ -25338,8 +25692,8 @@ impl Interpreter { crate::object::materialize_stack_at(&self.frame_stack, idx); } } - match fr.back.borrow().as_ref() { - Some(parent) => Ok(Object::Frame(parent.clone())), + match fr.back_frame() { + Some(parent) => Ok(Object::Frame(parent)), None => Ok(Object::None), } } @@ -25402,11 +25756,11 @@ impl Interpreter { ))), }, Object::Traceback(tb) => match name { - "tb_frame" => Ok(Object::Frame(tb.frame.clone())), + "tb_frame" => Ok(Object::Frame(tb.frame())), "tb_lineno" => Ok(Object::Int(i64::from(tb.lineno))), "tb_lasti" => Ok(match tb.raw_lasti { Some(v) => Object::Int(v), - None => Object::Int(i64::from(tb.frame.code.cpython_lasti(tb.lasti))), + None => Object::Int(i64::from(tb.code().cpython_lasti(tb.lasti))), }), "tb_next" => match tb.next.borrow().as_ref() { Some(n) => Ok(Object::Traceback(n.clone())), @@ -34777,7 +35131,7 @@ impl Interpreter { if line != 0 { py.last_line.set(Some(line)); } - *py.back.borrow_mut() = self.materialize_top_py_frame(); + py.set_back(self.materialize_top_py_frame()); self.push_materialized_frame(&py); let hook_result = self .fire_call_event(&py, crate::trace::EVENT_PY_THROW, exc.instance.clone()) @@ -34854,7 +35208,7 @@ impl Interpreter { frame.py_frame = Some(py.clone()); py }); - *py.back.borrow_mut() = self.materialize_top_py_frame(); + py.set_back(self.materialize_top_py_frame()); self.push_materialized_frame(&py); let exc_obj = match &err { RuntimeError::PyException(pe) => pe.instance.clone(), @@ -35002,13 +35356,12 @@ impl Interpreter { if let (Object::Instance(inst), Object::Frame(pf)) = (&pyexc.instance, &frame_obj) { - let tb = Rc::new(PyTraceback { - frame: pf.clone(), - lineno: pf.last_line.get().unwrap_or(1), - lasti: pf.lasti.get(), - raw_lasti: None, - next: RefCell::new(None), - }); + let tb = Rc::new(PyTraceback::new( + pf.clone(), + pf.last_line.get().unwrap_or(1), + pf.lasti.get(), + None, + )); inst.slot_set("__traceback__", Object::Traceback(tb)); } } @@ -35134,7 +35487,7 @@ impl Interpreter { py }); py.lasti.set(frame.pc); - *py.back.borrow_mut() = self.materialize_top_py_frame(); + py.set_back(self.materialize_top_py_frame()); self.push_materialized_frame(&py); let hook_result = self.fire_monitoring_event(&py, crate::trace::EVENT_EXCEPTION_HANDLED, inst); @@ -46479,17 +46832,18 @@ impl Interpreter { "TracebackType() lasti/lineno must be ints".to_owned(), )); }; - return Ok(Object::Traceback(Rc::new(crate::object::PyTraceback { - frame: frame.clone(), - lineno: u32::try_from(*lineno).unwrap_or(0), - lasti: u32::try_from(*lasti).unwrap_or(0), + let tb = crate::object::PyTraceback::new( + frame.clone(), + u32::try_from(*lineno).unwrap_or(0), + u32::try_from(*lasti).unwrap_or(0), // Explicit construction stores the given lasti // verbatim (test_raise TestTracebackType): it's // already a CPython-style byte offset, not a // WeavePy instruction index to remap on read. - raw_lasti: Some(*lasti), - next: RefCell::new(next), - }))); + Some(*lasti), + ); + *tb.next.borrow_mut() = next; + return Ok(Object::Traceback(Rc::new(tb))); } // `types.MethodType(func, obj)` — bind `func` to `obj`, // producing a callable bound method (CPython `method`). @@ -56971,6 +57325,7 @@ static SLOW_LEAF_OPS: [bool; 256] = { OpCode::BuildString, OpCode::BuildTuple, OpCode::Call, + OpCode::CheckExcMatch, OpCode::CompareOp, OpCode::CopyFreeVars, OpCode::CopyTop, @@ -56996,8 +57351,10 @@ static SLOW_LEAF_OPS: [bool; 256] = { OpCode::PopJumpIfFalse, OpCode::PopJumpIfNone, OpCode::PopJumpIfNotNone, + OpCode::PopExcept, OpCode::PopJumpIfTrue, OpCode::PopTop, + OpCode::PushExcInfo, OpCode::PushNull, OpCode::Resume, OpCode::StoreAttr, @@ -57809,12 +58166,11 @@ fn unbound_gen_method_sentinel(ty: &Rc, name: &str) -> Option<&'stat /// overwrite it. fn instance_has_nonnull_attr(instance: &Object, name: &str) -> bool { match instance { + // Read without publishing a `__dict__` the instance never needed + // (every fresh exception asks). Object::Instance(i) => { matches!(i.slot_get(name), Some(v) if !matches!(v, Object::None)) - || matches!( - i.dict_cell().borrow().get(&crate::object::StrKey(name)), - Some(v) if !matches!(v, Object::None) - ) + || matches!(i.attr_get_str(name), Some(v) if !matches!(v, Object::None)) } _ => false, } @@ -71066,9 +71422,26 @@ for _ in range(10): fn jit_native_string_arguments_validate_and_preserve_values() { std::thread::spawn(|| { crate::tier2::force_enable_for_test(2); - let out = run(include_str!( + // The fixture's case functions are loop-free, so calls from a + // quiet loop run them as inline activations, which never enter + // compiled code. Dispatch every call through the general path + // (its exact-arity `CALL` enters compiled callees directly), as + // a frame the quiet loops leave does: the driver loop's frame + // used to, once its first caught exception materialized it. + let module = parse_module(include_str!( "../../../tests/regrtest/test_native_string_arguments.py" - )); + )) + .expect("parse"); + let code = compile_module(&module).expect("compile"); + let mut interp = Interpreter::new(); + interp.quiet_off = true; + let buf: Rc>> = Rc::new(RefCell::new(Vec::new())); + let writer: Stdout = crate::rc_unsize!(buf.clone() => RefCell); + interp.set_stdout(writer); + interp + .run_module(&code) + .unwrap_or_else(|error| panic!("run: {error}")); + let out = String::from_utf8(buf.borrow().clone()).expect("utf-8"); assert_eq!(out, "ok\n"); assert!( crate::tier2::direct_call_count_for_test() >= 1, diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 6c70c2b9..c2b4063b 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -450,6 +450,35 @@ pub struct PyFrame { /// CPython clearing the cell slots out of `f_localsplus` while /// the cell objects live on in the closures that captured them. pub cleared: Cell, + /// `f_back` while it is still a live activation nobody has looked at: + /// a frame object materialized as its activation leaves the spine + /// (for a traceback entry that outlives it) links its caller lazily, + /// so the caller's own frame object is built only if `f_back` is read + /// or the caller leaves the spine while this link lives. Resolved + /// into [`Self::back`] by [`Self::back_frame`]. + pub(crate) lazy_back: RefCell>, +} + +impl PyFrame { + /// `f_back`, materializing a lazy link (see [`Self::lazy_back`]). + pub fn back_frame(&self) -> Option> { + let shell = match self.lazy_back.borrow().as_ref() { + None => return self.back.borrow().clone(), + Some(lazy) => lazy.0.clone(), + }; + let py = materialize_shell(&shell); + *self.back.borrow_mut() = Some(py.clone()); + *self.lazy_back.borrow_mut() = None; + Some(py) + } + + /// Replace `f_back` (dropping any lazy link). + pub fn set_back(&self, back: Option>) { + *self.back.borrow_mut() = back; + if self.lazy_back.borrow().is_some() { + *self.lazy_back.borrow_mut() = None; + } + } } impl fmt::Debug for PyFrame { @@ -465,6 +494,25 @@ impl fmt::Debug for PyFrame { impl Drop for PyFrame { fn drop(&mut self) { + // A dead frame's locals die with its frame object, and CPython's + // `frame_dealloc` releases them from the top of the value stack + // down, before `f_back`: finalizers observe that order (the + // innermost frame of a dropped traceback frees its last local + // first). + if let Some(locals) = self.locals_mirror.get_mut().take() { + if Rc::strong_count(&locals) == 1 { + if let Ok(mut values) = locals.try_borrow_mut() { + while let Some(value) = values.pop() { + drop(values); + drop(value); + values = match locals.try_borrow_mut() { + Ok(values) => values, + Err(_) => break, + }; + } + } + } + } // Iteratively tear down the `back` chain. A plain recursive drop // would drop this frame's `back` Arc, whose own drop drops the // *next* frame's `back`, and so on — one native stack frame per @@ -921,6 +969,10 @@ pub struct FrameShell { pub has_materialized: std::sync::atomic::AtomicBool, /// The real Python frame object, once someone asked for it. pub materialized: RefCell>>, + /// Live traceback entries that recorded this activation without + /// materializing it (see `PyTraceback::new_lazy`). The pop + /// materializes a shell that still has some. Zero for a parked shell. + pub tb_refs: std::sync::atomic::AtomicU32, } /// Metadata in a live frame shell. Recycled shells leave these slots @@ -1044,6 +1096,7 @@ impl FrameShell { lasti: std::sync::atomic::AtomicU32::new(py.lasti.get()), has_materialized: std::sync::atomic::AtomicBool::new(true), materialized: RefCell::new(Some(py.clone())), + tb_refs: std::sync::atomic::AtomicU32::new(0), } } @@ -1064,11 +1117,23 @@ impl FrameShell { } pub fn materialize(&self, back: Option>) -> Rc { + let py = self.materialize_departing(back); + // RFC 0065 (WS1): a freshly materialized frame must kick its + // dispatch loop off the quiet path so the per-instruction + // `lasti`-cell sync resumes. + crate::hot_gates::bump_loop_gen(); + py + } + + /// [`Self::materialize`] for an activation that is leaving the spine: + /// no dispatch loop will run it again, so none needs kicking off its + /// quiet path. + pub fn materialize_departing(&self, back: Option>) -> Rc { if let Some(existing) = self.materialized.borrow().as_ref() { + // (`materialize` still bumps: the new holder may keep it, and + // a quiet loop running this activation re-derives, finding it + // observed if so.) self.refresh_materialized(existing); - // The new holder may keep it: a quiet loop running this - // activation re-derives, and finds it observed if so. - crate::hot_gates::bump_loop_gen(); return existing.clone(); } let py = Rc::new(PyFrame { @@ -1094,14 +1159,11 @@ impl FrameShell { on_stack: Cell::new(1), extra_locals: RefCell::new(None), cleared: Cell::new(false), + lazy_back: RefCell::new(None), }); *self.materialized.borrow_mut() = Some(py.clone()); self.has_materialized .store(true, std::sync::atomic::Ordering::Release); - // RFC 0065 (WS1): a freshly materialized frame must kick its - // dispatch loop off the quiet path so the per-instruction - // `lasti`-cell sync resumes. - crate::hot_gates::bump_loop_gen(); py } @@ -1172,7 +1234,7 @@ pub fn materialize_stack_at(stack: &FrameStack, idx: usize) -> Option { shell.refresh_materialized(&py); refreshed = true; - *py.back.borrow_mut() = back; + py.set_back(back); py } None => { @@ -1205,7 +1267,10 @@ pub fn materialize_stack_top(stack: &FrameStack) -> Option> { /// unwind machinery and chained outward through [`Self::next`]. #[derive(Debug)] pub struct PyTraceback { - pub frame: Rc, + /// The frame this entry records: the frame object, or the live + /// activation's spine shell until something asks for the object + /// (see [`TbFrame`]). Read through [`Self::frame`]. + frame: RefCell, pub lineno: u32, pub lasti: u32, /// `types.TracebackType(…)` stores the caller-supplied `tb_lasti` @@ -1215,6 +1280,143 @@ pub struct PyTraceback { pub next: RefCell>>, } +/// A traceback entry's frame. +/// +/// CPython creates the frame object for every traceback entry, but its +/// frame objects are cheap; WeavePy's materialization links the whole +/// `f_back` chain and takes the running activation off its quiet path. +/// An exception caught and dropped in the frame that raised it (the +/// `try: d[k] except KeyError:` idiom) never shows anyone that frame, +/// so an entry recorded while its activation runs keeps the spine +/// shell instead and materializes from it on first use. While the +/// activation is live, materializing then is what materializing at the +/// raise would have produced by now (the object tracks the running +/// frame either way); when the activation leaves the spine with lazy +/// entries still alive, `pop_frame_shell` materializes it on the way +/// out, so the object exists exactly as CPython's would. +#[derive(Debug)] +enum TbFrame { + Ready(Rc), + Lazy(LazyShellRef), +} + +/// A hold on a live activation's shell standing for its frame object +/// until someone asks for it: a lazy traceback entry (see [`TbFrame`]) or +/// a frame's lazy `f_back` (see [`PyFrame::lazy_back`]). Counted in +/// [`FrameShell::tb_refs`] so the activation's pop knows to materialize. +#[derive(Debug)] +pub(crate) struct LazyShellRef(Rc); + +impl LazyShellRef { + pub(crate) fn new(shell: Rc) -> Self { + shell + .tb_refs + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Self(shell) + } +} + +impl Drop for LazyShellRef { + fn drop(&mut self) { + self.0 + .tb_refs + .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + } +} + +impl PyTraceback { + /// An entry for an existing frame object. + pub fn new(frame: Rc, lineno: u32, lasti: u32, raw_lasti: Option) -> Self { + Self { + frame: RefCell::new(TbFrame::Ready(frame)), + lineno, + lasti, + raw_lasti, + next: RefCell::new(None), + } + } + + /// An entry for the activation `shell` stands for, which must be + /// live on this thread's spine and not yet materialized (see + /// [`TbFrame`]). + pub fn new_lazy(shell: Rc, lineno: u32, lasti: u32) -> Self { + Self { + frame: RefCell::new(TbFrame::Lazy(LazyShellRef::new(shell))), + lineno, + lasti, + raw_lasti: None, + next: RefCell::new(None), + } + } + + /// `tb_frame`: the frame object, materialized on first use. + pub fn frame(&self) -> Rc { + let shell = match &*self.frame.borrow() { + TbFrame::Ready(py) => return py.clone(), + TbFrame::Lazy(lazy) => lazy.0.clone(), + }; + let py = materialize_shell(&shell); + *self.frame.borrow_mut() = TbFrame::Ready(py.clone()); + py + } + + /// The frame object if one exists already (no materialization). + pub fn frame_if_ready(&self) -> Option> { + match &*self.frame.borrow() { + TbFrame::Ready(py) => Some(py.clone()), + TbFrame::Lazy(lazy) => lazy.0.materialized.borrow().clone(), + } + } + + /// The entry's code object, without materializing its frame. + pub fn code(&self) -> Rc { + match &*self.frame.borrow() { + TbFrame::Ready(py) => py.code.clone(), + TbFrame::Lazy(lazy) => (*lazy.0.code).clone(), + } + } +} + +/// The frame object for `shell`, a lazy traceback entry's activation: +/// the one it already has, else materialized where it sits on this +/// thread's spine (linking `f_back` like any other materialization). +pub(crate) fn materialize_shell(shell: &Rc) -> Rc { + if let Some(handles) = crate::vm_singletons::current_thread_handles() { + let idx = handles + .frame_stack + .borrow() + .iter() + .rposition(|s| Rc::ptr_eq(s, shell)); + if let Some(idx) = idx { + if let Some(py) = materialize_stack_at(&handles.frame_stack, idx) { + return py; + } + } + } + if let Some(py) = shell.materialized.borrow().as_ref() { + shell.refresh_materialized(py); + return py.clone(); + } + // Live on another thread's spine (an exception from + // `sys._current_exceptions()`): materialize it there, as + // `sys._current_frames()` does; peer threads are parked under the GIL. + for (_, stack, _) in crate::stdlib::faulthandler_mod::thread_snapshots() { + let idx = stack + .try_borrow() + .ok() + .and_then(|s| s.iter().rposition(|s| Rc::ptr_eq(s, shell))); + if let Some(idx) = idx { + if let Some(py) = materialize_stack_at(&stack, idx) { + return py; + } + } + } + // A greenlet's parked spine: the shell's slots are valid while the + // activation lives, but the `f_back` chain belongs to that stack. Its + // pop decrements `on_stack` as for any materialization. + shell.materialize(None) +} + impl Drop for PyTraceback { fn drop(&mut self) { // Same unbounded-recursion hazard as `PyFrame::drop`: a traceback diff --git a/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs b/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs index 6575d610..f3129168 100644 --- a/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs +++ b/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs @@ -208,6 +208,7 @@ fn frame_new(args: &[Object]) -> Result { on_stack: Cell::new(0), extra_locals: RefCell::new(None), cleared: Cell::new(false), + lazy_back: RefCell::new(None), }); Ok(Object::Frame(frame)) } diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 3ef874ee..0db40f75 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -5896,10 +5896,13 @@ fn call_with_activation_shell( lasti: std::sync::atomic::AtomicU32::new(jf.deopt_pc), has_materialized: std::sync::atomic::AtomicBool::new(false), materialized: GilRefCell::new(None), + tb_refs: std::sync::atomic::AtomicU32::new(0), }); interp.frame_stack.borrow_mut().push(shell); let out = f(interp); - interp.frame_stack.borrow_mut().pop(); + // The general pop: a traceback entry or a callee frame's `f_back` may + // hold the shell lazily, and gets its frame object on the way out. + interp.pop_frame_shell(); out } From 32ff811481aef3c56bda8b539c64f7aef5cc910b Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:02:35 -0700 Subject: [PATCH 62/96] perf: trim the remaining per-exception bookkeeping - A fresh exception raised while nothing is being handled settles its (empty) implicit context without probing the instance's own `__context__`. - A traceback entry replaces the chain head in one probe of the slot table instead of a read and a write. - `ValueError("x")` and other classes outside the special exception families get their `args` slot built in place; no exception keeps the `message` mirror (CPython has none; `str()` and the Rust display derive the text from `args`). - An inline activation whose shell exists only because it caught an exception returns through the core loop's `core_return`, and one whose traceback outlives it (`except E as e: return e`) finishes in `inline_finish` instead of the general loop: the shell's pop builds the frame object, and `settle_exited_frame_object` gives it the general epilogue's locals handling. Instructions retired per iteration (/usr/bin/time -l slopes): try: d[5] except KeyError 8,102 -> 7,451 (CPython 2,301) same, raised in a callee 12,578 -> 11,684 (CPython 3,593) raise ValueError; except as e 14,171 -> 12,506 (CPython 2,687) --- crates/weavepy-vm/src/lib.rs | 166 ++++++++++++++++++++++++++++------- 1 file changed, 135 insertions(+), 31 deletions(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 45179b82..acc40a40 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -8245,10 +8245,12 @@ impl Interpreter { ) if !shell .has_materialized .load(std::sync::atomic::Ordering::Relaxed) - && shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) == 0 && self.exc_info_len() <= exc_depth => { self.pop_frame_shell(); + // A lazy traceback entry still holding the shell had its + // frame object built by the pop. + Self::settle_exited_frame_object(&shell, frame, true); self.recycle_frame_shell(shell); Ok(FrameOutcome::Returned(v)) } @@ -9849,10 +9851,12 @@ impl Interpreter { ) if !shell .has_materialized .load(std::sync::atomic::Ordering::Relaxed) - && shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) == 0 && self.exc_info_len() <= exc_depth_on_entry => { self.pop_frame_shell(); + // A lazy traceback entry still holding the shell had its + // frame object built by the pop. + Self::settle_exited_frame_object(&shell, frame, true); self.recycle_frame_shell(shell); FrameOutcome::Returned(v) } @@ -15325,11 +15329,15 @@ impl Interpreter { let Some(top) = inl.last_mut() else { return false; }; - // A generator resume, or an activation that needed its shell, - // finishes through `quiet_run`; so does a return into a caller - // whose protocol would yield straight away. + // A generator resume, or an activation whose shell the full + // epilogue would act on, finishes through `quiet_run`; so does a + // return into a caller whose protocol would yield straight away. if top.gen.is_some() - || top.act.shell.is_some() + || top + .act + .shell + .as_deref() + .is_some_and(|s| !self.shell_pops_quietly(s, top.exc_depth)) || self.gil_countdown <= 2 || crate::hot_gates::loop_gen() != snap_gen { @@ -15346,6 +15354,16 @@ impl Interpreter { if depth == sw.entry_depth { sw.entry_dead = true; } + // An activation that caught an exception got its shell then (see + // `quiet_catch`): retire it as `inline_finish` does. Such a frame + // is not clean (the handler path set its exception fields). + let had_shell = match done.act.shell.take() { + Some(shell) => { + self.retire_quiet_shell(shell, frame, cur_pc); + true + } + None => false, + }; // `inline_finish`'s shell-less return, its common case in line: no // leftover operands or cells, and locals only this activation // holds. They are released here; an object that dies with them @@ -15356,7 +15374,7 @@ impl Interpreter { let mut tmp = None; let (cframe, clast, cshell, entry); if clean && done.init_inst.is_none() { - done.clean = true; + done.clean = !had_shell; // SAFETY: as above. for v in unsafe { (*frame.locals.as_ptr()).drain(..) } { drop_hot(v); @@ -15379,7 +15397,7 @@ impl Interpreter { entry = QuietEntry::Returned { cur_pc: call_pc }; } else { let result = if clean { - done.clean = true; + done.clean = !had_shell; // SAFETY: as above. for v in unsafe { (*frame.locals.as_ptr()).drain(..) } { drop_hot(v); @@ -15395,6 +15413,7 @@ impl Interpreter { }, ) }; + done.clean &= !had_shell; // SAFETY: the caller is the innermost remaining activation. (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; // SAFETY: as above. @@ -23866,13 +23885,19 @@ impl Interpreter { /// (`context_settled`); a fresh Rust-raised error has empty traceback /// and unset context/cause. fn chain_fresh_exception(&self, exc: &mut PyException) { - if !exc.context_settled + let fresh = !exc.context_settled && exc.context.is_none() && exc.cause.is_none() && exc.traceback.is_empty() - && !exc.suppress_tb_once - && !instance_has_nonnull_attr(&exc.instance, "__context__") - { + && !exc.suppress_tb_once; + // Nothing is being handled: there is no context to chain (the + // decision `attach_implicit_context` would settle the same way), + // so the instance's own `__context__` need not be probed. + if fresh && self.exc_info_len() == 0 { + exc.context_settled = true; + return; + } + if fresh && !instance_has_nonnull_attr(&exc.instance, "__context__") { self.attach_implicit_context(exc); if exc.context.is_some() { Self::sync_exc_attrs(exc); @@ -23943,19 +23968,7 @@ impl Interpreter { Ok(_) => RuntimeError::Internal("quiet unwind found a handler".to_owned()), }; self.pop_frame_shell(); - if shell - .has_materialized - .load(std::sync::atomic::Ordering::Relaxed) - { - // An exceptional exit leaves a frame object that outlives the - // activation (its traceback) at the last instruction, with - // its `f_locals` brought up to date. - let py = shell.materialized.borrow().clone(); - if let Some(py) = py { - shell.refresh_materialized(&py); - py.invalidate_locals(); - } - } + Self::settle_exited_frame_object(&shell, frame, false); if self.exc_info_len() > exc_depth { self.exc_info_stack.borrow_mut().truncate(exc_depth); } @@ -23963,6 +23976,67 @@ impl Interpreter { err } + /// Whether an inline activation returning with `shell` needs nothing + /// of the full epilogue: no frame object yet and no handled-exception + /// entries left above its entry depth (`inline_finish`'s second arm). + #[inline] + fn shell_pops_quietly(&self, shell: &crate::object::FrameShell, exc_depth: usize) -> bool { + !shell + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) + && self.exc_info_len() <= exc_depth + } + + /// Retire the shell of a returning inline activation that + /// [`Self::shell_pops_quietly`] accepted, its return at `cur_pc`. + #[cold] + #[inline(never)] + fn retire_quiet_shell( + &mut self, + shell: Rc, + frame: &Frame, + cur_pc: usize, + ) { + if shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) != 0 { + // See `sync_lazy_tb_lasti`. + shell + .lasti + .store(cur_pc as u32, std::sync::atomic::Ordering::Relaxed); + } + self.pop_frame_shell(); + Self::settle_exited_frame_object(&shell, frame, true); + self.recycle_frame_shell(shell); + } + + /// `run_activation`'s epilogue for the frame object of an activation + /// that just left the spine (`shell` popped), for the lean finishes: + /// leave it at the last executed instruction, and give a returned + /// frame's object that outlives the activation its own locals + /// (CPython's `take_ownership`), else bring its `f_locals` current. + fn settle_exited_frame_object( + shell: &crate::object::FrameShell, + frame: &Frame, + returned: bool, + ) { + if !shell + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) + { + return; + } + let Some(py) = shell.materialized.borrow().clone() else { + return; + }; + shell.refresh_materialized(&py); + // The shell's cell and `py`; see `run_activation`. + let internal = 2 + usize::from(frame.py_frame.as_ref().is_some_and(|p| Rc::ptr_eq(p, &py))); + if returned && Rc::strong_count(&py) > internal { + py.take_ownership_of_locals(); + } else { + py.invalidate_locals(); + } + } + /// Look up a handler for `exc` at the current pc. If found, /// truncate the stack and jump to the handler. Otherwise propagate. fn handle_exception( @@ -24125,11 +24199,25 @@ impl Interpreter { // *head*; `tb_next` walks toward the raise site. Each // propagation prepends the current frame's `tb` to the // existing chain. + // One probe of the slot table: replace an existing chain's head in + // place, else add the slot. if let Object::Instance(inst) = &exc.instance { - if let Some(Object::Traceback(prev_tb)) = inst.slot_get("__traceback__") { - *new_tb.next.borrow_mut() = Some(prev_tb); + let new_tb = Object::Traceback(new_tb); + inst.note_slot_store(&new_tb); + let mut slots = inst.slots.borrow_mut(); + match slots.get_mut("__traceback__") { + Some(slot) => { + let prev = std::mem::replace(slot, new_tb); + if let (Object::Traceback(prev_tb), Object::Traceback(head)) = (&prev, &*slot) { + *head.next.borrow_mut() = Some(prev_tb.clone()); + } + drop(slots); + drop(prev); + } + None => { + slots.insert("__traceback__", new_tb); + } } - inst.slot_set("__traceback__", Object::Traceback(new_tb)); } } @@ -47466,6 +47554,23 @@ impl Interpreter { // class against its attribute version (the MRO walk compared // nine names per entry on every construction). let families = exc_family_flags(&cls); + // The common case (`ValueError("x")`, a plain user subclass): no + // family shapes construction, so `args` is the only slot, built + // in place without the per-slot probes below. + if families & !EXC_CUSTOM_INIT == 0 { + let inst = PyInstance::new(cls); + let args_tuple = Object::new_tuple(args.to_vec()); + inst.note_slot_store(&args_tuple); + let mut entries = Vec::with_capacity(4); + entries.push((crate::types::slot_key("args"), args_tuple)); + *inst.slots.borrow_mut() = crate::types::SlotStorage::from_entries(entries); + let obj = Object::Instance(Rc::new(inst)); + // See the GC note at the end. + if args.iter().any(|x| !crate::gc_trace::is_atomic(x)) { + crate::gc_trace::track(&obj); + } + return obj; + } let mro_has = |name: &str| families & exc_family_bit(name) != 0; let is_stop_iteration = mro_has("StopIteration"); let inst = PyInstance::new(cls.clone()); @@ -47484,10 +47589,9 @@ impl Interpreter { } else { Object::new_tuple(args.to_vec()) }; + // (No `message` mirror: CPython has none, and `str()` and the Rust + // display derive the text from `args`.) inst.slot_set("args", args_tuple); - if let Some(first) = args.first() { - inst.slot_set("message", first.clone()); - } // PEP 380: `StopIteration.value` is the first constructor arg // (or None). Generator `return` goes through // `stop_iteration_with`, but user code constructs From 62f9ba7383d6e4f2adfd4c9e5a9127fdfa306ac8 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:08:47 -0700 Subject: [PATCH 63/96] perf: run re.Pattern and re.Match methods natively `re.Pattern`, `re.Match` and a new `_sre.SRE_Scanner` stay Python classes, now with `__slots__`, and `_sre.install` replaces their hot methods with native bodies over a shared slot layout: `match`, `fullmatch`, `search`, `findall`, `finditer`, `scanner`, `sub`, `subn` and `split`, and `group`, `__getitem__`, `groups`, `groupdict`, `start`, `end` and `span`. Patterns, matches and scanners are built natively. A native serves exact `str` or `bytes` subjects with plain arguments, and its pure half is a registered leaf; a callable replacement or a template not yet parsed takes the native full body; anything else calls the Python method it replaced. The engine matches an ASCII `str` or a `bytes` subject over its bytes in place instead of decoding it, its search uses the INFO block's literal prefix and leading charset as CPython's does, a choice point outside a repeat saves only `lastmark`/`lastindex`, the stack-growth check runs at every eighth level, and the Unicode classes test ASCII first. `re.search(...)` and the other module functions look `_cache2` up natively (`_sre.compile_cached`) before calling `_compile`. Behavior changes, all toward CPython: a callable replacement that returns None inserts nothing, a negative `count` substitutes nothing, a replacement with no backslash is used without template parsing (bytes subclasses work), `re.Pattern()`/`re.Match()` raise "cannot create", a group that did not take part reports (-1, -1), and `finditer` over a `str`/`bytes` returns an `SRE_Scanner`. Instructions retired per call (before -> after; CPython 3.14): search+group 78242 -> 19404 (12322), match 34032 -> 6165 (3228), fullmatch 33631 -> 6278 (3338), findall 1387634 -> 56087 (52283), finditer 198194 -> 40932 (15894), sub str 190071 -> 12765 (12257), sub func 296798 -> 52715 (23800), split 143064 -> 22358 (14079), module re.search 66003 -> 22701 (13892), groups/span/start/end 180722 -> 12153 (6717), sub template 340668 -> 10244 (15120). --- crates/weavepy-vm/src/stdlib/mod.rs | 1 + .../weavepy-vm/src/stdlib/python/re_engine.py | 181 +- .../weavepy-vm/src/stdlib/python/re_init.py | 25 +- crates/weavepy-vm/src/stdlib/sre_mod.rs | 545 ++++-- crates/weavepy-vm/src/stdlib/sre_objects.rs | 1533 +++++++++++++++++ 5 files changed, 2077 insertions(+), 208 deletions(-) create mode 100644 crates/weavepy-vm/src/stdlib/sre_objects.rs diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index 1d96968f..b2a81217 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -103,6 +103,7 @@ pub mod signal_mod; pub mod socket_mod; pub mod sqlite3_native; pub mod sre_mod; +pub(crate) mod sre_objects; pub mod statistics_accel; pub mod struct_mod; pub mod subprocess_mod; diff --git a/crates/weavepy-vm/src/stdlib/python/re_engine.py b/crates/weavepy-vm/src/stdlib/python/re_engine.py index 4e97d43d..60ad9e7c 100644 --- a/crates/weavepy-vm/src/stdlib/python/re_engine.py +++ b/crates/weavepy-vm/src/stdlib/python/re_engine.py @@ -2,11 +2,14 @@ # WeavePy: user-visible Pattern / Match objects for the re package. # # CPython implements `re.Pattern` and `re.Match` as C types inside the -# `_sre` extension. WeavePy instead keeps `_sre` as a pure-data -# backtracking core (compile + exec returning group spans) and builds -# the user-facing objects here, in Python. Doing so means callable -# `re.sub` replacements, `finditer`, `Scanner`, etc. all run on the -# normal interpreter without the engine ever re-entering the VM. +# `_sre` extension. WeavePy declares them here as Python classes with +# `__slots__`, and `_sre.install` (at the end of this module) replaces +# their hot methods with native bodies that work on the slot storage. +# The natives serve exact `str`/`bytes` subjects with plain arguments; +# everything else calls the Python methods below, which remain the +# reference implementation. Patterns, matches and scanners are always +# built by `_sre` (`make_pattern`, `exec_match`), never by calling the +# classes. # # Behaviour (group semantics, greedy/lazy scanning, empty-match # handling, split/sub/subn rules) follows CPython 3.13 exactly. @@ -37,8 +40,9 @@ def compile_pattern(pattern, flags, code, groups, groupindex, indexgroup): """Build a Pattern. Called by re._compiler.compile().""" - handle = _sre.compile(code, groups) - return Pattern(handle, pattern, flags, groups, groupindex, indexgroup) + # Issue 14260: a non-modifiable mapping, like CPython's C getter. + return _sre.make_pattern(Pattern, pattern, flags, code, groups, + _MappingProxy(groupindex), indexgroup) # Code-point length is O(n) for WeavePy's UTF-8 `str`, and regex-driven @@ -109,6 +113,12 @@ def _clamp_span(string, pos, endpos): class Pattern: __module__ = 're' + # The native methods read these by position (see `_sre.install`). + # `_code` is the compiled code's handle; `_indexgroup` maps a group + # number to its name or None. + __slots__ = ('_code', 'pattern', 'flags', 'groups', 'groupindex', + '_indexgroup', '__weakref__') + # PEP 585: `re.Pattern[str]` / `re.Pattern[bytes]` yield a # `types.GenericAlias` (CPython exposes this on the C `Pattern` type). __class_getitem__ = classmethod(_GenericAlias) @@ -118,15 +128,8 @@ class Pattern: def __init_subclass__(cls, /, **kwargs): raise TypeError("type 're.Pattern' is not an acceptable base type") - def __init__(self, handle, pattern, flags, groups, groupindex, indexgroup): - self._handle = handle - self.pattern = pattern - self.flags = flags - self.groups = groups - # Issue 14260: a non-modifiable mapping, like CPython's C getter. - self.groupindex = _MappingProxy(groupindex) - # tuple: group number (1-based) -> name or None - self._indexgroup = indexgroup + def __new__(cls, *args, **kwargs): + raise TypeError("cannot create 're.Pattern' instances") # CPython `_sre` pattern rich compare (bpo-35966): equal iff the # pattern string/bytes and effective flags agree; hash must follow. @@ -152,19 +155,22 @@ def __hash__(self): # -- internal -------------------------------------------------------- - def _exec(self, string, pos, endpos, mode, must_advance): + def _exec(self, string, pos, endpos, mode, must_advance, mpos=None): # CPython `_sre` rejects a str/bytes mismatch between the pattern and # the subject before matching (test_fnmatch.test_mix_bytes_str, plus - # the analogous re tests). `self.pattern` is the original str/bytes. + # the analogous re tests). `self.pattern` is the original str/bytes, + # or None for a `re.Scanner` pattern, which accepts either. if isinstance(self.pattern, str): if not isinstance(string, str): raise TypeError( "cannot use a string pattern on a bytes-like object") - elif isinstance(string, str): + elif self.pattern is not None and isinstance(string, str): raise TypeError( "cannot use a bytes pattern on a string-like object") - return _sre.exec(self._handle, string, pos, endpos, mode, - 1 if must_advance else 0) + # A Match reporting `mpos` (default `pos`) and `endpos`, or None. + return _sre.exec_match(self, string, pos, endpos, mode, + 1 if must_advance else 0, + pos if mpos is None else mpos) def _iter(self, string, pos, endpos, guard=None): # `guard` is a live buffer export pinning a mutable subject @@ -175,11 +181,12 @@ def _iter(self, string, pos, endpos, guard=None): must_advance = False opos, oendpos = pos, endpos while pos <= endpos: - r = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance) - if r is None: + m = self._exec(string, pos, endpos, _MODE_SEARCH, + must_advance, opos) + if m is None: break - start, end = r[0], r[1] - yield Match(self, string, opos, oendpos, r) + start, end = m._marks[0], m._marks[1] + yield m must_advance = start == end pos = end finally: @@ -190,24 +197,15 @@ def _iter(self, string, pos, endpos, guard=None): def match(self, string, pos=0, endpos=None): p, e = _clamp_span(string, pos, endpos) - r = self._exec(string, p, e, _MODE_MATCH, False) - if r is None: - return None - return Match(self, string, p, e, r) + return self._exec(string, p, e, _MODE_MATCH, False) def fullmatch(self, string, pos=0, endpos=None): p, e = _clamp_span(string, pos, endpos) - r = self._exec(string, p, e, _MODE_FULLMATCH, False) - if r is None: - return None - return Match(self, string, p, e, r) + return self._exec(string, p, e, _MODE_FULLMATCH, False) def search(self, string, pos=0, endpos=None): p, e = _clamp_span(string, pos, endpos) - r = self._exec(string, p, e, _MODE_SEARCH, False) - if r is None: - return None - return Match(self, string, p, e, r) + return self._exec(string, p, e, _MODE_SEARCH, False) def findall(self, string, pos=0, endpos=None): g = self.groups @@ -242,11 +240,19 @@ def subn(self, repl, string, count=0): return self._subx(repl, string, count) def _subx(self, repl, string, count): + # CPython substitutes nothing for a negative count. if count < 0: count = 0 + limit = 0 + else: + limit = None empty = _plain_slice(string, 0, 0) if callable(repl): filt = repl + elif _is_literal(repl): + # Like CPython, a replacement with no backslash is used as is, + # without parsing it as a template. + filt = lambda m, _l=repl: _l else: template = _compile_template(self, repl) if len(template) == 1 and not isinstance(template[0], int): @@ -262,15 +268,17 @@ def _subx(self, repl, string, count): endpos = _subject_len(string) must_advance = False while pos <= endpos: - if count and n >= count: + if (count and n >= count) or limit == 0: break - r = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance) - if r is None: + m = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance, 0) + if m is None: break - start, end = r[0], r[1] + start, end = m._marks[0], m._marks[1] out.append(_plain_slice(string, last, start)) - m = Match(self, string, 0, endpos, r) - out.append(filt(m)) + item = filt(m) + # CPython skips a None replacement. + if item is not None: + out.append(item) last = end n += 1 must_advance = start == end @@ -291,11 +299,10 @@ def split(self, string, maxsplit=0): while pos <= endpos: if maxsplit and n >= maxsplit: break - r = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance) - if r is None: + m = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance, 0) + if m is None: break - start, end = r[0], r[1] - m = Match(self, string, 0, endpos, r) + start, end = m._marks[0], m._marks[1] out.append(_plain_slice(string, last, start)) for i in range(1, g + 1): out.append(m.group(i)) @@ -362,6 +369,11 @@ def _flags_repr(flags): class Match: __module__ = 're' + # The native methods read these by position (see `_sre.install`). + # `_marks` holds the match span, then each group's span, with -1 for a + # group that did not take part; `_lastindex` is -1 for none. + __slots__ = ('re', 'string', 'pos', 'endpos', '_marks', '_lastindex') + # PEP 585: `re.Match[str]` / `re.Match[bytes]` yield a `types.GenericAlias`. __class_getitem__ = classmethod(_GenericAlias) @@ -370,22 +382,13 @@ class Match: def __init_subclass__(cls, /, **kwargs): raise TypeError("type 're.Match' is not an acceptable base type") - def __init__(self, pattern, string, pos, endpos, r): - self.re = pattern - self.string = string - self.pos = pos - self.endpos = endpos - self._start = r[0] - self._end = r[1] - self._lastindex_raw = r[2] - self._marks = r[3] + def __new__(cls, *args, **kwargs): + raise TypeError("cannot create 're.Match' instances") # -- group span helpers --------------------------------------------- def _span_of(self, idx): - if idx == 0: - return (self._start, self._end) - i = (idx - 1) * 2 + i = idx * 2 return (self._marks[i], self._marks[i + 1]) def _index(self, group): @@ -439,14 +442,11 @@ def span(self, group=0): @property def regs(self): - spans = [(self._start, self._end)] - for i in range(1, self.re.groups + 1): - spans.append(self._span_of(i)) - return tuple(spans) + return tuple(self._span_of(i) for i in range(self.re.groups + 1)) @property def lastindex(self): - li = self._lastindex_raw + li = self._lastindex return None if li < 0 else li @property @@ -469,9 +469,10 @@ def __deepcopy__(self, memo): return self def __repr__(self): - text = _plain_slice(self.string, self._start, self._end) + start, end = self._marks[0], self._marks[1] + text = _plain_slice(self.string, start, end) return "" % ( - self._start, self._end, text) + start, end, text) # CPython's Pattern/Match are C static types whose `tp_name` carries the @@ -481,6 +482,23 @@ def __repr__(self): __weavepy_set_tp_name__(Match, "re.Match") +class SRE_Scanner: + """The scanner `finditer` and `Pattern.scanner` return for an exact + `str` or `bytes` subject. Built by `_sre` only; `match`, `search` and + the iterator protocol are native (see `_sre.install`).""" + + __module__ = '_sre' + __slots__ = ('pattern', '_string', '_start', '_pos', '_endpos', + '_must_advance') + + def __new__(cls, *args, **kwargs): + raise TypeError("cannot create '_sre.SRE_Scanner' instances") + + def __init_subclass__(cls, /, **kwargs): + raise TypeError( + "type '_sre.SRE_Scanner' is not an acceptable base type") + + class _Scanner: def __init__(self, pattern, string, pos, endpos): self.pattern = pattern @@ -499,14 +517,14 @@ def search(self): def _run(self, mode): if self._pos > self._endpos: return None - r = _sre.exec(self.pattern._handle, self._string, self._pos, - self._endpos, mode, 1 if self._must_advance else 0) - if r is None: - if mode == _MODE_MATCH: - return None + m = _sre.exec_match(self.pattern, self._string, self._pos, + self._endpos, mode, + 1 if self._must_advance else 0, self._opos) + if m is None: + # Exhausted, like CPython's scanner. + self._pos = self._endpos + 1 return None - start, end = r[0], r[1] - m = Match(self.pattern, self._string, self._opos, self._oendpos, r) + start, end = m._marks[0], m._marks[1] self._must_advance = start == end self._pos = end return m @@ -516,13 +534,24 @@ def _run(self, mode): # Replacement-template handling # --------------------------------------------------------------------------- +def _is_literal(repl): + """Whether `repl` is a str or bytes-like object with no backslash + (CPython's `pattern_subx` check).""" + if isinstance(repl, str): + return str.find(repl, '\\') < 0 + try: + return b'\\' not in memoryview(repl).tobytes() + except TypeError: + return False + + def _parse_template(pattern, repl): return _parser.parse_template(repl, pattern) def _compile_template(pattern, repl): try: - key = (pattern._handle, repl) + key = (pattern._code, repl) return _template_cache[key] except KeyError: pass @@ -553,3 +582,7 @@ def _expand_template(template, match): def clear_template_cache(): _template_cache.clear() + _sre.clear_templates(Pattern) + + +_sre.install(Pattern, Match, SRE_Scanner, _compile_template) diff --git a/crates/weavepy-vm/src/stdlib/python/re_init.py b/crates/weavepy-vm/src/stdlib/python/re_init.py index 62980985..bba3a862 100644 --- a/crates/weavepy-vm/src/stdlib/python/re_init.py +++ b/crates/weavepy-vm/src/stdlib/python/re_init.py @@ -45,6 +45,9 @@ from . import _engine import functools import _sre +# WeavePy: `_compile(pattern, flags)` answered natively from `_cache2` on a +# hit (see `_compile_front` below); a miss calls `_compile`. +from _sre import compile_cached as _compile_cached # public symbols @@ -83,17 +86,17 @@ class RegexFlag(enum.IntFlag, boundary=enum.KEEP): def match(pattern, string, flags=0): """Try to apply the pattern at the start of the string, returning a Match object, or None if no match was found.""" - return _compile(pattern, flags).match(string) + return _compile_cached(_compile_front, pattern, flags).match(string) def fullmatch(pattern, string, flags=0): """Try to apply the pattern to all of the string, returning a Match object, or None if no match was found.""" - return _compile(pattern, flags).fullmatch(string) + return _compile_cached(_compile_front, pattern, flags).fullmatch(string) def search(pattern, string, flags=0): """Scan through string looking for a match to the pattern, returning a Match object, or None if no match was found.""" - return _compile(pattern, flags).search(string) + return _compile_cached(_compile_front, pattern, flags).search(string) class _ZeroSentinel(int): pass @@ -128,7 +131,7 @@ def sub(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel if flags is _zero_sentinel: flags = 0 - return _compile(pattern, flags).sub(repl, string, count) + return _compile_cached(_compile_front, pattern, flags).sub(repl, string, count) sub.__text_signature__ = '(pattern, repl, string, count=0, flags=0)' def subn(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel): @@ -162,7 +165,7 @@ def subn(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentine if flags is _zero_sentinel: flags = 0 - return _compile(pattern, flags).subn(repl, string, count) + return _compile_cached(_compile_front, pattern, flags).subn(repl, string, count) subn.__text_signature__ = '(pattern, repl, string, count=0, flags=0)' def split(pattern, string, *args, maxsplit=_zero_sentinel, flags=_zero_sentinel): @@ -195,7 +198,7 @@ def split(pattern, string, *args, maxsplit=_zero_sentinel, flags=_zero_sentinel) if flags is _zero_sentinel: flags = 0 - return _compile(pattern, flags).split(string, maxsplit) + return _compile_cached(_compile_front, pattern, flags).split(string, maxsplit) split.__text_signature__ = '(pattern, string, maxsplit=0, flags=0)' def findall(pattern, string, flags=0): @@ -206,18 +209,18 @@ def findall(pattern, string, flags=0): has more than one group. Empty matches are included in the result.""" - return _compile(pattern, flags).findall(string) + return _compile_cached(_compile_front, pattern, flags).findall(string) def finditer(pattern, string, flags=0): """Return an iterator over all non-overlapping matches in the string. For each match, the iterator returns a Match object. Empty matches are included in the result.""" - return _compile(pattern, flags).finditer(string) + return _compile_cached(_compile_front, pattern, flags).finditer(string) def compile(pattern, flags=0): "Compile a regular expression pattern, returning a Pattern object." - return _compile(pattern, flags) + return _compile_cached(_compile_front, pattern, flags) def purge(): "Clear the regular expression caches" @@ -299,6 +302,10 @@ def _compile(pattern, flags): _cache2[key] = p return p +# What `_compile_cached` needs: the FIFO cache it reads, the full lookup it +# falls back to, and the flag type `_compile` normalizes. +_compile_front = (_cache2, _compile, RegexFlag) + # register myself for pickling import copyreg diff --git a/crates/weavepy-vm/src/stdlib/sre_mod.rs b/crates/weavepy-vm/src/stdlib/sre_mod.rs index 986fa982..d39ef159 100644 --- a/crates/weavepy-vm/src/stdlib/sre_mod.rs +++ b/crates/weavepy-vm/src/stdlib/sre_mod.rs @@ -86,6 +86,11 @@ const OP_LITERAL_UNI_IGNORE: u32 = 40; const OP_NOT_LITERAL_UNI_IGNORE: u32 = 41; const OP_RANGE_UNI_IGNORE: u32 = 42; +// INFO block flags. +const SRE_INFO_PREFIX: u32 = 1; +const SRE_INFO_LITERAL: u32 = 2; +const SRE_INFO_CHARSET: u32 = 4; + // AT codes. const AT_BEGINNING: u32 = 0; const AT_BEGINNING_LINE: u32 = 1; @@ -134,9 +139,13 @@ const MAX_DEPTH: u32 = 200_000; // Compiled-pattern registry // --------------------------------------------------------------------------- -struct CompiledCode { - code: Vec, - groups: usize, +pub(crate) struct CompiledCode { + pub(crate) code: Vec, + pub(crate) groups: usize, + /// Whether the pattern was a `str` (`Some(true)`) or a bytes-like + /// object (`Some(false)`); `None` for a pattern compiled from a parsed + /// tree (`re.Scanner`), which, like CPython's, accepts either subject. + pub(crate) is_str: Option, } /// Process-global compiled-pattern registry. @@ -148,13 +157,49 @@ struct CompiledCode { /// thread tries to `exec` it (e.g. `threading`/`logging` module-level /// regexes used from a `Thread.run`). Handles are indices that only /// ever grow, so they stay valid across threads; `CompiledCode` is -/// immutable after compilation, so sharing it is race-free. -fn registry() -> &'static parking_lot::Mutex>> { - static REGISTRY: std::sync::OnceLock>>> = +/// immutable after compilation, so sharing it is race-free. Entries are +/// never removed, so each one is leaked once and shared by reference. +fn registry() -> &'static parking_lot::Mutex> { + static REGISTRY: std::sync::OnceLock>> = std::sync::OnceLock::new(); REGISTRY.get_or_init(|| parking_lot::Mutex::new(Vec::new())) } +thread_local! { + /// This thread's copy of the registry's prefix, so a lookup takes no + /// lock (entries never change once registered). + static LOCAL_CODES: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; +} + +/// Register compiled code; the handle indexes the registry. +pub(crate) fn register_code(code: Vec, groups: usize, is_str: Option) -> i64 { + let cc: &'static CompiledCode = Box::leak(Box::new(CompiledCode { + code, + groups, + is_str, + })); + let mut reg = registry().lock(); + reg.push(cc); + (reg.len() - 1) as i64 +} + +/// The compiled code behind `handle`. +#[inline] +pub(crate) fn compiled(handle: i64) -> Option<&'static CompiledCode> { + let idx = usize::try_from(handle).ok()?; + LOCAL_CODES.with(|local| { + if let Some(cc) = local.borrow().get(idx) { + return Some(*cc); + } + let mut local = local.borrow_mut(); + let reg = registry().lock(); + let have = local.len(); + local.extend_from_slice(reg.get(have..)?); + local.get(idx).copied() + }) +} + // --------------------------------------------------------------------------- // Case-folding helpers // --------------------------------------------------------------------------- @@ -260,6 +305,9 @@ fn loc_word(ch: u32) -> bool { } fn uni_digit(ch: u32) -> bool { + if ch < 128 { + return ascii_digit(ch); + } match char::from_u32(ch) { // Py_UNICODE_ISDECIMAL — decimal digits: ASCII `0`-`9` plus the // Unicode Decimal_Number (Nd) category for non-ASCII scripts. @@ -294,6 +342,10 @@ fn nd_digit(c: char) -> bool { } fn uni_space(ch: u32) -> bool { + if ch < 128 { + // `char::is_whitespace` over ASCII. + return matches!(ch, 0x09..=0x0d | 0x20); + } match char::from_u32(ch) { Some(c) => c.is_whitespace(), None => false, @@ -301,8 +353,8 @@ fn uni_space(ch: u32) -> bool { } fn uni_word(ch: u32) -> bool { - if ch == u32::from(b'_') { - return true; + if ch < 128 { + return ascii_word(ch); } match char::from_u32(ch) { Some(c) => c.is_alphanumeric(), @@ -345,9 +397,22 @@ fn category(chcode: u32, ch: u32) -> bool { // The matcher // --------------------------------------------------------------------------- +/// A subject code unit: a byte (`bytes`, or an ASCII `str` matched over +/// its UTF-8 bytes in place) or a code point. +pub(crate) trait SreChar: Copy + Into {} +impl SreChar for u8 {} +impl SreChar for u32 {} + +#[inline(always)] +fn cu(c: C) -> u32 { + c.into() +} + +/// Saved capture state (CPython's `LASTMARK_SAVE`, plus `MARK_PUSH` of +/// the marks themselves while a `REPEAT` is active). #[derive(Clone)] struct MarkSnapshot { - marks: Vec, + marks: Option>, lastmark: isize, lastindex: isize, } @@ -360,8 +425,8 @@ struct RepeatCtx { prev: Option, } -struct Matcher<'a> { - s: &'a [u32], +struct Matcher<'a, C: SreChar> { + s: &'a [C], code: &'a [u32], beginning: usize, start: usize, @@ -482,8 +547,9 @@ fn code_range_has_mark(code: &[u32], start: usize, end: usize) -> bool { false } -impl<'a> Matcher<'a> { - fn new(s: &'a [u32], code: &'a [u32], groups: usize) -> Self { +impl<'a, C: SreChar> Matcher<'a, C> { + /// A matcher over `s`; `marks` holds `2 * groups` entries, all `-1`. + fn new(s: &'a [C], code: &'a [u32], marks: Vec) -> Self { Matcher { s, code, @@ -491,7 +557,7 @@ impl<'a> Matcher<'a> { start: 0, end: s.len(), ptr: 0, - marks: vec![-1; groups * 2], + marks, lastmark: -1, lastindex: -1, must_advance: false, @@ -512,10 +578,16 @@ impl<'a> Matcher<'a> { self.lastindex = -1; } + /// Save the capture state before a choice point. Outside any `REPEAT` + /// only `lastmark`/`lastindex` are saved, as in CPython: a failed + /// alternative can only have set marks above the saved `lastmark` + /// (each group's marks appear once in the code, and nothing loops + /// back outside a repeat), and restoring `lastmark` invalidates + /// those. Inside a repeat the marks themselves are saved too. #[inline] fn snapshot(&self) -> MarkSnapshot { MarkSnapshot { - marks: self.marks.clone(), + marks: self.cur_repeat.is_some().then(|| self.marks.clone()), lastmark: self.lastmark, lastindex: self.lastindex, } @@ -523,7 +595,9 @@ impl<'a> Matcher<'a> { #[inline] fn restore(&mut self, snap: &MarkSnapshot) { - self.marks.clone_from(&snap.marks); + if let Some(marks) = &snap.marks { + self.marks.clone_from(marks); + } self.lastmark = snap.lastmark; self.lastindex = snap.lastindex; } @@ -728,9 +802,9 @@ impl<'a> Matcher<'a> { let s = self.s; match atcode { AT_BEGINNING | AT_BEGINNING_STRING => ptr == self.beginning, - AT_BEGINNING_LINE => ptr == self.beginning || is_linebreak(s[ptr - 1]), - AT_END => (self.end - ptr == 1 && is_linebreak(s[ptr])) || ptr == self.end, - AT_END_LINE => ptr == self.end || is_linebreak(s[ptr]), + AT_BEGINNING_LINE => ptr == self.beginning || is_linebreak(cu(s[ptr - 1])), + AT_END => (self.end - ptr == 1 && is_linebreak(cu(s[ptr]))) || ptr == self.end, + AT_END_LINE => ptr == self.end || is_linebreak(cu(s[ptr])), AT_END_STRING => ptr == self.end, AT_BOUNDARY => self.word_boundary(ptr, ascii_word), // 3.14 (gh-124130): `\B` matches the empty string again (the @@ -757,8 +831,8 @@ impl<'a> Matcher<'a> { /// its exact complement (CPython `SRE_AT_NON_BOUNDARY`). #[inline] fn word_boundary_raw(&self, ptr: usize, is_word: fn(u32) -> bool) -> bool { - let thatp = ptr > self.beginning && is_word(self.s[ptr - 1]); - let thisp = ptr < self.end && is_word(self.s[ptr]); + let thatp = ptr > self.beginning && is_word(cu(self.s[ptr - 1])); + let thisp = ptr < self.end && is_word(cu(self.s[ptr])); thisp != thatp } @@ -859,14 +933,14 @@ impl<'a> Matcher<'a> { let counted = match op { OP_IN => { let mut p = ptr; - while p < end && self.charset(pat + 2, s[p]) { + while p < end && self.charset(pat + 2, cu(s[p])) { p += 1; } p - ptr } OP_ANY => { let mut p = ptr; - while p < end && !is_linebreak(s[p]) { + while p < end && !is_linebreak(cu(s[p])) { p += 1; } p - ptr @@ -875,7 +949,7 @@ impl<'a> Matcher<'a> { OP_LITERAL => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && s[p] == chr { + while p < end && cu(s[p]) == chr { p += 1; } p - ptr @@ -883,7 +957,7 @@ impl<'a> Matcher<'a> { OP_NOT_LITERAL => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && s[p] != chr { + while p < end && cu(s[p]) != chr { p += 1; } p - ptr @@ -891,7 +965,7 @@ impl<'a> Matcher<'a> { OP_LITERAL_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && lower_ascii(s[p]) == chr { + while p < end && lower_ascii(cu(s[p])) == chr { p += 1; } p - ptr @@ -899,7 +973,7 @@ impl<'a> Matcher<'a> { OP_LITERAL_UNI_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && lower_unicode(s[p]) == chr { + while p < end && lower_unicode(cu(s[p])) == chr { p += 1; } p - ptr @@ -907,7 +981,7 @@ impl<'a> Matcher<'a> { OP_LITERAL_LOC_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && char_loc_ignore(chr, s[p]) { + while p < end && char_loc_ignore(chr, cu(s[p])) { p += 1; } p - ptr @@ -915,7 +989,7 @@ impl<'a> Matcher<'a> { OP_NOT_LITERAL_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && lower_ascii(s[p]) != chr { + while p < end && lower_ascii(cu(s[p])) != chr { p += 1; } p - ptr @@ -923,7 +997,7 @@ impl<'a> Matcher<'a> { OP_NOT_LITERAL_UNI_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && lower_unicode(s[p]) != chr { + while p < end && lower_unicode(cu(s[p])) != chr { p += 1; } p - ptr @@ -931,7 +1005,7 @@ impl<'a> Matcher<'a> { OP_NOT_LITERAL_LOC_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && !char_loc_ignore(chr, s[p]) { + while p < end && !char_loc_ignore(chr, cu(s[p])) { p += 1; } p - ptr @@ -967,10 +1041,15 @@ impl<'a> Matcher<'a> { } // Segmented stack growth: linear-depth backtracking (one native // frame per repetition) must not overflow the OS stack before - // reaching MAX_DEPTH. - let r = stacker::maybe_grow(128 * 1024, 4 * 1024 * 1024, || { + // reaching MAX_DEPTH. Checking at every eighth level is enough: + // eight frames stay far inside the 128 KiB red zone. + let r = if self.depth % 8 == 1 { + stacker::maybe_grow(128 * 1024, 4 * 1024 * 1024, || { + self.do_match_inner(pat, toplevel) + }) + } else { self.do_match_inner(pat, toplevel) - }); + }; self.depth -= 1; r } @@ -1011,14 +1090,14 @@ impl<'a> Matcher<'a> { pat += 1; } OP_LITERAL => { - if ptr >= end || self.s[ptr] != code[pat] { + if ptr >= end || cu(self.s[ptr]) != code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_NOT_LITERAL => { - if ptr >= end || self.s[ptr] == code[pat] { + if ptr >= end || cu(self.s[ptr]) == code[pat] { return Ok(false); } pat += 1; @@ -1041,14 +1120,14 @@ impl<'a> Matcher<'a> { pat += 1; } OP_CATEGORY => { - if ptr >= end || !category(code[pat], self.s[ptr]) { + if ptr >= end || !category(code[pat], cu(self.s[ptr])) { return Ok(false); } pat += 1; ptr += 1; } OP_ANY => { - if ptr >= end || is_linebreak(self.s[ptr]) { + if ptr >= end || is_linebreak(cu(self.s[ptr])) { return Ok(false); } ptr += 1; @@ -1060,70 +1139,70 @@ impl<'a> Matcher<'a> { ptr += 1; } OP_IN => { - if ptr >= end || !self.charset(pat + 1, self.s[ptr]) { + if ptr >= end || !self.charset(pat + 1, cu(self.s[ptr])) { return Ok(false); } pat += code[pat] as usize; ptr += 1; } OP_LITERAL_IGNORE => { - if ptr >= end || lower_ascii(self.s[ptr]) != code[pat] { + if ptr >= end || lower_ascii(cu(self.s[ptr])) != code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_LITERAL_UNI_IGNORE => { - if ptr >= end || lower_unicode(self.s[ptr]) != code[pat] { + if ptr >= end || lower_unicode(cu(self.s[ptr])) != code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_LITERAL_LOC_IGNORE => { - if ptr >= end || !char_loc_ignore(code[pat], self.s[ptr]) { + if ptr >= end || !char_loc_ignore(code[pat], cu(self.s[ptr])) { return Ok(false); } pat += 1; ptr += 1; } OP_NOT_LITERAL_IGNORE => { - if ptr >= end || lower_ascii(self.s[ptr]) == code[pat] { + if ptr >= end || lower_ascii(cu(self.s[ptr])) == code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_NOT_LITERAL_UNI_IGNORE => { - if ptr >= end || lower_unicode(self.s[ptr]) == code[pat] { + if ptr >= end || lower_unicode(cu(self.s[ptr])) == code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_NOT_LITERAL_LOC_IGNORE => { - if ptr >= end || char_loc_ignore(code[pat], self.s[ptr]) { + if ptr >= end || char_loc_ignore(code[pat], cu(self.s[ptr])) { return Ok(false); } pat += 1; ptr += 1; } OP_IN_IGNORE => { - if ptr >= end || !self.charset(pat + 1, lower_ascii(self.s[ptr])) { + if ptr >= end || !self.charset(pat + 1, lower_ascii(cu(self.s[ptr]))) { return Ok(false); } pat += code[pat] as usize; ptr += 1; } OP_IN_UNI_IGNORE => { - if ptr >= end || !self.charset(pat + 1, lower_unicode(self.s[ptr])) { + if ptr >= end || !self.charset(pat + 1, lower_unicode(cu(self.s[ptr]))) { return Ok(false); } pat += code[pat] as usize; ptr += 1; } OP_IN_LOC_IGNORE => { - if ptr >= end || !self.charset_loc_ignore(pat + 1, self.s[ptr]) { + if ptr >= end || !self.charset_loc_ignore(pat + 1, cu(self.s[ptr])) { return Ok(false); } pat += code[pat] as usize; @@ -1137,13 +1216,13 @@ impl<'a> Matcher<'a> { while code[pat] != 0 { // Fast skip when the branch can't possibly match. if code[pat + 1] == OP_LITERAL - && (ptr >= end || self.s[ptr] != code[pat + 2]) + && (ptr >= end || cu(self.s[ptr]) != code[pat + 2]) { pat += code[pat] as usize; continue; } if code[pat + 1] == OP_IN - && (ptr >= end || !self.charset(pat + 3, self.s[ptr])) + && (ptr >= end || !self.charset(pat + 3, cu(self.s[ptr]))) { pat += code[pat] as usize; continue; @@ -1192,7 +1271,7 @@ impl<'a> Matcher<'a> { loop { while cnt >= pmin_i && { let pos = orig + cnt as usize; - pos >= end || self.s[pos] != chr + pos >= end || cu(self.s[pos]) != chr } { cnt -= 1; } @@ -1567,8 +1646,8 @@ impl<'a> Matcher<'a> { if ptr >= end { return false; } - let a = self.s[ptr]; - let b = self.s[p]; + let a = cu(self.s[ptr]); + let b = cu(self.s[p]); let eq = match kind { GroupRefKind::Exact => a == b, GroupRefKind::Ascii => lower_ascii(a) == lower_ascii(b), @@ -1587,46 +1666,177 @@ impl<'a> Matcher<'a> { /// `SRE(search)` — scan for the leftmost match at or after /// `self.start`. Returns the start position of the match on success. + /// + /// Like CPython's, it uses the pattern's `INFO` block: a literal + /// prefix is found with its overlap table (and is the whole match + /// for a literal pattern), a leading character set skips the + /// positions that cannot start a match, and the minimum width stops + /// the scan early. fn search(&mut self) -> Result, RuntimeError> { - // Determine where the real pattern starts (after any INFO block) - // for the anchored-pattern fast reject. - let mut p = 0usize; - let mut min = 0usize; - if self.code[0] == OP_INFO { - min = self.code[3] as usize; - p = 1 + self.code[1] as usize; - } - let anchored = self.code.get(p) == Some(&OP_AT) - && matches!( - self.code.get(p + 1).copied(), - Some(AT_BEGINNING) | Some(AT_BEGINNING_STRING) - ); - + let code = self.code; let mut ptr = self.start; - let mut first = true; - loop { - if min != 0 && self.end.saturating_sub(ptr) < min { + let mut end = self.end; + if ptr > end { + return Ok(None); + } + let mut pat = 0usize; + let mut flags = 0u32; + let mut prefix_len = 0usize; + let mut prefix_skip = 0usize; + let mut charset = None; + if code[0] == OP_INFO { + // <1=skip> <2=flags> <3=min> <4=max> <5=prefix info> + flags = code[2]; + let min = code[3] as usize; + if min != 0 && end - ptr < min { return Ok(None); } - self.start = ptr; - self.ptr = ptr; - self.reset_capture(); - let matched = self.do_match(0, true)?; - if first { - self.must_advance = false; - first = false; + if min > 1 { + // Leave at least one character, so the literal search + // works. + end -= min - 1; + if end <= ptr { + end = ptr; + } } - if matched { - return Ok(Some(ptr)); + if flags & SRE_INFO_PREFIX != 0 { + // + prefix_len = code[5] as usize; + prefix_skip = code[6] as usize; + } else if flags & SRE_INFO_CHARSET != 0 { + charset = Some(5); } - if anchored { - return Ok(None); + pat = 1 + code[1] as usize; + } + + if prefix_len == 1 { + // The pattern starts with a literal character. + let c = code[7]; + let end = self.end; + self.must_advance = false; + while ptr < end { + while cu(self.s[ptr]) != c { + ptr += 1; + if ptr >= end { + return Ok(None); + } + } + self.start = ptr; + self.ptr = ptr + prefix_skip; + if flags & SRE_INFO_LITERAL != 0 { + return Ok(Some(ptr)); + } + if self.do_match(pat + 2 * prefix_skip, false)? { + return Ok(Some(ptr)); + } + ptr += 1; + self.reset_capture(); } - if ptr >= self.end { + return Ok(None); + } + + if prefix_len > 1 { + // The pattern starts with a known prefix: use the overlap + // table to skip forward as fast as possible. + let prefix = &code[7..7 + prefix_len]; + let overlap = &code[7 + prefix_len - 1..]; + let end = self.end; + if prefix_len > end - ptr { return Ok(None); } + while ptr < end { + let c = prefix[0]; + loop { + let ch = cu(self.s[ptr]); + ptr += 1; + if ch == c { + break; + } + if ptr >= end { + return Ok(None); + } + } + if ptr >= end { + return Ok(None); + } + let mut i = 1usize; + self.must_advance = false; + loop { + if cu(self.s[ptr]) == prefix[i] { + i += 1; + if i != prefix_len { + ptr += 1; + if ptr >= end { + return Ok(None); + } + continue; + } + // A potential match. + let start = ptr - (prefix_len - 1); + self.start = start; + self.ptr = ptr - (prefix_len - prefix_skip - 1); + if flags & SRE_INFO_LITERAL != 0 { + return Ok(Some(start)); + } + if self.do_match(pat + 2 * prefix_skip, false)? { + return Ok(Some(start)); + } + // Close, but no match: try again. + ptr += 1; + if ptr >= end { + return Ok(None); + } + self.reset_capture(); + } + i = overlap[i] as usize; + if i == 0 { + break; + } + } + } + return Ok(None); + } + + if let Some(set) = charset { + // The pattern starts with a character from a known set. + let end = self.end; + self.must_advance = false; + loop { + while ptr < end && !self.charset(set, cu(self.s[ptr])) { + ptr += 1; + } + if ptr >= end { + return Ok(None); + } + self.start = ptr; + self.ptr = ptr; + if self.do_match(pat, false)? { + return Ok(Some(ptr)); + } + ptr += 1; + self.reset_capture(); + } + } + + // The general case. + self.start = ptr; + self.ptr = ptr; + let mut matched = self.do_match(pat, true)?; + self.must_advance = false; + if !matched + && code[pat] == OP_AT + && (code[pat + 1] == AT_BEGINNING || code[pat + 1] == AT_BEGINNING_STRING) + { + return Ok(None); + } + while !matched && ptr < end { ptr += 1; + self.reset_capture(); + self.start = ptr; + self.ptr = ptr; + matched = self.do_match(pat, false)?; } + Ok(matched.then_some(ptr)) } } @@ -1764,7 +1974,7 @@ fn subject_cache_key(obj: &Object) -> Option { /// Decode the subject into code points, reusing the thread-local cache when /// the same immutable subject is matched repeatedly. -fn decode_subject_cached(obj: &Object) -> Result>, RuntimeError> { +pub(crate) fn decode_subject_cached(obj: &Object) -> Result>, RuntimeError> { let Some(key) = subject_cache_key(obj) else { return Ok(Rc::new(subject_to_vec(obj)?)); }; @@ -1790,44 +2000,79 @@ fn sre_compile(args: &[Object]) -> Result { .ok_or_else(|| type_error("_sre.compile: code"))?, )?; let groups = arg_i64(args, 1, "groups")?.max(0) as usize; - let handle = { - let mut reg = registry().lock(); - reg.push(Rc::new(CompiledCode { code, groups })); - reg.len() - 1 + Ok(Object::Int(register_code(code, groups, None))) +} + +/// `_sre.compile_code(pattern, code, groups)`: [`sre_compile`] for a +/// pattern whose source (`str`, bytes-like, or `None`) fixes the subject +/// type it accepts. +pub(crate) fn compile_code( + pattern: &Object, + code: &Object, + groups: &Object, +) -> Result { + let code = codeseq_to_vec(code)?; + let groups = groups + .as_i64() + .ok_or_else(|| type_error("_sre: expected int for groups"))? + .max(0) as usize; + let is_str = match pattern { + Object::None => None, + Object::Str(_) | Object::WStr(_) => Some(true), + other => Some(matches!( + other.native_value(), + Some(Object::Str(_) | Object::WStr(_)) + )), }; - Ok(Object::Int(handle as i64)) + Ok(register_code(code, groups, is_str)) } -/// `_sre.exec(handle, string, pos, endpos, mode, must_advance)`. +/// One `SRE(match)` / `SRE(search)` over `subject[pos..endpos]`. /// -/// Returns `None` on no match, otherwise a tuple -/// `(start, end, lastindex, marks)` where `marks` is a tuple of -/// `2 * groups` code-point indices (`-1` for an unset group). -fn sre_exec(args: &[Object]) -> Result { - let handle = arg_i64(args, 0, "handle")? as usize; - let cc = registry().lock().get(handle).cloned(); - let cc = cc.ok_or_else(|| value_error("_sre.exec: invalid pattern handle"))?; - let subject = - decode_subject_cached(args.get(1).ok_or_else(|| type_error("_sre.exec: string"))?)?; - let slen = subject.len() as i64; - let pos = arg_i64(args, 2, "pos")?.clamp(0, slen) as usize; - let endpos = arg_i64(args, 3, "endpos")?.clamp(0, slen) as usize; - let mode = arg_i64(args, 4, "mode")?; - let must_advance = args - .get(5) - .map(|o| o.as_i64().unwrap_or(0) != 0) - .unwrap_or(false); - +/// `mode` 0 searches, 1 matches at `pos`, 2 must match all of +/// `[pos, endpos)`. On a match, `marks` receives the match span followed +/// by each group's span, with `-1` for a group that did not take part +/// (normalized the way CPython's `pattern_new_match` does), and the +/// result is `Some(lastindex)` (`-1` for none). +pub(crate) fn exec_marks( + cc: &CompiledCode, + subject: &[C], + pos: usize, + endpos: usize, + mode: i64, + must_advance: bool, + marks: &mut Vec, +) -> Result, RuntimeError> { + marks.clear(); if pos > endpos { - return Ok(Object::None); + return Ok(None); } - - let mut m = Matcher::new(&subject[..], &cc.code, cc.groups); + // The matcher's mark array comes from this thread's spare (taken, so + // a match started while this one runs gets its own). + let mut buf = SPARE_MARKS.with(|s| std::mem::take(&mut *s.borrow_mut())); + buf.clear(); + buf.resize(cc.groups * 2, -1); + let mut m = Matcher::new(subject, &cc.code, buf); m.end = endpos; m.start = pos; m.ptr = pos; m.must_advance = must_advance; + let r = run_exec(&mut m, pos, mode, cc.groups, marks); + SPARE_MARKS.with(|s| *s.borrow_mut() = std::mem::take(&mut m.marks)); + r +} + +thread_local! { + static SPARE_MARKS: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; +} +fn run_exec( + m: &mut Matcher<'_, C>, + pos: usize, + mode: i64, + groups: usize, + marks: &mut Vec, +) -> Result, RuntimeError> { let (mstart, ok) = match mode { // 1 = match (anchored at pos) 1 => { @@ -1846,26 +2091,61 @@ fn sre_exec(args: &[Object]) -> Result { None => (0, false), }, }; - if !ok { - return Ok(Object::None); + return Ok(None); } - - let mend = m.ptr; - let mut marks_out: Vec = Vec::with_capacity(cc.groups * 2); - for i in 0..cc.groups * 2 { - let v = if (i as isize) <= m.lastmark { - m.marks[i] + marks.reserve(2 + groups * 2); + marks.push(mstart as i64); + marks.push(m.ptr as i64); + for g in 0..groups { + let j = 2 * g; + if (j + 1) as isize <= m.lastmark && m.marks[j] >= 0 && m.marks[j + 1] >= 0 { + marks.push(m.marks[j] as i64); + marks.push(m.marks[j + 1] as i64); } else { - -1 - }; - marks_out.push(Object::Int(v as i64)); + marks.push(-1); + marks.push(-1); + } } + Ok(Some(m.lastindex)) +} + +/// `_sre.exec(handle, string, pos, endpos, mode, must_advance)`. +/// +/// Returns `None` on no match, otherwise a tuple +/// `(start, end, lastindex, marks)` where `marks` is a tuple of +/// `2 * groups` code-point indices (`-1` for an unset group). +fn sre_exec(args: &[Object]) -> Result { + let handle = arg_i64(args, 0, "handle")?; + let cc = compiled(handle).ok_or_else(|| value_error("_sre.exec: invalid pattern handle"))?; + let subject = + decode_subject_cached(args.get(1).ok_or_else(|| type_error("_sre.exec: string"))?)?; + let slen = subject.len() as i64; + let pos = arg_i64(args, 2, "pos")?.clamp(0, slen) as usize; + let endpos = arg_i64(args, 3, "endpos")?.clamp(0, slen) as usize; + let mode = arg_i64(args, 4, "mode")?; + let must_advance = args + .get(5) + .map(|o| o.as_i64().unwrap_or(0) != 0) + .unwrap_or(false); + let mut marks = Vec::new(); + let Some(lastindex) = exec_marks( + cc, + &subject[..], + pos, + endpos, + mode, + must_advance, + &mut marks, + )? + else { + return Ok(Object::None); + }; Ok(Object::new_tuple_array([ - Object::Int(mstart as i64), - Object::Int(mend as i64), - Object::Int(m.lastindex as i64), - Object::new_tuple(marks_out), + Object::Int(marks[0]), + Object::Int(marks[1]), + Object::Int(lastindex as i64), + Object::new_tuple(marks[2..].iter().map(|&v| Object::Int(v)).collect()), ])) } @@ -1944,6 +2224,21 @@ pub fn build(_cache: &ModuleCache) -> Rc { b("compile", sre_compile), ); d.insert(DictKey(Object::from_static("exec")), b("exec", sre_exec)); + for (name, f) in [ + ( + "install", + super::sre_objects::install as fn(&[Object]) -> Result, + ), + ("make_pattern", super::sre_objects::make_pattern), + ("exec_match", super::sre_objects::exec_match), + ("clear_templates", super::sre_objects::clear_templates), + ] { + d.insert(DictKey(Object::from_static(name)), b(name, f)); + } + d.insert( + DictKey(Object::from_static("compile_cached")), + super::sre_objects::compile_cached_builtin(), + ); d.insert( DictKey(Object::from_static("ascii_tolower")), b("ascii_tolower", sre_ascii_tolower), diff --git a/crates/weavepy-vm/src/stdlib/sre_objects.rs b/crates/weavepy-vm/src/stdlib/sre_objects.rs new file mode 100644 index 00000000..5b9d2539 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/sre_objects.rs @@ -0,0 +1,1533 @@ +//! Native method bodies for `re.Pattern`, `re.Match` and the scanner. +//! +//! The classes are Python classes in the frozen `re._engine` module, with +//! `__slots__` storage, so their reprs, attributes, pickling and the rarer +//! paths stay in Python. `_sre.install` (called at the end of the module +//! body) saves the Python methods it replaces, puts native bodies in the +//! class dicts, and hangs this module's [`State`] on each class. Every +//! pattern, match and scanner is built natively over a shared slot layout +//! (see `SlotStorage::from_layout`), so a native reads all of an instance's +//! fields after one layout check. +//! +//! A native serves the common shapes: an exact `str` or `bytes` subject of +//! the pattern's type, plain `int` positions and counts, `int` or `str` +//! group names, and `str`/`bytes` replacement templates. Those fast halves +//! run no Python code, so they are registered as leaf entries the dispatch +//! loop calls inline. A callable replacement, or a template not parsed +//! yet, takes the native full body, which calls into the interpreter. +//! Anything else (other subject types, `__index__` positions, unexpected +//! keywords, and every error message the Python code owns) calls the +//! Python method the native replaced, so behavior is unchanged. +//! +//! The instances are never cycle-collector tracked, like `datetime`'s +//! natively built objects: a match holds its pattern and subject, a +//! pattern holds strings, ints and its name tables, and none of them can +//! reach back to a match. + +use std::collections::HashMap; + +use crate::error::{index_error, stop_iteration, type_error, RuntimeError}; +use crate::object::{BuiltinFn, DictData, DictKey, Object, StrKey}; +use crate::shared_value::{SharedSlice, SharedStr}; +use crate::sync::{Rc, RefCell, Weak}; +use crate::types::{PyInstance, SlotStorage, TypeObject}; + +use super::sre_mod::{self, CompiledCode}; + +// Slot order of the natively built instances (the `__slots__` order the +// classes declare). +const PATTERN_SLOTS: [&str; 6] = [ + "_code", + "pattern", + "flags", + "groups", + "groupindex", + "_indexgroup", +]; +const P_CODE: usize = 0; +const P_GROUPINDEX: usize = 4; + +const MATCH_SLOTS: [&str; 6] = ["re", "string", "pos", "endpos", "_marks", "_lastindex"]; +const M_RE: usize = 0; +const M_STRING: usize = 1; +const M_MARKS: usize = 4; + +const SCANNER_SLOTS: [&str; 6] = [ + "pattern", + "_string", + "_start", + "_pos", + "_endpos", + "_must_advance", +]; +const S_PATTERN: usize = 0; +const S_STRING: usize = 1; +const S_START: usize = 2; +const S_POS: usize = 3; +const S_ENDPOS: usize = 4; +const S_MUST: usize = 5; + +/// A parsed replacement template: literals and group references. +enum Piece { + Lit(Object), + Group(usize), +} + +/// One interpreter's `re` classes (held weakly: the natives live in their +/// dicts), the instance layouts, and the Python methods the natives +/// replaced. +pub(crate) struct State { + match_cls: Weak, + scanner_cls: Weak, + pattern_layout: SharedSlice, + match_layout: SharedSlice, + scanner_layout: SharedSlice, + /// `"Pattern.search"` and so on: the replaced Python method. + orig: HashMap<&'static str, Object>, + /// `re._engine._compile_template(pattern, repl)`. + compile_template: Object, + /// Per pattern handle, the last replacement template it expanded. + templates: parking_lot::Mutex>)>>, +} + +#[inline] +fn state_of(cls: &TypeObject) -> Option<&State> { + cls.native_ext.get()?.downcast_ref::() +} + +type Ext = Rc; + +/// The state hanging on `inst`'s class, held without borrowing the class +/// cell (a callable replacement may run any Python code). +#[inline] +fn class_ext(inst: &PyInstance) -> Option { + inst.class.borrow().native_ext.get().cloned() +} + +fn with_interp( + f: impl FnOnce(&mut crate::Interpreter) -> Result, +) -> Result { + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| type_error("re: no active interpreter"))?; + // SAFETY: published by the enclosing VM frame on this thread. + f(unsafe { &mut *ptr }) +} + +fn build(cls: Rc, layout: &SharedSlice, values: Vec) -> Object { + let mut i = PyInstance::new(cls); + i.slots = RefCell::new(SlotStorage::from_layout(layout.clone(), values)); + Object::Instance(Rc::new(i)) +} + +// --------------------------------------------------------------------- +// Subjects. + +/// Whether `o` is a subject the fast paths serve for `cc`: an exact `str` +/// or `bytes` of the type the pattern accepts. +#[inline] +fn fast_subject(o: &Object, cc: &CompiledCode) -> bool { + match o { + Object::Str(_) => cc.is_str != Some(false), + Object::Bytes(_) => cc.is_str != Some(true), + _ => false, + } +} + +fn empty_like(o: &Object) -> Object { + match o { + Object::Bytes(_) => Object::new_bytes(Vec::new()), + _ => Object::Str(SharedStr::from_ascii("")), + } +} + +/// `s[a:b]` in code points, or `None` when the span is out of range. +fn str_slice(s: &SharedStr, a: usize, b: usize) -> Option { + if b <= a { + return (a <= SharedStr::char_count(s)).then(|| Object::Str(SharedStr::from_ascii(""))); + } + if SharedStr::is_ascii(s) { + let bytes = s.as_bytes(); + if b > bytes.len() { + return None; + } + if b - a == 1 { + return Some(Object::from_char(char::from(bytes[a]))); + } + return Some(Object::Str(SharedStr::from_ascii(&s[a..b]))); + } + let (mut ba, mut bb) = (None, None); + for (n, (i, _)) in s.char_indices().enumerate() { + if n == a { + ba = Some(i); + } + if n == b { + bb = Some(i); + break; + } + } + let ba = ba?; + let bb = match bb { + Some(i) => i, + None if b == SharedStr::char_count(s) => s.len(), + None => return None, + }; + Some(Object::Str(SharedStr::with_count(&s[ba..bb], b - a))) +} + +/// The subject's `[a:b]` slice (CPython's `getslice`), for an exact `str` +/// or `bytes`; `None` for any other subject or an out-of-range span. +fn subject_slice(o: &Object, a: usize, b: usize) -> Option { + match o { + Object::Str(s) => str_slice(s, a, b), + Object::Bytes(bs) => { + if b > bs.len() || a > b { + return None; + } + if a == b { + return Some(Object::new_bytes(Vec::new())); + } + Some(Object::Bytes(SharedSlice::from(&bs[a..b]))) + } + _ => None, + } +} + +/// The code units a subject is matched over: the bytes of a `bytes` or +/// of an ASCII `str` (borrowed in place), or the code points of any other +/// subject. +enum Units<'a> { + Narrow(&'a [u8]), + Wide(Rc>), +} + +/// A subject ready for matching, with cheap slicing for `str`/`bytes`. +struct Subject<'a> { + obj: &'a Object, + units: Units<'a>, +} + +impl<'a> Subject<'a> { + fn new(obj: &'a Object) -> Result { + let units = match obj { + Object::Str(s) if SharedStr::is_ascii(s) => Units::Narrow(s.as_bytes()), + Object::Bytes(b) => Units::Narrow(&b[..]), + _ => Units::Wide(sre_mod::decode_subject_cached(obj)?), + }; + Ok(Self { obj, units }) + } + + fn len(&self) -> usize { + match &self.units { + Units::Narrow(b) => b.len(), + Units::Wide(w) => w.len(), + } + } + + /// One match attempt (see [`sre_mod::exec_marks`]). + fn exec( + &self, + cc: &CompiledCode, + pos: usize, + endpos: usize, + mode: i64, + must_advance: bool, + marks: &mut Vec, + ) -> Result, RuntimeError> { + match &self.units { + Units::Narrow(b) => sre_mod::exec_marks(cc, b, pos, endpos, mode, must_advance, marks), + Units::Wide(w) => { + sre_mod::exec_marks(cc, &w[..], pos, endpos, mode, must_advance, marks) + } + } + } + + /// `[a:b]` as a new object (the subject itself for the whole range). + fn slice(&self, a: usize, b: usize) -> Object { + let (a, b) = (a.min(self.len()), b.min(self.len())); + if a == 0 && b == self.len() { + return self.obj.clone(); + } + match (self.obj, &self.units) { + (Object::Str(_), _) if b <= a => Object::Str(SharedStr::from_ascii("")), + (Object::Str(s), Units::Narrow(_)) => { + if b - a == 1 { + Object::from_char(char::from(s.as_bytes()[a])) + } else { + Object::Str(SharedStr::from_ascii(&s[a..b])) + } + } + (Object::Str(_), Units::Wide(w)) => { + let mut t = String::with_capacity(b - a); + push_cps(&mut t, &w[a..b]); + Object::Str(SharedStr::with_count(&t, b - a)) + } + (Object::Bytes(_), _) if b <= a => Object::new_bytes(Vec::new()), + (Object::Bytes(bs), _) => Object::Bytes(SharedSlice::from(&bs[a..b])), + _ => Object::None, + } + } + + /// Group `g`'s text from `marks`, or `default` when it did not match. + fn group(&self, marks: &[i64], g: usize, default: &Object) -> Object { + let (a, b) = (marks[2 * g], marks[2 * g + 1]); + if a < 0 || b < 0 { + default.clone() + } else { + self.slice(a as usize, b as usize) + } + } +} + +fn push_cps(t: &mut String, cps: &[u32]) { + for &c in cps { + t.push(char::from_u32(c).unwrap_or('\u{FFFD}')); + } +} + +/// An output buffer for `sub`: text for a `str` subject, bytes for a +/// `bytes` one, or a list of parts to `join` through the interpreter once +/// a replacement of another type turns up. +enum Sink { + Str(String), + Bytes(Vec), + Parts(Vec), +} + +impl Sink { + fn new(subject: &Subject<'_>) -> Self { + match subject.obj { + Object::Bytes(_) => Sink::Bytes(Vec::new()), + _ => Sink::Str(String::new()), + } + } + + fn push_span(&mut self, subject: &Subject<'_>, a: usize, b: usize) { + if b <= a { + return; + } + match (self, subject.obj, &subject.units) { + (Sink::Str(t), Object::Str(s), Units::Narrow(_)) => t.push_str(&s[a..b]), + (Sink::Str(t), Object::Str(_), Units::Wide(w)) => push_cps(t, &w[a..b]), + (Sink::Bytes(v), Object::Bytes(bs), _) => v.extend_from_slice(&bs[a..b]), + (Sink::Parts(parts), _, _) => parts.push(subject.slice(a, b)), + _ => {} + } + } + + fn push_obj(&mut self, o: &Object) { + match (&mut *self, o) { + (Sink::Str(t), Object::Str(s)) => t.push_str(s), + (Sink::Bytes(v), Object::Bytes(b)) => v.extend_from_slice(b), + (Sink::Parts(parts), _) => parts.push(o.clone()), + _ => { + let done = std::mem::replace(self, Sink::Parts(Vec::new())).finish_plain(); + if let Sink::Parts(parts) = self { + parts.push(done); + parts.push(o.clone()); + } + } + } + } + + fn finish_plain(self) -> Object { + match self { + Sink::Str(t) => Object::Str(SharedStr::from(t.as_str())), + Sink::Bytes(v) => Object::new_bytes(v), + Sink::Parts(_) => Object::None, + } + } + + fn finish(self, empty: &Object) -> Result { + match self { + Sink::Parts(parts) => with_interp(|interp| { + let join = interp.load_attr_public(empty, "join")?; + interp.call_object(join, &[Object::new_list(parts)], &[]) + }), + plain => Ok(plain.finish_plain()), + } + } +} + +// --------------------------------------------------------------------- +// Receivers and arguments. + +/// The pattern's compiled code and handle. +fn pattern_code(st: &State, inst: &PyInstance) -> Option<(&'static CompiledCode, i64)> { + let slots = inst.slots.borrow(); + let v = slots.values_for_layout(&st.pattern_layout)?; + let Object::Int(h) = v[P_CODE] else { + return None; + }; + Some((sre_mod::compiled(h)?, h)) +} + +/// A position argument: a plain `int` (others take the Python path). +#[inline] +fn pos_arg(o: Option<&Object>, default: i64) -> Option { + match o { + None => Some(default), + Some(Object::Int(i)) => Some(*i), + Some(Object::Bool(b)) => Some(i64::from(*b)), + _ => None, + } +} + +#[inline] +fn clamp(pos: i64, endpos: i64, len: usize) -> (usize, usize) { + let n = len as i64; + (pos.clamp(0, n) as usize, endpos.clamp(0, n) as usize) +} + +/// Lay keyword arguments out positionally after `args` (receiver first) +/// following `names`, when they fill the next positions exactly. +fn merge_kwargs(args: &[Object], kw: &[(String, Object)], names: &[&str]) -> Option> { + let given = args.len().checked_sub(1)?; + if given + kw.len() > names.len() { + return None; + } + let mut out = args.to_vec(); + for name in &names[given..given + kw.len()] { + let (_, v) = kw.iter().find(|(k, _)| k == name)?; + out.push(v.clone()); + } + Some(out) +} + +thread_local! { + static SPARE: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; +} + +/// A marks buffer from this thread's spare (taken, so a nested match, +/// from a callable replacement, gets its own). +fn take_marks() -> Vec { + SPARE.with(|s| std::mem::take(&mut *s.borrow_mut())) +} + +fn give_marks(v: Vec) { + SPARE.with(|s| { + let mut s = s.borrow_mut(); + if s.capacity() < v.capacity() { + *s = v; + } + }); +} + +fn new_match( + st: &State, + pattern: &Object, + string: &Object, + pos: usize, + endpos: usize, + marks: &[i64], + lastindex: isize, +) -> Result { + let cls = st + .match_cls + .upgrade() + .ok_or_else(|| type_error("re.Match is gone"))?; + let mut values = Vec::with_capacity(MATCH_SLOTS.len()); + values.push(pattern.clone()); + values.push(string.clone()); + values.push(Object::Int(pos as i64)); + values.push(Object::Int(endpos as i64)); + values.push(Object::Tuple( + crate::tuple_storage::TupleStorage::from_exact_iter(marks.iter().map(|&m| Object::Int(m))), + )); + values.push(Object::Int(lastindex as i64)); + Ok(build(cls, &st.match_layout, values)) +} + +// --------------------------------------------------------------------- +// Pattern methods. + +/// The receiver, subject and position arguments of `match`/`search`/ +/// `fullmatch`/`findall`/`finditer`/`scanner`, when the fast path serves +/// them. +struct ExecArgs<'a> { + recv: &'a Object, + string: &'a Object, + pos: i64, + endpos: i64, +} + +fn exec_args(args: &[Object]) -> Option> { + let [recv, string, rest @ ..] = args else { + return None; + }; + if rest.len() > 2 { + return None; + } + Some(ExecArgs { + recv, + string, + pos: pos_arg(rest.first(), 0)?, + endpos: pos_arg(rest.get(1), i64::MAX)?, + }) +} + +/// Run `f` with the pattern's state and code, when the receiver is a +/// natively built pattern and the subject a fast one. +fn with_pattern( + recv: &Object, + string: &Object, + f: impl FnOnce(&State, &'static CompiledCode, i64) -> R, +) -> Option { + let Object::Instance(inst) = recv else { + return None; + }; + let ext = class_ext(inst)?; + let st = ext.downcast_ref::()?; + let (cc, handle) = pattern_code(st, inst)?; + if !fast_subject(string, cc) { + return None; + } + Some(f(st, cc, handle)) +} + +fn pattern_exec(args: &[Object], mode: i64) -> Option> { + let a = exec_args(args)?; + with_pattern(a.recv, a.string, |st, cc, _| { + let subject = Subject::new(a.string)?; + let (p, e) = clamp(a.pos, a.endpos, subject.len()); + let mut marks = take_marks(); + let r = match subject.exec(cc, p, e, mode, false, &mut marks) { + Ok(None) => Ok(Object::None), + Ok(Some(li)) => new_match(st, a.recv, a.string, p, e, &marks, li), + Err(err) => Err(err), + }; + give_marks(marks); + r + }) +} + +fn pattern_match(args: &[Object]) -> Option> { + pattern_exec(args, 1) +} + +fn pattern_fullmatch(args: &[Object]) -> Option> { + pattern_exec(args, 2) +} + +fn pattern_search(args: &[Object]) -> Option> { + pattern_exec(args, 0) +} + +fn pattern_findall(args: &[Object]) -> Option> { + let a = exec_args(args)?; + with_pattern(a.recv, a.string, |_, cc, _| { + let subject = Subject::new(a.string)?; + let (mut p, e) = clamp(a.pos, a.endpos, subject.len()); + let empty = empty_like(a.string); + let mut out = Vec::new(); + let mut marks = Vec::new(); + let mut must_advance = false; + while p <= e { + if subject + .exec(cc, p, e, 0, must_advance, &mut marks)? + .is_none() + { + break; + } + let (s, en) = (marks[0] as usize, marks[1] as usize); + let item = match cc.groups { + 0 => subject.slice(s, en), + 1 => subject.group(&marks, 1, &empty), + n => Object::new_tuple((1..=n).map(|g| subject.group(&marks, g, &empty)).collect()), + }; + out.push(item); + must_advance = s == en; + p = en; + } + Ok(Object::new_list(out)) + }) +} + +/// `finditer` and `scanner`: a scanner over a fast subject. +fn pattern_scanner(args: &[Object]) -> Option> { + let a = exec_args(args)?; + with_pattern(a.recv, a.string, |st, _, _| { + let len = match a.string { + Object::Str(s) => SharedStr::char_count(s), + Object::Bytes(b) => b.len(), + _ => 0, + }; + let (p, e) = clamp(a.pos, a.endpos, len); + let cls = st + .scanner_cls + .upgrade() + .ok_or_else(|| type_error("re scanner type is gone"))?; + Ok(build( + cls, + &st.scanner_layout, + vec![ + a.recv.clone(), + a.string.clone(), + Object::Int(p as i64), + Object::Int(p as i64), + Object::Int(e as i64), + Object::Bool(false), + ], + )) + }) +} + +/// How `sub` produces each replacement. +enum Filter<'a> { + Literal(&'a Object), + Template(Rc>), + Callable(&'a Object), +} + +/// The cached template for `repl` (a `str` or `bytes` with a backslash). +fn cached_template(st: &State, handle: i64, repl: &Object) -> Option>> { + let cache = st.templates.lock(); + let (key, t) = cache.get(&handle)?; + let same = match (key, repl) { + (Object::Str(a), Object::Str(b)) => a == b, + (Object::Bytes(a), Object::Bytes(b)) => a[..] == b[..], + _ => false, + }; + same.then(|| t.clone()) +} + +/// Parse `repl` through `re._engine._compile_template` and cache it. +fn parse_template( + st: &State, + pattern: &Object, + handle: i64, + repl: &Object, +) -> Result>>, RuntimeError> { + let f = st.compile_template.clone(); + let parsed = with_interp(|i| i.call_object(f, &[pattern.clone(), repl.clone()], &[]))?; + let items: Vec = match &parsed { + Object::List(l) => l.borrow().clone(), + Object::Tuple(t) => t.to_vec(), + _ => return Ok(None), + }; + let mut pieces = Vec::with_capacity(items.len()); + for it in items { + match it { + Object::Int(g) if g >= 0 => pieces.push(Piece::Group(g as usize)), + Object::Str(ref s) if s.is_empty() => {} + Object::Bytes(ref b) if b.is_empty() => {} + o @ (Object::Str(_) | Object::Bytes(_)) => pieces.push(Piece::Lit(o)), + _ => return Ok(None), + } + } + let t = Rc::new(pieces); + let mut cache = st.templates.lock(); + if cache.len() >= 512 { + cache.clear(); + } + cache.insert(handle, (repl.clone(), t.clone())); + Ok(Some(t)) +} + +fn has_backslash(repl: &Object) -> bool { + match repl { + Object::Str(s) => s.as_bytes().contains(&b'\\'), + Object::Bytes(b) => b.contains(&b'\\'), + _ => true, + } +} + +/// The receiver, replacement, subject and count of `sub`/`subn`. +fn sub_args(args: &[Object]) -> Option<(&Object, &Object, &Object, i64)> { + let [recv, repl, string, rest @ ..] = args else { + return None; + }; + let count = match rest { + [] => 0, + [Object::Int(c)] => *c, + [Object::Bool(b)] => i64::from(*b), + _ => return None, + }; + Some((recv, repl, string, count)) +} + +/// Whether `repl` is a `str`/`bytes` replacement of the subject's type. +fn plain_repl(repl: &Object, string: &Object) -> bool { + matches!( + (repl, string), + (Object::Str(_), Object::Str(_)) | (Object::Bytes(_), Object::Bytes(_)) + ) +} + +#[allow(clippy::too_many_arguments)] +fn run_sub( + st: &State, + cc: &CompiledCode, + recv: &Object, + string: &Object, + filter: &Filter<'_>, + count: i64, + subn: bool, +) -> Result { + let subject = Subject::new(string)?; + let e = subject.len(); + let mut sink = Sink::new(&subject); + let mut marks = Vec::new(); + let (mut p, mut last, mut n) = (0usize, 0usize, 0i64); + let mut must_advance = false; + while p <= e && (count == 0 || n < count) { + let Some(li) = subject.exec(cc, p, e, 0, must_advance, &mut marks)? else { + break; + }; + let (s, en) = (marks[0] as usize, marks[1] as usize); + sink.push_span(&subject, last, s); + match filter { + Filter::Literal(l) => sink.push_obj(l), + Filter::Template(t) => { + for piece in t.iter() { + match piece { + Piece::Lit(l) => sink.push_obj(l), + Piece::Group(g) => { + let (a, b) = (marks[2 * g], marks[2 * g + 1]); + if a >= 0 && b >= 0 { + sink.push_span(&subject, a as usize, b as usize); + } + } + } + } + } + Filter::Callable(f) => { + let m = new_match(st, recv, string, 0, e, &marks, li)?; + let f = (*f).clone(); + let item = with_interp(|i| i.call_object(f, &[m], &[]))?; + if !matches!(item, Object::None) { + sink.push_obj(&item); + } + } + } + last = en; + n += 1; + must_advance = s == en; + p = en; + } + sink.push_span(&subject, last, e); + let result = if n == 0 && !matches!(sink, Sink::Parts(_)) { + // No replacement: CPython hands back the subject itself. + string.clone() + } else { + sink.finish(&empty_like(string))? + }; + Ok(if subn { + Object::new_tuple_array([result, Object::Int(n)]) + } else { + result + }) +} + +fn sub_fast(args: &[Object], subn: bool) -> Option> { + let (recv, repl, string, count) = sub_args(args)?; + if count < 0 || !plain_repl(repl, string) { + return None; + } + with_pattern(recv, string, |st, cc, handle| { + let filter = if has_backslash(repl) { + Filter::Template(cached_template(st, handle, repl)?) + } else { + Filter::Literal(repl) + }; + Some(run_sub(st, cc, recv, string, &filter, count, subn)) + })? +} + +/// `sub`'s full body: a callable replacement, or a template to parse. +fn sub_full(args: &[Object], subn: bool) -> Option> { + let (recv, repl, string, count) = sub_args(args)?; + if count < 0 { + return None; + } + let callable = !plain_repl(repl, string); + if callable && !crate::builtins::object_is_callable(repl) { + return None; + } + with_pattern(recv, string, |st, cc, handle| { + let filter = if callable { + Filter::Callable(repl) + } else if has_backslash(repl) { + match cached_template(st, handle, repl) { + Some(t) => Filter::Template(t), + None => match parse_template(st, recv, handle, repl) { + Ok(Some(t)) => Filter::Template(t), + Ok(None) => return None, + Err(e) => return Some(Err(e)), + }, + } + } else { + Filter::Literal(repl) + }; + Some(run_sub(st, cc, recv, string, &filter, count, subn)) + })? +} + +fn pattern_sub(args: &[Object]) -> Option> { + sub_fast(args, false) +} + +fn pattern_subn(args: &[Object]) -> Option> { + sub_fast(args, true) +} + +fn pattern_sub_full(args: &[Object]) -> Option> { + sub_full(args, false) +} + +fn pattern_subn_full(args: &[Object]) -> Option> { + sub_full(args, true) +} + +fn pattern_split(args: &[Object]) -> Option> { + let [recv, string, rest @ ..] = args else { + return None; + }; + let maxsplit = match rest { + [] => 0, + [Object::Int(m)] => *m, + [Object::Bool(b)] => i64::from(*b), + _ => return None, + }; + with_pattern(recv, string, |_, cc, _| { + let subject = Subject::new(string)?; + let e = subject.len(); + let mut out = Vec::new(); + let mut marks = Vec::new(); + let (mut p, mut last, mut n) = (0usize, 0usize, 0i64); + let mut must_advance = false; + while p <= e && (maxsplit == 0 || n < maxsplit) { + if subject + .exec(cc, p, e, 0, must_advance, &mut marks)? + .is_none() + { + break; + } + let (s, en) = (marks[0] as usize, marks[1] as usize); + out.push(subject.slice(last, s)); + for g in 1..=cc.groups { + out.push(subject.group(&marks, g, &Object::None)); + } + n += 1; + must_advance = s == en; + p = en; + last = en; + } + out.push(subject.slice(last, e)); + Ok(Object::new_list(out)) + }) +} + +// --------------------------------------------------------------------- +// Match methods. + +/// Run `f` over a natively built match's slot values and its marks. +fn with_match( + recv: &Object, + f: impl FnOnce(&State, &[Object], &[Object]) -> Option, +) -> Option { + let Object::Instance(inst) = recv else { + return None; + }; + let ext = class_ext(inst)?; + let st = ext.downcast_ref::()?; + let slots = inst.slots.borrow(); + let v = slots.values_for_layout(&st.match_layout)?; + let Object::Tuple(marks) = &v[M_MARKS] else { + return None; + }; + if marks.len() < 2 || marks.len() % 2 != 0 { + return None; + } + f(st, v, marks) +} + +enum GroupIdx { + Index(usize), + NoSuch, + Slow, +} + +/// CPython's `match_getindex` for an `int` or `str` group. +fn group_index(st: &State, v: &[Object], marks: &[Object], g: &Object) -> GroupIdx { + let groups = marks.len() / 2 - 1; + let i = match g { + Object::Int(i) => *i, + Object::Bool(b) => i64::from(*b), + Object::Str(name) => { + let Object::Instance(p) = &v[M_RE] else { + return GroupIdx::Slow; + }; + let slots = p.slots.borrow(); + let Some(pv) = slots.values_for_layout(&st.pattern_layout) else { + return GroupIdx::Slow; + }; + let Object::MappingProxy(d) = &pv[P_GROUPINDEX] else { + return GroupIdx::Slow; + }; + let found = match d.borrow().get(&StrKey(name)) { + Some(Object::Int(i)) => Some(*i), + Some(_) => return GroupIdx::Slow, + None => None, + }; + match found { + Some(i) => i, + None => return GroupIdx::NoSuch, + } + } + _ => return GroupIdx::Slow, + }; + if i < 0 || i as usize > groups { + GroupIdx::NoSuch + } else { + GroupIdx::Index(i as usize) + } +} + +/// Group `idx`'s span. +fn span_of(marks: &[Object], idx: usize) -> Option<(i64, i64)> { + match (&marks[2 * idx], &marks[2 * idx + 1]) { + (Object::Int(a), Object::Int(b)) => Some((*a, *b)), + _ => None, + } +} + +/// Group `idx`'s text, or `default` when it did not match. +fn group_text(v: &[Object], marks: &[Object], idx: usize, default: &Object) -> Option { + let (a, b) = span_of(marks, idx)?; + if a < 0 || b < 0 { + return Some(default.clone()); + } + subject_slice(&v[M_STRING], a as usize, b as usize) +} + +fn no_such_group() -> RuntimeError { + index_error("no such group") +} + +fn match_group(args: &[Object]) -> Option> { + let (recv, rest) = args.split_first()?; + with_match(recv, |st, v, marks| match rest { + [] => Some(Ok(group_text(v, marks, 0, &Object::None)?)), + [g] => match group_index(st, v, marks, g) { + GroupIdx::Index(i) => Some(Ok(group_text(v, marks, i, &Object::None)?)), + GroupIdx::NoSuch => Some(Err(no_such_group())), + GroupIdx::Slow => None, + }, + many => { + let mut out = Vec::with_capacity(many.len()); + for g in many { + match group_index(st, v, marks, g) { + GroupIdx::Index(i) => out.push(group_text(v, marks, i, &Object::None)?), + GroupIdx::NoSuch => return Some(Err(no_such_group())), + GroupIdx::Slow => return None, + } + } + Some(Ok(Object::new_tuple(out))) + } + }) +} + +fn match_getitem(args: &[Object]) -> Option> { + match args { + [_, _] => match_group(args), + _ => None, + } +} + +fn match_groups(args: &[Object]) -> Option> { + let (recv, rest) = args.split_first()?; + let default = match rest { + [] => &Object::None, + [d] => d, + _ => return None, + }; + with_match(recv, |_, v, marks| { + let n = marks.len() / 2 - 1; + let mut out = Vec::with_capacity(n); + for i in 1..=n { + out.push(group_text(v, marks, i, default)?); + } + Some(Ok(Object::new_tuple(out))) + }) +} + +fn match_groupdict(args: &[Object]) -> Option> { + let (recv, rest) = args.split_first()?; + let default = match rest { + [] => &Object::None, + [d] => d, + _ => return None, + }; + with_match(recv, |st, v, marks| { + let Object::Instance(p) = &v[M_RE] else { + return None; + }; + let slots = p.slots.borrow(); + let pv = slots.values_for_layout(&st.pattern_layout)?; + let Object::MappingProxy(d) = &pv[P_GROUPINDEX] else { + return None; + }; + let groups = marks.len() / 2 - 1; + let mut out = DictData::default(); + for (k, idx) in d.borrow().iter() { + let Object::Int(i) = idx else { + return None; + }; + if *i < 0 || *i as usize > groups { + return None; + } + out.insert(k.clone(), group_text(v, marks, *i as usize, default)?); + } + Some(Ok(Object::Dict(Rc::new(RefCell::new(out))))) + }) +} + +/// `start` (0), `end` (1) and `span` (2). +fn match_position(args: &[Object], which: u8) -> Option> { + let (recv, rest) = args.split_first()?; + with_match(recv, |st, v, marks| { + let idx = match rest { + [] => 0, + [g] => match group_index(st, v, marks, g) { + GroupIdx::Index(i) => i, + GroupIdx::NoSuch => return Some(Err(no_such_group())), + GroupIdx::Slow => return None, + }, + _ => return None, + }; + let (a, b) = span_of(marks, idx)?; + Some(Ok(match which { + 0 => Object::Int(a), + 1 => Object::Int(b), + _ => Object::new_tuple_array([Object::Int(a), Object::Int(b)]), + })) + }) +} + +fn match_start(args: &[Object]) -> Option> { + match_position(args, 0) +} + +fn match_end(args: &[Object]) -> Option> { + match_position(args, 1) +} + +fn match_span(args: &[Object]) -> Option> { + match_position(args, 2) +} + +// --------------------------------------------------------------------- +// The scanner (`finditer`, `Pattern.scanner`). + +/// One `match()` / `search()` step of a scanner; `iter` turns exhaustion +/// into `StopIteration` for `__next__`. +fn scanner_step(args: &[Object], mode: i64, iter: bool) -> Result { + let done = || { + if iter { + Err(stop_iteration()) + } else { + Ok(Object::None) + } + }; + let [Object::Instance(inst)] = args else { + return Err(type_error("SRE_Scanner method takes no arguments")); + }; + let bad = || type_error("descriptor requires a '_sre.SRE_Scanner' object"); + let ext = class_ext(inst).ok_or_else(bad)?; + let st = ext.downcast_ref::().ok_or_else(bad)?; + let (pattern, string, start, pos, endpos, must_advance) = { + let slots = inst.slots.borrow(); + let v = slots + .values_for_layout(&st.scanner_layout) + .ok_or_else(bad)?; + let int = |o: &Object| o.as_i64().unwrap_or(-1); + ( + v[S_PATTERN].clone(), + v[S_STRING].clone(), + int(&v[S_START]), + int(&v[S_POS]), + int(&v[S_ENDPOS]), + matches!(v[S_MUST], Object::Bool(true)), + ) + }; + if start < 0 || start > endpos { + return done(); + } + let Object::Instance(pinst) = &pattern else { + return Err(bad()); + }; + let (cc, _) = pattern_code(st, pinst).ok_or_else(bad)?; + if !fast_subject(&string, cc) { + return Err(bad()); + } + let subject = Subject::new(&string)?; + let endpos = (endpos as usize).min(subject.len()); + let start = start as usize; + let mut marks = Vec::new(); + let r = subject.exec(cc, start, endpos, mode, must_advance, &mut marks)?; + let mut slots = inst.slots.borrow_mut(); + let v = slots + .values_for_layout_mut(&st.scanner_layout) + .ok_or_else(bad)?; + let Some(li) = r else { + v[S_START] = Object::Int(endpos as i64 + 1); + return done(); + }; + v[S_START] = Object::Int(marks[1]); + v[S_MUST] = Object::Bool(marks[0] == marks[1]); + drop(slots); + new_match( + st, + &pattern, + &string, + pos.max(0) as usize, + endpos, + &marks, + li, + ) +} + +fn scanner_match(args: &[Object]) -> Result { + scanner_step(args, 1, false) +} + +fn scanner_search(args: &[Object]) -> Result { + scanner_step(args, 0, false) +} + +fn scanner_next(args: &[Object]) -> Result { + scanner_step(args, 0, true) +} + +fn scanner_iter(args: &[Object]) -> Result { + match args { + [it] => Ok(it.clone()), + _ => Err(type_error("__iter__ takes no arguments")), + } +} + +// --------------------------------------------------------------------- +// Module functions. + +type Fast = fn(&[Object]) -> Option>; + +/// One native method of `Pattern` (class 0) or `Match` (class 1). +struct Spec { + cls: usize, + name: &'static str, + key: &'static str, + fast: Fast, + /// The body that may call into the interpreter, tried after `fast`. + full: Option, + /// Parameter names after the receiver, for keyword arguments. + params: &'static [&'static str], +} + +const EXEC_PARAMS: &[&str] = &["string", "pos", "endpos"]; + +const SPECS: &[Spec] = &[ + Spec { + cls: 0, + name: "match", + key: "Pattern.match", + fast: pattern_match, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "fullmatch", + key: "Pattern.fullmatch", + fast: pattern_fullmatch, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "search", + key: "Pattern.search", + fast: pattern_search, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "findall", + key: "Pattern.findall", + fast: pattern_findall, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "finditer", + key: "Pattern.finditer", + fast: pattern_scanner, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "scanner", + key: "Pattern.scanner", + fast: pattern_scanner, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "sub", + key: "Pattern.sub", + fast: pattern_sub, + full: Some(pattern_sub_full as Fast), + params: &["repl", "string", "count"], + }, + Spec { + cls: 0, + name: "subn", + key: "Pattern.subn", + fast: pattern_subn, + full: Some(pattern_subn_full as Fast), + params: &["repl", "string", "count"], + }, + Spec { + cls: 0, + name: "split", + key: "Pattern.split", + fast: pattern_split, + full: None, + params: &["string", "maxsplit"], + }, + Spec { + cls: 1, + name: "group", + key: "Match.group", + fast: match_group, + full: None, + params: &[], + }, + Spec { + cls: 1, + name: "__getitem__", + key: "Match.__getitem__", + fast: match_getitem, + full: None, + params: &[], + }, + Spec { + cls: 1, + name: "groups", + key: "Match.groups", + fast: match_groups, + full: None, + params: &["default"], + }, + Spec { + cls: 1, + name: "groupdict", + key: "Match.groupdict", + fast: match_groupdict, + full: None, + params: &["default"], + }, + Spec { + cls: 1, + name: "start", + key: "Match.start", + fast: match_start, + full: None, + params: &["group"], + }, + Spec { + cls: 1, + name: "end", + key: "Match.end", + fast: match_end, + full: None, + params: &["group"], + }, + Spec { + cls: 1, + name: "span", + key: "Match.span", + fast: match_span, + full: None, + params: &["group"], + }, +]; + +fn dispatch( + st: &State, + spec: &Spec, + args: &[Object], + kw: &[(String, Object)], +) -> Result { + let merged; + let pos_args = if kw.is_empty() { + Some(args) + } else { + merged = merge_kwargs(args, kw, spec.params); + merged.as_deref() + }; + if let Some(a) = pos_args { + if let Some(r) = (spec.fast)(a) { + return r; + } + if let Some(r) = spec.full.and_then(|full| full(a)) { + return r; + } + } + let f = st + .orig + .get(spec.key) + .cloned() + .ok_or_else(|| type_error(format!("{} is not available", spec.key)))?; + with_interp(|i| i.call_object(f, args, kw)) +} + +fn interned(s: &str) -> SharedStr { + match crate::stdlib::sys::intern_name(s) { + Object::Str(s) => s, + _ => SharedStr::from(s), + } +} + +fn layout(names: &[&str]) -> SharedSlice { + names + .iter() + .map(|n| DictKey(Object::Str(interned(n)))) + .collect::>() + .into() +} + +fn native(name: &'static str, f: fn(&[Object]) -> Result) -> Rc { + Rc::new(BuiltinFn { + name, + binds_instance: true, + call: Box::new(f), + call_kw: None, + }) +} + +/// `_sre.install(Pattern, Match, SRE_Scanner, compile_template)`: put the +/// native bodies in the classes (see the module docs). +pub(crate) fn install(args: &[Object]) -> Result { + let [Object::Type(pattern), Object::Type(matchc), Object::Type(scanner), compile_template] = + args + else { + return Err(type_error( + "install() expects Pattern, Match, the scanner type and _compile_template", + )); + }; + if pattern.native_ext.get().is_some() { + return Ok(Object::None); + } + let classes = [pattern, matchc]; + let mut orig = HashMap::new(); + for spec in SPECS { + let v = classes[spec.cls] + .dict + .borrow() + .get(&StrKey(spec.name)) + .cloned() + .ok_or_else(|| type_error(format!("install(): no {}", spec.key)))?; + orig.insert(spec.key, v); + } + let state = Rc::new(State { + match_cls: Rc::downgrade(matchc), + scanner_cls: Rc::downgrade(scanner), + pattern_layout: layout(&PATTERN_SLOTS), + match_layout: layout(&MATCH_SLOTS), + scanner_layout: layout(&SCANNER_SLOTS), + orig, + compile_template: compile_template.clone(), + templates: parking_lot::Mutex::new(HashMap::new()), + }); + for cls in [pattern, matchc, scanner] { + let _ = cls + .native_ext + .set(crate::rc_unsize!(state.clone() => dyn std::any::Any + Send + Sync)); + } + for spec in SPECS { + let (st, st_kw) = (state.clone(), state.clone()); + // A method with no keyword parameters takes no keyword entry (the + // dispatch loop's native subscript requires that of + // `__getitem__`); the VM rejects keywords for it. + let call_kw: Option _ + Send + Sync>> = + if spec.params.is_empty() { + None + } else { + Some(Box::new(move |a: &[Object], kw: &[(String, Object)]| { + dispatch(&st_kw, spec, a, kw) + })) + }; + let b = Rc::new(BuiltinFn { + name: spec.name, + binds_instance: true, + call: Box::new(move |a: &[Object]| dispatch(&st, spec, a, &[])), + call_kw, + }); + crate::leaf_builtins::register_fast(&b, spec.fast); + classes[spec.cls].dict.borrow_mut().insert( + DictKey(Object::Str(interned(spec.name))), + Object::Builtin(b), + ); + } + for (name, f) in [ + ( + "match", + scanner_match as fn(&[Object]) -> Result, + ), + ("search", scanner_search), + ("__next__", scanner_next), + ("__iter__", scanner_iter), + ] { + let b = native(name, f); + crate::leaf_builtins::register(&b); + scanner + .dict + .borrow_mut() + .insert(DictKey(Object::Str(interned(name))), Object::Builtin(b)); + } + for cls in [pattern, matchc, scanner] { + cls.bump_attr_version(); + } + Ok(Object::None) +} + +/// `_sre.make_pattern(Pattern, pattern, flags, code, groups, groupindex, +/// indexgroup)`: compile `code` and build the pattern object. +pub(crate) fn make_pattern(args: &[Object]) -> Result { + let [Object::Type(cls), pattern, flags, code, groups, groupindex, indexgroup] = args else { + return Err(type_error("make_pattern() takes 7 arguments")); + }; + let st = state_of(cls).ok_or_else(|| type_error("make_pattern(): re is not installed"))?; + let handle = sre_mod::compile_code(pattern, code, groups)?; + Ok(build( + cls.clone(), + &st.pattern_layout, + vec![ + Object::Int(handle), + pattern.clone(), + flags.clone(), + groups.clone(), + groupindex.clone(), + indexgroup.clone(), + ], + )) +} + +/// `_sre.exec_match(pattern, string, pos, endpos, mode, must_advance, +/// mpos)`: one match attempt over any subject, as a `Match` reporting +/// `mpos` and `endpos`, or `None`. The Python paths' engine entry. +pub(crate) fn exec_match(args: &[Object]) -> Result { + let [pattern, string, pos, endpos, mode, must_advance, mpos] = args else { + return Err(type_error("exec_match() takes 7 arguments")); + }; + let Object::Instance(inst) = pattern else { + return Err(type_error("exec_match(): expected a pattern")); + }; + let ext = class_ext(inst).ok_or_else(|| type_error("exec_match(): expected a pattern"))?; + let st = ext + .downcast_ref::() + .ok_or_else(|| type_error("exec_match(): expected a pattern"))?; + let (cc, _) = + pattern_code(st, inst).ok_or_else(|| type_error("exec_match(): expected a pattern"))?; + let subject = Subject::new(string)?; + let int = |o: &Object| { + o.as_i64() + .ok_or_else(|| type_error("exec_match(): expected an int")) + }; + let (p, e) = clamp(int(pos)?, int(endpos)?, subject.len()); + let mut marks = Vec::new(); + let must = must_advance.as_i64().unwrap_or(0) != 0; + match subject.exec(cc, p, e, int(mode)?, must, &mut marks)? { + None => Ok(Object::None), + Some(li) => new_match( + st, + pattern, + string, + int(mpos)?.max(0) as usize, + e, + &marks, + li, + ), + } +} + +/// `_sre.clear_templates(Pattern)`: forget the parsed templates +/// (`re.purge()`). +pub(crate) fn clear_templates(args: &[Object]) -> Result { + if let [Object::Type(cls)] = args { + if let Some(st) = state_of(cls) { + st.templates.lock().clear(); + } + } + Ok(Object::None) +} + +// --------------------------------------------------------------------- +// The module-level cache (`re.search(pattern, ...)` and friends). + +/// The fast half of [`compile_cached`]: a hit in `re._cache2`. +fn compile_cached_fast(args: &[Object]) -> Option> { + let [Object::Tuple(front), pattern, flags] = args else { + return None; + }; + let [Object::Dict(cache), _, Object::Type(flag_type)] = &front[..] else { + return None; + }; + let bt = crate::builtin_types::builtin_types(); + let ty = match pattern { + Object::Str(_) => bt.str_.clone(), + Object::Bytes(_) => bt.bytes_.clone(), + _ => return None, + }; + // `_compile` keys on `flags.value` for a `RegexFlag`, and on the flags + // themselves otherwise (any other int subclass keeps the Python path). + let flags = match flags { + Object::Int(_) | Object::Bool(_) => flags.clone(), + Object::Instance(i) if Rc::ptr_eq(&i.cls(), flag_type) => match i.native.get() { + Some(v @ Object::Int(_)) => v.clone(), + _ => return None, + }, + _ => return None, + }; + let key = DictKey(Object::new_tuple_array([ + Object::Type(ty), + pattern.clone(), + flags, + ])); + let hit = cache.borrow().get(&key).cloned(); + hit.map(Ok) +} + +/// `_sre.compile_cached((cache, compile, RegexFlag), pattern, flags)`: +/// `re._compile(pattern, flags)`, answered from the `_cache2` dict when +/// the pattern is there (the hit `_compile` would return first), else by +/// calling `_compile`. +pub(crate) fn compile_cached(args: &[Object]) -> Result { + if let Some(r) = compile_cached_fast(args) { + return r; + } + let [Object::Tuple(front), pattern, flags] = args else { + return Err(type_error("compile_cached() takes 3 arguments")); + }; + let compile = front + .get(1) + .cloned() + .ok_or_else(|| type_error("compile_cached(): bad cache front"))?; + with_interp(|i| i.call_object(compile, &[pattern.clone(), flags.clone()], &[])) +} + +/// The `compile_cached` builtin, with its fast half registered as a leaf. +pub(crate) fn compile_cached_builtin() -> Object { + let b = Rc::new(BuiltinFn { + name: "compile_cached", + binds_instance: false, + call: Box::new(compile_cached), + call_kw: None, + }); + crate::leaf_builtins::register_fast(&b, compile_cached_fast); + Object::Builtin(b) +} From c46c23bbc63fe5fc9c157268cc6aa1df551b9b5c Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 17:00:51 -0700 Subject: [PATCH 64/96] perf: skip regex tails that cannot start, and count classes in one loop A greedy or lazy single-character repeat (`REPEAT_ONE`, `MIN_REPEAT_ONE`) now tries its tail only at positions whose character can start it: the tail's first consuming op, past any `MARK`s, is a literal or a set that the character must satisfy. CPython checks only a literal that follows directly, so `(\w+)@` retried the tail at every backtracking step. Counting a lone category (`\w+`, `\d*`, `\s+`) runs one loop per class instead of walking the set per character. A callable `sub` replacement is called directly (the builtin already runs with the interpreter published), and a match's slot values are built in one allocation. Instructions retired per call (before -> after; CPython 3.14): search+group 19404 -> 11822 (12322), module re.search 22701 -> 15192 (13892), findall 56087 -> 43579 (52283), split 22358 -> 19300 (14079), sub str 12765 -> 10647 (12257), sub func 52715 -> 48707 (23800), finditer 40932 -> 38429 (15894), match 6165 -> 5986 (3228). --- crates/weavepy-vm/src/stdlib/sre_mod.rs | 95 ++++++++++++++++++--- crates/weavepy-vm/src/stdlib/sre_objects.rs | 28 +++--- 2 files changed, 99 insertions(+), 24 deletions(-) diff --git a/crates/weavepy-vm/src/stdlib/sre_mod.rs b/crates/weavepy-vm/src/stdlib/sre_mod.rs index d39ef159..67c396b6 100644 --- a/crates/weavepy-vm/src/stdlib/sre_mod.rs +++ b/crates/weavepy-vm/src/stdlib/sre_mod.rs @@ -304,6 +304,7 @@ fn loc_word(ch: u32) -> bool { } } +#[inline] fn uni_digit(ch: u32) -> bool { if ch < 128 { return ascii_digit(ch); @@ -341,6 +342,7 @@ fn nd_digit(c: char) -> bool { ) } +#[inline] fn uni_space(ch: u32) -> bool { if ch < 128 { // `char::is_whitespace` over ASCII. @@ -352,6 +354,7 @@ fn uni_space(ch: u32) -> bool { } } +#[inline] fn uni_word(ch: u32) -> bool { if ch < 128 { return ascii_word(ch); @@ -369,6 +372,7 @@ fn uni_linebreak(ch: u32) -> bool { ) } +#[inline] fn category(chcode: u32, ch: u32) -> bool { match chcode { CAT_DIGIT => ascii_digit(ch), @@ -408,6 +412,24 @@ fn cu(c: C) -> u32 { c.into() } +/// The length of `s`'s prefix whose units all satisfy `pred`. +#[inline(always)] +fn scan(s: &[C], pred: impl Fn(u32) -> bool) -> usize { + s.iter().position(|&c| !pred(cu(c))).unwrap_or(s.len()) +} + +/// What a tail's first consuming op requires of the character it starts +/// at, after any `MARK`s (which consume nothing): a `REPEAT_ONE` or +/// `MIN_REPEAT_ONE` skips the positions whose character cannot start the +/// tail instead of trying the tail there. +#[derive(Clone, Copy)] +enum TailFirst { + Any, + Literal(u32), + /// A set (`IN`), by its code position. + Set(usize), +} + /// Saved capture state (CPython's `LASTMARK_SAVE`, plus `MARK_PUSH` of /// the marks themselves while a `REPEAT` is active). #[derive(Clone)] @@ -919,6 +941,31 @@ impl<'a, C: SreChar> Matcher<'a, C> { up != lo && self.charset(set, up) } + /// The first-character requirement of the tail at `tail` (see + /// [`TailFirst`]). + #[inline] + fn tail_first(&self, mut tail: usize) -> TailFirst { + let code = self.code; + while code[tail] == OP_MARK { + tail += 2; + } + match code[tail] { + OP_LITERAL => TailFirst::Literal(code[tail + 1]), + OP_IN => TailFirst::Set(tail + 2), + _ => TailFirst::Any, + } + } + + /// Whether the character at `pos` can start a tail requiring `first`. + #[inline] + fn can_start(&self, first: TailFirst, pos: usize) -> bool { + match first { + TailFirst::Any => true, + TailFirst::Literal(c) => pos < self.end && cu(self.s[pos]) == c, + TailFirst::Set(set) => pos < self.end && self.charset(set, cu(self.s[pos])), + } + } + /// `SRE(count)` — count repeated single-character matches of the /// item at `pat`, starting at `self.ptr`, up to `maxcount`. fn count(&mut self, pat: usize, maxcount: u32) -> Result { @@ -932,11 +979,33 @@ impl<'a, C: SreChar> Matcher<'a, C> { let op = code[pat]; let counted = match op { OP_IN => { - let mut p = ptr; - while p < end && self.charset(pat + 2, cu(s[p])) { - p += 1; + let set = pat + 2; + match (code[set], code.get(set + 2).copied()) { + // A lone category (`\w+`, `\d*`, `\s+`): one loop per + // class, with no set walk per character. + (OP_CATEGORY, Some(OP_FAILURE)) => { + let s = &s[ptr..end]; + match code[set + 1] { + CAT_UNI_WORD => scan(s, uni_word), + CAT_WORD => scan(s, ascii_word), + CAT_UNI_DIGIT => scan(s, uni_digit), + CAT_DIGIT => scan(s, ascii_digit), + CAT_UNI_SPACE => scan(s, uni_space), + CAT_SPACE => scan(s, ascii_space), + CAT_UNI_NOT_WORD => scan(s, |c| !uni_word(c)), + CAT_UNI_NOT_DIGIT => scan(s, |c| !uni_digit(c)), + CAT_UNI_NOT_SPACE => scan(s, |c| !uni_space(c)), + cat => scan(s, |c| category(cat, c)), + } + } + _ => { + let mut p = ptr; + while p < end && self.charset(set, cu(s[p])) { + p += 1; + } + p - ptr + } } - p - ptr } OP_ANY => { let mut p = ptr; @@ -1266,13 +1335,10 @@ impl<'a, C: SreChar> Matcher<'a, C> { let save = self.snapshot(); let orig = ptr; let pmin_i = pmin as isize; - if code[tail] == OP_LITERAL { - let chr = code[tail + 1]; + let first = self.tail_first(tail); + if !matches!(first, TailFirst::Any) { loop { - while cnt >= pmin_i && { - let pos = orig + cnt as usize; - pos >= end || cu(self.s[pos]) != chr - } { + while cnt >= pmin_i && !self.can_start(first, orig + cnt as usize) { cnt -= 1; } if cnt < pmin_i { @@ -1331,15 +1397,18 @@ impl<'a, C: SreChar> Matcher<'a, C> { return Ok(true); } let save = self.snapshot(); + let first = self.tail_first(tail); loop { if !(pmax == MAXREPEAT || (cnt as u32) <= pmax) { break; } self.ptr = ptr; - if self.do_match(tail, toplevel)? { - return Ok(true); + if self.can_start(first, ptr) { + if self.do_match(tail, toplevel)? { + return Ok(true); + } + self.restore(&save); } - self.restore(&save); self.ptr = ptr; let r = self.count(pat + 3, 1)?; if r == 0 { diff --git a/crates/weavepy-vm/src/stdlib/sre_objects.rs b/crates/weavepy-vm/src/stdlib/sre_objects.rs index 5b9d2539..763def62 100644 --- a/crates/weavepy-vm/src/stdlib/sre_objects.rs +++ b/crates/weavepy-vm/src/stdlib/sre_objects.rs @@ -425,15 +425,16 @@ fn new_match( .match_cls .upgrade() .ok_or_else(|| type_error("re.Match is gone"))?; - let mut values = Vec::with_capacity(MATCH_SLOTS.len()); - values.push(pattern.clone()); - values.push(string.clone()); - values.push(Object::Int(pos as i64)); - values.push(Object::Int(endpos as i64)); - values.push(Object::Tuple( - crate::tuple_storage::TupleStorage::from_exact_iter(marks.iter().map(|&m| Object::Int(m))), - )); - values.push(Object::Int(lastindex as i64)); + let values = vec![ + pattern.clone(), + string.clone(), + Object::Int(pos as i64), + Object::Int(endpos as i64), + Object::Tuple(crate::tuple_storage::TupleStorage::from_exact_iter( + marks.iter().map(|&m| Object::Int(m)), + )), + Object::Int(lastindex as i64), + ]; Ok(build(cls, &st.match_layout, values)) } @@ -692,8 +693,13 @@ fn run_sub( } Filter::Callable(f) => { let m = new_match(st, recv, string, 0, e, &marks, li)?; - let f = (*f).clone(); - let item = with_interp(|i| i.call_object(f, &[m], &[]))?; + // A builtin call runs with the interpreter and its thread + // handles already published, so the callable is called + // directly (as `functools` does). + let item = with_interp(|i| { + let globals = i.builtins_dict(); + i.call(f, &[m], &[], &globals) + })?; if !matches!(item, Object::None) { sink.push_obj(&item); } From cea4bfdb26e49560e0a25d34ac400801acfc9b83 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:55:26 -0700 Subject: [PATCH 65/96] perf: test regex sets by table, and find short finditer matches up front Each character set in compiled code gets a membership table over the first 256 code points, built at compile time, so the matcher tests a Latin-1 character with one bit lookup instead of walking the set. A search whose pattern starts with a literal, a set, or a single-character repeat of one skips the positions that cannot start a match, as the INFO charset path already did. `finditer` over a `str` or `bytes` subject of at most 512 code units finds its matches when called and returns a native tuple iterator, which the dispatch loop steps and ends with no call and no `StopIteration`; longer subjects keep the lazy scanner. Match attempts hand their spans to the caller in place (no marks buffer), the matcher keeps up to 16 groups' marks inline, a match's spans tuple is built in one fixed-size allocation for up to five groups, handles resolve through a lock-free table instead of a thread-local copy, and the fast paths read the classes' state in place. Instructions retired per call (before -> after; CPython 3.14): split 19300 -> 10219 (14079), finditer 38429 -> 20650 (15894), findall 43579 -> 38234 (52283), match 5986 -> 5459 (3228), fullmatch 6033 -> 4494 (3338), groups/span/start/end 11891 -> 10559 (6717), sub str 10647 -> 9747 (12257), sub template 11881 -> 9471 (15120), search+group 11822 -> 11546 (12322). --- crates/weavepy-vm/src/stdlib/sre_mod.rs | 424 ++++++++++++++++---- crates/weavepy-vm/src/stdlib/sre_objects.rs | 317 +++++++++------ 2 files changed, 529 insertions(+), 212 deletions(-) diff --git a/crates/weavepy-vm/src/stdlib/sre_mod.rs b/crates/weavepy-vm/src/stdlib/sre_mod.rs index 67c396b6..f23b9b3f 100644 --- a/crates/weavepy-vm/src/stdlib/sre_mod.rs +++ b/crates/weavepy-vm/src/stdlib/sre_mod.rs @@ -142,6 +142,10 @@ const MAX_DEPTH: u32 = 200_000; pub(crate) struct CompiledCode { pub(crate) code: Vec, pub(crate) groups: usize, + /// Per code position, the index in `tables` of the Latin-1 membership + /// table of the character set starting there ([`NO_TABLE`] for none). + table_of: Vec, + tables: Vec<[u64; 4]>, /// Whether the pattern was a `str` (`Some(true)`) or a bytes-like /// object (`Some(false)`); `None` for a pattern compiled from a parsed /// tree (`re.Scanner`), which, like CPython's, accepts either subject. @@ -165,39 +169,188 @@ fn registry() -> &'static parking_lot::Mutex> { REGISTRY.get_or_init(|| parking_lot::Mutex::new(Vec::new())) } -thread_local! { - /// This thread's copy of the registry's prefix, so a lookup takes no - /// lock (entries never change once registered). - static LOCAL_CODES: std::cell::RefCell> = - const { std::cell::RefCell::new(Vec::new()) }; +/// Lock-free handle lookup: chunks of entries, allocated on demand and +/// never freed, published with release stores (registration is +/// serialized by the registry lock). +const CHUNK_BITS: usize = 10; +const CHUNK: usize = 1 << CHUNK_BITS; +const CHUNKS: usize = 4096; + +type Chunk = [std::sync::atomic::AtomicPtr; CHUNK]; + +static LOOKUP: [std::sync::atomic::AtomicPtr; CHUNKS] = + [const { std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()) }; CHUNKS]; + +fn publish(idx: usize, cc: &'static CompiledCode) { + use std::sync::atomic::{AtomicPtr, Ordering}; + let Some(slot) = LOOKUP.get(idx >> CHUNK_BITS) else { + return; + }; + let mut chunk = slot.load(Ordering::Acquire); + if chunk.is_null() { + let fresh: Box = Box::new([const { AtomicPtr::new(std::ptr::null_mut()) }; CHUNK]); + chunk = Box::into_raw(fresh); + slot.store(chunk, Ordering::Release); + } + // SAFETY: chunks are never freed. + let chunk = unsafe { &*chunk }; + chunk[idx & (CHUNK - 1)].store(std::ptr::from_ref(cc).cast_mut(), Ordering::Release); } /// Register compiled code; the handle indexes the registry. pub(crate) fn register_code(code: Vec, groups: usize, is_str: Option) -> i64 { + let (table_of, tables) = set_tables(&code); let cc: &'static CompiledCode = Box::leak(Box::new(CompiledCode { code, groups, + table_of, + tables, is_str, })); let mut reg = registry().lock(); reg.push(cc); - (reg.len() - 1) as i64 + let idx = reg.len() - 1; + publish(idx, cc); + idx as i64 +} + +const NO_TABLE: u32 = u32::MAX; + +/// Membership tables over the first 256 code points for the character +/// sets in `code`, so the matcher tests a Latin-1 character with one bit +/// lookup instead of walking the set. A table is built for every position +/// that follows an `IN`-family opcode word (and for the `INFO` block's +/// charset): the opcode value can also occur as data, but a table built +/// at a position that is not a set start is never consulted, since the +/// matcher only asks about real set starts. A position whose set does not +/// evaluate cleanly gets no table and keeps the full walk. +fn set_tables(code: &[u32]) -> (Vec, Vec<[u64; 4]>) { + let mut table_of = vec![NO_TABLE; code.len()]; + let mut tables = Vec::new(); + let mut starts: Vec = Vec::new(); + if code.first() == Some(&OP_INFO) && code.get(2).is_some_and(|f| f & SRE_INFO_CHARSET != 0) { + starts.push(5); + } + for (i, &op) in code.iter().enumerate() { + // ... , the skip landing past FAILURE. + if matches!(op, OP_IN | OP_IN_IGNORE | OP_IN_UNI_IGNORE) + && code + .get(i + 1) + .is_some_and(|&skip| skip >= 2 && code.get(i + skip as usize) == Some(&OP_FAILURE)) + { + starts.push(i + 2); + } + } + for set in starts { + if set >= code.len() || table_of[set] != NO_TABLE { + continue; + } + let mut t = [0u64; 4]; + let mut clean = true; + for ch in 0..256u32 { + match charset_checked(code, set, ch) { + Some(true) => t[(ch >> 6) as usize] |= 1 << (ch & 63), + Some(false) => {} + None => { + clean = false; + break; + } + } + } + if clean { + table_of[set] = tables.len() as u32; + tables.push(t); + } + } + (table_of, tables) +} + +/// `SRE(charset)` over possibly malformed code: `None` when the walk +/// leaves the code or meets an opcode a set cannot hold. +fn charset_checked(code: &[u32], mut set: usize, ch: u32) -> Option { + let at = |i: usize| code.get(i).copied(); + let mut ok = true; + // Real sets are short (the compiler folds long ones into bitmaps). + for _ in 0..256 { + let op = at(set)?; + set += 1; + match op { + OP_FAILURE => return Some(!ok), + OP_LITERAL => { + if ch == at(set)? { + return Some(ok); + } + set += 1; + } + OP_CATEGORY => { + if category(at(set)?, ch) { + return Some(ok); + } + set += 1; + } + OP_CHARSET => { + let word = at(set + (ch / 32) as usize)?; + at(set + 7)?; + if ch < 256 && word & (1u32 << (ch & 31)) != 0 { + return Some(ok); + } + set += 8; + } + OP_RANGE => { + if at(set)? <= ch && ch <= at(set + 1)? { + return Some(ok); + } + set += 2; + } + OP_RANGE_UNI_IGNORE => { + let (lo, hi) = (at(set)?, at(set + 1)?); + if lo <= ch && ch <= hi { + return Some(ok); + } + let uch = upper_unicode(ch); + if lo <= uch && uch <= hi { + return Some(ok); + } + set += 2; + } + OP_NEGATE => ok = !ok, + OP_BIGCHARSET => { + let count = at(set)? as usize; + set += 1; + let byte_index = (ch >> 8) as usize; + let word = at(set + byte_index / 4)?; + let block = ((word >> ((byte_index % 4) * 8)) & 0xff) as usize; + set += 64; + let bit = block * 256 + (ch as usize & 255); + if at(set + bit / 32)? & (1u32 << (bit & 31)) != 0 { + return Some(ok); + } + set += count * 8; + } + _ => return None, + } + } + None } /// The compiled code behind `handle`. #[inline] pub(crate) fn compiled(handle: i64) -> Option<&'static CompiledCode> { + use std::sync::atomic::Ordering; let idx = usize::try_from(handle).ok()?; - LOCAL_CODES.with(|local| { - if let Some(cc) = local.borrow().get(idx) { - return Some(*cc); + if let Some(slot) = LOOKUP.get(idx >> CHUNK_BITS) { + let chunk = slot.load(Ordering::Acquire); + if !chunk.is_null() { + // SAFETY: chunks are never freed. + let cc = unsafe { &*chunk }[idx & (CHUNK - 1)].load(Ordering::Acquire); + if !cc.is_null() { + // SAFETY: registered entries are leaked, never freed. + return Some(unsafe { &*cc }); + } } - let mut local = local.borrow_mut(); - let reg = registry().lock(); - let have = local.len(); - local.extend_from_slice(reg.get(have..)?); - local.get(idx).copied() - }) + } + // Beyond the lookup table's reach. + registry().lock().get(idx).copied() } // --------------------------------------------------------------------------- @@ -447,14 +600,59 @@ struct RepeatCtx { prev: Option, } +/// The matcher's capture marks: inline for up to 16 groups. +enum MarkBuf { + Inline { len: usize, buf: [isize; 32] }, + Heap(Vec), +} + +impl MarkBuf { + /// `n` marks, all unset (`-1`). + #[inline] + fn new(n: usize) -> Self { + if n <= 32 { + MarkBuf::Inline { + len: n, + buf: [-1; 32], + } + } else { + MarkBuf::Heap(vec![-1; n]) + } + } +} + +impl std::ops::Deref for MarkBuf { + type Target = [isize]; + #[inline] + fn deref(&self) -> &[isize] { + match self { + MarkBuf::Inline { len, buf } => &buf[..*len], + MarkBuf::Heap(v) => v, + } + } +} + +impl std::ops::DerefMut for MarkBuf { + #[inline] + fn deref_mut(&mut self) -> &mut [isize] { + match self { + MarkBuf::Inline { len, buf } => &mut buf[..*len], + MarkBuf::Heap(v) => v, + } + } +} + struct Matcher<'a, C: SreChar> { s: &'a [C], code: &'a [u32], + /// The compiled code's set tables (see [`set_tables`]). + table_of: &'a [u32], + tables: &'a [[u64; 4]], beginning: usize, start: usize, end: usize, ptr: usize, - marks: Vec, + marks: MarkBuf, lastmark: isize, lastindex: isize, must_advance: bool, @@ -571,10 +769,12 @@ fn code_range_has_mark(code: &[u32], start: usize, end: usize) -> bool { impl<'a, C: SreChar> Matcher<'a, C> { /// A matcher over `s`; `marks` holds `2 * groups` entries, all `-1`. - fn new(s: &'a [C], code: &'a [u32], marks: Vec) -> Self { + fn new(s: &'a [C], cc: &'a CompiledCode, marks: MarkBuf) -> Self { Matcher { s, - code, + code: &cc.code, + table_of: &cc.table_of, + tables: &cc.tables, beginning: 0, start: 0, end: s.len(), @@ -609,7 +809,7 @@ impl<'a, C: SreChar> Matcher<'a, C> { #[inline] fn snapshot(&self) -> MarkSnapshot { MarkSnapshot { - marks: self.cur_repeat.is_some().then(|| self.marks.clone()), + marks: self.cur_repeat.is_some().then(|| self.marks.to_vec()), lastmark: self.lastmark, lastindex: self.lastindex, } @@ -618,7 +818,7 @@ impl<'a, C: SreChar> Matcher<'a, C> { #[inline] fn restore(&mut self, snap: &MarkSnapshot) { if let Some(marks) = &snap.marks { - self.marks.clone_from(marks); + self.marks.copy_from_slice(marks); } self.lastmark = snap.lastmark; self.lastindex = snap.lastindex; @@ -859,7 +1059,21 @@ impl<'a, C: SreChar> Matcher<'a, C> { } /// `SRE(charset)` — is `ch` a member of the set starting at `set`? - fn charset(&self, mut set: usize, ch: u32) -> bool { + #[inline] + fn charset(&self, set: usize, ch: u32) -> bool { + if ch < 256 { + if let Some(&t) = self.table_of.get(set) { + if t != NO_TABLE { + let bits = self.tables[t as usize][(ch >> 6) as usize]; + return bits & (1 << (ch & 63)) != 0; + } + } + } + self.charset_walk(set, ch) + } + + /// [`Self::charset`] by walking the set. + fn charset_walk(&self, mut set: usize, ch: u32) -> bool { let code = self.code; let mut ok = true; loop { @@ -952,6 +1166,15 @@ impl<'a, C: SreChar> Matcher<'a, C> { match code[tail] { OP_LITERAL => TailFirst::Literal(code[tail + 1]), OP_IN => TailFirst::Set(tail + 2), + // A single-character repeat of at least one item starts with + // that item: . + OP_REPEAT_ONE | OP_MIN_REPEAT_ONE | OP_POSSESSIVE_REPEAT_ONE if code[tail + 2] >= 1 => { + match code[tail + 4] { + OP_LITERAL => TailFirst::Literal(code[tail + 5]), + OP_IN => TailFirst::Set(tail + 6), + _ => TailFirst::Any, + } + } _ => TailFirst::Any, } } @@ -1887,7 +2110,28 @@ impl<'a, C: SreChar> Matcher<'a, C> { } } - // The general case. + // The general case. When the pattern's first consuming op + // requires a known character, the positions without one are + // skipped (a match attempt there would fail at that op). + let first = self.tail_first(pat); + if !matches!(first, TailFirst::Any) { + let start = ptr; + loop { + if self.can_start(first, ptr) { + self.start = ptr; + self.ptr = ptr; + if self.do_match(pat, ptr == start)? { + return Ok(Some(ptr)); + } + self.reset_capture(); + } + self.must_advance = false; + if ptr >= end { + return Ok(None); + } + ptr += 1; + } + } self.start = ptr; self.ptr = ptr; let mut matched = self.do_match(pat, true)?; @@ -2096,52 +2340,74 @@ pub(crate) fn compile_code( Ok(register_code(code, groups, is_str)) } +/// A successful match: its span, and the capture state its group spans +/// are read from. +pub(crate) struct Found<'m> { + pub(crate) start: usize, + pub(crate) end: usize, + /// The last matched group (`-1` for none). + pub(crate) lastindex: isize, + groups: usize, + marks: &'m [isize], + lastmark: isize, +} + +impl Found<'_> { + /// The number of capturing groups. + #[inline] + pub(crate) fn groups(&self) -> usize { + self.groups + } + + /// Group `g`'s span (group 0 is the match), `(-1, -1)` for a group + /// that did not take part, normalized the way CPython's + /// `pattern_new_match` does. + #[inline] + pub(crate) fn span(&self, g: usize) -> (i64, i64) { + if g == 0 { + return (self.start as i64, self.end as i64); + } + let j = 2 * (g - 1); + if (j + 1) as isize <= self.lastmark && self.marks[j] >= 0 && self.marks[j + 1] >= 0 { + (self.marks[j] as i64, self.marks[j + 1] as i64) + } else { + (-1, -1) + } + } + + /// Entry `i` of the flat list of spans (`span(i / 2)`'s start or end). + #[inline] + pub(crate) fn mark(&self, i: usize) -> i64 { + let (a, b) = self.span(i / 2); + if i % 2 == 0 { + a + } else { + b + } + } +} + /// One `SRE(match)` / `SRE(search)` over `subject[pos..endpos]`. /// /// `mode` 0 searches, 1 matches at `pos`, 2 must match all of -/// `[pos, endpos)`. On a match, `marks` receives the match span followed -/// by each group's span, with `-1` for a group that did not take part -/// (normalized the way CPython's `pattern_new_match` does), and the -/// result is `Some(lastindex)` (`-1` for none). -pub(crate) fn exec_marks( +/// `[pos, endpos)`. On a match the result is `f` applied to it. +pub(crate) fn exec_found( cc: &CompiledCode, subject: &[C], pos: usize, endpos: usize, mode: i64, must_advance: bool, - marks: &mut Vec, -) -> Result, RuntimeError> { - marks.clear(); + f: impl FnOnce(&Found<'_>) -> R, +) -> Result, RuntimeError> { if pos > endpos { return Ok(None); } - // The matcher's mark array comes from this thread's spare (taken, so - // a match started while this one runs gets its own). - let mut buf = SPARE_MARKS.with(|s| std::mem::take(&mut *s.borrow_mut())); - buf.clear(); - buf.resize(cc.groups * 2, -1); - let mut m = Matcher::new(subject, &cc.code, buf); + let mut m = Matcher::new(subject, cc, MarkBuf::new(cc.groups * 2)); m.end = endpos; m.start = pos; m.ptr = pos; m.must_advance = must_advance; - let r = run_exec(&mut m, pos, mode, cc.groups, marks); - SPARE_MARKS.with(|s| *s.borrow_mut() = std::mem::take(&mut m.marks)); - r -} - -thread_local! { - static SPARE_MARKS: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; -} - -fn run_exec( - m: &mut Matcher<'_, C>, - pos: usize, - mode: i64, - groups: usize, - marks: &mut Vec, -) -> Result, RuntimeError> { let (mstart, ok) = match mode { // 1 = match (anchored at pos) 1 => { @@ -2163,20 +2429,14 @@ fn run_exec( if !ok { return Ok(None); } - marks.reserve(2 + groups * 2); - marks.push(mstart as i64); - marks.push(m.ptr as i64); - for g in 0..groups { - let j = 2 * g; - if (j + 1) as isize <= m.lastmark && m.marks[j] >= 0 && m.marks[j + 1] >= 0 { - marks.push(m.marks[j] as i64); - marks.push(m.marks[j + 1] as i64); - } else { - marks.push(-1); - marks.push(-1); - } - } - Ok(Some(m.lastindex)) + Ok(Some(f(&Found { + start: mstart, + end: m.ptr, + lastindex: m.lastindex, + groups: cc.groups, + marks: &m.marks, + lastmark: m.lastmark, + }))) } /// `_sre.exec(handle, string, pos, endpos, mode, must_advance)`. @@ -2197,25 +2457,19 @@ fn sre_exec(args: &[Object]) -> Result { .get(5) .map(|o| o.as_i64().unwrap_or(0) != 0) .unwrap_or(false); - let mut marks = Vec::new(); - let Some(lastindex) = exec_marks( - cc, - &subject[..], - pos, - endpos, - mode, - must_advance, - &mut marks, - )? - else { - return Ok(Object::None); - }; - Ok(Object::new_tuple_array([ - Object::Int(marks[0]), - Object::Int(marks[1]), - Object::Int(lastindex as i64), - Object::new_tuple(marks[2..].iter().map(|&v| Object::Int(v)).collect()), - ])) + let found = exec_found(cc, &subject[..], pos, endpos, mode, must_advance, |f| { + Object::new_tuple_array([ + Object::Int(f.start as i64), + Object::Int(f.end as i64), + Object::Int(f.lastindex as i64), + Object::new_tuple( + (2..2 + 2 * f.groups()) + .map(|i| Object::Int(f.mark(i))) + .collect(), + ), + ]) + })?; + Ok(found.unwrap_or(Object::None)) } fn sre_ascii_tolower(args: &[Object]) -> Result { diff --git a/crates/weavepy-vm/src/stdlib/sre_objects.rs b/crates/weavepy-vm/src/stdlib/sre_objects.rs index 763def62..bab10dd8 100644 --- a/crates/weavepy-vm/src/stdlib/sre_objects.rs +++ b/crates/weavepy-vm/src/stdlib/sre_objects.rs @@ -32,7 +32,7 @@ use crate::shared_value::{SharedSlice, SharedStr}; use crate::sync::{Rc, RefCell, Weak}; use crate::types::{PyInstance, SlotStorage, TypeObject}; -use super::sre_mod::{self, CompiledCode}; +use super::sre_mod::{self, CompiledCode, Found}; // Slot order of the natively built instances (the `__slots__` order the // classes declare). @@ -104,6 +104,18 @@ fn class_ext(inst: &PyInstance) -> Option { inst.class.borrow().native_ext.get().cloned() } +/// Run `f` with the state on `inst`'s class, read in place. `f` must run +/// no Python code (nothing can then reassign `__class__` and free the +/// class while it runs). +#[inline] +fn with_state(inst: &PyInstance, f: impl FnOnce(&State) -> R) -> Option { + if crate::gil::free_threading_enabled() { + let ext = class_ext(inst)?; + return Some(f(ext.downcast_ref::()?)); + } + Some(f(state_of(inst.cls_raw())?)) +} + fn with_interp( f: impl FnOnce(&mut crate::Interpreter) -> Result, ) -> Result { @@ -223,21 +235,19 @@ impl<'a> Subject<'a> { } } - /// One match attempt (see [`sre_mod::exec_marks`]). - fn exec( + /// One match attempt (see [`sre_mod::exec_found`]). + fn exec( &self, cc: &CompiledCode, pos: usize, endpos: usize, mode: i64, must_advance: bool, - marks: &mut Vec, - ) -> Result, RuntimeError> { + f: impl FnOnce(&Found<'_>) -> R, + ) -> Result, RuntimeError> { match &self.units { - Units::Narrow(b) => sre_mod::exec_marks(cc, b, pos, endpos, mode, must_advance, marks), - Units::Wide(w) => { - sre_mod::exec_marks(cc, &w[..], pos, endpos, mode, must_advance, marks) - } + Units::Narrow(b) => sre_mod::exec_found(cc, b, pos, endpos, mode, must_advance, f), + Units::Wide(w) => sre_mod::exec_found(cc, &w[..], pos, endpos, mode, must_advance, f), } } @@ -268,8 +278,8 @@ impl<'a> Subject<'a> { } /// Group `g`'s text from `marks`, or `default` when it did not match. - fn group(&self, marks: &[i64], g: usize, default: &Object) -> Object { - let (a, b) = (marks[2 * g], marks[2 * g + 1]); + fn group(&self, found: &Found<'_>, g: usize, default: &Object) -> Object { + let (a, b) = found.span(g); if a < 0 || b < 0 { default.clone() } else { @@ -393,23 +403,22 @@ fn merge_kwargs(args: &[Object], kw: &[(String, Object)], names: &[&str]) -> Opt Some(out) } -thread_local! { - static SPARE: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; -} - -/// A marks buffer from this thread's spare (taken, so a nested match, -/// from a callable replacement, gets its own). -fn take_marks() -> Vec { - SPARE.with(|s| std::mem::take(&mut *s.borrow_mut())) -} - -fn give_marks(v: Vec) { - SPARE.with(|s| { - let mut s = s.borrow_mut(); - if s.capacity() < v.capacity() { - *s = v; - } - }); +/// The flat spans tuple a match keeps (`_marks`), built in one +/// fixed-size allocation for the common group counts. +fn marks_tuple(found: &Found<'_>) -> Object { + use crate::tuple_storage::TupleStorage; + let n = 2 * (found.groups() + 1); + macro_rules! sized { + ($($k:literal),*) => { + match n { + $($k => TupleStorage::from_array::<$k>(std::array::from_fn(|i| { + Object::Int(found.mark(i)) + })),)* + _ => TupleStorage::from_exact_iter((0..n).map(|i| Object::Int(found.mark(i)))), + } + }; + } + Object::Tuple(sized!(2, 4, 6, 8, 10, 12)) } fn new_match( @@ -418,8 +427,7 @@ fn new_match( string: &Object, pos: usize, endpos: usize, - marks: &[i64], - lastindex: isize, + found: &Found<'_>, ) -> Result { let cls = st .match_cls @@ -430,10 +438,8 @@ fn new_match( string.clone(), Object::Int(pos as i64), Object::Int(endpos as i64), - Object::Tuple(crate::tuple_storage::TupleStorage::from_exact_iter( - marks.iter().map(|&m| Object::Int(m)), - )), - Object::Int(lastindex as i64), + marks_tuple(found), + Object::Int(found.lastindex as i64), ]; Ok(build(cls, &st.match_layout, values)) } @@ -467,11 +473,31 @@ fn exec_args(args: &[Object]) -> Option> { } /// Run `f` with the pattern's state and code, when the receiver is a -/// natively built pattern and the subject a fast one. +/// natively built pattern and the subject a fast one. `f` runs no Python +/// code, so the state is read through the class in place. fn with_pattern( recv: &Object, string: &Object, f: impl FnOnce(&State, &'static CompiledCode, i64) -> R, +) -> Option { + let Object::Instance(inst) = recv else { + return None; + }; + with_state(inst, |st| { + let (cc, handle) = pattern_code(st, inst)?; + if !fast_subject(string, cc) { + return None; + } + Some(f(st, cc, handle)) + })? +} + +/// [`with_pattern`] for a body that may run Python code: the state is +/// held by its own reference, not through the class. +fn with_pattern_held( + recv: &Object, + string: &Object, + f: impl FnOnce(&State, &'static CompiledCode, i64) -> R, ) -> Option { let Object::Instance(inst) = recv else { return None; @@ -490,14 +516,11 @@ fn pattern_exec(args: &[Object], mode: i64) -> Option Ok(Object::None), - Ok(Some(li)) => new_match(st, a.recv, a.string, p, e, &marks, li), - Err(err) => Err(err), - }; - give_marks(marks); - r + subject + .exec(cc, p, e, mode, false, |f| { + new_match(st, a.recv, a.string, p, e, f) + })? + .unwrap_or(Ok(Object::None)) }) } @@ -519,23 +542,20 @@ fn pattern_findall(args: &[Object]) -> Option> { let subject = Subject::new(a.string)?; let (mut p, e) = clamp(a.pos, a.endpos, subject.len()); let empty = empty_like(a.string); - let mut out = Vec::new(); - let mut marks = Vec::new(); + let mut out = Vec::with_capacity(8); let mut must_advance = false; while p <= e { - if subject - .exec(cc, p, e, 0, must_advance, &mut marks)? - .is_none() - { + let Some((s, en)) = subject.exec(cc, p, e, 0, must_advance, |f| { + out.push(match cc.groups { + 0 => subject.slice(f.start, f.end), + 1 => subject.group(f, 1, &empty), + n => Object::new_tuple((1..=n).map(|g| subject.group(f, g, &empty)).collect()), + }); + (f.start, f.end) + })? + else { break; - } - let (s, en) = (marks[0] as usize, marks[1] as usize); - let item = match cc.groups { - 0 => subject.slice(s, en), - 1 => subject.group(&marks, 1, &empty), - n => Object::new_tuple((1..=n).map(|g| subject.group(&marks, g, &empty)).collect()), }; - out.push(item); must_advance = s == en; p = en; } @@ -572,6 +592,49 @@ fn pattern_scanner(args: &[Object]) -> Option> { }) } +/// Subjects up to this many code units have their matches found when +/// `finditer` is called. +const EAGER_FINDITER: usize = 512; + +/// `finditer`: over a short subject, an iterator over the matches found +/// up front (a native tuple iterator, which the dispatch loop steps and +/// ends without a call or a `StopIteration`); over a longer one, a lazy +/// scanner. Only an immutable `str` or `bytes` subject is served, so +/// finding the matches early cannot observe a different subject. +fn pattern_finditer(args: &[Object]) -> Option> { + let a = exec_args(args)?; + let len = match a.string { + Object::Str(s) => SharedStr::char_count(s), + Object::Bytes(b) => b.len(), + _ => return None, + }; + if len > EAGER_FINDITER { + return pattern_scanner(args); + } + with_pattern(a.recv, a.string, |st, cc, _| { + let subject = Subject::new(a.string)?; + let (mut p, e) = clamp(a.pos, a.endpos, subject.len()); + let opos = p; + let mut out = Vec::with_capacity(8); + let mut must_advance = false; + while p <= e { + let Some((m, s, en)) = subject.exec(cc, p, e, 0, must_advance, |f| { + (new_match(st, a.recv, a.string, opos, e, f), f.start, f.end) + })? + else { + break; + }; + out.push(m?); + must_advance = s == en; + p = en; + } + let items = crate::tuple_storage::TupleStorage::from_vec(out); + Ok(Object::Iter(Rc::new(RefCell::new( + crate::object::PyIterator::Tuple { items, index: 0 }, + )))) + }) +} + /// How `sub` produces each replacement. enum Filter<'a> { Literal(&'a Object), @@ -667,32 +730,42 @@ fn run_sub( let subject = Subject::new(string)?; let e = subject.len(); let mut sink = Sink::new(&subject); - let mut marks = Vec::new(); let (mut p, mut last, mut n) = (0usize, 0usize, 0i64); let mut must_advance = false; while p <= e && (count == 0 || n < count) { - let Some(li) = subject.exec(cc, p, e, 0, must_advance, &mut marks)? else { - break; - }; - let (s, en) = (marks[0] as usize, marks[1] as usize); - sink.push_span(&subject, last, s); - match filter { - Filter::Literal(l) => sink.push_obj(l), - Filter::Template(t) => { - for piece in t.iter() { - match piece { - Piece::Lit(l) => sink.push_obj(l), - Piece::Group(g) => { - let (a, b) = (marks[2 * g], marks[2 * g + 1]); - if a >= 0 && b >= 0 { - sink.push_span(&subject, a as usize, b as usize); + // The literal and template replacements are written while the + // match is at hand; a callable gets a Match object, called after. + let Some((s, en, m)) = subject.exec(cc, p, e, 0, must_advance, |f| { + sink.push_span(&subject, last, f.start); + let m = match filter { + Filter::Literal(l) => { + sink.push_obj(l); + None + } + Filter::Template(t) => { + for piece in t.iter() { + match piece { + Piece::Lit(l) => sink.push_obj(l), + Piece::Group(g) => { + let (a, b) = f.span(*g); + if a >= 0 && b >= 0 { + sink.push_span(&subject, a as usize, b as usize); + } } } } + None } - } - Filter::Callable(f) => { - let m = new_match(st, recv, string, 0, e, &marks, li)?; + Filter::Callable(_) => Some(new_match(st, recv, string, 0, e, f)), + }; + (f.start, f.end, m) + })? + else { + break; + }; + match (filter, m) { + (Filter::Callable(f), Some(m)) => { + let m = m?; // A builtin call runs with the interpreter and its thread // handles already published, so the callable is called // directly (as `functools` does). @@ -704,6 +777,7 @@ fn run_sub( sink.push_obj(&item); } } + _ => {} } last = en; n += 1; @@ -749,7 +823,7 @@ fn sub_full(args: &[Object], subn: bool) -> Option> if callable && !crate::builtins::object_is_callable(repl) { return None; } - with_pattern(recv, string, |st, cc, handle| { + with_pattern_held(recv, string, |st, cc, handle| { let filter = if callable { Filter::Callable(repl) } else if has_backslash(repl) { @@ -797,22 +871,20 @@ fn pattern_split(args: &[Object]) -> Option> { with_pattern(recv, string, |_, cc, _| { let subject = Subject::new(string)?; let e = subject.len(); - let mut out = Vec::new(); - let mut marks = Vec::new(); + let mut out = Vec::with_capacity(8); let (mut p, mut last, mut n) = (0usize, 0usize, 0i64); let mut must_advance = false; while p <= e && (maxsplit == 0 || n < maxsplit) { - if subject - .exec(cc, p, e, 0, must_advance, &mut marks)? - .is_none() - { + let Some((s, en)) = subject.exec(cc, p, e, 0, must_advance, |f| { + out.push(subject.slice(last, f.start)); + for g in 1..=cc.groups { + out.push(subject.group(f, g, &Object::None)); + } + (f.start, f.end) + })? + else { break; - } - let (s, en) = (marks[0] as usize, marks[1] as usize); - out.push(subject.slice(last, s)); - for g in 1..=cc.groups { - out.push(subject.group(&marks, g, &Object::None)); - } + }; n += 1; must_advance = s == en; p = en; @@ -834,17 +906,17 @@ fn with_match( let Object::Instance(inst) = recv else { return None; }; - let ext = class_ext(inst)?; - let st = ext.downcast_ref::()?; - let slots = inst.slots.borrow(); - let v = slots.values_for_layout(&st.match_layout)?; - let Object::Tuple(marks) = &v[M_MARKS] else { - return None; - }; - if marks.len() < 2 || marks.len() % 2 != 0 { - return None; - } - f(st, v, marks) + with_state(inst, |st| { + let slots = inst.slots.borrow(); + let v = slots.values_for_layout(&st.match_layout)?; + let Object::Tuple(marks) = &v[M_MARKS] else { + return None; + }; + if marks.len() < 2 || marks.len() % 2 != 0 { + return None; + } + f(st, v, marks) + })? } enum GroupIdx { @@ -1069,28 +1141,26 @@ fn scanner_step(args: &[Object], mode: i64, iter: bool) -> Result Result { @@ -1168,7 +1238,7 @@ const SPECS: &[Spec] = &[ cls: 0, name: "finditer", key: "Pattern.finditer", - fast: pattern_scanner, + fast: pattern_finditer, full: None, params: EXEC_PARAMS, }, @@ -1445,20 +1515,13 @@ pub(crate) fn exec_match(args: &[Object]) -> Result { .ok_or_else(|| type_error("exec_match(): expected an int")) }; let (p, e) = clamp(int(pos)?, int(endpos)?, subject.len()); - let mut marks = Vec::new(); let must = must_advance.as_i64().unwrap_or(0) != 0; - match subject.exec(cc, p, e, int(mode)?, must, &mut marks)? { - None => Ok(Object::None), - Some(li) => new_match( - st, - pattern, - string, - int(mpos)?.max(0) as usize, - e, - &marks, - li, - ), - } + let mpos = int(mpos)?.max(0) as usize; + subject + .exec(cc, p, e, int(mode)?, must, |f| { + new_match(st, pattern, string, mpos, e, f) + })? + .unwrap_or(Ok(Object::None)) } /// `_sre.clear_templates(Pattern)`: forget the parsed templates From a21df4d1baee90feb22ca2c23f7e79a076d8f4ac Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:05:25 -0700 Subject: [PATCH 66/96] perf: skip regex start positions for case-folded literals and branches The first-character requirement a search, a single-character repeat's tail, and a `BRANCH` alternative are screened with now also covers case-insensitive literals and sets (`LITERAL_IGNORE`, `LITERAL_UNI_IGNORE`, `IN_IGNORE`, `IN_UNI_IGNORE`) and a `BRANCH` all of whose alternatives have one, so `re.compile("QUICK|jump", re.I)` no longer tries both alternatives at every position. The branch checks stay out of line, so the matcher's hot loop keeps its shape. The matcher's first stack-growth check now comes at the eighth level of recursion, so a shallow match never reads the stack limit. Instructions retired per call (before -> after; CPython 3.14): findall with re.I (two alternatives, 160-char text) 63871 -> 26557 (44724); search+group 11546 -> 11168 (12322), match 5459 -> 5377 (3228), findall 38234 -> 37589 (52283), split 10219 -> 9748 (14079). --- crates/weavepy-vm/src/stdlib/sre_mod.rs | 135 +++++++++++++++++++----- 1 file changed, 108 insertions(+), 27 deletions(-) diff --git a/crates/weavepy-vm/src/stdlib/sre_mod.rs b/crates/weavepy-vm/src/stdlib/sre_mod.rs index f23b9b3f..3992a119 100644 --- a/crates/weavepy-vm/src/stdlib/sre_mod.rs +++ b/crates/weavepy-vm/src/stdlib/sre_mod.rs @@ -579,8 +579,17 @@ fn scan(s: &[C], pred: impl Fn(u32) -> bool) -> usize { enum TailFirst { Any, Literal(u32), - /// A set (`IN`), by its code position. + /// A literal compared after ASCII (`LITERAL_IGNORE`) or Unicode + /// (`LITERAL_UNI_IGNORE`) lowercasing. + LiteralIgnore(u32), + LiteralUniIgnore(u32), + /// A set (`IN`), by its code position, and its lowercasing forms. Set(usize), + SetIgnore(usize), + SetUniIgnore(usize), + /// A `BRANCH` (by its code position) whose every alternative has a + /// requirement: any one of them. + Branch(usize), } /// Saved capture state (CPython's `LASTMARK_SAVE`, plus `MARK_PUSH` of @@ -1157,35 +1166,112 @@ impl<'a, C: SreChar> Matcher<'a, C> { /// The first-character requirement of the tail at `tail` (see /// [`TailFirst`]). - #[inline] - fn tail_first(&self, mut tail: usize) -> TailFirst { - let code = self.code; + #[inline(always)] + fn tail_first(&self, tail: usize) -> TailFirst { + match Self::simple_first(self.code, tail) { + (TailFirst::Any, op) if self.code[op] == OP_BRANCH => self.branch_first(op), + (first, _) => first, + } + } + + /// [`Self::tail_first`] without the `BRANCH` case, with the position + /// of the op it looked at (past any `MARK`s). + #[inline(always)] + fn simple_first(code: &[u32], mut tail: usize) -> (TailFirst, usize) { while code[tail] == OP_MARK { tail += 2; } - match code[tail] { - OP_LITERAL => TailFirst::Literal(code[tail + 1]), - OP_IN => TailFirst::Set(tail + 2), + let first = match code[tail] { // A single-character repeat of at least one item starts with // that item: . OP_REPEAT_ONE | OP_MIN_REPEAT_ONE | OP_POSSESSIVE_REPEAT_ONE if code[tail + 2] >= 1 => { - match code[tail + 4] { - OP_LITERAL => TailFirst::Literal(code[tail + 5]), - OP_IN => TailFirst::Set(tail + 6), - _ => TailFirst::Any, - } + Self::item_first(code, tail + 4) + } + _ => Self::item_first(code, tail), + }; + (first, tail) + } + + /// [`Self::tail_first`] of the `BRANCH` at `op`: + /// ` ( )* <0>`. + #[inline(never)] + fn branch_first(&self, op: usize) -> TailFirst { + let code = self.code; + let mut alt = op + 1; + while code[alt] != 0 { + // (A nested branch counts as no requirement.) + if matches!(Self::simple_first(code, alt + 1).0, TailFirst::Any) { + return TailFirst::Any; + } + alt += code[alt] as usize; + } + TailFirst::Branch(op) + } + + /// Whether some alternative of the `BRANCH` at `op` can start at `pos`. + #[inline(never)] + fn branch_can_start(&self, op: usize, pos: usize) -> bool { + let code = self.code; + let mut alt = op + 1; + while code[alt] != 0 { + if self.simple_can_start(Self::simple_first(code, alt + 1).0, pos) { + return true; } + alt += code[alt] as usize; + } + false + } + + /// Whether the alternative whose body starts at `body` can start at + /// `pos` (kept out of line: the `BRANCH` arm is in the matcher's hot + /// loop). + #[inline(never)] + fn alternative_can_start(&self, body: usize, pos: usize) -> bool { + self.can_start(self.tail_first(body), pos) + } + + /// The requirement of the single-character op at `op`. + #[inline(always)] + fn item_first(code: &[u32], op: usize) -> TailFirst { + match code[op] { + OP_LITERAL => TailFirst::Literal(code[op + 1]), + OP_LITERAL_IGNORE => TailFirst::LiteralIgnore(code[op + 1]), + OP_LITERAL_UNI_IGNORE => TailFirst::LiteralUniIgnore(code[op + 1]), + OP_IN => TailFirst::Set(op + 2), + OP_IN_IGNORE => TailFirst::SetIgnore(op + 2), + OP_IN_UNI_IGNORE => TailFirst::SetUniIgnore(op + 2), _ => TailFirst::Any, } } /// Whether the character at `pos` can start a tail requiring `first`. - #[inline] + #[inline(always)] fn can_start(&self, first: TailFirst, pos: usize) -> bool { + if let TailFirst::Branch(op) = first { + return self.branch_can_start(op, pos); + } + self.simple_can_start(first, pos) + } + + /// [`Self::can_start`] for a requirement other than a `BRANCH`. + #[inline(always)] + fn simple_can_start(&self, first: TailFirst, pos: usize) -> bool { + if matches!(first, TailFirst::Any) { + return true; + } + if pos >= self.end { + return false; + } + let ch = cu(self.s[pos]); match first { TailFirst::Any => true, - TailFirst::Literal(c) => pos < self.end && cu(self.s[pos]) == c, - TailFirst::Set(set) => pos < self.end && self.charset(set, cu(self.s[pos])), + TailFirst::Literal(c) => ch == c, + TailFirst::LiteralIgnore(c) => lower_ascii(ch) == c, + TailFirst::LiteralUniIgnore(c) => lower_unicode(ch) == c, + TailFirst::Set(set) => self.charset(set, ch), + TailFirst::SetIgnore(set) => self.charset(set, lower_ascii(ch)), + TailFirst::SetUniIgnore(set) => self.charset(set, lower_unicode(ch)), + TailFirst::Branch(_) => true, } } @@ -1334,8 +1420,10 @@ impl<'a, C: SreChar> Matcher<'a, C> { // Segmented stack growth: linear-depth backtracking (one native // frame per repetition) must not overflow the OS stack before // reaching MAX_DEPTH. Checking at every eighth level is enough: - // eight frames stay far inside the 128 KiB red zone. - let r = if self.depth % 8 == 1 { + // eight frames stay far inside the 128 KiB red zone. (The first + // check is at the eighth level, so a shallow match, the usual + // case, never asks for the stack pointer.) + let r = if self.depth % 8 == 0 { stacker::maybe_grow(128 * 1024, 4 * 1024 * 1024, || { self.do_match_inner(pat, toplevel) }) @@ -1506,16 +1594,9 @@ impl<'a, C: SreChar> Matcher<'a, C> { OP_BRANCH => { let save = self.snapshot(); while code[pat] != 0 { - // Fast skip when the branch can't possibly match. - if code[pat + 1] == OP_LITERAL - && (ptr >= end || cu(self.s[ptr]) != code[pat + 2]) - { - pat += code[pat] as usize; - continue; - } - if code[pat + 1] == OP_IN - && (ptr >= end || !self.charset(pat + 3, cu(self.s[ptr]))) - { + // Fast skip when the branch can't possibly match: + // its first consuming op rejects the character. + if !self.alternative_can_start(pat + 1, ptr) { pat += code[pat] as usize; continue; } From b33a0a4fedcea60edff040dbcaa18130311511c8 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 19:20:51 -0700 Subject: [PATCH 67/96] perf: switch to plain Python callees and back with less bookkeeping A Python-to-Python call the core loop runs in place paid for every general case on every call: the caller's activation looked up through a temporary shell, the call-site probes, the out-of-line slot and binding helpers, the warm-compile counter's thread-local threshold read, and on return a full slot reset, a generic release of the function and an entry value matched after the fact. Calls dominate deltablue's planning and every call-heavy micro benchmark. - A plain function (positional parameters only, no cells or closure) that its call site's shape names, called with exactly its arity, now binds in one pass: the arguments and the callable move into a pooled slot, the caller waits on the pending list through its own handles, and the threshold is read only while the counter can still reach it. - A clean return parks the slot with only the code handle to release, decrements a function something else still holds instead of releasing it generically, and runs the caller's entry protocol directly. - The core loop builds the native code's state only when it enters native code, not on every reload. Measured (retired instructions, min of 3): a call of a one-line function from an interpreted loop 2,432 -> 2,160 per iteration, a method call 2,731 -> 2,471; deltablue 34.8M -> 33.7M per unit. The five fixtures are unchanged within noise. --- crates/weavepy-vm/src/lib.rs | 225 +++++++++++++++++++++++++--- crates/weavepy-vm/src/tier2.rs | 28 ++-- tests/regrtest/test_inline_calls.py | 143 ++++++++++++++++++ 3 files changed, 361 insertions(+), 35 deletions(-) create mode 100644 tests/regrtest/test_inline_calls.py diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index acc40a40..95e28034 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -7962,6 +7962,23 @@ impl Interpreter { } } + /// [`Self::inline_park`] for a clean slot whose operands and locals + /// the caller already released (the in-loop return's common case): + /// only the code handle and the slot's own fields remain. + #[inline(always)] + fn inline_park_clean(&mut self, mut act: Box) { + debug_assert!(act.clean && act.frame.stack.is_empty()); + // SAFETY: as `inline_park`: the code handle is owned while active. + unsafe { drop(std::ptr::read(&raw const act.frame.code)) }; + act.parked = true; + act.clean = false; + act.guard = None; + act.caller_pending = None; + if self.inline_pool.len() < INLINE_POOL_CAP { + self.inline_pool.push(act); + } + } + /// `FOR_ITER` over a suspended generator at `pc` of `frame`, as an /// inline activation: the generator's boxed frame runs in place (the /// `generator_send_lean` shape, without a nested native activation). @@ -10338,22 +10355,10 @@ impl Interpreter { ext.frame_jit.warm(code, ext, nlocals, frame_jit::Heat::Call); } } + // The native code's view of the activation, built at its first + // entry (only the native copy of this loop enters it). #[cfg(feature = "jit")] - let mut nst = frame_jit::State { - locals: lbase, - stack: base, - len, - cap, - pc, - last, - countdown: std::ptr::addr_of_mut!(self.gil_countdown), - snap_gen, - maybe_dead: pending_work, - interp: std::ptr::from_ref(self), - out: 0, - depth_cell: sw.depth_cell, - err: None, - }; + let mut nst: Option = None; // The pc the native code last returned at (the arms run it). #[cfg(feature = "jit")] let mut handed = usize::MAX; @@ -10361,12 +10366,27 @@ impl Interpreter { #[cfg(feature = "jit")] if NATIVE && pc != handed { if let Some(native) = native.filter(|n| n.enters_at(pc)) { + let nst = nst.get_or_insert_with(|| frame_jit::State { + locals: lbase, + stack: base, + len, + cap, + pc, + last, + countdown: std::ptr::addr_of_mut!(self.gil_countdown), + snap_gen, + maybe_dead: pending_work, + interp: std::ptr::from_ref(self), + out: 0, + depth_cell: sw.depth_cell, + err: None, + }); nst.len = len; nst.pc = pc; nst.last = last; // SAFETY: the running activation's state, as this // loop holds it (its locals count checked above). - let status = unsafe { native.run(&mut nst) }; + let status = unsafe { native.run(nst) }; len = nst.len; pc = nst.pc; last = nst.last; @@ -12709,6 +12729,9 @@ impl Interpreter { // SAFETY: see `CoreSwitch`: the running activation is synced and // unborrowed here. Self::unpack_bound_callee(unsafe { &mut *sw.cur }, pc); + if self.core_call_plain(sw, pc) { + return true; + } let mut tmp = None; // SAFETY: see `CoreSwitch`: the running activation is the // innermost; its handles are live and unborrowed here. @@ -12752,6 +12775,133 @@ impl Interpreter { true } + /// [`Self::core_call`]'s common case, with every check the site's + /// [`CallSlot`] settles done once: a plain function (no cells, no + /// closure, positional parameters only) that the site saw before, + /// called with exactly its arity, switched to in a pooled slot. The + /// general path's work, without its probes: `false` touches nothing. + #[inline(always)] + fn core_call_plain(&mut self, sw: &mut CoreSwitch, pc: usize) -> bool { + // SAFETY: see `CoreSwitch`: the running activation is synced and + // unborrowed here. + let frame = unsafe { &mut *sw.cur }; + let Some(ins) = frame.code.instructions.get(pc) else { + return false; + }; + let argc = ins.arg as usize; + let n = frame.stack.len(); + let Some(callee_slot) = n.checked_sub(argc + 2) else { + return false; + }; + let Object::Function(f) = &frame.stack[callee_slot] else { + return false; + }; + let fp: *const crate::object::PyFunction = Rc::as_ptr(f); + let has_self = !matches!(frame.stack[callee_slot + 1], Object::Unbound); + // SAFETY: GIL-serialized raw read of the function's code cell (see + // `PyFunction::code`); only compared, then cloned below. + let code_rc: &Rc = unsafe { &*f.code.as_ptr() }; + let Some(slot) = code_call_slot(&frame.code, pc) else { + return false; + }; + if slot.hit(fp, Rc::as_ptr(code_rc)) != Some((0, has_self)) + || !code_rc.cellvars.is_empty() + || !code_rc.freevars.is_empty() + || !f.closure.is_empty() + || Self::has_extended_params(code_rc) + || !Self::lean_code_ok(code_rc) + || crate::gil::free_threading_enabled() + { + return false; + } + let Some(mut act) = self.inline_pool.pop() else { + return false; + }; + let crate::recursion::Enter::Ok(guard) = crate::recursion::enter_with(sw.depth_cell) + else { + self.inline_pool.push(act); + return false; + }; + // Committed. The arguments move into the slot's (empty, sole-owned) + // locals; the callable and the empty self slot leave the stack. + let code = code_rc.clone(); + let nlocals = code.varnames.len(); + // SAFETY: a parked slot's locals storage is its own, and empty. + let locals = unsafe { &mut *act.frame.locals.as_ptr() }; + let first = if has_self { callee_slot + 1 } else { callee_slot + 2 }; + let nargs = n - first; + locals.reserve(nlocals.max(nargs)); + // SAFETY: the `nargs` operands above `first` move into the locals + // (reserved above) and the stack forgets them; the callable moves + // out of its slot, and an empty self slot owns nothing. + let callable = unsafe { + let (src, dst) = (frame.stack.as_ptr().add(first), locals.as_mut_ptr()); + for k in 0..nargs { + dst.add(k).write(src.add(k).read()); + } + locals.set_len(nargs); + let callable = frame.stack.as_ptr().add(callee_slot).read(); + frame.stack.set_len(callee_slot); + callable + }; + fill_unbound(locals, nlocals); + frame.pc = pc as u32 + 1; + // SAFETY: the slot is parked (see `inline_bind_cells`): its handles + // are stale copies, overwritten without a drop. `fp` stays alive + // while the slot holds `callable`. + unsafe { + let fr: &mut Frame = &mut act.frame; + std::ptr::write(&raw mut fr.code, code); + std::ptr::write( + &raw mut fr.cells, + borrowed_rc(crate::object::empty_cells_ref()), + ); + std::ptr::write(&raw mut fr.globals, borrowed_rc(&(*fp).globals)); + std::ptr::write(&raw mut fr.builtins, borrowed_rc(&(*fp).builtins)); + std::ptr::write(&raw mut act.callable, callable); + } + act.frame.pc = 0; + act.parked = false; + act.call_pc = pc; + // The caller waits on the pending list (see `lean_pending_enter`). + // SAFETY: see `CoreSwitch`: the caller's handles are live, and + // nothing else borrows them here. + act.caller_pending = unsafe { + let depth = (*sw.inl).len(); + if depth == 0 { + self.lean_pending_enter(frame, &mut *sw.root_shell, pc) + } else if depth == sw.entry_depth && !sw.entry_dead { + self.lean_pending_enter(frame, &mut *sw.entry_shell, pc) + } else { + let inl: &mut Vec> = &mut *sw.inl; + let caller: *mut InlineAct = &raw mut *inl[depth - 1]; + self.lean_pending_enter(frame, &mut QuietShell::Lazy(&mut (*caller).act), pc) + } + }; + act.exc_depth = self.exc_info_len(); + act.guard = Some(guard); + // As `inline_bind_cells`: a hot body is compiled once, so native + // callers can take the direct lanes. + #[cfg(feature = "jit")] + { + let n = act.frame.code.jit_hint.bump_lean_entries(); + if n <= crate::tier2::LEAN_WARM_COMPILE_THRESHOLD_CAP + && n == crate::tier2::lean_warm_at() + && !crate::tier2::jit_off_for_process() + && !act.frame.code.jit_hint.is_not_jitable() + { + crate::tier2::warm_compile(self, &mut act.frame); + } + } + let callee: *mut Frame = &raw mut *act.frame; + // SAFETY: as above. + unsafe { (*sw.inl).push(act) }; + sw.cur = callee; + sw.scratch = usize::MAX; + sw.last = &raw mut sw.scratch; + true + } + /// A bound method over a plain function with an empty self slot, /// called at `pc` of `frame`, becomes that function with the receiver /// as self (CPython's `CALL_BOUND_METHOD_EXACT_ARGS`), so the cached @@ -15388,13 +15538,46 @@ impl Interpreter { // SAFETY: the callable is moved out exactly once; the parked // slot treats the field as stale (see `inline_deliver`). let callable = unsafe { std::ptr::read(&raw const done.callable) }; - self.inline_park(done); + self.inline_park_clean(done); // SAFETY: as above. - unsafe { - (*cframe).stack.push(v); - self.release(callable); + unsafe { (*cframe).stack.push(v) }; + match callable { + // A function something else still holds (its class, its + // module) can't die here: a plain decrement. + Object::Function(f) if Rc::strong_count(&f) > 1 => drop(f), + callable => self.release(callable), + } + sw.cur = cframe; + sw.last = if clast == &raw mut sw.scratch { + sw.scratch = usize::MAX; + &raw mut sw.scratch + } else { + clast + }; + // The caller's `Returned` entry protocol (`quiet_frame`), as + // below. + // SAFETY: as above. + unsafe { *sw.last = call_pc }; + if self.gil_countdown <= 2 { + sw.pending = Some(CoreExit::Stop(LeafStop::Breaker)); + return true; + } + self.gil_countdown -= 1; + if crate::hot_gates::loop_gen() != snap_gen { + sw.pending = Some(CoreExit::Stop(LeafStop::Breaker)); + return true; + } + // SAFETY: see `quiet_run`. + if unsafe { (*sw.maybe_dead).get() } { + // SAFETY: as above. + unsafe { + self.flush_lean(&mut *cframe, &mut *cshell.cast::>()); + } + if self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen { + sw.pending = Some(CoreExit::Stop(LeafStop::Breaker)); + } } - entry = QuietEntry::Returned { cur_pc: call_pc }; + return true; } else { let result = if clean { done.clean = !had_shell; diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 0db40f75..eb69854b 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -8411,20 +8411,6 @@ unsafe extern "C" fn wpjit_get_iter(frame: *mut JitFrame, pin: i64) -> i64 { } } -/// The `wpjit_iter_next` helper (RFC 0071 WS4): one step of a -/// [`weavepy_jit::TTerm`]`::ForIter` loop over a pinned identity -/// iterable. **Runs Python code** for a generator source (the resume -/// executes the generator body — possibly natively, through -/// `try_enter_resume`), so burned-in resolutions are revalidated after -/// a dirty step; builtin iterators step without running Python and -/// skip the revalidation. Statuses per [`weavepy_jit::IterNextHelper`]: -/// `0` element in the lane, `1` exhausted, `2` deopt at the header -/// (nothing consumed), `3` element consumed but outside the lane (raw -/// object pinned, resume at the fused store), `4` raised. -/// -/// # Safety -/// -/// Same contract as [`wpjit_call_py`]. /// One step of a list, tuple or range iterator, without the generic /// dispatch: the next item, or `None` once exhausted (the iterator then /// detached as [`PyIterator::next_value`] leaves it). `None` outside for @@ -8441,6 +8427,20 @@ fn builtin_seq_step(cell: &Rc>) -> Option< } } +/// The `wpjit_iter_next` helper (RFC 0071 WS4): one step of a +/// [`weavepy_jit::TTerm`]`::ForIter` loop over a pinned identity +/// iterable. **Runs Python code** for a generator source (the resume +/// executes the generator body — possibly natively, through +/// `try_enter_resume`), so burned-in resolutions are revalidated after +/// a dirty step; builtin iterators step without running Python and +/// skip the revalidation. Statuses per [`weavepy_jit::IterNextHelper`]: +/// `0` element in the lane, `1` exhausted, `2` deopt at the header +/// (nothing consumed), `3` element consumed but outside the lane (raw +/// object pinned, resume at the fused store), `4` raised. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_py`]. unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i64) -> i64 { // SAFETY: see wpjit_call_py — same live-buffer contract. let jf = unsafe { &mut *frame }; diff --git a/tests/regrtest/test_inline_calls.py b/tests/regrtest/test_inline_calls.py new file mode 100644 index 00000000..5b0d3319 --- /dev/null +++ b/tests/regrtest/test_inline_calls.py @@ -0,0 +1,143 @@ +"""Plain Python calls the core loop switches to in place. + +A call of a plain function (positional parameters only, no cells) that +its call site has seen before runs in a pooled activation, and its return +hands the result straight back. Each loop here runs long enough for the +sites to settle; then the callee, its arguments or its frame change, and +every call must still behave as the interpreter's general call does. +""" + +import sys +import unittest + +WARM = 3000 + + +def add(a, b): + return [a + b][0] + + +def ident(x): + return [x][0] + + +class Box: + def __init__(self, v): + self.v = v + + def get(self): + return [self.v][0] + + def put(self, v): + self.v = [v][0] + return self + + +def loop_add(n): + t = 0 + for i in range(n): + t += add(i, 1) + return t + + +def loop_methods(n): + b = Box(0) + t = 0 + for i in range(n): + t += b.put(i).get() + return t + + +def depth(n): + if n == 0: + return [sys._getframe(0).f_code.co_name, sys._getframe(1).f_code.co_name] + return depth(n - 1) + + +class InlineCallTests(unittest.TestCase): + def test_results(self): + self.assertEqual(loop_add(WARM), sum(range(WARM)) + WARM) + self.assertEqual(loop_methods(WARM), sum(range(WARM))) + + def test_rebinding_the_callee(self): + global add + saved = add + try: + self.assertEqual(loop_add(WARM), sum(range(WARM)) + WARM) + add = lambda a, b: a - b + self.assertEqual(loop_add(10), sum(range(10)) - 10) + + def add(a, b=100): + return a + b + + self.assertEqual(loop_add(10), sum(range(10)) + 10) + finally: + add = saved + + def test_code_replaced(self): + def f(x): + return [x][0] + + def g(x): + return [x * 2][0] + + def run(n): + t = 0 + for i in range(n): + t += f(i) + return t + + self.assertEqual(run(WARM), sum(range(WARM))) + f.__code__ = g.__code__ + self.assertEqual(run(10), 2 * sum(range(10))) + + def test_frames_and_errors(self): + for _ in range(WARM): + ident(1) + self.assertEqual(depth(5), ["depth", "depth"]) + + def boom(x): + return [x][1] + + def caller(n): + for i in range(n): + boom(i) + + try: + caller(5) + except IndexError as e: + tb = e.__traceback__ + else: + self.fail("no IndexError") + names = [] + while tb: + names.append(tb.tb_frame.f_code.co_name) + tb = tb.tb_next + self.assertEqual(names[-2:], ["caller", "boom"]) + + def test_recursion_limit(self): + def down(n): + return [down(n + 1)][0] + + with self.assertRaises(RecursionError): + down(0) + self.assertEqual(loop_add(10), sum(range(10)) + 10) + + def test_finalizers_run_on_return(self): + seen = [] + + class Note: + def __del__(self): + seen.append(1) + + def make(i): + n = Note() + return [i][0] + + for i in range(WARM): + make(i) + self.assertEqual(len(seen), WARM) + + +if __name__ == "__main__": + unittest.main() From 0ce6ba97f86602f2f0716d4c25474a52221004cd Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 20:42:04 -0700 Subject: [PATCH 68/96] perf: store a compiled effect leaf's scalar field in line A compiled effect leaf whose only store is its last operation (a setter, or deltablue's `execute` writing `v2.value`) called the `h_store` helper for it, which re-derived everything the interpreter's indexed store checks: the site's field slot, the class version, the split values, and the old value's ownership. When nothing after the store can decline, the native code now does the interpreter's indexed store itself for a scalar written over a scalar: it checks the receiver is a plain instance whose class has the site's recorded version, whose split values use the class's keys, are borrowed by nobody and hold the site's position, and that no dict watcher or shared-cell mode is active; then it writes the tag and payload. Any other case calls the helper as before. Measured (retired instructions, min of 3): deltablue's execute phase 8.44M -> 8.09M per unit, the full fixture 33.67M -> 33.37M per unit. The planning phase is unchanged within noise. --- crates/weavepy-vm/src/leaf_plan/jit.rs | 110 ++++++++++++++++++++++++- crates/weavepy-vm/src/lib.rs | 13 +++ tests/regrtest/test_jit_plan_stores.py | 56 +++++++++++++ 3 files changed, 178 insertions(+), 1 deletion(-) diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index 8d6b2867..e6120090 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -1253,6 +1253,98 @@ impl Lower<'_> { self.b.ins().jump(done, &[t.into(), p.into()]); } + /// The interpreter's indexed store (`core_store_attr` through the + /// site's [`crate::FieldSlot`] at `slot`) in line, for a scalar value + /// over a scalar field: the receiver register is a plain instance whose + /// class has the slot's version and whose values, split over its + /// class's names and borrowed by nobody, hold the slot's position. Its + /// old value owns nothing, and neither does the new one (no write + /// barrier, no release). Jumps to `done` stored, or to `slow` untouched. + fn inline_store( + &mut self, + l: &weavepy_jit::ObjLayout, + (tr, pr): (Value, Value), + (tv, pv): (Value, Value), + slot: i64, + done: Block, + slow: Block, + ) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + // The value: a scalar register. + let is_i = self.b.ins().icmp_imm(IntCC::Equal, tv, T_I as i64); + let is_f = self.b.ins().icmp_imm(IntCC::Equal, tv, T_F as i64); + let is_b = self.b.ins().icmp_imm(IntCC::Equal, tv, T_B as i64); + let is_n = self.b.ins().icmp_imm(IntCC::Equal, tv, T_N as i64); + let any = self.b.ins().bor(is_i, is_f); + let any = self.b.ins().bor(any, is_b); + let any = self.b.ins().bor(any, is_n); + let not_scalar = self.b.ins().icmp_imm(IntCC::Equal, any, 0); + let not_ref = self.b.ins().icmp_imm(IntCC::NotEqual, tr, T_R as i64); + let bad = self.b.ins().bor(not_scalar, not_ref); + self.miss_if(bad, slow); + let otag = self.b.ins().uload8(types::I32, f, pr, 0); + let not_inst = self.b.ins().icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + self.miss_if(not_inst, slow); + let inst = self.b.ins().load(ptr, f, pr, 8); + let cls = self.b.ins().load(ptr, f, inst, l.inst_class); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let at = self.b.ins().iconst(ptr, slot); + let want = self.b.ins().load(types::I64, f, at, crate::FIELD_SLOT_VER as i32); + let idx = self.b.ins().uload32(f, at, crate::FIELD_SLOT_IDX as i32); + let lazy = self.b.ins().load(ptr, f, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, f, flag, 0); + let watched = self.b.ins().iconst(ptr, l.dict_watchers as i64); + let watched = self.b.ins().uload8(types::I64, f, watched, 0); + let borrow = self.b.ins().sload32(f, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, f, inst, l.inst_split_block); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let busy = self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().bor(other, watched); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, slow); + let keys = self.b.ins().load(ptr, f, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, f, cls, l.type_shared_keys); + let len = self.b.ins().uload32(f, block, l.split_len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let bad = self.b.ins().bor(foreign, absent); + self.miss_if(bad, slow); + let off = self.b.ins().ishl_imm(idx, 4); + let v = self.b.ins().iadd(block, off); + let v = self.b.ins().iadd_imm(v, i64::from(l.split_values)); + // The old value: a scalar, which owns nothing. + let old = self.b.ins().uload8(types::I32, f, v, 0); + let mut owns = self.b.ins().iconst(types::I8, 1); + for tag in [l.tag_int, l.tag_float, l.tag_bool, l.tag_none] { + let hit = self.b.ins().icmp_imm(IntCC::Equal, old, i64::from(tag)); + let clear = self.b.ins().bxor_imm(hit, 1); + owns = self.b.ins().band(owns, clear); + } + self.miss_if(owns, slow); + // The new value's tag byte and payload (a bool's byte at offset 1). + let t_int = self.b.ins().iconst(types::I32, i64::from(l.tag_int)); + let t_float = self.b.ins().iconst(types::I32, i64::from(l.tag_float)); + let t_bool = self.b.ins().iconst(types::I32, i64::from(l.tag_bool)); + let t_none = self.b.ins().iconst(types::I32, i64::from(l.tag_none)); + let tag = self.b.ins().select(is_i, t_int, t_none); + let tag = self.b.ins().select(is_f, t_float, tag); + let tag = self.b.ins().select(is_b, t_bool, tag); + self.b.ins().istore8(f, tag, v, 0); + let byte = self.b.ins().band_imm(pv, 1); + self.b.ins().istore8(f, byte, v, 1); + self.b.ins().store(f, pv, v, 8); + self.b.ins().jump(done, &[]); + } + /// A class attribute the site's stamp remembers, in line: the receiver /// register `(t, p)` is a class at the stamp's version, which resolved /// the name to a scalar (see `class_attr_fill`). Jumps to `done` with @@ -1916,13 +2008,29 @@ impl Lower<'_> { } let (tr, pr) = self.get(fl, recv); let (tv, pv) = self.get(fl, val); + let last = Self::infallible_after(fl.plan, i); + // The plan's only store, with nothing after it that can + // decline: a scalar over a scalar field lands in line. + let done = self.b.create_block(); + if last && !fl.may_store[i] { + if let (Some(l), Some(slot)) = ( + crate::tier2::published_obj_layout(), + crate::code_field_slot(fl.code, usize::from(pc)), + ) { + let slow = self.b.create_block(); + self.inline_store(l, (tr, pr), (tv, pv), slot as i64, done, slow); + self.b.switch_to_block(slow); + } + } let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); - let last = self.u32c(u32::from(Self::infallible_after(fl.plan, i))); + let last = self.u32c(u32::from(last)); let st = self.call( h_store as *const () as usize, &[self.ctx, tr, pr, tv, pv, pn, last], ); self.check(st); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); } } Some(true) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 95e28034..fd69f80c 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -62296,8 +62296,21 @@ impl StampSlot { /// is process-unique, so while both match, the value at that position of /// a split instance *is* the attribute: no inline-cache decode and no /// name comparison. +#[repr(transparent)] struct FieldSlot(std::cell::UnsafeCell<(u64, u32)>); +/// Where a [`FieldSlot`] keeps its class version (`u64`) and split +/// position (`u32`), for native code that reads it in line. +pub(crate) const FIELD_SLOT_VER: usize = std::mem::offset_of!((u64, u32), 0); +pub(crate) const FIELD_SLOT_IDX: usize = std::mem::offset_of!((u64, u32), 1); + +/// The [`FieldSlot`] of the attribute site at `pc` of `code`, if the code +/// has any (native code reads it in line). +pub(crate) fn code_field_slot(code: &CodeObject, pc: usize) -> Option<*const u8> { + let slot = code_vm_ext(code)?.field_slots.get()?.get(pc)?; + Some(std::ptr::from_ref(slot).cast()) +} + // SAFETY: as `StampSlot`. unsafe impl Send for FieldSlot {} unsafe impl Sync for FieldSlot {} diff --git a/tests/regrtest/test_jit_plan_stores.py b/tests/regrtest/test_jit_plan_stores.py index ad8a8806..6ae2cb9a 100644 --- a/tests/regrtest/test_jit_plan_stores.py +++ b/tests/regrtest/test_jit_plan_stores.py @@ -102,6 +102,62 @@ class Slotted: drive(n, 1) self.assertEqual(s.v, 3) + def test_scalar_stores_in_place(self): + class Src: + def __init__(self, x): + self.x = x + + class Dst: + def __init__(self): + self.v = 0 + self.w = 1 + + class Copy: + def __init__(self, a, b): + self.a = a + self.b = b + + def put(self): + self.b.v = self.a.x + + src, dst = Src(1), Dst() + c = Copy(src, dst) + drive(c, WARM) + self.assertEqual(dst.v, 1) + for value in (2.5, True, None, -7, 2**62): + src.x = value + drive(c, 1) + self.assertIs(type(dst.v), type(value)) + self.assertEqual(dst.v, value) + # A heap value, over a scalar and then over itself. + src.x = [1] + drive(c, 2) + self.assertIs(dst.v, src.x) + src.x = 3 + drive(c, 1) + self.assertEqual(dst.v, 3) + # Another instance of the class, without the field yet. + fresh = Dst.__new__(Dst) + c.b = fresh + drive(c, 1) + self.assertEqual(vars(fresh), {"v": 3}) + # An instance whose dictionary was handed out. + other = Dst() + d = other.__dict__ + c.b = other + src.x = 9 + drive(c, 1) + self.assertEqual(d["v"], 9) + # The class changes under the cached version. + c.b = dst + Dst.extra = 1 + src.x = 11 + drive(c, 1) + self.assertEqual(dst.v, 11) + Dst.v = property(lambda self: -1, lambda self, value: None) + drive(c, 1) + self.assertEqual(dst.v, -1) + if __name__ == "__main__": unittest.main() From 7136380229461fe503cd1c10cf5a53f1f8dde6f5 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:47:22 -0700 Subject: [PATCH 69/96] perf: make functions in the leaf burst and keep young ones off the GC's books A `def` or `lambda` that runs in a loop still paid for two things per function: - The burst's slow arms build functions (and fold the closure and defaults attributes that follow), but `MAKE_FUNCTION` was missing from their opcode table, so the core loop handed every one to the full step. The table now lists it, along with a closure tuple's `LOAD_FAST` of a cell and a prologue `MAKE_CELL`, which the slow arms now run too. - Every function was registered with the collector at birth (an index entry and a handle), and its globals dict re-checked for tracking. Functions now join the young set instances already use: held weakly until a collection, a reflective API or a finalization pass flushes the live ones to the collector, together with their globals dicts. `gc.is_tracked` answers for a young function as for a young instance. Measured (retired instructions per call, a local call of each case): `lambda x: x + 1` built 5,555 -> 3,964 (CPython 1,138); built and called 6,268 -> 4,426; a nested `def` 7,843 -> 5,177; the closure case `mk(i)(1)` 15,271 -> 12,430. Deltablue and the five fixtures are unchanged within noise. --- crates/weavepy-vm/src/gc_trace.rs | 95 +++++++++++++++++++------ crates/weavepy-vm/src/lib.rs | 25 +++++++ crates/weavepy-vm/src/stdlib/gc_real.rs | 2 + 3 files changed, 102 insertions(+), 20 deletions(-) diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 9d9de59f..e90f89bd 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -389,6 +389,11 @@ pub struct GcState { /// collector the ones still alive ([`GcState::flush_young`]), so it /// sees exactly the population eager tracking would have shown it. young: RefCell>>, + /// Functions born since the last collection, held weakly as + /// [`Self::young`] holds instances (a `def` or `lambda` run in a loop + /// makes one per pass). A flush also hands the collector each live + /// one's globals dict, which [`track`] would have tracked at birth. + young_fns: RefCell>>, /// Frozen handles. `gc.freeze()` moves all tracked objects /// here; they are skipped by future collections until /// `gc.unfreeze()` runs. @@ -437,6 +442,7 @@ impl GcState { deferred: RefCell::new(Vec::new()), deferred_limit: AtomicUsize::new(DEFERRED_FLOOR), young: RefCell::new(Vec::new()), + young_fns: RefCell::new(Vec::new()), frozen: RefCell::new(Vec::new()), garbage: RefCell::new(Vec::new()), callbacks: RefCell::new(Vec::new()), @@ -474,6 +480,7 @@ impl GcState { RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).counts)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).deferred)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).young)); + RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).young_fns)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).frozen)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).garbage)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).callbacks)); @@ -521,20 +528,34 @@ impl GcState { return; } if let Object::Instance(inst) = obj { - if self.nurse(inst) { + if self.nurse(&self.young, inst) { return; } } self.track_now(obj); } - /// Register a newborn instance in the young set (see [`Self::young`]). - /// `false` leaves it to eager registration. - fn nurse(&self, inst: &crate::Rc) -> bool { + /// [`Self::track`] for a function: young (see [`Self::young_fns`]), or + /// registered at once together with its globals dict. + fn track_function(&self, obj: &Object, f: &crate::Rc) { + if self.nurse(&self.young_fns, f) { + return; + } + self.track(&Object::Dict(f.globals.clone())); + self.track_now(obj); + } + + /// Register a newborn instance (or function) in its young set (see + /// [`Self::young`]). `false` leaves it to eager registration. + fn nurse( + &self, + set: &RefCell>>, + obj: &crate::Rc, + ) -> bool { let over = { // SAFETY: nothing below runs code while the set is borrowed // (a dead predecessor's release only frees memory). - let Some(young) = (unsafe { self.young.peek_mut() }) else { + let Some(young) = (unsafe { set.peek_mut() }) else { // A flush is walking the set, or cells are shared. return false; }; @@ -547,14 +568,14 @@ impl GcState { young.swap_remove(i); } } - young.push(crate::Rc::downgrade(inst)); + young.push(crate::Rc::downgrade(obj)); young.len() >= YOUNG_CAP }; // As in `track_now`: an id recycled from a finalized object starts // un-finalized (a finalizer that runs from here on is recorded). // SAFETY: a read that runs no code. if unsafe { self.finalized_ids.peek() }.is_none_or(|f| !f.is_empty()) { - let id = crate::Rc::as_ptr(inst) as usize as ObjectId; + let id = crate::Rc::as_ptr(obj) as usize as ObjectId; self.finalized_ids.borrow_mut().remove(&id); } self.note_gen0_alloc(); @@ -564,8 +585,20 @@ impl GcState { true } - /// Register the young instances still alive with the collector. + /// Register the young instances and functions still alive with the + /// collector. fn flush_young(&self) { + if let Ok(mut fns) = self.young_fns.try_borrow_mut() { + if !fns.is_empty() { + let fns = std::mem::take(&mut *fns); + for w in fns { + if let Some(f) = w.upgrade() { + self.track(&Object::Dict(f.globals.clone())); + self.register(&Object::Function(f), false); + } + } + } + } let young = match self.young.try_borrow_mut() { Ok(mut young) if !young.is_empty() => std::mem::take(&mut *young), _ => return, @@ -577,22 +610,30 @@ impl GcState { } } - /// The young instances still alive (the dead ones are dropped). + /// The young instances and functions still alive (the dead ones are + /// dropped). fn young_live(&self) -> usize { + let fns = self.young_fns.try_borrow_mut().map_or(0, |mut fns| { + fns.retain(|w| w.strong_count() > 0); + fns.len() + }); let Ok(mut young) = self.young.try_borrow_mut() else { - return 0; + return fns; }; young.retain(|w| w.strong_count() > 0); - young.len() + young.len() + fns } - /// Whether the instance `id` is in the young set. + /// Whether the instance or function `id` is in a young set. fn is_young(&self, id: ObjectId) -> bool { - self.young.try_borrow().is_ok_and(|young| { - young - .iter() - .any(|w| w.as_ptr() as usize as ObjectId == id && w.strong_count() > 0) - }) + fn holds(set: &RefCell>>, id: ObjectId) -> bool { + set.try_borrow().is_ok_and(|young| { + young + .iter() + .any(|w| w.as_ptr() as usize as ObjectId == id && w.strong_count() > 0) + }) + } + holds(&self.young, id) || holds(&self.young_fns, id) } /// [`Self::track`] for a container its builder knows holds only @@ -753,6 +794,19 @@ impl GcState { /// Stop tracking `obj`. Backs the explicit `gc._untrack(obj)` extension /// and the C-API `PyObject_GC_UnTrack`. pub fn untrack_id(&self, id: ObjectId) { + // (From the newest: an untrack usually follows its birth closely.) + if let Ok(mut fns) = self.young_fns.try_borrow_mut() { + if let Some(i) = fns + .iter() + .rposition(|w| w.as_ptr() as usize as ObjectId == id) + { + fns.swap_remove(i); + let mut counts = self.counts.borrow_mut(); + counts[0] = counts[0].saturating_sub(1); + self.sync_gen0_gauge(counts[0], None); + return; + } + } if let Ok(mut young) = self.young.try_borrow_mut() { if let Some(i) = young .iter() @@ -896,7 +950,8 @@ impl GcState { self.index.borrow().contains_key(&id) } - /// [`Self::is_tracked`] for an instance, which may be young. + /// [`Self::is_tracked`] for an instance or function, which may be + /// young. pub fn is_tracked_instance(&self, id: ObjectId) -> bool { self.is_tracked(id) || self.is_young(id) } @@ -2320,12 +2375,12 @@ pub fn track(obj: &Object) { // that never went through BuildMap) would otherwise never be a // candidate. CPython tracks every dict; we pair the tracking with // the objects that make the dict cycle-capable. + // (A young function's dict is tracked when the function is flushed.) if let Object::Module(m) = obj { let dict = Object::Dict(m.dict.clone()); with_state(|s| s.track(&dict)); } else if let Object::Function(f) = obj { - let dict = Object::Dict(f.globals.clone()); - with_state(|s| s.track(&dict)); + return with_state(|s| s.track_function(obj, f)); } with_state(|s| s.track(obj)); } diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index fd69f80c..1d75e9ca 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -16724,6 +16724,27 @@ impl Interpreter { last = pc + k; pc += 1 + k; } + // A closure tuple's cell (`LOAD_FAST` of a cell variable in + // CPython 3.14's listing). + OpCode::LoadClosure | OpCode::LoadClosureBorrow => { + let Some(cell) = cells.get(ins.arg as usize) else { + break; + }; + stack.push(Object::Cell(cell.clone())); + last = pc; + pc += 1; + } + // A prologue `MAKE_CELL`: the activation's cells are built + // with it (a later one, for an inlined comprehension, takes + // the full handler's fresh cell). + OpCode::MakeCell + if instrs[..pc] + .iter() + .all(|i| matches!(i.op, OpCode::MakeCell | OpCode::CopyFreeVars)) => + { + last = pc; + pc += 1; + } OpCode::LoadConst => { let Some(v) = consts.get(ins.arg as usize) else { break; @@ -57625,6 +57646,8 @@ static SLOW_LEAF_OPS: [bool; 256] = { OpCode::JumpForward, OpCode::ListAppend, OpCode::LoadAttr, + OpCode::LoadClosure, + OpCode::LoadClosureBorrow, OpCode::LoadConst, OpCode::LoadDeref, OpCode::LoadFast, @@ -57633,6 +57656,8 @@ static SLOW_LEAF_OPS: [bool; 256] = { OpCode::LoadMethodAttr, OpCode::LoadSmallInt, OpCode::LoadSuperAttr, + OpCode::MakeCell, + OpCode::MakeFunction, OpCode::Nop, OpCode::NotTaken, OpCode::PopJumpIfFalse, diff --git a/crates/weavepy-vm/src/stdlib/gc_real.rs b/crates/weavepy-vm/src/stdlib/gc_real.rs index 8108c34d..0238fb1f 100644 --- a/crates/weavepy-vm/src/stdlib/gc_real.rs +++ b/crates/weavepy-vm/src/stdlib/gc_real.rs @@ -506,6 +506,8 @@ fn object_is_tracked(target: &Object) -> bool { inst.ensure_gc_tracked(); gc_trace::with_state(|s| s.is_tracked_instance(id_of(target))) } + // A function may still be young. + Object::Function(_) => gc_trace::with_state(|s| s.is_tracked_instance(id_of(target))), // A list/dict/set born holding only scalars has its tracking // deferred; CPython tracks it from birth, so answer as CPython // does and hand it to the collector from here on. From 569aba44e93efbcd46795aa3f019f0e9a306ee94 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 21:49:36 -0700 Subject: [PATCH 70/96] perf: run cell-variable, `*args` and `with` calls inline in the core loop Three common call shapes still left the core loop for the general interpreter, each paying a full frame or a quiet-loop round trip: - A function with cell variables (one that defines a closure over its locals, like `mk(n)` in `mk(n)(1)`) never ran as an inline activation: the lean paths only borrow the callee's cells. An inline activation now builds and owns its fresh cells as the full frame does (a parameter captured by a closure moves into its cell) and releases them when its slot parks. - A `*args` callee bound through the general inline call every time, because a call site's cached shape excluded it. The cached shape now admits `*args` (not keyword-only parameters or `**kwargs`), and the cached call builds the tuple of surplus positionals. - `with`'s `LOAD_SPECIAL` always took the full step, which allocated a bound method. The core loop now loads a plain function `__enter__` or `__exit__` from the site's method slot, keyed by the class version (special lookup reads only the type), and pushes it under the instance, as CPython does; the helper fills the slot. The burst's slow arms also hand a call, return, cell read, list build or `in` test back to the core loop instead of to the full step. Measured (retired instructions per call, a local call of each case): `with CM() as c` 10,759 -> 5,692 (CPython 2,280); `with cm` on an existing manager 9,539 -> 4,426; `mk(i)(1)` 12,430 -> 8,885 (CPython 2,502); a call of a function with a cell variable 10,526 -> 6,920; a `*args` call 3,842 -> 3,515. Deltablue and the five fixtures are unchanged within noise. --- crates/weavepy-vm/src/lib.rs | 247 +++++++++++++++++++++++++--- tests/regrtest/test_inline_calls.py | 154 ++++++++++++++++- 2 files changed, 375 insertions(+), 26 deletions(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 1d75e9ca..8e4301c7 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -7824,6 +7824,11 @@ impl Interpreter { let Object::Function(f) = &callable else { unreachable!("inline callees are plain functions") }; + if !code.cellvars.is_empty() { + // SAFETY: a bound slot's locals storage is its own. + let cells = fresh_cells(f, &code, unsafe { &mut *act.frame.locals.as_ptr() }); + return self.inline_bind_with(frame, shell, pc, act, code, callable, guard, cells, true); + } let cells: *const Rc>>> = f.lean_cells_ref(&code).expect("checked by the shape"); self.inline_bind_cells(frame, shell, pc, act, code, callable, guard, cells) @@ -7843,26 +7848,48 @@ impl Interpreter { callable: Object, guard: crate::recursion::Guard, cells: *const Rc>>>, + ) -> Box { + // SAFETY: the caller's contract — `callable` (moved into the + // slot below) or the process keeps it alive. + let cells = borrowed_rc(unsafe { &*cells }); + self.inline_bind_with(frame, shell, pc, act, code, callable, guard, cells, false) + } + + /// [`Self::inline_bind_cells`] with the frame's cells handle built: + /// borrowed (`owned` false: the callee's lean cells, see + /// [`borrowed_rc`]) or the activation's own fresh cells, which the + /// slot releases when it parks. + #[inline] + #[allow(clippy::too_many_arguments)] + fn inline_bind_with( + &mut self, + frame: &mut Frame, + shell: &mut QuietShell<'_>, + pc: usize, + mut act: Box, + code: Rc, + callable: Object, + guard: crate::recursion::Guard, + cells: Rc>>>, + owned: bool, ) -> Box { let Object::Function(f) = &callable else { unreachable!("inline callees are plain functions") }; - // SAFETY: the caller's contract — `callable` (moved into the - // slot below) or the process keeps it alive. - let cells = unsafe { &*cells }; // SAFETY: the slot is parked: its code, cells, globals and // builtins fields hold stale non-owning copies, overwritten here // without a drop. The new handles are `f`'s (borrowed: the slot - // holds `callable`, so `f` outlives the activation) and the code - // (owned). + // holds `callable`, so `f` outlives the activation), the cells + // as given, and the code (owned). unsafe { let fr: &mut Frame = &mut act.frame; std::ptr::write(&raw mut fr.code, code); - std::ptr::write(&raw mut fr.cells, borrowed_rc(cells)); + std::ptr::write(&raw mut fr.cells, cells); std::ptr::write(&raw mut fr.globals, borrowed_rc(&f.globals)); std::ptr::write(&raw mut fr.builtins, borrowed_rc(&f.builtins)); std::ptr::write(&raw mut act.callable, callable); } + act.owns_cells = owned; act.frame.pc = 0; act.parked = false; act.call_pc = pc; @@ -7949,8 +7976,14 @@ impl Interpreter { } // SAFETY: the code handle is owned while active; release it and // leave the stale copy (never read again until `inline_bind` - // overwrites it; `InlineAct::drop` restores an owned one). + // overwrites it; `InlineAct::drop` restores an owned one). Owned + // cells are released the same way. unsafe { drop(std::ptr::read(&raw const fr.code)) }; + if act.owns_cells { + act.owns_cells = false; + // SAFETY: as for the code handle. + unsafe { drop(std::ptr::read(&raw const act.frame.cells)) }; + } act.parked = true; act.clean = false; act.act.shell = None; @@ -7970,6 +8003,11 @@ impl Interpreter { debug_assert!(act.clean && act.frame.stack.is_empty()); // SAFETY: as `inline_park`: the code handle is owned while active. unsafe { drop(std::ptr::read(&raw const act.frame.code)) }; + if act.owns_cells { + act.owns_cells = false; + // SAFETY: as `inline_park`. + unsafe { drop(std::ptr::read(&raw const act.frame.cells)) }; + } act.parked = true; act.clean = false; act.guard = None; @@ -8242,7 +8280,7 @@ impl Interpreter { // operands or cells, the locals are released here. done.clean = true; if frame.stack.is_empty() - && frame.code.cellvars.is_empty() + && (frame.code.cellvars.is_empty() || done.owns_cells) && Rc::strong_count(&frame.locals) == 1 { // SAFETY: sole owner; nothing else reaches the vector. @@ -8601,6 +8639,17 @@ impl Interpreter { /// code, and how many trailing parameters its compiled defaults fill. #[inline] fn lean_call_shape(f: &PyFunction, eff_argc: usize) -> Option<(Rc, usize)> { + Self::lean_call_shape_cells(f, eff_argc, false) + } + + /// [`Self::lean_call_shape`] for an inline activation, which with + /// `cells` may also own cell variables (see [`fresh_cells`]). + #[inline] + fn lean_call_shape_cells( + f: &PyFunction, + eff_argc: usize, + cells: bool, + ) -> Option<(Rc, usize)> { let code = f.code(); let total = code.arg_count as usize; if code.is_generator @@ -8619,7 +8668,13 @@ impl Interpreter { { return None; } - f.lean_cells_ref(&code)?; + if cells && !code.cellvars.is_empty() { + if !closure_cells_ok(f, &code) { + return None; + } + } else { + f.lean_cells_ref(&code)?; + } let missing = if eff_argc < total { if f.defaults.len() < total - eff_argc { return None; @@ -8650,6 +8705,14 @@ impl Interpreter { code.has_varargs || code.has_varkeywords || code.kwonly_count != 0 } + /// Whether `code` binds keyword-only parameters or `**kwargs` (the + /// extended parameters a positional call's cached shape can't bind; + /// `*args` it can). + #[inline(always)] + fn has_keyword_params(code: &CodeObject) -> bool { + code.has_varkeywords || code.kwonly_count != 0 + } + /// How many trailing positional defaults a call of `f` (running /// `code`) with `given` positional arguments, self included, takes /// from `f.defaults`. `None` when too many arguments come, too few @@ -8826,13 +8889,13 @@ impl Interpreter { _ => None, }; let (code, missing) = match &bound { - Some((f, _)) => Self::lean_call_shape(f, argc + 1)?, + Some((f, _)) => Self::lean_call_shape_cells(f, argc + 1, true)?, None => { let Object::Function(f) = &frame.stack[callee_slot] else { return None; }; let has_self = !matches!(frame.stack[self_slot], Object::Unbound); - Self::lean_call_shape(f, argc + usize::from(has_self))? + Self::lean_call_shape_cells(f, argc + usize::from(has_self), true)? } }; // Past the recursion limit the nested lean path raises. @@ -10098,6 +10161,7 @@ impl Interpreter { } let done = match ins.op { OpCode::Call => self.leaf_core_call(frame, ins, pc), + OpCode::LoadSpecial => Self::leaf_load_special(frame, ins.arg, pc), _ => self.leaf_core_attr(frame, ins, pc), }; match done { @@ -10113,6 +10177,35 @@ impl Interpreter { done } + /// `LOAD_SPECIAL` at `pc` of an instance whose class resolves the + /// method to a plain function: the function and the instance as its + /// self (what the full handler's bound method unpacks to at the call), + /// remembered in the site's method slot for the core loop. Declines + /// any other owner or class attribute, touching nothing. + fn leaf_load_special(frame: &mut Frame, special: u32, pc: usize) -> CoreAttr { + use weavepy_compiler::bytecode::{SPECIAL_ENTER, SPECIAL_EXIT}; + let name = match special { + SPECIAL_ENTER => "__enter__", + SPECIAL_EXIT => "__exit__", + _ => return CoreAttr::Decline, + }; + let Some(Object::Instance(inst)) = frame.stack.last() else { + return CoreAttr::Decline; + }; + let cls = inst.cls_raw(); + let ver = cls.attr_version.get(); + let Some(Object::Function(f)) = cls.lookup(name) else { + return CoreAttr::Decline; + }; + if let Some(ms) = code_method_slot(&frame.code, pc as u32) { + ms.set(ver, &f); + } + let owner = frame.stack.pop().expect("checked above"); + frame.stack.push(Object::Function(f)); + frame.stack.push(owner); + CoreAttr::Done + } + /// The core loop's `LOAD_FAST x` (at `pc`, of the heap local `other`) /// followed by `x.m()`: a native method the site /// cached for the receiver's class, a leaf method the site verified, @@ -12422,6 +12515,38 @@ impl Interpreter { last = pc; pc += 1; } + // `with`'s `__enter__` / `__exit__` of an instance: the + // plain function its class resolved at this site, under + // the instance (special lookup reads only the type, so + // the class version alone keys it). The helper fills + // the site. + OpCode::LoadSpecial => { + if len == 0 || len == cap { + break Some(CoreExit::Helper); + } + // SAFETY: `len > 0`. + let top = unsafe { base.add(len - 1) }; + let Object::Instance(inst) = (unsafe { &*top }) else { + break Some(CoreExit::Helper); + }; + let ver = inst.cls_raw().attr_version.get(); + let Some(f) = mslots!(cold_mslots, ext) + .get(pc) + .and_then(|ms| ms.get_held(ver)) + else { + break Some(CoreExit::Helper); + }; + // SAFETY: `len < cap`; the instance moves up into + // the self slot. + unsafe { + let recv = top.read(); + top.write(Object::Function(f)); + base.add(len).write(recv); + } + len += 1; + last = pc; + pc += 1; + } // The method-form load's per-site function (the helper's // `leaf_load_method` / class-receiver hits): an instance // receiver moves up into the self slot under the function; @@ -12749,7 +12874,7 @@ impl Interpreter { if let (Some(act), Some((func, has_self, eff_argc))) = (&act, shape) { let code = &act.frame.code; if let Some(slot) = code_call_slot(&frame.code, pc) - .filter(|_| !Self::has_extended_params(code)) + .filter(|_| !Self::has_keyword_params(code)) { slot.set(CallShape { func, @@ -13140,7 +13265,7 @@ impl Interpreter { _ => return None, }; let eff_argc = items.len() + usize::from(receiver.is_some()); - let (code, missing) = Self::lean_call_shape(f, eff_argc)?; + let (code, missing) = Self::lean_call_shape_cells(f, eff_argc, true)?; // Past the recursion limit the nested lean path raises. let crate::recursion::Enter::Ok(guard) = crate::recursion::enter() else { return None; @@ -14052,9 +14177,7 @@ impl Interpreter { // `PyFunction::code`); only compared, then cloned below. let code_rc: &Rc = unsafe { &*f.code.as_ptr() }; let (missing, slot_self) = slot.hit(Rc::as_ptr(f), Rc::as_ptr(code_rc))?; - if slot_self != has_self - || !Self::lean_code_ok(code_rc) - || Self::has_extended_params(code_rc) + if slot_self != has_self || !Self::lean_code_ok(code_rc) || Self::has_keyword_params(code_rc) { return None; } @@ -14065,7 +14188,15 @@ impl Interpreter { { return None; } - let cells: *const Rc>>> = f.lean_cells_ref(code_rc)?; + let own_cells = !code_rc.cellvars.is_empty(); + let cells: *const Rc>>> = if own_cells { + if !closure_cells_ok(f, code_rc) { + return None; + } + std::ptr::null() + } else { + f.lean_cells_ref(code_rc)? + }; let crate::recursion::Enter::Ok(guard) = crate::recursion::enter_with(depth_cell) else { return None; }; @@ -14095,14 +14226,37 @@ impl Interpreter { } frame.stack.pop().expect("callee slot checked above") }; + // A `*args` callee's surplus positionals become its tuple, which + // follows the positional parameters (there are no keyword ones). + let star = code.has_varargs.then(|| { + let npos = code.arg_count as usize; + if locals.len() > npos { + Object::Tuple(crate::tuple_storage::TupleStorage::from_exact_iter( + locals.drain(npos..), + )) + } else { + // The interned empty tuple. + Object::new_tuple(Vec::new()) + } + }); if missing > 0 { let Object::Function(f) = &callable else { unreachable!("checked above") }; locals.extend(f.defaults[f.defaults.len() - missing..].iter().cloned()); } + locals.extend(star); fill_unbound(locals, nlocals); frame.pc = pc as u32 + 1; + if own_cells { + let Object::Function(f) = &callable else { + unreachable!("checked above") + }; + let cells = fresh_cells(f, &code, locals); + return Some(self.inline_bind_with( + frame, shell, pc, act, code, callable, guard, cells, true, + )); + } // The cells handle is `f`'s (now `callable`'s) or the shared empty // vector: moving the callable leaves it where it was. Some(self.inline_bind_cells(frame, shell, pc, act, code, callable, guard, cells)) @@ -15519,7 +15673,7 @@ impl Interpreter { // holds. They are released here; an object that dies with them // queues its finalizer, which runs below before the caller goes on. let clean = frame.stack.is_empty() - && frame.code.cellvars.is_empty() + && (frame.code.cellvars.is_empty() || done.owns_cells) && Rc::strong_count(&frame.locals) == 1; let mut tmp = None; let (cframe, clast, cshell, entry); @@ -56656,6 +56810,9 @@ struct InlineAct { /// the activation runs its `__init__`, and the caller receives the /// instance instead of the (`None`) result. init_inst: Option, + /// The frame's `cells` handle is the activation's own (a callee with + /// cell variables, see [`fresh_cells`]), not borrowed from the callee. + owns_cells: bool, } impl InlineAct { @@ -56704,6 +56861,7 @@ impl InlineAct { gen_frame: std::ptr::null_mut(), exhaust_arg: 0, init_inst: None, + owns_cells: false, }); act.act.frame = std::ptr::from_mut::(&mut act.frame); // SAFETY: parked slots hold a stale code copy (see the type docs): @@ -56720,6 +56878,11 @@ unsafe impl Send for InlineAct {} impl Drop for InlineAct { fn drop(&mut self) { + if self.owns_cells { + // SAFETY: owned cells are released once (the frame's own drop + // forgets the handle). + unsafe { drop(std::ptr::read(&raw const self.frame.cells)) }; + } if self.parked { // SAFETY: the stale copies are overwritten without a drop, so // the frame's and callable's own drops release valid values. @@ -57103,7 +57266,9 @@ struct LeanKwShape { /// handles are *borrowed* from the callee function — which the caller /// keeps alive for the whole activation, and whose handles are /// immutable (a lean frame has no cell variables, so nothing rebinds -/// `cells`) — so building one costs no reference-count traffic. The +/// `cells`; an inline activation with them owns its `cells`, see +/// `InlineAct::owns_cells`) — so building one costs no reference-count +/// traffic. The /// borrowed handles are forgotten, never dropped; the rest is released /// on drop, or recycled by [`Interpreter::retire_lean_frame`]. Anything /// that keeps one of these handles beyond the activation (a shell or a @@ -57165,6 +57330,44 @@ impl Drop for LeanFrame { } } +/// Whether `f`'s closure is exactly cells for `code`'s free variables +/// (what [`fresh_cells`] appends). +#[inline] +fn closure_cells_ok(f: &PyFunction, code: &CodeObject) -> bool { + f.closure.len() == code.freevars.len() + && f.closure.iter().all(|c| matches!(c, Object::Cell(_))) +} + +/// The cells of a call of `f` running `code`, which has cell variables, +/// as `make_frame` builds them: a fresh cell per cell variable, holding +/// the bound local of that name (moved out of `locals`) or nothing, then +/// `f`'s closure cells (checked by [`closure_cells_ok`]). +fn fresh_cells( + f: &PyFunction, + code: &CodeObject, + locals: &mut [Object], +) -> Rc>>> { + let mut cells: Vec>> = + Vec::with_capacity(code.cellvars.len() + code.freevars.len()); + for name in &code.cellvars { + let initial = code + .varnames + .iter() + .position(|n| n == name) + .and_then(|i| locals.get_mut(i)) + .map_or(Object::Unbound, |slot| std::mem::replace(slot, Object::Unbound)); + cells.push(Rc::new(RefCell::new(initial))); + } + if crate::stdlib::tracemalloc_real::is_tracking() { + crate::stdlib::tracemalloc_real::track_new_cells(&cells); + } + cells.extend(f.closure.iter().filter_map(|c| match c { + Object::Cell(c) => Some(c.clone()), + _ => None, + })); + Rc::new(cells) +} + /// An `Rc` sharing `r`'s allocation *without* a count of its own: it must /// be forgotten, never dropped, and must not outlive `r`'s owner. #[inline] @@ -57715,6 +57918,7 @@ static CORE_LEAF_OPS: [bool; 256] = { OpCode::LoadAttr, OpCode::StoreAttr, OpCode::LoadMethodAttr, + OpCode::LoadSpecial, OpCode::CopyTop, OpCode::Swap, OpCode::GetIter, @@ -57724,6 +57928,11 @@ static CORE_LEAF_OPS: [bool; 256] = { OpCode::PopJumpIfNone, OpCode::PopJumpIfNotNone, OpCode::CallEx, + OpCode::Call, + OpCode::ReturnValue, + OpCode::LoadDeref, + OpCode::BuildList, + OpCode::ContainsOp, ]; let mut i = 0; while i < ops.len() { diff --git a/tests/regrtest/test_inline_calls.py b/tests/regrtest/test_inline_calls.py index 5b0d3319..f421fb9e 100644 --- a/tests/regrtest/test_inline_calls.py +++ b/tests/regrtest/test_inline_calls.py @@ -1,10 +1,12 @@ -"""Plain Python calls the core loop switches to in place. - -A call of a plain function (positional parameters only, no cells) that -its call site has seen before runs in a pooled activation, and its return -hands the result straight back. Each loop here runs long enough for the -sites to settle; then the callee, its arguments or its frame change, and -every call must still behave as the interpreter's general call does. +"""Python calls the core loop switches to in place. + +A call of a function that its call site has seen before runs in a pooled +activation (with its own cells when it has cell variables, and its +`*args` tuple built by the site), and its return hands the result +straight back; `with` loads its `__enter__` and `__exit__` from the site. +Each loop here runs long enough for the sites to settle; then the callee, +its arguments or its frame change, and every call must still behave as +the interpreter's general call does. """ import sys @@ -123,6 +125,144 @@ def down(n): down(0) self.assertEqual(loop_add(10), sum(range(10)) + 10) + def test_cell_variables(self): + def adder(n): + def add(x): + return x + n + return add + + def counter(start): + count = start + + def bump(): + nonlocal count + count += 1 + return count + bump() + return [bump, count] + + def shadow(a, b): + def get(): + return a, b + a = a * 2 + return get + + def peek(n): + def inner(): + return n + return sorted(sys._getframe().f_locals) + + def fails(n): + def inner(): + return n + return [inner][n] + + made = [adder(i) for i in range(WARM)] + self.assertEqual([f(1) for f in made[:3]], [1, 2, 3]) + self.assertEqual(sum(f(0) for f in made), sum(range(WARM))) + for i in range(WARM): + bump, seen = counter(i) + self.assertEqual(seen, i + 1) + self.assertEqual(bump(), i + 2) + self.assertEqual(shadow(i, 1)(), (2 * i, 1)) + self.assertEqual(peek(3), ["inner", "n"]) + for i in range(WARM): + self.assertEqual(fails(0)(), 0) + with self.assertRaises(IndexError): + fails(1) + + def test_cell_variable_finalizers(self): + seen = [] + + class Note: + def __del__(self): + seen.append(1) + + def hold(i): + n = Note() + + def keep(): + return n + return i + + def escape(i): + n = Note() + + def keep(): + return n + return keep + + for i in range(WARM): + hold(i) + self.assertEqual(len(seen), WARM) + kept = [escape(i) for i in range(100)] + self.assertEqual(len(seen), WARM) + del kept + self.assertEqual(len(seen), WARM + 100) + + def test_with_statements(self): + log = [] + + class CM: + def __enter__(self): + return self + + def __exit__(self, *exc): + log.append(exc[0]) + return False + + def use(cm): + with cm as c: + return c + + cm = CM() + for _ in range(WARM): + self.assertIs(use(cm), cm) + self.assertEqual(log, [None] * WARM) + log.clear() + # Special lookup reads the class, never the instance. + cm.__enter__ = lambda: "instance" + self.assertIs(use(cm), cm) + # A rebound class attribute takes effect at once. + CM.__enter__ = lambda self: "rebound" + self.assertEqual(use(cm), "rebound") + CM.__enter__ = staticmethod(lambda: "static") + self.assertEqual(use(cm), "static") + + class Suppress(CM): + def __exit__(self, kind, value, tb): + log.append(kind) + return True + + def boom(cm): + with cm: + raise KeyError(1) + return "suppressed" + + for _ in range(WARM): + self.assertEqual(boom(Suppress()), "suppressed") + self.assertEqual(log[-1], KeyError) + with self.assertRaises(KeyError): + boom(CM()) + + def test_star_args(self): + def va(*args): + return args + + def some(a, b=2, *rest): + return a, b, rest + + def run(n): + t = 0 + for i in range(n): + t += len(va(i, i, i)) + len(va()) + some(i)[1] + len(some(i, 1, 2, 3)[2]) + return t + + self.assertEqual(run(WARM), WARM * 7) + self.assertEqual(va(1, 2), (1, 2)) + self.assertEqual(some(1, 5, 6), (1, 5, (6,))) + self.assertIs(type(va()), tuple) + def test_finalizers_run_on_return(self): seen = [] From 574aadfde6e8cd1d0e00b390b62af484ac4da1d0 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:39:04 -0700 Subject: [PATCH 71/96] perf: keep several classes' leaf methods per call site and enter warm plans directly A frameless leaf call (a method like deltablue's `output()`, or a plain function like `return d[0]`) paid for its layers twice over: - A local receiver's `x.m(...)` site remembered the verified leaf callee of one class only. A loop over mixed receivers (constraints of several kinds) replaced it at every class change, re-resolving the method and taking the slower pure-method path each time. The site now keeps the callees of up to three earlier classes beside the current one, as the method slots already do, and is checked before the other method probes. - Once a callee's warm-up is past and its plan has native code, the call enters that code directly, with the recursion depth the core loop already holds, instead of through the evaluation layers (the shape probes, the plan lookup, the plan runner's own checks and a thread-local depth read). The tiny shapes the evaluator answers itself (`return None`, `return x`) skip the probe. Measured (retired instructions per call): a method call over receivers of four classes from a compiled loop 1,532 -> 1,168 (CPython 881), over two classes 1,581 -> 1,181, over four from an interpreted loop 1,705 -> 1,304 (CPython 1,265), over one class 1,167 -> 1,119; a local call of `return d.get(5)` 1,526 -> 1,479 and of `return d[0]` 1,172 -> 1,124. Calls of the tiny shapes are unchanged; deltablue 33.36M -> 33.18M per unit (32.53M -> 32.07M once warm, 40 -> 120 units). --- crates/weavepy-vm/src/leaf_plan.rs | 36 +++++ crates/weavepy-vm/src/leaf_plan/jit.rs | 39 ++++- crates/weavepy-vm/src/lib.rs | 189 +++++++++++++++++++------ tests/regrtest/test_inline_calls.py | 57 +++++++- 4 files changed, 275 insertions(+), 46 deletions(-) diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index e91a69bd..71974a17 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -949,6 +949,42 @@ impl Interpreter { .into_object() } + /// A frameless call's evaluation of the general leaf `code` (see + /// `Interpreter::pure_leaf_call`) straight in its plan's native + /// code, at recursion depth `depth`, once the callee's warm-up is past + /// (the frameless call bookkeeping is then only the hit count): + /// `None` when the plan has no native code, or the callee one of the + /// tiny shapes `leaf_eval` answers itself; otherwise the result, or + /// `None` inside having done nothing observable. + #[cfg(feature = "jit")] + #[inline] + pub(crate) fn leaf_site_native( + &self, + code: &CodeObject, + f: &crate::object::PyFunction, + args: &[*const Object], + depth: usize, + ) -> Option> { + let ext = crate::code_vm_ext(code)?; + if ext.pure_leaf.load(std::sync::atomic::Ordering::Relaxed) >= 3 + || code.jit_hint.lean_entries() <= crate::tier2::LEAN_WARM_COMPILE_THRESHOLD_CAP + { + return None; + } + let plan = ext.leaf_plan.get()?.as_deref()?; + if args.len() != usize::from(plan.nargs) || args.len() != crate::leaf_arity(code) { + return None; + } + let r = jit::run_compiled::(self, code, ext, plan, f, args, depth)? + .and_then(LeafRet::into_object); + if r.is_some() { + code.jit_hint.note_leaf_hit(); + } else { + code.jit_hint.note_leaf_miss(); + } + Some(r) + } + /// [`Self::leaf_run`] with its result borrowed when the caller may /// (see [`LeafRet`]). #[inline(never)] diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index e6120090..5f3312de 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -295,9 +295,46 @@ pub(super) fn run( .as_ref()? } }; + enter::( + interp, + native, + (code, ext, plan, f), + args, + nest, + crate::recursion::current_depth(), + ) +} + +/// [`run`] for a plan whose native code is already compiled, with the +/// caller's recursion `depth` in hand: `None` when there is no native +/// code (yet), otherwise the evaluation's outcome. +#[inline] +pub(super) fn run_compiled( + interp: &Interpreter, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + f: &PyFunction, + args: &[*const Object], + depth: usize, +) -> Option> { + let native = plan.native[usize::from(EFFECT)].get()?.as_ref()?; + enter::(interp, native, (code, ext, plan, f), args, 0, depth) +} + +/// One evaluation of `native` (see [`run`]), at recursion depth `depth`. +#[inline(always)] +fn enter( + interp: &Interpreter, + native: &Native, + (code, ext, plan, f): (&CodeObject, &CodeConstObjects, &LeafPlan, &PyFunction), + args: &[*const Object], + nest: u8, + depth: usize, +) -> Option> { // In-line callees run where the interpreter would nest frames: near // the recursion limit, it decides. - if crate::recursion::current_depth() + usize::from(nest) + usize::from(native.depth) + 1 + if depth + usize::from(nest) + usize::from(native.depth) + 1 >= crate::recursion::recursion_limit() { return None; diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 8e4301c7..5bf665b4 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -10237,27 +10237,8 @@ impl Interpreter { .map_or(&[][..], |s| &s[..]); match other { Object::Instance(_) | Object::Module(_) => { - // `x.m(...)` of a pure leaf method with simple arguments: its - // result, evaluated on the borrowed operands. - if !simple_args_prefix(&code.instructions, pc + 2) { - return None; - } - // A native method the site cached for this class version: - // straight to its fused call (the Python leaf probes below - // would miss). - if let (Object::Instance(i), Some(ms)) = (other, mslots.get(pc + 1)) { - if ms - .peek_inst_builtin(i.cls_raw().attr_version.get()) - .is_some() - { - if let Some(hit) = self.core_native_method( - code, other, pc + 1, next.arg, mslots, lbase, nlocals, consts, - ) { - return Some(hit); - } - } - } - // A site that verified its callee for this class version. + // A site that verified its callee for this class version + // (its call checks the argument shape itself). let mut missed = false; if let (Object::Instance(i), Some(ext)) = (other, ext) { if let Some(site) = leaf_site_hit(ext, pc + 1, i.cls_raw().attr_version.get()) { @@ -10279,6 +10260,26 @@ impl Interpreter { } } } + // `x.m(...)` of a pure leaf method with simple arguments: its + // result, evaluated on the borrowed operands. + if !simple_args_prefix(&code.instructions, pc + 2) { + return None; + } + // A native method the site cached for this class version: + // straight to its fused call (the Python leaf probes below + // would miss). + if let (Object::Instance(i), Some(ms)) = (other, mslots.get(pc + 1)) { + if ms + .peek_inst_builtin(i.cls_raw().attr_version.get()) + .is_some() + { + if let Some(hit) = self.core_native_method( + code, other, pc + 1, next.arg, mslots, lbase, nlocals, consts, + ) { + return Some(hit); + } + } + } let pure_site = !missed && match (other, mslots.get(pc + 1)) { (Object::Instance(i), Some(ms)) => ms @@ -14634,11 +14635,9 @@ impl Interpreter { for (k, o) in f.defaults[f.defaults.len() - missing..].iter().enumerate() { args[nargs + k] = o; } - if effect { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - } else { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - } + // SAFETY: this thread's own depth cell. + let depth = unsafe { (*depth_cell).get() }; + self.pure_leaf_call(code_rc, f, &args[..total], effect, depth) } /// The core loop's fused `LOAD_FAST x; LOAD_ATTR m (method); (code_rc, f, &args[..total]) - } else { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - }; + // SAFETY: as above. + let depth = unsafe { (*depth_cell).get() }; + let r = self.pure_leaf_call(code_rc, f, &args[..total], effect, depth); let Some(v) = r else { if let Some(ext) = code_vm_ext(code) { leaf_site_set(ext, code.instructions.len(), attr_pc, None); @@ -14939,11 +14936,9 @@ impl Interpreter { for (k, o) in f.defaults[f.defaults.len() - missing..].iter().enumerate() { args[nargs + k] = o; } - if effect { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - } else { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - } + // SAFETY: this thread's own depth cell. + let depth = unsafe { (*depth_cell).get() }; + self.pure_leaf_call(code_rc, f, &args[..total], effect, depth) } /// [`Self::core_pure_call`] for a `CALL_KW` at `pc`: `ops` is the @@ -15269,6 +15264,40 @@ impl Interpreter { r } + /// [`Self::pure_leaf_eval`] (with `effect`, of an effect leaf) at + /// recursion depth `depth`: a warm callee's compiled plan is entered + /// directly (see [`Self::leaf_site_native`]). + #[inline(always)] + fn pure_leaf_call( + &self, + code: &CodeObject, + f: &crate::object::PyFunction, + args: &[*const Object], + effect: bool, + depth: usize, + ) -> Option { + // (The tiny shapes `leaf_eval` answers itself skip the probe.) + #[cfg(feature = "jit")] + if code_vm_ext(code) + .is_some_and(|e| e.pure_leaf.load(std::sync::atomic::Ordering::Relaxed) < 3) + { + let fast = if effect { + self.leaf_site_native::(code, f, args, depth) + } else { + self.leaf_site_native::(code, f, args, depth) + }; + if let Some(r) = fast { + return r; + } + } + let _ = depth; + if effect { + self.pure_leaf_eval::(code, f, args) + } else { + self.pure_leaf_eval::(code, f, args) + } + } + /// [`Self::pure_leaf_eval`] for a constructor's leaf `__init__` on the /// fresh instance `args[0]` (see `leaf_run`'s `FRESH`): `None` may /// leave that instance half-initialized, for the caller to discard. @@ -62573,8 +62602,23 @@ impl FieldSlot { /// `ver`, the method is `func`, whose code was `code`, a pure (or, with /// `effect`, an effect) leaf taking exactly the site's arguments. The /// class holds the function at that version; `func` is weak all the same, -/// as [`MethodSlot`]'s are. -struct LeafSite(std::cell::UnsafeCell>); +/// as [`MethodSlot`]'s are. A site that sees several classes (a loop over +/// a list of mixed constraint kinds) keeps the ones it saw before in its +/// second cell, as [`MethodSlot`] keeps earlier resolutions. +struct LeafSite( + std::cell::UnsafeCell>, + std::cell::UnsafeCell>>, +); + +/// How many earlier callees a polymorphic [`LeafSite`] keeps. +const POLY_LEAF_SITES: usize = 3; + +/// A [`LeafSite`]'s earlier callees, by class version, replaced +/// round-robin. +struct PolyLeafSites { + entries: [Option; POLY_LEAF_SITES], + next: usize, +} struct LeafSiteData { ver: u64, @@ -62589,7 +62633,60 @@ unsafe impl Sync for LeafSite {} impl LeafSite { const fn empty() -> Self { - Self(std::cell::UnsafeCell::new(None)) + Self( + std::cell::UnsafeCell::new(None), + std::cell::UnsafeCell::new(None), + ) + } + + /// An earlier callee verified under `ver` (see `leaf_site_hit`). + #[cold] + #[inline(never)] + fn poly_hit( + &self, + ver: u64, + ) -> Option<(*const crate::object::PyFunction, *const CodeObject, bool)> { + // SAFETY: GIL-serialized; the borrow ends before any refill. + let poly = unsafe { &*self.1.get() }.as_deref()?; + poly.entries + .iter() + .flatten() + .find(|d| d.ver == ver && d.func.strong_count() > 0) + .map(|d| (d.func.as_ptr(), d.code, d.effect)) + } + + /// Record `data` as the site's callee, keeping the one it replaces + /// (a live callee under another class version) among the earlier + /// ones; `None` forgets them all. + fn set(&self, data: Option) { + // SAFETY: GIL-serialized; no reference into the site is live. + let (primary, poly) = unsafe { (&mut *self.0.get(), &mut *self.1.get()) }; + let Some(data) = data else { + *primary = None; + *poly = None; + return; + }; + if let Some(old) = primary.take() { + if old.ver != data.ver && old.func.strong_count() > 0 { + let poly = poly.get_or_insert_with(|| { + Box::new(PolyLeafSites { + entries: [const { None }; POLY_LEAF_SITES], + next: 0, + }) + }); + // The new callee's own entry, or the oldest, makes room. + let i = poly + .entries + .iter() + .position(|e| e.as_ref().is_some_and(|e| e.ver == old.ver || e.ver == data.ver)) + .unwrap_or(poly.next); + if i == poly.next { + poly.next = (i + 1) % POLY_LEAF_SITES; + } + poly.entries[i] = Some(old); + } + } + *primary = Some(data); } } @@ -62601,10 +62698,15 @@ fn leaf_site_hit( pc: usize, ver: u64, ) -> Option<(*const crate::object::PyFunction, *const CodeObject, bool)> { + let site = ext.leaf_sites.get()?.get(pc)?; // SAFETY: GIL-serialized; the borrow ends before any refill. - let site = unsafe { &*ext.leaf_sites.get()?.get(pc)?.0.get() }.as_ref()?; - (site.ver == ver && site.func.strong_count() > 0) - .then(|| (site.func.as_ptr(), site.code, site.effect)) + match unsafe { &*site.0.get() } { + Some(d) if d.ver == ver && d.func.strong_count() > 0 => { + Some((d.func.as_ptr(), d.code, d.effect)) + } + Some(_) => site.poly_hit(ver), + None => None, + } } /// Whether the method call whose `LOAD_ATTR` is at `pc` last resolved to @@ -62643,8 +62745,7 @@ fn leaf_site_set(ext: &CodeConstObjects, ninstrs: usize, pc: usize, data: Option .leaf_sites .get_or_init(|| (0..ninstrs).map(|_| LeafSite::empty()).collect()); if let Some(site) = sites.get(pc) { - // SAFETY: GIL-serialized; no reference into the site is live. - unsafe { *site.0.get() = data }; + site.set(data); } } diff --git a/tests/regrtest/test_inline_calls.py b/tests/regrtest/test_inline_calls.py index f421fb9e..0fc1254f 100644 --- a/tests/regrtest/test_inline_calls.py +++ b/tests/regrtest/test_inline_calls.py @@ -3,7 +3,8 @@ A call of a function that its call site has seen before runs in a pooled activation (with its own cells when it has cell variables, and its `*args` tuple built by the site), and its return hands the result -straight back; `with` loads its `__enter__` and `__exit__` from the site. +straight back; `with` loads its `__enter__` and `__exit__` from the site, +and a method call site remembers the leaf methods of several classes. Each loop here runs long enough for the sites to settle; then the callee, its arguments or its frame change, and every call must still behave as the interpreter's general call does. @@ -263,6 +264,60 @@ def run(n): self.assertEqual(some(1, 5, 6), (1, 5, (6,))) self.assertIs(type(va()), tuple) + def test_polymorphic_leaf_sites(self): + class Base: + FORWARD = 1 + + def __init__(self, k): + self.k = k + self.direction = 1 + + def output(self): + return self.k if self.direction == Base.FORWARD else -self.k + + class A(Base): + pass + + class B(Base): + pass + + class C: + def __init__(self, k): + self.k = k + + def output(self): + return self.k * 10 + + class D(C): + pass + + objs = [A(1), B(2), C(3), D(4)] + + def total(n): + t = 0 + for i in range(n): + for o in objs: + t += o.output() + return t + + self.assertEqual(total(WARM), WARM * (1 + 2 + 30 + 40)) + # A class's method changes: only its instances see the new one. + B.output = lambda self: 100 + self.assertEqual(total(10), 10 * (1 + 100 + 30 + 40)) + # A function's code changes under the same class version. + def other(self): + return 7 + C.output.__code__ = other.__code__ + self.assertEqual(total(10), 10 * (1 + 100 + 7 + 7)) + # An instance attribute shadows its class's method. + objs[0].output = lambda: 1000 + self.assertEqual(total(10), 10 * (1000 + 100 + 7 + 7)) + del objs[0].output + objs[1].direction = 0 + self.assertEqual(total(10), 10 * (1 + 100 + 7 + 7)) + del B.output + self.assertEqual(total(10), 10 * (1 - 2 + 7 + 7)) + def test_finalizers_run_on_return(self): seen = [] From ed6f038ed92a559b7a6eecf2ecbeb36eac8d5fae Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 22:39:20 -0700 Subject: [PATCH 72/96] perf: store `__slots__` members in line and trim the inline call path Two costs every constructor or call of these shapes paid: - A `__slots__` class's `__init__` (`self.x = x; self.y = y`) never ran as a frameless leaf: the leaf store path only knew instance dicts, so each evaluation declined until the body was demoted to an inline activation, whose stores then each left the core loop for the helper. The core loop's store, and a leaf's buffered or constructor store, now write a validated slot member (the site's `StoreAttrSlot`, keyed by the class version) directly, as the helper does. - The inline call and return pushed the activation, the parked slot and the result through out-of-line `Vec::push` calls, emptied the locals through a `Drain`, and called two small helpers out of line. They now push with the room check in line and release the locals in place. Measured (retired instructions per call, a local call of each case): `P(i, 2)` for a two-slot class 6,219 -> 4,027 (CPython 1,460); `p = P(i, 2); return p.x + p.y` 7,098 -> 4,891 (CPython 1,762); a call of a one-line non-leaf function from a loop 2,179 -> 2,044, a method call 2,493 -> 2,362; `with cm` 4,426 -> 4,213. Deltablue 33.18M -> 32.62M per unit (32.07M -> 31.50M once warm, 40 -> 120 units). --- crates/weavepy-vm/src/lib.rs | 170 +++++++++++++++++++++---- tests/regrtest/test_jit_plan_stores.py | 43 +++++++ 2 files changed, 189 insertions(+), 24 deletions(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 5bf665b4..c7172976 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -7991,7 +7991,7 @@ impl Interpreter { act.caller_pending = None; act.init_inst = None; if self.inline_pool.len() < INLINE_POOL_CAP { - self.inline_pool.push(act); + push_fast(&mut self.inline_pool, act); } } @@ -8013,7 +8013,7 @@ impl Interpreter { act.guard = None; act.caller_pending = None; if self.inline_pool.len() < INLINE_POOL_CAP { - self.inline_pool.push(act); + push_fast(&mut self.inline_pool, act); } } @@ -9355,7 +9355,7 @@ impl Interpreter { /// the call didn't use (its locals must be empty again). fn inline_unslot(&mut self, act: Box) { if self.inline_pool.len() < INLINE_POOL_CAP { - self.inline_pool.push(act); + push_fast(&mut self.inline_pool, act); } } @@ -9647,7 +9647,7 @@ impl Interpreter { /// `lasti` from the advanced `pc`); one that has its shell keeps the /// mirror current itself, as the general path's pre-step store does. /// Returns the pending entry for [`Self::lean_pending_exit`]. - #[inline] + #[inline(always)] fn lean_pending_enter( &mut self, frame: &mut Frame, @@ -9782,7 +9782,7 @@ impl Interpreter { /// warms that compile after a few entries): a native entry is then /// worth its framing — a recursive body calls its native self /// directly from there. - #[inline] + #[inline(always)] fn lean_code_ok(code: &CodeObject) -> bool { if code.wire.as_ref().is_some_and(|w| w.exec_error.is_some()) { return false; @@ -12894,7 +12894,7 @@ impl Interpreter { }; let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -13021,7 +13021,7 @@ impl Interpreter { } let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -13117,7 +13117,7 @@ impl Interpreter { }; let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -13144,7 +13144,7 @@ impl Interpreter { }; let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -13173,7 +13173,7 @@ impl Interpreter { }; let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -13217,7 +13217,7 @@ impl Interpreter { }; let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -13613,7 +13613,7 @@ impl Interpreter { act.init_inst = Some(inst); let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -15553,7 +15553,7 @@ impl Interpreter { }; let gen_frame = act.gen_frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = gen_frame; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -15614,7 +15614,7 @@ impl Interpreter { // SAFETY: the consumer is the innermost remaining activation. let (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; // SAFETY: as above. - unsafe { (*cframe).stack.push(v) }; + unsafe { push_fast(&mut (*cframe).stack, v) }; sw.cur = cframe; sw.last = if clast == &raw mut sw.scratch { sw.scratch = usize::MAX; @@ -15709,9 +15709,8 @@ impl Interpreter { if clean && done.init_inst.is_none() { done.clean = !had_shell; // SAFETY: as above. - for v in unsafe { (*frame.locals.as_ptr()).drain(..) } { - drop_hot(v); - } + // SAFETY: as above. + unsafe { release_locals(&mut *frame.locals.as_ptr()) }; drop(done.guard.take()); // SAFETY: the caller is the innermost remaining activation. (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; @@ -15723,7 +15722,7 @@ impl Interpreter { let callable = unsafe { std::ptr::read(&raw const done.callable) }; self.inline_park_clean(done); // SAFETY: as above. - unsafe { (*cframe).stack.push(v) }; + unsafe { push_fast(&mut (*cframe).stack, v) }; match callable { // A function something else still holds (its class, its // module) can't die here: a plain decrement. @@ -15765,9 +15764,8 @@ impl Interpreter { let result = if clean { done.clean = !had_shell; // SAFETY: as above. - for v in unsafe { (*frame.locals.as_ptr()).drain(..) } { - drop_hot(v); - } + // SAFETY: as above. + unsafe { release_locals(&mut *frame.locals.as_ptr()) }; drop(done.guard.take()); Ok(v) } else { @@ -16358,10 +16356,11 @@ impl Interpreter { } /// The core loop's `STORE_ATTR` into an instance: an indexed store - /// into the receiver's `__dict__` (the warm shape), or the single - /// probe a `StoreAttrNewKey` site's insert-or-overwrite needs — the + /// into the receiver's `__dict__` (the warm shape), the single probe + /// a `StoreAttrNewKey` site's insert-or-overwrite needs — the /// constructor shape, which would otherwise leave the loop for the - /// helper on every `self.x = …`. Nothing here runs Python code. + /// helper on every `self.x = …` — or a `__slots__` member's store. + /// Nothing here runs Python code. /// /// On `true` the value *moved* into the dict: the caller drops its /// stack slot without dropping the value. @@ -16434,10 +16433,86 @@ impl Interpreter { IC::StoreAttrNewKey { ver } => { Self::core_store_new_attr(code, inst, cls, attr_pc, name_idx, ver, value) } + IC::StoreAttrSlot { key_idx, ver } if cls.attr_version.get() == ver => { + Self::core_store_slot(code, inst, key_idx, name_idx, value) + } _ => false, } } + /// A validated `__slots__` member's store (the site's + /// `StoreAttrSlot`, whose class version the caller checked), as + /// `leaf_store_attr` makes it: over the member's droppable value, or + /// as its first value (the constructor shape). `true` moved `value` + /// in; `false` touched nothing. + #[inline(never)] + fn core_store_slot( + code: &CodeObject, + inst: &PyInstance, + key_idx: u32, + name_idx: u32, + value: &Object, + ) -> bool { + let Some(name) = code.names.get(name_idx as usize) else { + return false; + }; + let Ok(mut slots) = inst.slots.try_borrow_mut() else { + return false; + }; + match slots.get_index_mut(key_idx as usize) { + Some((key, slot)) + if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => + { + if !Self::core_droppable(slot) { + return false; + } + // SAFETY: the value moves out of the caller's slot (it + // forgets it on `true`); the displaced one is droppable. + let v = unsafe { std::ptr::read(value) }; + inst.note_slot_store(&v); + drop(std::mem::replace(slot, v)); + true + } + _ => { + // Held elsewhere (another layout): the full store decides. + if slots.get(name).is_some() { + return false; + } + // SAFETY: as above. + let v = unsafe { std::ptr::read(value) }; + inst.note_slot_store(&v); + match code_name_obj(code, name_idx) { + Some(Object::Str(shared)) => slots.insert_shared(shared, v), + _ => slots.insert(name, v), + }; + true + } + } + } + + /// [`Self::core_store_slot`]'s verdict, without storing. + fn core_store_slot_ready( + code: &CodeObject, + inst: &PyInstance, + key_idx: u32, + name_idx: u32, + ) -> bool { + let Some(name) = code.names.get(name_idx as usize) else { + return false; + }; + let Ok(slots) = inst.slots.try_borrow() else { + return false; + }; + match slots.get_index(key_idx as usize) { + Some((key, old)) + if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => + { + Self::core_droppable(old) + } + _ => slots.get(name).is_none(), + } + } + /// [`Self::core_store_attr_ready`] through the site's [`FieldSlot`], /// for the next of an effect leaf's stores to one receiver in order /// (`cursor` as [`PyInstance::split_store_ready`]): `None` when the @@ -16471,6 +16546,10 @@ impl Interpreter { let cls = inst.cls_raw(); let ver = match code.caches.get(attr_pc as u32) { IC::StoreAttrInstance { ver, .. } | IC::StoreAttrNewKey { ver } => ver, + IC::StoreAttrSlot { key_idx, ver } => { + return cls.attr_version.get() == ver + && Self::core_store_slot_ready(code, inst, key_idx, name_idx); + } _ => return false, }; if cls.native_kind.get() != 0 @@ -57397,6 +57476,49 @@ fn fresh_cells( Rc::new(cells) } +/// `v.push(x)` with the room check in line (the call paths push one +/// activation, slot or result per call, and the out-of-line `push` showed +/// up in their profiles). +#[inline(always)] +fn push_fast(v: &mut Vec, x: T) { + let n = v.len(); + if n < v.capacity() { + // SAFETY: `n < capacity`: the slot is allocated and unused. + unsafe { + v.as_mut_ptr().add(n).write(x); + v.set_len(n + 1); + } + } else { + push_grow(v, x); + } +} + +#[cold] +#[inline(never)] +fn push_grow(v: &mut Vec, x: T) { + v.push(x); +} + +/// Empty a finished activation's `locals`, releasing each value (an +/// object that dies queues its finalizer, as any release does). +/// +/// # Safety +/// +/// Nothing else borrows `locals`. +#[inline(always)] +unsafe fn release_locals(locals: &mut Vec) { + let n = locals.len(); + let base = locals.as_ptr(); + // The vector forgets its values first: each is read out exactly once. + // SAFETY: the first `n` values are initialized. + unsafe { + locals.set_len(0); + for k in 0..n { + drop_hot(base.add(k).read()); + } + } +} + /// An `Rc` sharing `r`'s allocation *without* a count of its own: it must /// be forgotten, never dropped, and must not outlive `r`'s owner. #[inline] diff --git a/tests/regrtest/test_jit_plan_stores.py b/tests/regrtest/test_jit_plan_stores.py index 6ae2cb9a..6634b9bd 100644 --- a/tests/regrtest/test_jit_plan_stores.py +++ b/tests/regrtest/test_jit_plan_stores.py @@ -102,6 +102,49 @@ class Slotted: drive(n, 1) self.assertEqual(s.v, 3) + def test_slots_constructor(self): + import gc + + class P: + __slots__ = ("x", "y") + + def __init__(self, x, y): + self.x = x + self.y = y + + class Q(P): + pass + + def build(n): + if n < 0: + return locals() + t = 0 + for i in range(n): + p = P(i, 2) + t += p.x + p.y + return t + + self.assertEqual(build(WARM), sum(range(WARM)) + 2 * WARM) + p = P([], "s") + self.assertEqual((p.x, p.y), ([], "s")) + self.assertTrue(gc.is_tracked(p)) + del p.x + with self.assertRaises(AttributeError): + p.x + p.__init__(1, 2) + self.assertEqual((p.x, p.y), (1, 2)) + q = Q(3, 4) + self.assertEqual((q.x, q.y), (3, 4)) + q.z = 5 + self.assertEqual(vars(q), {"z": 5}) + loop = P(0, 0) + loop.x = loop + del loop + gc.collect() + P.y = property(lambda self: "prop", lambda self, v: None) + r = P(1, 2) + self.assertEqual((r.x, r.y), (1, "prop")) + def test_scalar_stores_in_place(self): class Src: def __init__(self, x): From 720ce883653ec370b5f30abae1bc2fe86513c15c Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:19:36 -0700 Subject: [PATCH 73/96] perf: share a new function's name slots and run closures over free variables frameless Two more costs of closures and functions made in a loop: - Every new function copied its code's template of name slots (`__module__`, `__name__`, `__qualname__`): a small dict built, three keys and values cloned, then freed with the function. A function now holds the shared template as a seed and copies it into its own slots only when something reads or writes them; most functions never do. - A function with free variables (`lambda x: x * k`, a nested `def` that reads its maker's locals) was never a leaf, so each call took an inline activation. Leaf plans now read a free variable straight from the function's closure cell (a leaf has no `STORE_DEREF`, so nothing can rebind it during the evaluation; an empty cell declines to the full path, which raises), and such functions are frameless leaves. Measured (retired instructions per call, a local call of each case): a call of an existing closure `g(i)` 2,894 -> 1,466 (CPython 920); `mk(i)(1)` 8,544 -> 6,574 (CPython 2,502); building a nested `def` 4,926 -> 4,152; `lambda` built 3,514 -> 2,806, built and called 4,275 -> 3,490 (CPython 1,549). Importing argparse, typing, dataclasses, email.message, json, unittest, logging and decimal 727M -> 716M. Deltablue is unchanged within noise. --- crates/weavepy-vm/src/builtins.rs | 5 +- crates/weavepy-vm/src/gc_trace.rs | 8 ++- crates/weavepy-vm/src/leaf_plan.rs | 32 ++++++++- crates/weavepy-vm/src/leaf_plan/jit.rs | 21 ++++++ crates/weavepy-vm/src/lib.rs | 54 +++++++++------ crates/weavepy-vm/src/object.rs | 37 +++++++++-- crates/weavepy-vm/src/stdlib/pickle_accel.rs | 2 +- tests/regrtest/test_function_creation.py | 69 ++++++++++++++++++++ 8 files changed, 199 insertions(+), 29 deletions(-) diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 48349b6e..6fcf5745 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -5272,7 +5272,7 @@ fn attr_delete(obj: &Object, name: &str) -> Result<(), RuntimeError> { } Object::Function(f) => { if crate::object::is_function_slot(name) { - f.slots + f.slots() .borrow_mut() .shift_remove(&crate::object::DictKey(Object::from_str(name))); } else { @@ -10455,7 +10455,8 @@ fn b_mark_iterable_coroutine(args: &[Object]) -> Result { // Shared, not copied: `func.__dict__` mutations stay visible on // both, matching CPython where the function object is the same. attrs: RefCell::new(Some(f.attrs())), - slots: RefCell::new(f.slots.borrow().clone()), + slots_raw: RefCell::new(f.slots().borrow().clone()), + slot_seed: RefCell::new(None), closure_cells: std::sync::OnceLock::new(), // The copied slot store carries any override along. defaults_override: crate::object::OverrideFlag::new(f.defaults_maybe_overridden()), diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index e90f89bd..9e2d795e 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -2048,7 +2048,8 @@ pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) { } } } - if let Ok(slots) = f.slots.try_borrow() { + // (An unplanted seed holds only the shared name strings.) + if let Ok(slots) = f.slots_raw.try_borrow() { for (k, v) in slots.iter() { visit(&k.0); visit(v); @@ -2259,9 +2260,12 @@ pub fn clear_object_fields(obj: &Object) -> bool { attrs.clear(); } } - if let Ok(mut slots) = f.slots.try_borrow_mut() { + if let Ok(mut slots) = f.slots_raw.try_borrow_mut() { slots.clear(); } + if let Ok(mut seed) = f.slot_seed.try_borrow_mut() { + seed.take(); + } } Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { // Dropping the suspended frame box breaks the cycle diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index 71974a17..0b46cc61 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -132,6 +132,12 @@ enum Op { dst: u8, pc: u16, }, + /// `regs[dst]` = the closure cell `idx`'s value (a free variable: a + /// leaf has no cell variables of its own). + Deref { + dst: u8, + idx: u8, + }, Attr { dst: u8, src: u8, @@ -395,7 +401,14 @@ impl Builder<'_> { fn step(&mut self, pc: usize, ins: weavepy_compiler::Instruction) -> Option { let arg = ins.arg; match ins.op { - OpCode::Resume | OpCode::Nop | OpCode::NotTaken => {} + OpCode::Resume | OpCode::Nop | OpCode::NotTaken | OpCode::CopyFreeVars => {} + OpCode::LoadDeref => { + let dst = self.push_new()?; + self.ops.push(Op::Deref { + dst, + idx: u8::try_from(arg).ok()?, + }); + } OpCode::LoadFast | OpCode::LoadFastBorrow | OpCode::LoadFastCheck => { self.load_local(arg)?; } @@ -899,6 +912,22 @@ pub(crate) enum LeafRet { Owned(Object), } +/// The value of `f`'s closure cell `idx`, borrowed: nothing an evaluation +/// runs can rebind a cell (a leaf has no `STORE_DEREF`), and the function +/// holds its closure. `None` for an empty cell (the read raises). +#[inline] +pub(super) fn free_var(f: &crate::object::PyFunction, idx: u8) -> Option { + let Object::Cell(cell) = f.closure.get(usize::from(idx))? else { + return None; + }; + let p: *const Object = cell.as_ptr(); + // SAFETY: see above. + if matches!(unsafe { &*p }, Object::Unbound) { + return None; + } + Some(norm(p)) +} + impl LeafRet { /// The result as an owned object (a borrowed value is cloned). #[inline(always)] @@ -1062,6 +1091,7 @@ impl Interpreter { // SAFETY: `k` names a plan constant (the translation). Op::Const { dst, k } => set!(dst, unsafe { *consts.get_unchecked(usize::from(k)) }), Op::Global { dst, pc } => set!(dst, self.plan_global(code, stamps, f, pc)?), + Op::Deref { dst, idx } => set!(dst, free_var(f, idx)?), Op::Attr { dst, src, pc, name } => { let v = self.plan_attr::( code, diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index 5f3312de..25a266ea 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -422,6 +422,18 @@ unsafe extern "C" fn h_global( 0 } +unsafe extern "C" fn h_deref(ctx: *mut Ctx<'static>, fr: *const Frame, idx: u32) -> u32 { + // SAFETY: called by native code with its context and a live frame. + let (c, (_, _, f)) = unsafe { (cx(ctx), (*fr).parts()) }; + match super::free_var(f, idx as u8) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + unsafe extern "C" fn h_attr( ctx: *mut Ctx<'static>, fr: *const Frame, @@ -1697,6 +1709,15 @@ impl Lower<'_> { let (t, p) = (self.b.block_params(done)[0], self.b.block_params(done)[1]); self.set(fl, dst, t, p); } + Op::Deref { dst, idx } => { + let idx = self.u32c(u32::from(idx)); + let st = self.call(h_deref as *const () as usize, &[self.ctx, fl.fr, idx]); + self.check(st); + let flags = MemFlags::trusted(); + let t = self.b.ins().load(types::I64, flags, self.ctx, 0); + let p = self.b.ins().load(types::I64, flags, self.ctx, 8); + self.set(fl, dst, t, p); + } Op::Attr { dst, src, pc, name } => { let (t, p) = self.get(fl, src); let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index c7172976..5340c256 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -23149,7 +23149,7 @@ impl Interpreter { } } 0x04 => { - f.slots + f.slots() .borrow_mut() .insert(DictKey(Object::from_static("__annotations__")), value); } @@ -23174,7 +23174,7 @@ impl Interpreter { Object::from_str(format!("{owner_qualname}.__annotate__")), ); } - f.slots + f.slots() .borrow_mut() .insert(DictKey(Object::from_static("__annotate__")), value); } @@ -40576,11 +40576,11 @@ impl Interpreter { // stored dict; either way the deferred `__annotate__` // is dropped so the next read doesn't recompute. "__annotations__" => { - f.slots + f.slots() .borrow_mut() .shift_remove(&DictKey(Object::from_static("__annotate__"))); if matches!(value, Object::None) { - f.slots + f.slots() .borrow_mut() .shift_remove(&DictKey(Object::from_static("__annotations__"))); return Ok(()); @@ -40599,7 +40599,7 @@ impl Interpreter { if !crate::builtins::object_is_callable(&value) { return Err(type_error("__annotate__ must be callable or None")); } - f.slots + f.slots() .borrow_mut() .shift_remove(&DictKey(Object::from_static("__annotations__"))); f.set_slot(name, value); @@ -41498,7 +41498,7 @@ impl Interpreter { // `del f.__annotations__` clears both the dict and the // deferred `__annotate__` (the setter with NULL). "__annotations__" => { - let mut slots = f.slots.borrow_mut(); + let mut slots = f.slots().borrow_mut(); slots.shift_remove(&DictKey(Object::from_static("__annotations__"))); slots.shift_remove(&DictKey(Object::from_static("__annotate__"))); return Ok(()); @@ -41518,7 +41518,7 @@ impl Interpreter { // CPython allows deleting the nullable slots // (`__doc__`, `__annotations__`, …): the value // resets so the computed default resurfaces. - f.slots + f.slots() .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); return Ok(()); @@ -44722,7 +44722,8 @@ impl Interpreter { kw_defaults, closure, attrs: RefCell::new(None), - slots: RefCell::new(DictData::default()), + slots_raw: RefCell::new(DictData::default()), + slot_seed: RefCell::new(None), closure_cells: std::sync::OnceLock::new(), defaults_override: crate::object::OverrideFlag::new(false), }))) @@ -48800,7 +48801,7 @@ impl Interpreter { // The pinned slot objects are shared, as CPython's // `gi_name` shares the function's `__name__`. let (gen_name, gen_qualname) = { - let slots = f.slots.borrow(); + let slots = f.slots().borrow(); let attr_str = |attr: &'static str| match slots.get(&crate::object::StrKey(attr)) { Some(o @ Object::Str(_)) => Some(o.clone()), @@ -62402,7 +62403,7 @@ struct CodeConstObjects { /// The getset slots of a function made from this code (see /// [`function_slots`]): the module name they were built for, and the /// entries, copied into each new function. - fn_slots: std::sync::OnceLock<(Option, crate::object::DictMap)>, + fn_slots: std::sync::OnceLock<(Option, Rc)>, /// Whether this code is a *pure leaf* (see [`code_is_pure_leaf`]): /// `0` not yet decided, `1` no, `2` general leaf, `3` argument return, /// `4` constant return, `5` small-int return, `6` attribute return, @@ -63638,8 +63639,9 @@ fn code_pure_leaf_decide(code: &CodeObject, ext: &CodeConstObjects) -> bool { && !code.is_async_generator && !code.is_iterable_coroutine && !code.is_class_body + // (Free variables are read through the function's closure; cell + // variables would need cells of the evaluation's own.) && code.cellvars.is_empty() - && code.freevars.is_empty() && !code.has_varargs && code.kwonly_count == 0 && leaf_arity(code) <= 8 @@ -63657,6 +63659,8 @@ fn code_pure_leaf_decide(code: &CodeObject, ext: &CodeConstObjects) -> bool { OpCode::Resume | OpCode::Nop | OpCode::NotTaken + | OpCode::CopyFreeVars + | OpCode::LoadDeref | OpCode::LoadFast | OpCode::LoadFastBorrow | OpCode::LoadFastCheck @@ -64084,7 +64088,7 @@ fn new_function( // staticmethod wrappers) return the *same* object, which // `assertIs(wrapper.__name__, func.__name__)` in test_decorators // relies on. - let mut slots = function_slots(&code, globals); + let (mut slots, seed) = function_slots(&code, globals, annotations.is_none()); if let Some(ann) = annotations { slots.insert(DictKey(fn_slot_keys()[3].clone()), ann); } @@ -64100,7 +64104,8 @@ fn new_function( kw_defaults, closure, attrs: RefCell::new(None), - slots: RefCell::new(slots), + slots_raw: RefCell::new(slots), + slot_seed: RefCell::new(seed), closure_cells: std::sync::OnceLock::new(), defaults_override: crate::object::OverrideFlag::new(false), }; @@ -64117,9 +64122,15 @@ fn new_function( /// The getset slots a new function made from `code` in `globals` starts /// with: `__module__` (the globals' `__name__`, when set), `__name__` and /// `__qualname__`. Built once per code object (for the module name it -/// first sees) and copied into each function after, so two functions made -/// from one `def` share their name objects, as CPython's do. -fn function_slots(code: &CodeObject, globals: &Rc>) -> DictData { +/// first sees) and shared by each function after, so two functions made +/// from one `def` share their name objects, as CPython's do: with +/// `seeded`, as the function's seed (see `PyFunction::slot_seed`) beside +/// empty slots, otherwise copied. +fn function_slots( + code: &CodeObject, + globals: &Rc>, + seeded: bool, +) -> (DictData, Option>) { let keys = fn_slot_keys(); let module = globals.borrow().get(&DictKey(keys[1].clone())).cloned(); let build = |module: Option| { @@ -64149,12 +64160,17 @@ fn function_slots(code: &CodeObject, globals: &Rc>) -> DictDat _ => false, }; if let Some(ext) = code_vm_ext(code) { - let (built_for, map) = ext.fn_slots.get_or_init(|| (module.clone(), build(module.clone()))); + let (built_for, map) = ext + .fn_slots + .get_or_init(|| (module.clone(), Rc::new(build(module.clone())))); if same(built_for, &module) { - return DictData::from(map.clone()); + if seeded { + return (DictData::default(), Some(map.clone())); + } + return (DictData::from((**map).clone()), None); } } - DictData::from(build(module)) + (DictData::from(build(module)), None) } fn fn_slot_keys() -> &'static [Object; 4] { diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index c2b4063b..4d672c75 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -4585,8 +4585,13 @@ pub struct PyFunction { /// data descriptors on the `function` type, so `f.__name__ = x` /// must never appear in `f.__dict__` (functools.update_wrapper /// copies `__dict__` and asserts the wrapper's annotations are - /// untouched by the wrapped function's slots). - pub slots: RefCell, + /// untouched by the wrapped function's slots). Read through + /// [`Self::slots`], which first copies in [`Self::slot_seed`]. + pub slots_raw: RefCell, + /// The slots a new function starts with, shared by every function + /// one `def` makes (see `new_function`), until the first access to + /// [`Self::slots`] copies them in: most functions never read theirs. + pub slot_seed: RefCell>>, /// The frame `cells` vector for a function whose code has free /// variables but no cell variables: exactly `closure`'s cells, built /// once (the closure is immutable) for the lean call paths. @@ -4722,11 +4727,35 @@ impl PyFunction { .cloned() } + /// The function's getset slots (see [`Self::slots_raw`]), with the + /// seed it was made with copied in. + #[inline] + pub fn slots(&self) -> &RefCell { + // SAFETY: a read of the seed's presence with nothing running. + if unsafe { (*self.slot_seed.as_ptr()).is_some() } { + self.plant_slot_seed(); + } + &self.slots_raw + } + + #[cold] + #[inline(never)] + fn plant_slot_seed(&self) { + let Some(seed) = self.slot_seed.borrow_mut().take() else { + return; + }; + let mut slots = self.slots_raw.borrow_mut(); + // Every store goes through `slots()`, which plants the seed + // first: nothing is there yet to keep. + debug_assert!(slots.is_empty()); + *slots = DictData::from((*seed).clone()); + } + /// Read a slot value if one has been stored (explicitly assigned or /// stamped at definition time). Computed fallbacks live at the /// attribute-access sites. pub fn slot(&self, name: &str) -> Option { - self.slots + self.slots() .borrow() .get(&crate::object::StrKey(name)) .cloned() @@ -4738,7 +4767,7 @@ impl PyFunction { .0 .store(true, std::sync::atomic::Ordering::Relaxed); } - self.slots + self.slots() .borrow_mut() .insert(DictKey(Object::from_str(name)), value); } diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs index 446bfa2e..f449ad1c 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs @@ -72,7 +72,7 @@ impl FunctionGuard { if !function.defaults_maybe_overridden() { return true; } - let slots = function.slots.borrow(); + let slots = function.slots().borrow(); !slots.contains_key(&StrKey("__defaults__")) && !slots.contains_key(&StrKey("__kwdefaults__")) } diff --git a/tests/regrtest/test_function_creation.py b/tests/regrtest/test_function_creation.py index afdbc349..31933ed2 100644 --- a/tests/regrtest/test_function_creation.py +++ b/tests/regrtest/test_function_creation.py @@ -80,6 +80,75 @@ def h(a: int, *, b=i) -> int: self.assertEqual(hs[7].__kwdefaults__, {"b": 7}) self.assertEqual(hs[7].__annotations__, {"a": int, "return": int}) + def test_name_slots_read_and_written_later(self): + import functools + import pickle + + fs = [make_adder(i) for i in range(50)] + + @functools.wraps(fs[0]) + def wrapper(*args): + return fs[0](*args) + + self.assertEqual(wrapper.__name__, "add") + self.assertEqual(wrapper.__qualname__, "make_adder..add") + self.assertEqual(wrapper.__module__, __name__) + self.assertIs(wrapper.__wrapped__, fs[0]) + fs[1].__module__ = "elsewhere" + self.assertEqual(fs[1].__module__, "elsewhere") + self.assertEqual(fs[2].__module__, __name__) + fs[3].__doc__ = "doc" + self.assertEqual((fs[3].__doc__, fs[4].__doc__), ("doc", None)) + self.assertIs(pickle.loads(pickle.dumps(make_adder)), make_adder) + g = [make_lambda(k) for k in range(3)] + self.assertEqual({h.__name__ for h in g}, {""}) + self.assertEqual(g[0].__qualname__, "make_lambda..") + + def test_free_variable_reads(self): + def counter(): + n = 0 + + def get(): + return n + + def bump(): + nonlocal n + n += 1 + + return get, bump + + get, bump = counter() + for _ in range(5000): + self.assertEqual(get(), 0) + bump() + self.assertEqual(get(), 1) + + def late(): + def read(): + return value + + out = [read] + try: + read() + except NameError as e: + out.append(type(e)) + value = 7 + out.append(read()) + return out + + for _ in range(3000): + r = late() + self.assertEqual(r[1:], [NameError, 7]) + + def scaled(k): + return lambda x: x * k + len(box) + + box = [1, 2] + fs = [scaled(i) for i in range(4000)] + self.assertEqual(sum(f(1) for f in fs), sum(range(4000)) + 2 * 4000) + box.append(3) + self.assertEqual(fs[5](2), 13) + def test_fresh_functions_share_call_shapes(self): def caller(n): t = 0 From dde35a23d2693992f9b7e713e9398f5c8d4636ed Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:36:58 -0700 Subject: [PATCH 74/96] perf: keep the frame compiler to tier 2's start-up and import budget Tier 2 compiles code during start-up and imports only after sixteen times its normal threshold, since that code is mostly run once; the frame compiler had no such budget. Importing several modules that each compile regular expressions made `re._compiler._optimize_charset` hot, and compiling its 559 instructions cost about 100M instructions, a tenth of the whole import. Code that gets hot while start-up or an import runs now compiles only after sixteen times the frame compiler's threshold too. `import typing, json, dataclasses, argparse, logging, asyncio, inspect, textwrap, http.client, unittest` in one process: 978M -> 874M instructions (python3.14: 447M). Single-module imports are unchanged. --- crates/weavepy-vm/src/frame_jit.rs | 13 +++++++++++++ crates/weavepy-vm/src/tier2.rs | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs index ae4ec55c..e36f4982 100644 --- a/crates/weavepy-vm/src/frame_jit.rs +++ b/crates/weavepy-vm/src/frame_jit.rs @@ -185,6 +185,9 @@ impl Native { pub(crate) struct Slot { native: std::sync::OnceLock>>, heat: AtomicU32, + /// How many times the code got hot while start-up or an import ran + /// (see `try_compile`). + deferred: AtomicU32, } impl Slot { @@ -221,6 +224,16 @@ impl Slot { let _ = self.native.set(None); return; } + // Code that start-up or an import runs is mostly run once (a + // regular expression compiler's loops, a module's set-up), and a + // compile costs as much as thousands of interpreted iterations: + // there, code compiles only after sustained work, as tier 2's + // budget has it. + let d = self.deferred.load(Ordering::Relaxed).saturating_add(1); + if !crate::tier2::frame_compile_allowed(d.saturating_mul(hot()), hot()) { + self.deferred.store(d, Ordering::Relaxed); + return; + } // Loops the tier-2 compiler may still take keep their back edges' // consultation in the core loop (a generator body the fast steps // run never reaches it). diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index eb69854b..29a197f6 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -887,6 +887,12 @@ fn compile_allowed(counter: u32, threshold: u32) -> bool { || counter >= threshold.saturating_mul(16) } +/// [`compile_allowed`] for the frame compiler (see `frame_jit`), which +/// keeps tier 2's start-up and import budgets. +pub(crate) fn frame_compile_allowed(counter: u32, threshold: u32) -> bool { + compile_allowed(counter, threshold) +} + /// True when no thread's JIT can be enabled (see [`JIT_PROCESS_GATE`]). #[inline] pub(crate) fn jit_off_for_process() -> bool { From f7448fec49884da414ba370c435f7c6b90ebefac Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:08:00 -0700 Subject: [PATCH 75/96] perf: build the opcode shim's stack-effect tables on first use `_opcode` (imported by `opcode`, and so by `dis`, `inspect` and `asyncio`) built a lambda per opcode for its popped and pushed counts and a set per flag predicate when it was imported, though most programs never ask for a stack effect. The generator now emits the stack tables behind `_stack_tables()`, built on the first `stack_effect` call, with a plain int for every count that doesn't depend on the oparg, and the predicates test literal frozensets of the opcodes that have each flag. `_weave_flowgraph` reads the tables the same way. Import-only instructions: opcode 27M -> 13M (python3.14 3M), dis 57M -> 43M (20M). The regenerated module answers all 21,210 checks of every predicate and stack effect the same as before. --- .../weavepy-vm/src/stdlib/python/_opcode.py | 1428 ++++++++++------- .../src/stdlib/python/_weave_flowgraph.py | 13 +- tools/gen_opcode_shim.py | 99 +- 3 files changed, 931 insertions(+), 609 deletions(-) diff --git a/crates/weavepy-vm/src/stdlib/python/_opcode.py b/crates/weavepy-vm/src/stdlib/python/_opcode.py index 6b21ab9c..ffeda74f 100644 --- a/crates/weavepy-vm/src/stdlib/python/_opcode.py +++ b/crates/weavepy-vm/src/stdlib/python/_opcode.py @@ -284,584 +284,831 @@ _names['STORE_FAST_MAYBE_NULL']: 545, } -# `_PyOpcode_Deopt`: specialized form -> base instruction. -_DEOPT = { - _names['BINARY_OP_ADD_FLOAT']: _names['BINARY_OP'], - _names['BINARY_OP_ADD_INT']: _names['BINARY_OP'], - _names['BINARY_OP_ADD_UNICODE']: _names['BINARY_OP'], - _names['BINARY_OP_EXTEND']: _names['BINARY_OP'], - _names['BINARY_OP_INPLACE_ADD_UNICODE']: _names['BINARY_OP'], - _names['BINARY_OP_MULTIPLY_FLOAT']: _names['BINARY_OP'], - _names['BINARY_OP_MULTIPLY_INT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_DICT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_GETITEM']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_LIST_INT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_LIST_SLICE']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_STR_INT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_TUPLE_INT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBTRACT_FLOAT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBTRACT_INT']: _names['BINARY_OP'], - _names['CALL_ALLOC_AND_ENTER_INIT']: _names['CALL'], - _names['CALL_BOUND_METHOD_EXACT_ARGS']: _names['CALL'], - _names['CALL_BOUND_METHOD_GENERAL']: _names['CALL'], - _names['CALL_BUILTIN_CLASS']: _names['CALL'], - _names['CALL_BUILTIN_FAST']: _names['CALL'], - _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: _names['CALL'], - _names['CALL_BUILTIN_O']: _names['CALL'], - _names['CALL_ISINSTANCE']: _names['CALL'], - _names['CALL_KW_BOUND_METHOD']: _names['CALL_KW'], - _names['CALL_KW_NON_PY']: _names['CALL_KW'], - _names['CALL_KW_PY']: _names['CALL_KW'], - _names['CALL_LEN']: _names['CALL'], - _names['CALL_LIST_APPEND']: _names['CALL'], - _names['CALL_METHOD_DESCRIPTOR_FAST']: _names['CALL'], - _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: _names['CALL'], - _names['CALL_METHOD_DESCRIPTOR_NOARGS']: _names['CALL'], - _names['CALL_METHOD_DESCRIPTOR_O']: _names['CALL'], - _names['CALL_NON_PY_GENERAL']: _names['CALL'], - _names['CALL_PY_EXACT_ARGS']: _names['CALL'], - _names['CALL_PY_GENERAL']: _names['CALL'], - _names['CALL_STR_1']: _names['CALL'], - _names['CALL_TUPLE_1']: _names['CALL'], - _names['CALL_TYPE_1']: _names['CALL'], - _names['COMPARE_OP_FLOAT']: _names['COMPARE_OP'], - _names['COMPARE_OP_INT']: _names['COMPARE_OP'], - _names['COMPARE_OP_STR']: _names['COMPARE_OP'], - _names['CONTAINS_OP_DICT']: _names['CONTAINS_OP'], - _names['CONTAINS_OP_SET']: _names['CONTAINS_OP'], - _names['FOR_ITER_GEN']: _names['FOR_ITER'], - _names['FOR_ITER_LIST']: _names['FOR_ITER'], - _names['FOR_ITER_RANGE']: _names['FOR_ITER'], - _names['FOR_ITER_TUPLE']: _names['FOR_ITER'], - _names['JUMP_BACKWARD_JIT']: _names['JUMP_BACKWARD'], - _names['JUMP_BACKWARD_NO_JIT']: _names['JUMP_BACKWARD'], - _names['LOAD_ATTR_CLASS']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_INSTANCE_VALUE']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_METHOD_LAZY_DICT']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_METHOD_NO_DICT']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_METHOD_WITH_VALUES']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_MODULE']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_PROPERTY']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_SLOT']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_WITH_HINT']: _names['LOAD_ATTR'], - _names['LOAD_CONST_IMMORTAL']: _names['LOAD_CONST'], - _names['LOAD_CONST_MORTAL']: _names['LOAD_CONST'], - _names['LOAD_GLOBAL_BUILTIN']: _names['LOAD_GLOBAL'], - _names['LOAD_GLOBAL_MODULE']: _names['LOAD_GLOBAL'], - _names['LOAD_SUPER_ATTR_ATTR']: _names['LOAD_SUPER_ATTR'], - _names['LOAD_SUPER_ATTR_METHOD']: _names['LOAD_SUPER_ATTR'], - _names['RESUME_CHECK']: _names['RESUME'], - _names['SEND_GEN']: _names['SEND'], - _names['STORE_ATTR_INSTANCE_VALUE']: _names['STORE_ATTR'], - _names['STORE_ATTR_SLOT']: _names['STORE_ATTR'], - _names['STORE_ATTR_WITH_HINT']: _names['STORE_ATTR'], - _names['STORE_SUBSCR_DICT']: _names['STORE_SUBSCR'], - _names['STORE_SUBSCR_LIST_INT']: _names['STORE_SUBSCR'], - _names['TO_BOOL_ALWAYS_TRUE']: _names['TO_BOOL'], - _names['TO_BOOL_BOOL']: _names['TO_BOOL'], - _names['TO_BOOL_INT']: _names['TO_BOOL'], - _names['TO_BOOL_LIST']: _names['TO_BOOL'], - _names['TO_BOOL_NONE']: _names['TO_BOOL'], - _names['TO_BOOL_STR']: _names['TO_BOOL'], - _names['UNPACK_SEQUENCE_LIST']: _names['UNPACK_SEQUENCE'], - _names['UNPACK_SEQUENCE_TUPLE']: _names['UNPACK_SEQUENCE'], - _names['UNPACK_SEQUENCE_TWO_TUPLE']: _names['UNPACK_SEQUENCE'], -} +# The tables only `stack_effect` reads, built on its first call (most +# importers of `opcode` and `dis` never compute a stack effect). +_STACK_TABLES = None -# `_PyOpcode_num_popped(opcode, oparg)`. -_POPPED = { - _names['ANNOTATIONS_PLACEHOLDER']: lambda oparg: 0, - _names['BINARY_OP']: lambda oparg: 2, - _names['BINARY_OP_ADD_FLOAT']: lambda oparg: 2, - _names['BINARY_OP_ADD_INT']: lambda oparg: 2, - _names['BINARY_OP_ADD_UNICODE']: lambda oparg: 2, - _names['BINARY_OP_EXTEND']: lambda oparg: 2, - _names['BINARY_OP_INPLACE_ADD_UNICODE']: lambda oparg: 2, - _names['BINARY_OP_MULTIPLY_FLOAT']: lambda oparg: 2, - _names['BINARY_OP_MULTIPLY_INT']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_DICT']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_GETITEM']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_LIST_INT']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_LIST_SLICE']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_STR_INT']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_TUPLE_INT']: lambda oparg: 2, - _names['BINARY_OP_SUBTRACT_FLOAT']: lambda oparg: 2, - _names['BINARY_OP_SUBTRACT_INT']: lambda oparg: 2, - _names['BINARY_SLICE']: lambda oparg: 3, - _names['BUILD_INTERPOLATION']: lambda oparg: 2 + (oparg & 1), - _names['BUILD_LIST']: lambda oparg: oparg, - _names['BUILD_MAP']: lambda oparg: oparg*2, - _names['BUILD_SET']: lambda oparg: oparg, - _names['BUILD_SLICE']: lambda oparg: oparg, - _names['BUILD_STRING']: lambda oparg: oparg, - _names['BUILD_TEMPLATE']: lambda oparg: 2, - _names['BUILD_TUPLE']: lambda oparg: oparg, - _names['CACHE']: lambda oparg: 0, - _names['CALL']: lambda oparg: 2 + oparg, - _names['CALL_ALLOC_AND_ENTER_INIT']: lambda oparg: 2 + oparg, - _names['CALL_BOUND_METHOD_EXACT_ARGS']: lambda oparg: 2 + oparg, - _names['CALL_BOUND_METHOD_GENERAL']: lambda oparg: 2 + oparg, - _names['CALL_BUILTIN_CLASS']: lambda oparg: 2 + oparg, - _names['CALL_BUILTIN_FAST']: lambda oparg: 2 + oparg, - _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: lambda oparg: 2 + oparg, - _names['CALL_BUILTIN_O']: lambda oparg: 2 + oparg, - _names['CALL_FUNCTION_EX']: lambda oparg: 4, - _names['CALL_INTRINSIC_1']: lambda oparg: 1, - _names['CALL_INTRINSIC_2']: lambda oparg: 2, - _names['CALL_ISINSTANCE']: lambda oparg: 2 + oparg, - _names['CALL_KW']: lambda oparg: 3 + oparg, - _names['CALL_KW_BOUND_METHOD']: lambda oparg: 3 + oparg, - _names['CALL_KW_NON_PY']: lambda oparg: 3 + oparg, - _names['CALL_KW_PY']: lambda oparg: 3 + oparg, - _names['CALL_LEN']: lambda oparg: 3, - _names['CALL_LIST_APPEND']: lambda oparg: 3, - _names['CALL_METHOD_DESCRIPTOR_FAST']: lambda oparg: 2 + oparg, - _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: lambda oparg: 2 + oparg, - _names['CALL_METHOD_DESCRIPTOR_NOARGS']: lambda oparg: 2 + oparg, - _names['CALL_METHOD_DESCRIPTOR_O']: lambda oparg: 2 + oparg, - _names['CALL_NON_PY_GENERAL']: lambda oparg: 2 + oparg, - _names['CALL_PY_EXACT_ARGS']: lambda oparg: 2 + oparg, - _names['CALL_PY_GENERAL']: lambda oparg: 2 + oparg, - _names['CALL_STR_1']: lambda oparg: 3, - _names['CALL_TUPLE_1']: lambda oparg: 3, - _names['CALL_TYPE_1']: lambda oparg: 3, - _names['CHECK_EG_MATCH']: lambda oparg: 2, - _names['CHECK_EXC_MATCH']: lambda oparg: 2, - _names['CLEANUP_THROW']: lambda oparg: 3, - _names['COMPARE_OP']: lambda oparg: 2, - _names['COMPARE_OP_FLOAT']: lambda oparg: 2, - _names['COMPARE_OP_INT']: lambda oparg: 2, - _names['COMPARE_OP_STR']: lambda oparg: 2, - _names['CONTAINS_OP']: lambda oparg: 2, - _names['CONTAINS_OP_DICT']: lambda oparg: 2, - _names['CONTAINS_OP_SET']: lambda oparg: 2, - _names['CONVERT_VALUE']: lambda oparg: 1, - _names['COPY']: lambda oparg: 1 + (oparg-1), - _names['COPY_FREE_VARS']: lambda oparg: 0, - _names['DELETE_ATTR']: lambda oparg: 1, - _names['DELETE_DEREF']: lambda oparg: 0, - _names['DELETE_FAST']: lambda oparg: 0, - _names['DELETE_GLOBAL']: lambda oparg: 0, - _names['DELETE_NAME']: lambda oparg: 0, - _names['DELETE_SUBSCR']: lambda oparg: 2, - _names['DICT_MERGE']: lambda oparg: 5 + (oparg - 1), - _names['DICT_UPDATE']: lambda oparg: 2 + (oparg - 1), - _names['END_ASYNC_FOR']: lambda oparg: 2, - _names['END_FOR']: lambda oparg: 1, - _names['END_SEND']: lambda oparg: 2, - _names['ENTER_EXECUTOR']: lambda oparg: 0, - _names['EXIT_INIT_CHECK']: lambda oparg: 1, - _names['EXTENDED_ARG']: lambda oparg: 0, - _names['FORMAT_SIMPLE']: lambda oparg: 1, - _names['FORMAT_WITH_SPEC']: lambda oparg: 2, - _names['FOR_ITER']: lambda oparg: 1, - _names['FOR_ITER_GEN']: lambda oparg: 1, - _names['FOR_ITER_LIST']: lambda oparg: 1, - _names['FOR_ITER_RANGE']: lambda oparg: 1, - _names['FOR_ITER_TUPLE']: lambda oparg: 1, - _names['GET_AITER']: lambda oparg: 1, - _names['GET_ANEXT']: lambda oparg: 1, - _names['GET_AWAITABLE']: lambda oparg: 1, - _names['GET_ITER']: lambda oparg: 1, - _names['GET_LEN']: lambda oparg: 1, - _names['GET_YIELD_FROM_ITER']: lambda oparg: 1, - _names['IMPORT_FROM']: lambda oparg: 1, - _names['IMPORT_NAME']: lambda oparg: 2, - _names['INSTRUMENTED_CALL']: lambda oparg: 2 + oparg, - _names['INSTRUMENTED_CALL_FUNCTION_EX']: lambda oparg: 4, - _names['INSTRUMENTED_CALL_KW']: lambda oparg: 3 + oparg, - _names['INSTRUMENTED_END_ASYNC_FOR']: lambda oparg: 2, - _names['INSTRUMENTED_END_FOR']: lambda oparg: 2, - _names['INSTRUMENTED_END_SEND']: lambda oparg: 2, - _names['INSTRUMENTED_FOR_ITER']: lambda oparg: 1, - _names['INSTRUMENTED_INSTRUCTION']: lambda oparg: 0, - _names['INSTRUMENTED_JUMP_BACKWARD']: lambda oparg: 0, - _names['INSTRUMENTED_JUMP_FORWARD']: lambda oparg: 0, - _names['INSTRUMENTED_LINE']: lambda oparg: 0, - _names['INSTRUMENTED_LOAD_SUPER_ATTR']: lambda oparg: 3, - _names['INSTRUMENTED_NOT_TAKEN']: lambda oparg: 0, - _names['INSTRUMENTED_POP_ITER']: lambda oparg: 1, - _names['INSTRUMENTED_POP_JUMP_IF_FALSE']: lambda oparg: 1, - _names['INSTRUMENTED_POP_JUMP_IF_NONE']: lambda oparg: 1, - _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE']: lambda oparg: 1, - _names['INSTRUMENTED_POP_JUMP_IF_TRUE']: lambda oparg: 1, - _names['INSTRUMENTED_RESUME']: lambda oparg: 0, - _names['INSTRUMENTED_RETURN_VALUE']: lambda oparg: 1, - _names['INSTRUMENTED_YIELD_VALUE']: lambda oparg: 1, - _names['INTERPRETER_EXIT']: lambda oparg: 1, - _names['IS_OP']: lambda oparg: 2, - _names['JUMP']: lambda oparg: 0, - _names['JUMP_BACKWARD']: lambda oparg: 0, - _names['JUMP_BACKWARD_JIT']: lambda oparg: 0, - _names['JUMP_BACKWARD_NO_INTERRUPT']: lambda oparg: 0, - _names['JUMP_BACKWARD_NO_JIT']: lambda oparg: 0, - _names['JUMP_FORWARD']: lambda oparg: 0, - _names['JUMP_IF_FALSE']: lambda oparg: 1, - _names['JUMP_IF_TRUE']: lambda oparg: 1, - _names['JUMP_NO_INTERRUPT']: lambda oparg: 0, - _names['LIST_APPEND']: lambda oparg: 2 + (oparg-1), - _names['LIST_EXTEND']: lambda oparg: 2 + (oparg-1), - _names['LOAD_ATTR']: lambda oparg: 1, - _names['LOAD_ATTR_CLASS']: lambda oparg: 1, - _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: lambda oparg: 1, - _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: lambda oparg: 1, - _names['LOAD_ATTR_INSTANCE_VALUE']: lambda oparg: 1, - _names['LOAD_ATTR_METHOD_LAZY_DICT']: lambda oparg: 1, - _names['LOAD_ATTR_METHOD_NO_DICT']: lambda oparg: 1, - _names['LOAD_ATTR_METHOD_WITH_VALUES']: lambda oparg: 1, - _names['LOAD_ATTR_MODULE']: lambda oparg: 1, - _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: lambda oparg: 1, - _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: lambda oparg: 1, - _names['LOAD_ATTR_PROPERTY']: lambda oparg: 1, - _names['LOAD_ATTR_SLOT']: lambda oparg: 1, - _names['LOAD_ATTR_WITH_HINT']: lambda oparg: 1, - _names['LOAD_BUILD_CLASS']: lambda oparg: 0, - _names['LOAD_CLOSURE']: lambda oparg: 0, - _names['LOAD_COMMON_CONSTANT']: lambda oparg: 0, - _names['LOAD_CONST']: lambda oparg: 0, - _names['LOAD_CONST_IMMORTAL']: lambda oparg: 0, - _names['LOAD_CONST_MORTAL']: lambda oparg: 0, - _names['LOAD_DEREF']: lambda oparg: 0, - _names['LOAD_FAST']: lambda oparg: 0, - _names['LOAD_FAST_AND_CLEAR']: lambda oparg: 0, - _names['LOAD_FAST_BORROW']: lambda oparg: 0, - _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW']: lambda oparg: 0, - _names['LOAD_FAST_CHECK']: lambda oparg: 0, - _names['LOAD_FAST_LOAD_FAST']: lambda oparg: 0, - _names['LOAD_FROM_DICT_OR_DEREF']: lambda oparg: 1, - _names['LOAD_FROM_DICT_OR_GLOBALS']: lambda oparg: 1, - _names['LOAD_GLOBAL']: lambda oparg: 0, - _names['LOAD_GLOBAL_BUILTIN']: lambda oparg: 0, - _names['LOAD_GLOBAL_MODULE']: lambda oparg: 0, - _names['LOAD_LOCALS']: lambda oparg: 0, - _names['LOAD_NAME']: lambda oparg: 0, - _names['LOAD_SMALL_INT']: lambda oparg: 0, - _names['LOAD_SPECIAL']: lambda oparg: 1, - _names['LOAD_SUPER_ATTR']: lambda oparg: 3, - _names['LOAD_SUPER_ATTR_ATTR']: lambda oparg: 3, - _names['LOAD_SUPER_ATTR_METHOD']: lambda oparg: 3, - _names['MAKE_CELL']: lambda oparg: 0, - _names['MAKE_FUNCTION']: lambda oparg: 1, - _names['MAP_ADD']: lambda oparg: 3 + (oparg - 1), - _names['MATCH_CLASS']: lambda oparg: 3, - _names['MATCH_KEYS']: lambda oparg: 2, - _names['MATCH_MAPPING']: lambda oparg: 1, - _names['MATCH_SEQUENCE']: lambda oparg: 1, - _names['NOP']: lambda oparg: 0, - _names['NOT_TAKEN']: lambda oparg: 0, - _names['POP_BLOCK']: lambda oparg: 0, - _names['POP_EXCEPT']: lambda oparg: 1, - _names['POP_ITER']: lambda oparg: 1, - _names['POP_JUMP_IF_FALSE']: lambda oparg: 1, - _names['POP_JUMP_IF_NONE']: lambda oparg: 1, - _names['POP_JUMP_IF_NOT_NONE']: lambda oparg: 1, - _names['POP_JUMP_IF_TRUE']: lambda oparg: 1, - _names['POP_TOP']: lambda oparg: 1, - _names['PUSH_EXC_INFO']: lambda oparg: 1, - _names['PUSH_NULL']: lambda oparg: 0, - _names['RAISE_VARARGS']: lambda oparg: oparg, - _names['RERAISE']: lambda oparg: 1 + oparg, - _names['RESERVED']: lambda oparg: 0, - _names['RESUME']: lambda oparg: 0, - _names['RESUME_CHECK']: lambda oparg: 0, - _names['RETURN_GENERATOR']: lambda oparg: 0, - _names['RETURN_VALUE']: lambda oparg: 1, - _names['SEND']: lambda oparg: 2, - _names['SEND_GEN']: lambda oparg: 2, - _names['SETUP_ANNOTATIONS']: lambda oparg: 0, - _names['SETUP_CLEANUP']: lambda oparg: 0, - _names['SETUP_FINALLY']: lambda oparg: 0, - _names['SETUP_WITH']: lambda oparg: 0, - _names['SET_ADD']: lambda oparg: 2 + (oparg-1), - _names['SET_FUNCTION_ATTRIBUTE']: lambda oparg: 2, - _names['SET_UPDATE']: lambda oparg: 2 + (oparg-1), - _names['STORE_ATTR']: lambda oparg: 2, - _names['STORE_ATTR_INSTANCE_VALUE']: lambda oparg: 2, - _names['STORE_ATTR_SLOT']: lambda oparg: 2, - _names['STORE_ATTR_WITH_HINT']: lambda oparg: 2, - _names['STORE_DEREF']: lambda oparg: 1, - _names['STORE_FAST']: lambda oparg: 1, - _names['STORE_FAST_LOAD_FAST']: lambda oparg: 1, - _names['STORE_FAST_MAYBE_NULL']: lambda oparg: 1, - _names['STORE_FAST_STORE_FAST']: lambda oparg: 2, - _names['STORE_GLOBAL']: lambda oparg: 1, - _names['STORE_NAME']: lambda oparg: 1, - _names['STORE_SLICE']: lambda oparg: 4, - _names['STORE_SUBSCR']: lambda oparg: 3, - _names['STORE_SUBSCR_DICT']: lambda oparg: 3, - _names['STORE_SUBSCR_LIST_INT']: lambda oparg: 3, - _names['SWAP']: lambda oparg: 2 + (oparg-2), - _names['TO_BOOL']: lambda oparg: 1, - _names['TO_BOOL_ALWAYS_TRUE']: lambda oparg: 1, - _names['TO_BOOL_BOOL']: lambda oparg: 1, - _names['TO_BOOL_INT']: lambda oparg: 1, - _names['TO_BOOL_LIST']: lambda oparg: 1, - _names['TO_BOOL_NONE']: lambda oparg: 1, - _names['TO_BOOL_STR']: lambda oparg: 1, - _names['UNARY_INVERT']: lambda oparg: 1, - _names['UNARY_NEGATIVE']: lambda oparg: 1, - _names['UNARY_NOT']: lambda oparg: 1, - _names['UNPACK_EX']: lambda oparg: 1, - _names['UNPACK_SEQUENCE']: lambda oparg: 1, - _names['UNPACK_SEQUENCE_LIST']: lambda oparg: 1, - _names['UNPACK_SEQUENCE_TUPLE']: lambda oparg: 1, - _names['UNPACK_SEQUENCE_TWO_TUPLE']: lambda oparg: 1, - _names['WITH_EXCEPT_START']: lambda oparg: 5, - _names['YIELD_VALUE']: lambda oparg: 1, -} -# `_PyOpcode_num_pushed(opcode, oparg)`. -_PUSHED = { - _names['ANNOTATIONS_PLACEHOLDER']: lambda oparg: 0, - _names['BINARY_OP']: lambda oparg: 1, - _names['BINARY_OP_ADD_FLOAT']: lambda oparg: 1, - _names['BINARY_OP_ADD_INT']: lambda oparg: 1, - _names['BINARY_OP_ADD_UNICODE']: lambda oparg: 1, - _names['BINARY_OP_EXTEND']: lambda oparg: 1, - _names['BINARY_OP_INPLACE_ADD_UNICODE']: lambda oparg: 0, - _names['BINARY_OP_MULTIPLY_FLOAT']: lambda oparg: 1, - _names['BINARY_OP_MULTIPLY_INT']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_DICT']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_GETITEM']: lambda oparg: 0, - _names['BINARY_OP_SUBSCR_LIST_INT']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_LIST_SLICE']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_STR_INT']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_TUPLE_INT']: lambda oparg: 1, - _names['BINARY_OP_SUBTRACT_FLOAT']: lambda oparg: 1, - _names['BINARY_OP_SUBTRACT_INT']: lambda oparg: 1, - _names['BINARY_SLICE']: lambda oparg: 1, - _names['BUILD_INTERPOLATION']: lambda oparg: 1, - _names['BUILD_LIST']: lambda oparg: 1, - _names['BUILD_MAP']: lambda oparg: 1, - _names['BUILD_SET']: lambda oparg: 1, - _names['BUILD_SLICE']: lambda oparg: 1, - _names['BUILD_STRING']: lambda oparg: 1, - _names['BUILD_TEMPLATE']: lambda oparg: 1, - _names['BUILD_TUPLE']: lambda oparg: 1, - _names['CACHE']: lambda oparg: 0, - _names['CALL']: lambda oparg: 1, - _names['CALL_ALLOC_AND_ENTER_INIT']: lambda oparg: 0, - _names['CALL_BOUND_METHOD_EXACT_ARGS']: lambda oparg: 0, - _names['CALL_BOUND_METHOD_GENERAL']: lambda oparg: 0, - _names['CALL_BUILTIN_CLASS']: lambda oparg: 1, - _names['CALL_BUILTIN_FAST']: lambda oparg: 1, - _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: lambda oparg: 1, - _names['CALL_BUILTIN_O']: lambda oparg: 1, - _names['CALL_FUNCTION_EX']: lambda oparg: 1, - _names['CALL_INTRINSIC_1']: lambda oparg: 1, - _names['CALL_INTRINSIC_2']: lambda oparg: 1, - _names['CALL_ISINSTANCE']: lambda oparg: 1, - _names['CALL_KW']: lambda oparg: 1, - _names['CALL_KW_BOUND_METHOD']: lambda oparg: 0, - _names['CALL_KW_NON_PY']: lambda oparg: 1, - _names['CALL_KW_PY']: lambda oparg: 0, - _names['CALL_LEN']: lambda oparg: 1, - _names['CALL_LIST_APPEND']: lambda oparg: 0, - _names['CALL_METHOD_DESCRIPTOR_FAST']: lambda oparg: 1, - _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: lambda oparg: 1, - _names['CALL_METHOD_DESCRIPTOR_NOARGS']: lambda oparg: 1, - _names['CALL_METHOD_DESCRIPTOR_O']: lambda oparg: 1, - _names['CALL_NON_PY_GENERAL']: lambda oparg: 1, - _names['CALL_PY_EXACT_ARGS']: lambda oparg: 0, - _names['CALL_PY_GENERAL']: lambda oparg: 0, - _names['CALL_STR_1']: lambda oparg: 1, - _names['CALL_TUPLE_1']: lambda oparg: 1, - _names['CALL_TYPE_1']: lambda oparg: 1, - _names['CHECK_EG_MATCH']: lambda oparg: 2, - _names['CHECK_EXC_MATCH']: lambda oparg: 2, - _names['CLEANUP_THROW']: lambda oparg: 2, - _names['COMPARE_OP']: lambda oparg: 1, - _names['COMPARE_OP_FLOAT']: lambda oparg: 1, - _names['COMPARE_OP_INT']: lambda oparg: 1, - _names['COMPARE_OP_STR']: lambda oparg: 1, - _names['CONTAINS_OP']: lambda oparg: 1, - _names['CONTAINS_OP_DICT']: lambda oparg: 1, - _names['CONTAINS_OP_SET']: lambda oparg: 1, - _names['CONVERT_VALUE']: lambda oparg: 1, - _names['COPY']: lambda oparg: 2 + (oparg-1), - _names['COPY_FREE_VARS']: lambda oparg: 0, - _names['DELETE_ATTR']: lambda oparg: 0, - _names['DELETE_DEREF']: lambda oparg: 0, - _names['DELETE_FAST']: lambda oparg: 0, - _names['DELETE_GLOBAL']: lambda oparg: 0, - _names['DELETE_NAME']: lambda oparg: 0, - _names['DELETE_SUBSCR']: lambda oparg: 0, - _names['DICT_MERGE']: lambda oparg: 4 + (oparg - 1), - _names['DICT_UPDATE']: lambda oparg: 1 + (oparg - 1), - _names['END_ASYNC_FOR']: lambda oparg: 0, - _names['END_FOR']: lambda oparg: 0, - _names['END_SEND']: lambda oparg: 1, - _names['ENTER_EXECUTOR']: lambda oparg: 0, - _names['EXIT_INIT_CHECK']: lambda oparg: 0, - _names['EXTENDED_ARG']: lambda oparg: 0, - _names['FORMAT_SIMPLE']: lambda oparg: 1, - _names['FORMAT_WITH_SPEC']: lambda oparg: 1, - _names['FOR_ITER']: lambda oparg: 2, - _names['FOR_ITER_GEN']: lambda oparg: 1, - _names['FOR_ITER_LIST']: lambda oparg: 2, - _names['FOR_ITER_RANGE']: lambda oparg: 2, - _names['FOR_ITER_TUPLE']: lambda oparg: 2, - _names['GET_AITER']: lambda oparg: 1, - _names['GET_ANEXT']: lambda oparg: 2, - _names['GET_AWAITABLE']: lambda oparg: 1, - _names['GET_ITER']: lambda oparg: 1, - _names['GET_LEN']: lambda oparg: 2, - _names['GET_YIELD_FROM_ITER']: lambda oparg: 1, - _names['IMPORT_FROM']: lambda oparg: 2, - _names['IMPORT_NAME']: lambda oparg: 1, - _names['INSTRUMENTED_CALL']: lambda oparg: 1, - _names['INSTRUMENTED_CALL_FUNCTION_EX']: lambda oparg: 1, - _names['INSTRUMENTED_CALL_KW']: lambda oparg: 1, - _names['INSTRUMENTED_END_ASYNC_FOR']: lambda oparg: 0, - _names['INSTRUMENTED_END_FOR']: lambda oparg: 1, - _names['INSTRUMENTED_END_SEND']: lambda oparg: 1, - _names['INSTRUMENTED_FOR_ITER']: lambda oparg: 2, - _names['INSTRUMENTED_INSTRUCTION']: lambda oparg: 0, - _names['INSTRUMENTED_JUMP_BACKWARD']: lambda oparg: 0, - _names['INSTRUMENTED_JUMP_FORWARD']: lambda oparg: 0, - _names['INSTRUMENTED_LINE']: lambda oparg: 0, - _names['INSTRUMENTED_LOAD_SUPER_ATTR']: lambda oparg: 1 + (oparg & 1), - _names['INSTRUMENTED_NOT_TAKEN']: lambda oparg: 0, - _names['INSTRUMENTED_POP_ITER']: lambda oparg: 0, - _names['INSTRUMENTED_POP_JUMP_IF_FALSE']: lambda oparg: 0, - _names['INSTRUMENTED_POP_JUMP_IF_NONE']: lambda oparg: 0, - _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE']: lambda oparg: 0, - _names['INSTRUMENTED_POP_JUMP_IF_TRUE']: lambda oparg: 0, - _names['INSTRUMENTED_RESUME']: lambda oparg: 0, - _names['INSTRUMENTED_RETURN_VALUE']: lambda oparg: 1, - _names['INSTRUMENTED_YIELD_VALUE']: lambda oparg: 1, - _names['INTERPRETER_EXIT']: lambda oparg: 0, - _names['IS_OP']: lambda oparg: 1, - _names['JUMP']: lambda oparg: 0, - _names['JUMP_BACKWARD']: lambda oparg: 0, - _names['JUMP_BACKWARD_JIT']: lambda oparg: 0, - _names['JUMP_BACKWARD_NO_INTERRUPT']: lambda oparg: 0, - _names['JUMP_BACKWARD_NO_JIT']: lambda oparg: 0, - _names['JUMP_FORWARD']: lambda oparg: 0, - _names['JUMP_IF_FALSE']: lambda oparg: 1, - _names['JUMP_IF_TRUE']: lambda oparg: 1, - _names['JUMP_NO_INTERRUPT']: lambda oparg: 0, - _names['LIST_APPEND']: lambda oparg: 1 + (oparg-1), - _names['LIST_EXTEND']: lambda oparg: 1 + (oparg-1), - _names['LOAD_ATTR']: lambda oparg: 1 + (oparg&1), - _names['LOAD_ATTR_CLASS']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: lambda oparg: 1, - _names['LOAD_ATTR_INSTANCE_VALUE']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_METHOD_LAZY_DICT']: lambda oparg: 2, - _names['LOAD_ATTR_METHOD_NO_DICT']: lambda oparg: 2, - _names['LOAD_ATTR_METHOD_WITH_VALUES']: lambda oparg: 2, - _names['LOAD_ATTR_MODULE']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: lambda oparg: 1, - _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: lambda oparg: 1, - _names['LOAD_ATTR_PROPERTY']: lambda oparg: 0, - _names['LOAD_ATTR_SLOT']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_WITH_HINT']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_BUILD_CLASS']: lambda oparg: 1, - _names['LOAD_CLOSURE']: lambda oparg: 1, - _names['LOAD_COMMON_CONSTANT']: lambda oparg: 1, - _names['LOAD_CONST']: lambda oparg: 1, - _names['LOAD_CONST_IMMORTAL']: lambda oparg: 1, - _names['LOAD_CONST_MORTAL']: lambda oparg: 1, - _names['LOAD_DEREF']: lambda oparg: 1, - _names['LOAD_FAST']: lambda oparg: 1, - _names['LOAD_FAST_AND_CLEAR']: lambda oparg: 1, - _names['LOAD_FAST_BORROW']: lambda oparg: 1, - _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW']: lambda oparg: 2, - _names['LOAD_FAST_CHECK']: lambda oparg: 1, - _names['LOAD_FAST_LOAD_FAST']: lambda oparg: 2, - _names['LOAD_FROM_DICT_OR_DEREF']: lambda oparg: 1, - _names['LOAD_FROM_DICT_OR_GLOBALS']: lambda oparg: 1, - _names['LOAD_GLOBAL']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_GLOBAL_BUILTIN']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_GLOBAL_MODULE']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_LOCALS']: lambda oparg: 1, - _names['LOAD_NAME']: lambda oparg: 1, - _names['LOAD_SMALL_INT']: lambda oparg: 1, - _names['LOAD_SPECIAL']: lambda oparg: 2, - _names['LOAD_SUPER_ATTR']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_SUPER_ATTR_ATTR']: lambda oparg: 1, - _names['LOAD_SUPER_ATTR_METHOD']: lambda oparg: 2, - _names['MAKE_CELL']: lambda oparg: 0, - _names['MAKE_FUNCTION']: lambda oparg: 1, - _names['MAP_ADD']: lambda oparg: 1 + (oparg - 1), - _names['MATCH_CLASS']: lambda oparg: 1, - _names['MATCH_KEYS']: lambda oparg: 3, - _names['MATCH_MAPPING']: lambda oparg: 2, - _names['MATCH_SEQUENCE']: lambda oparg: 2, - _names['NOP']: lambda oparg: 0, - _names['NOT_TAKEN']: lambda oparg: 0, - _names['POP_BLOCK']: lambda oparg: 0, - _names['POP_EXCEPT']: lambda oparg: 0, - _names['POP_ITER']: lambda oparg: 0, - _names['POP_JUMP_IF_FALSE']: lambda oparg: 0, - _names['POP_JUMP_IF_NONE']: lambda oparg: 0, - _names['POP_JUMP_IF_NOT_NONE']: lambda oparg: 0, - _names['POP_JUMP_IF_TRUE']: lambda oparg: 0, - _names['POP_TOP']: lambda oparg: 0, - _names['PUSH_EXC_INFO']: lambda oparg: 2, - _names['PUSH_NULL']: lambda oparg: 1, - _names['RAISE_VARARGS']: lambda oparg: 0, - _names['RERAISE']: lambda oparg: oparg, - _names['RESERVED']: lambda oparg: 0, - _names['RESUME']: lambda oparg: 0, - _names['RESUME_CHECK']: lambda oparg: 0, - _names['RETURN_GENERATOR']: lambda oparg: 1, - _names['RETURN_VALUE']: lambda oparg: 1, - _names['SEND']: lambda oparg: 2, - _names['SEND_GEN']: lambda oparg: 1, - _names['SETUP_ANNOTATIONS']: lambda oparg: 0, - _names['SETUP_CLEANUP']: lambda oparg: 2, - _names['SETUP_FINALLY']: lambda oparg: 1, - _names['SETUP_WITH']: lambda oparg: 1, - _names['SET_ADD']: lambda oparg: 1 + (oparg-1), - _names['SET_FUNCTION_ATTRIBUTE']: lambda oparg: 1, - _names['SET_UPDATE']: lambda oparg: 1 + (oparg-1), - _names['STORE_ATTR']: lambda oparg: 0, - _names['STORE_ATTR_INSTANCE_VALUE']: lambda oparg: 0, - _names['STORE_ATTR_SLOT']: lambda oparg: 0, - _names['STORE_ATTR_WITH_HINT']: lambda oparg: 0, - _names['STORE_DEREF']: lambda oparg: 0, - _names['STORE_FAST']: lambda oparg: 0, - _names['STORE_FAST_LOAD_FAST']: lambda oparg: 1, - _names['STORE_FAST_MAYBE_NULL']: lambda oparg: 0, - _names['STORE_FAST_STORE_FAST']: lambda oparg: 0, - _names['STORE_GLOBAL']: lambda oparg: 0, - _names['STORE_NAME']: lambda oparg: 0, - _names['STORE_SLICE']: lambda oparg: 0, - _names['STORE_SUBSCR']: lambda oparg: 0, - _names['STORE_SUBSCR_DICT']: lambda oparg: 0, - _names['STORE_SUBSCR_LIST_INT']: lambda oparg: 0, - _names['SWAP']: lambda oparg: 2 + (oparg-2), - _names['TO_BOOL']: lambda oparg: 1, - _names['TO_BOOL_ALWAYS_TRUE']: lambda oparg: 1, - _names['TO_BOOL_BOOL']: lambda oparg: 1, - _names['TO_BOOL_INT']: lambda oparg: 1, - _names['TO_BOOL_LIST']: lambda oparg: 1, - _names['TO_BOOL_NONE']: lambda oparg: 1, - _names['TO_BOOL_STR']: lambda oparg: 1, - _names['UNARY_INVERT']: lambda oparg: 1, - _names['UNARY_NEGATIVE']: lambda oparg: 1, - _names['UNARY_NOT']: lambda oparg: 1, - _names['UNPACK_EX']: lambda oparg: 1 + (oparg & 0xFF) + (oparg >> 8), - _names['UNPACK_SEQUENCE']: lambda oparg: oparg, - _names['UNPACK_SEQUENCE_LIST']: lambda oparg: oparg, - _names['UNPACK_SEQUENCE_TUPLE']: lambda oparg: oparg, - _names['UNPACK_SEQUENCE_TWO_TUPLE']: lambda oparg: 2, - _names['WITH_EXCEPT_START']: lambda oparg: 6, - _names['YIELD_VALUE']: lambda oparg: 1, -} +def _stack_tables(): + global _STACK_TABLES + if _STACK_TABLES is not None: + return _STACK_TABLES + # `_PyOpcode_Deopt`: specialized form -> base instruction. + deopt = { + _names['BINARY_OP_ADD_FLOAT']: _names['BINARY_OP'], + _names['BINARY_OP_ADD_INT']: _names['BINARY_OP'], + _names['BINARY_OP_ADD_UNICODE']: _names['BINARY_OP'], + _names['BINARY_OP_EXTEND']: _names['BINARY_OP'], + _names['BINARY_OP_INPLACE_ADD_UNICODE']: _names['BINARY_OP'], + _names['BINARY_OP_MULTIPLY_FLOAT']: _names['BINARY_OP'], + _names['BINARY_OP_MULTIPLY_INT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_DICT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_GETITEM']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_LIST_INT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_LIST_SLICE']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_STR_INT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_TUPLE_INT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBTRACT_FLOAT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBTRACT_INT']: _names['BINARY_OP'], + _names['CALL_ALLOC_AND_ENTER_INIT']: _names['CALL'], + _names['CALL_BOUND_METHOD_EXACT_ARGS']: _names['CALL'], + _names['CALL_BOUND_METHOD_GENERAL']: _names['CALL'], + _names['CALL_BUILTIN_CLASS']: _names['CALL'], + _names['CALL_BUILTIN_FAST']: _names['CALL'], + _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: _names['CALL'], + _names['CALL_BUILTIN_O']: _names['CALL'], + _names['CALL_ISINSTANCE']: _names['CALL'], + _names['CALL_KW_BOUND_METHOD']: _names['CALL_KW'], + _names['CALL_KW_NON_PY']: _names['CALL_KW'], + _names['CALL_KW_PY']: _names['CALL_KW'], + _names['CALL_LEN']: _names['CALL'], + _names['CALL_LIST_APPEND']: _names['CALL'], + _names['CALL_METHOD_DESCRIPTOR_FAST']: _names['CALL'], + _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: _names['CALL'], + _names['CALL_METHOD_DESCRIPTOR_NOARGS']: _names['CALL'], + _names['CALL_METHOD_DESCRIPTOR_O']: _names['CALL'], + _names['CALL_NON_PY_GENERAL']: _names['CALL'], + _names['CALL_PY_EXACT_ARGS']: _names['CALL'], + _names['CALL_PY_GENERAL']: _names['CALL'], + _names['CALL_STR_1']: _names['CALL'], + _names['CALL_TUPLE_1']: _names['CALL'], + _names['CALL_TYPE_1']: _names['CALL'], + _names['COMPARE_OP_FLOAT']: _names['COMPARE_OP'], + _names['COMPARE_OP_INT']: _names['COMPARE_OP'], + _names['COMPARE_OP_STR']: _names['COMPARE_OP'], + _names['CONTAINS_OP_DICT']: _names['CONTAINS_OP'], + _names['CONTAINS_OP_SET']: _names['CONTAINS_OP'], + _names['FOR_ITER_GEN']: _names['FOR_ITER'], + _names['FOR_ITER_LIST']: _names['FOR_ITER'], + _names['FOR_ITER_RANGE']: _names['FOR_ITER'], + _names['FOR_ITER_TUPLE']: _names['FOR_ITER'], + _names['JUMP_BACKWARD_JIT']: _names['JUMP_BACKWARD'], + _names['JUMP_BACKWARD_NO_JIT']: _names['JUMP_BACKWARD'], + _names['LOAD_ATTR_CLASS']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_INSTANCE_VALUE']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_METHOD_LAZY_DICT']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_METHOD_NO_DICT']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_METHOD_WITH_VALUES']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_MODULE']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_PROPERTY']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_SLOT']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_WITH_HINT']: _names['LOAD_ATTR'], + _names['LOAD_CONST_IMMORTAL']: _names['LOAD_CONST'], + _names['LOAD_CONST_MORTAL']: _names['LOAD_CONST'], + _names['LOAD_GLOBAL_BUILTIN']: _names['LOAD_GLOBAL'], + _names['LOAD_GLOBAL_MODULE']: _names['LOAD_GLOBAL'], + _names['LOAD_SUPER_ATTR_ATTR']: _names['LOAD_SUPER_ATTR'], + _names['LOAD_SUPER_ATTR_METHOD']: _names['LOAD_SUPER_ATTR'], + _names['RESUME_CHECK']: _names['RESUME'], + _names['SEND_GEN']: _names['SEND'], + _names['STORE_ATTR_INSTANCE_VALUE']: _names['STORE_ATTR'], + _names['STORE_ATTR_SLOT']: _names['STORE_ATTR'], + _names['STORE_ATTR_WITH_HINT']: _names['STORE_ATTR'], + _names['STORE_SUBSCR_DICT']: _names['STORE_SUBSCR'], + _names['STORE_SUBSCR_LIST_INT']: _names['STORE_SUBSCR'], + _names['TO_BOOL_ALWAYS_TRUE']: _names['TO_BOOL'], + _names['TO_BOOL_BOOL']: _names['TO_BOOL'], + _names['TO_BOOL_INT']: _names['TO_BOOL'], + _names['TO_BOOL_LIST']: _names['TO_BOOL'], + _names['TO_BOOL_NONE']: _names['TO_BOOL'], + _names['TO_BOOL_STR']: _names['TO_BOOL'], + _names['UNPACK_SEQUENCE_LIST']: _names['UNPACK_SEQUENCE'], + _names['UNPACK_SEQUENCE_TUPLE']: _names['UNPACK_SEQUENCE'], + _names['UNPACK_SEQUENCE_TWO_TUPLE']: _names['UNPACK_SEQUENCE'], + } + # `_PyOpcode_num_popped(opcode, oparg)`: the count, or a function of + # `oparg` for the opcodes whose count depends on it. + popped = { + _names['ANNOTATIONS_PLACEHOLDER']: 0, + _names['BINARY_OP']: 2, + _names['BINARY_OP_ADD_FLOAT']: 2, + _names['BINARY_OP_ADD_INT']: 2, + _names['BINARY_OP_ADD_UNICODE']: 2, + _names['BINARY_OP_EXTEND']: 2, + _names['BINARY_OP_INPLACE_ADD_UNICODE']: 2, + _names['BINARY_OP_MULTIPLY_FLOAT']: 2, + _names['BINARY_OP_MULTIPLY_INT']: 2, + _names['BINARY_OP_SUBSCR_DICT']: 2, + _names['BINARY_OP_SUBSCR_GETITEM']: 2, + _names['BINARY_OP_SUBSCR_LIST_INT']: 2, + _names['BINARY_OP_SUBSCR_LIST_SLICE']: 2, + _names['BINARY_OP_SUBSCR_STR_INT']: 2, + _names['BINARY_OP_SUBSCR_TUPLE_INT']: 2, + _names['BINARY_OP_SUBTRACT_FLOAT']: 2, + _names['BINARY_OP_SUBTRACT_INT']: 2, + _names['BINARY_SLICE']: 3, + _names['BUILD_INTERPOLATION']: lambda oparg: 2 + (oparg & 1), + _names['BUILD_LIST']: lambda oparg: oparg, + _names['BUILD_MAP']: lambda oparg: oparg*2, + _names['BUILD_SET']: lambda oparg: oparg, + _names['BUILD_SLICE']: lambda oparg: oparg, + _names['BUILD_STRING']: lambda oparg: oparg, + _names['BUILD_TEMPLATE']: 2, + _names['BUILD_TUPLE']: lambda oparg: oparg, + _names['CACHE']: 0, + _names['CALL']: lambda oparg: 2 + oparg, + _names['CALL_ALLOC_AND_ENTER_INIT']: lambda oparg: 2 + oparg, + _names['CALL_BOUND_METHOD_EXACT_ARGS']: lambda oparg: 2 + oparg, + _names['CALL_BOUND_METHOD_GENERAL']: lambda oparg: 2 + oparg, + _names['CALL_BUILTIN_CLASS']: lambda oparg: 2 + oparg, + _names['CALL_BUILTIN_FAST']: lambda oparg: 2 + oparg, + _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: lambda oparg: 2 + oparg, + _names['CALL_BUILTIN_O']: lambda oparg: 2 + oparg, + _names['CALL_FUNCTION_EX']: 4, + _names['CALL_INTRINSIC_1']: 1, + _names['CALL_INTRINSIC_2']: 2, + _names['CALL_ISINSTANCE']: lambda oparg: 2 + oparg, + _names['CALL_KW']: lambda oparg: 3 + oparg, + _names['CALL_KW_BOUND_METHOD']: lambda oparg: 3 + oparg, + _names['CALL_KW_NON_PY']: lambda oparg: 3 + oparg, + _names['CALL_KW_PY']: lambda oparg: 3 + oparg, + _names['CALL_LEN']: 3, + _names['CALL_LIST_APPEND']: 3, + _names['CALL_METHOD_DESCRIPTOR_FAST']: lambda oparg: 2 + oparg, + _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: lambda oparg: 2 + oparg, + _names['CALL_METHOD_DESCRIPTOR_NOARGS']: lambda oparg: 2 + oparg, + _names['CALL_METHOD_DESCRIPTOR_O']: lambda oparg: 2 + oparg, + _names['CALL_NON_PY_GENERAL']: lambda oparg: 2 + oparg, + _names['CALL_PY_EXACT_ARGS']: lambda oparg: 2 + oparg, + _names['CALL_PY_GENERAL']: lambda oparg: 2 + oparg, + _names['CALL_STR_1']: 3, + _names['CALL_TUPLE_1']: 3, + _names['CALL_TYPE_1']: 3, + _names['CHECK_EG_MATCH']: 2, + _names['CHECK_EXC_MATCH']: 2, + _names['CLEANUP_THROW']: 3, + _names['COMPARE_OP']: 2, + _names['COMPARE_OP_FLOAT']: 2, + _names['COMPARE_OP_INT']: 2, + _names['COMPARE_OP_STR']: 2, + _names['CONTAINS_OP']: 2, + _names['CONTAINS_OP_DICT']: 2, + _names['CONTAINS_OP_SET']: 2, + _names['CONVERT_VALUE']: 1, + _names['COPY']: lambda oparg: 1 + (oparg-1), + _names['COPY_FREE_VARS']: 0, + _names['DELETE_ATTR']: 1, + _names['DELETE_DEREF']: 0, + _names['DELETE_FAST']: 0, + _names['DELETE_GLOBAL']: 0, + _names['DELETE_NAME']: 0, + _names['DELETE_SUBSCR']: 2, + _names['DICT_MERGE']: lambda oparg: 5 + (oparg - 1), + _names['DICT_UPDATE']: lambda oparg: 2 + (oparg - 1), + _names['END_ASYNC_FOR']: 2, + _names['END_FOR']: 1, + _names['END_SEND']: 2, + _names['ENTER_EXECUTOR']: 0, + _names['EXIT_INIT_CHECK']: 1, + _names['EXTENDED_ARG']: 0, + _names['FORMAT_SIMPLE']: 1, + _names['FORMAT_WITH_SPEC']: 2, + _names['FOR_ITER']: 1, + _names['FOR_ITER_GEN']: 1, + _names['FOR_ITER_LIST']: 1, + _names['FOR_ITER_RANGE']: 1, + _names['FOR_ITER_TUPLE']: 1, + _names['GET_AITER']: 1, + _names['GET_ANEXT']: 1, + _names['GET_AWAITABLE']: 1, + _names['GET_ITER']: 1, + _names['GET_LEN']: 1, + _names['GET_YIELD_FROM_ITER']: 1, + _names['IMPORT_FROM']: 1, + _names['IMPORT_NAME']: 2, + _names['INSTRUMENTED_CALL']: lambda oparg: 2 + oparg, + _names['INSTRUMENTED_CALL_FUNCTION_EX']: 4, + _names['INSTRUMENTED_CALL_KW']: lambda oparg: 3 + oparg, + _names['INSTRUMENTED_END_ASYNC_FOR']: 2, + _names['INSTRUMENTED_END_FOR']: 2, + _names['INSTRUMENTED_END_SEND']: 2, + _names['INSTRUMENTED_FOR_ITER']: 1, + _names['INSTRUMENTED_INSTRUCTION']: 0, + _names['INSTRUMENTED_JUMP_BACKWARD']: 0, + _names['INSTRUMENTED_JUMP_FORWARD']: 0, + _names['INSTRUMENTED_LINE']: 0, + _names['INSTRUMENTED_LOAD_SUPER_ATTR']: 3, + _names['INSTRUMENTED_NOT_TAKEN']: 0, + _names['INSTRUMENTED_POP_ITER']: 1, + _names['INSTRUMENTED_POP_JUMP_IF_FALSE']: 1, + _names['INSTRUMENTED_POP_JUMP_IF_NONE']: 1, + _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE']: 1, + _names['INSTRUMENTED_POP_JUMP_IF_TRUE']: 1, + _names['INSTRUMENTED_RESUME']: 0, + _names['INSTRUMENTED_RETURN_VALUE']: 1, + _names['INSTRUMENTED_YIELD_VALUE']: 1, + _names['INTERPRETER_EXIT']: 1, + _names['IS_OP']: 2, + _names['JUMP']: 0, + _names['JUMP_BACKWARD']: 0, + _names['JUMP_BACKWARD_JIT']: 0, + _names['JUMP_BACKWARD_NO_INTERRUPT']: 0, + _names['JUMP_BACKWARD_NO_JIT']: 0, + _names['JUMP_FORWARD']: 0, + _names['JUMP_IF_FALSE']: 1, + _names['JUMP_IF_TRUE']: 1, + _names['JUMP_NO_INTERRUPT']: 0, + _names['LIST_APPEND']: lambda oparg: 2 + (oparg-1), + _names['LIST_EXTEND']: lambda oparg: 2 + (oparg-1), + _names['LOAD_ATTR']: 1, + _names['LOAD_ATTR_CLASS']: 1, + _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: 1, + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: 1, + _names['LOAD_ATTR_INSTANCE_VALUE']: 1, + _names['LOAD_ATTR_METHOD_LAZY_DICT']: 1, + _names['LOAD_ATTR_METHOD_NO_DICT']: 1, + _names['LOAD_ATTR_METHOD_WITH_VALUES']: 1, + _names['LOAD_ATTR_MODULE']: 1, + _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: 1, + _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: 1, + _names['LOAD_ATTR_PROPERTY']: 1, + _names['LOAD_ATTR_SLOT']: 1, + _names['LOAD_ATTR_WITH_HINT']: 1, + _names['LOAD_BUILD_CLASS']: 0, + _names['LOAD_CLOSURE']: 0, + _names['LOAD_COMMON_CONSTANT']: 0, + _names['LOAD_CONST']: 0, + _names['LOAD_CONST_IMMORTAL']: 0, + _names['LOAD_CONST_MORTAL']: 0, + _names['LOAD_DEREF']: 0, + _names['LOAD_FAST']: 0, + _names['LOAD_FAST_AND_CLEAR']: 0, + _names['LOAD_FAST_BORROW']: 0, + _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW']: 0, + _names['LOAD_FAST_CHECK']: 0, + _names['LOAD_FAST_LOAD_FAST']: 0, + _names['LOAD_FROM_DICT_OR_DEREF']: 1, + _names['LOAD_FROM_DICT_OR_GLOBALS']: 1, + _names['LOAD_GLOBAL']: 0, + _names['LOAD_GLOBAL_BUILTIN']: 0, + _names['LOAD_GLOBAL_MODULE']: 0, + _names['LOAD_LOCALS']: 0, + _names['LOAD_NAME']: 0, + _names['LOAD_SMALL_INT']: 0, + _names['LOAD_SPECIAL']: 1, + _names['LOAD_SUPER_ATTR']: 3, + _names['LOAD_SUPER_ATTR_ATTR']: 3, + _names['LOAD_SUPER_ATTR_METHOD']: 3, + _names['MAKE_CELL']: 0, + _names['MAKE_FUNCTION']: 1, + _names['MAP_ADD']: lambda oparg: 3 + (oparg - 1), + _names['MATCH_CLASS']: 3, + _names['MATCH_KEYS']: 2, + _names['MATCH_MAPPING']: 1, + _names['MATCH_SEQUENCE']: 1, + _names['NOP']: 0, + _names['NOT_TAKEN']: 0, + _names['POP_BLOCK']: 0, + _names['POP_EXCEPT']: 1, + _names['POP_ITER']: 1, + _names['POP_JUMP_IF_FALSE']: 1, + _names['POP_JUMP_IF_NONE']: 1, + _names['POP_JUMP_IF_NOT_NONE']: 1, + _names['POP_JUMP_IF_TRUE']: 1, + _names['POP_TOP']: 1, + _names['PUSH_EXC_INFO']: 1, + _names['PUSH_NULL']: 0, + _names['RAISE_VARARGS']: lambda oparg: oparg, + _names['RERAISE']: lambda oparg: 1 + oparg, + _names['RESERVED']: 0, + _names['RESUME']: 0, + _names['RESUME_CHECK']: 0, + _names['RETURN_GENERATOR']: 0, + _names['RETURN_VALUE']: 1, + _names['SEND']: 2, + _names['SEND_GEN']: 2, + _names['SETUP_ANNOTATIONS']: 0, + _names['SETUP_CLEANUP']: 0, + _names['SETUP_FINALLY']: 0, + _names['SETUP_WITH']: 0, + _names['SET_ADD']: lambda oparg: 2 + (oparg-1), + _names['SET_FUNCTION_ATTRIBUTE']: 2, + _names['SET_UPDATE']: lambda oparg: 2 + (oparg-1), + _names['STORE_ATTR']: 2, + _names['STORE_ATTR_INSTANCE_VALUE']: 2, + _names['STORE_ATTR_SLOT']: 2, + _names['STORE_ATTR_WITH_HINT']: 2, + _names['STORE_DEREF']: 1, + _names['STORE_FAST']: 1, + _names['STORE_FAST_LOAD_FAST']: 1, + _names['STORE_FAST_MAYBE_NULL']: 1, + _names['STORE_FAST_STORE_FAST']: 2, + _names['STORE_GLOBAL']: 1, + _names['STORE_NAME']: 1, + _names['STORE_SLICE']: 4, + _names['STORE_SUBSCR']: 3, + _names['STORE_SUBSCR_DICT']: 3, + _names['STORE_SUBSCR_LIST_INT']: 3, + _names['SWAP']: lambda oparg: 2 + (oparg-2), + _names['TO_BOOL']: 1, + _names['TO_BOOL_ALWAYS_TRUE']: 1, + _names['TO_BOOL_BOOL']: 1, + _names['TO_BOOL_INT']: 1, + _names['TO_BOOL_LIST']: 1, + _names['TO_BOOL_NONE']: 1, + _names['TO_BOOL_STR']: 1, + _names['UNARY_INVERT']: 1, + _names['UNARY_NEGATIVE']: 1, + _names['UNARY_NOT']: 1, + _names['UNPACK_EX']: 1, + _names['UNPACK_SEQUENCE']: 1, + _names['UNPACK_SEQUENCE_LIST']: 1, + _names['UNPACK_SEQUENCE_TUPLE']: 1, + _names['UNPACK_SEQUENCE_TWO_TUPLE']: 1, + _names['WITH_EXCEPT_START']: 5, + _names['YIELD_VALUE']: 1, + } + # `_PyOpcode_num_pushed(opcode, oparg)`, likewise. + pushed = { + _names['ANNOTATIONS_PLACEHOLDER']: 0, + _names['BINARY_OP']: 1, + _names['BINARY_OP_ADD_FLOAT']: 1, + _names['BINARY_OP_ADD_INT']: 1, + _names['BINARY_OP_ADD_UNICODE']: 1, + _names['BINARY_OP_EXTEND']: 1, + _names['BINARY_OP_INPLACE_ADD_UNICODE']: 0, + _names['BINARY_OP_MULTIPLY_FLOAT']: 1, + _names['BINARY_OP_MULTIPLY_INT']: 1, + _names['BINARY_OP_SUBSCR_DICT']: 1, + _names['BINARY_OP_SUBSCR_GETITEM']: 0, + _names['BINARY_OP_SUBSCR_LIST_INT']: 1, + _names['BINARY_OP_SUBSCR_LIST_SLICE']: 1, + _names['BINARY_OP_SUBSCR_STR_INT']: 1, + _names['BINARY_OP_SUBSCR_TUPLE_INT']: 1, + _names['BINARY_OP_SUBTRACT_FLOAT']: 1, + _names['BINARY_OP_SUBTRACT_INT']: 1, + _names['BINARY_SLICE']: 1, + _names['BUILD_INTERPOLATION']: 1, + _names['BUILD_LIST']: 1, + _names['BUILD_MAP']: 1, + _names['BUILD_SET']: 1, + _names['BUILD_SLICE']: 1, + _names['BUILD_STRING']: 1, + _names['BUILD_TEMPLATE']: 1, + _names['BUILD_TUPLE']: 1, + _names['CACHE']: 0, + _names['CALL']: 1, + _names['CALL_ALLOC_AND_ENTER_INIT']: 0, + _names['CALL_BOUND_METHOD_EXACT_ARGS']: 0, + _names['CALL_BOUND_METHOD_GENERAL']: 0, + _names['CALL_BUILTIN_CLASS']: 1, + _names['CALL_BUILTIN_FAST']: 1, + _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: 1, + _names['CALL_BUILTIN_O']: 1, + _names['CALL_FUNCTION_EX']: 1, + _names['CALL_INTRINSIC_1']: 1, + _names['CALL_INTRINSIC_2']: 1, + _names['CALL_ISINSTANCE']: 1, + _names['CALL_KW']: 1, + _names['CALL_KW_BOUND_METHOD']: 0, + _names['CALL_KW_NON_PY']: 1, + _names['CALL_KW_PY']: 0, + _names['CALL_LEN']: 1, + _names['CALL_LIST_APPEND']: 0, + _names['CALL_METHOD_DESCRIPTOR_FAST']: 1, + _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: 1, + _names['CALL_METHOD_DESCRIPTOR_NOARGS']: 1, + _names['CALL_METHOD_DESCRIPTOR_O']: 1, + _names['CALL_NON_PY_GENERAL']: 1, + _names['CALL_PY_EXACT_ARGS']: 0, + _names['CALL_PY_GENERAL']: 0, + _names['CALL_STR_1']: 1, + _names['CALL_TUPLE_1']: 1, + _names['CALL_TYPE_1']: 1, + _names['CHECK_EG_MATCH']: 2, + _names['CHECK_EXC_MATCH']: 2, + _names['CLEANUP_THROW']: 2, + _names['COMPARE_OP']: 1, + _names['COMPARE_OP_FLOAT']: 1, + _names['COMPARE_OP_INT']: 1, + _names['COMPARE_OP_STR']: 1, + _names['CONTAINS_OP']: 1, + _names['CONTAINS_OP_DICT']: 1, + _names['CONTAINS_OP_SET']: 1, + _names['CONVERT_VALUE']: 1, + _names['COPY']: lambda oparg: 2 + (oparg-1), + _names['COPY_FREE_VARS']: 0, + _names['DELETE_ATTR']: 0, + _names['DELETE_DEREF']: 0, + _names['DELETE_FAST']: 0, + _names['DELETE_GLOBAL']: 0, + _names['DELETE_NAME']: 0, + _names['DELETE_SUBSCR']: 0, + _names['DICT_MERGE']: lambda oparg: 4 + (oparg - 1), + _names['DICT_UPDATE']: lambda oparg: 1 + (oparg - 1), + _names['END_ASYNC_FOR']: 0, + _names['END_FOR']: 0, + _names['END_SEND']: 1, + _names['ENTER_EXECUTOR']: 0, + _names['EXIT_INIT_CHECK']: 0, + _names['EXTENDED_ARG']: 0, + _names['FORMAT_SIMPLE']: 1, + _names['FORMAT_WITH_SPEC']: 1, + _names['FOR_ITER']: 2, + _names['FOR_ITER_GEN']: 1, + _names['FOR_ITER_LIST']: 2, + _names['FOR_ITER_RANGE']: 2, + _names['FOR_ITER_TUPLE']: 2, + _names['GET_AITER']: 1, + _names['GET_ANEXT']: 2, + _names['GET_AWAITABLE']: 1, + _names['GET_ITER']: 1, + _names['GET_LEN']: 2, + _names['GET_YIELD_FROM_ITER']: 1, + _names['IMPORT_FROM']: 2, + _names['IMPORT_NAME']: 1, + _names['INSTRUMENTED_CALL']: 1, + _names['INSTRUMENTED_CALL_FUNCTION_EX']: 1, + _names['INSTRUMENTED_CALL_KW']: 1, + _names['INSTRUMENTED_END_ASYNC_FOR']: 0, + _names['INSTRUMENTED_END_FOR']: 1, + _names['INSTRUMENTED_END_SEND']: 1, + _names['INSTRUMENTED_FOR_ITER']: 2, + _names['INSTRUMENTED_INSTRUCTION']: 0, + _names['INSTRUMENTED_JUMP_BACKWARD']: 0, + _names['INSTRUMENTED_JUMP_FORWARD']: 0, + _names['INSTRUMENTED_LINE']: 0, + _names['INSTRUMENTED_LOAD_SUPER_ATTR']: lambda oparg: 1 + (oparg & 1), + _names['INSTRUMENTED_NOT_TAKEN']: 0, + _names['INSTRUMENTED_POP_ITER']: 0, + _names['INSTRUMENTED_POP_JUMP_IF_FALSE']: 0, + _names['INSTRUMENTED_POP_JUMP_IF_NONE']: 0, + _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE']: 0, + _names['INSTRUMENTED_POP_JUMP_IF_TRUE']: 0, + _names['INSTRUMENTED_RESUME']: 0, + _names['INSTRUMENTED_RETURN_VALUE']: 1, + _names['INSTRUMENTED_YIELD_VALUE']: 1, + _names['INTERPRETER_EXIT']: 0, + _names['IS_OP']: 1, + _names['JUMP']: 0, + _names['JUMP_BACKWARD']: 0, + _names['JUMP_BACKWARD_JIT']: 0, + _names['JUMP_BACKWARD_NO_INTERRUPT']: 0, + _names['JUMP_BACKWARD_NO_JIT']: 0, + _names['JUMP_FORWARD']: 0, + _names['JUMP_IF_FALSE']: 1, + _names['JUMP_IF_TRUE']: 1, + _names['JUMP_NO_INTERRUPT']: 0, + _names['LIST_APPEND']: lambda oparg: 1 + (oparg-1), + _names['LIST_EXTEND']: lambda oparg: 1 + (oparg-1), + _names['LOAD_ATTR']: lambda oparg: 1 + (oparg&1), + _names['LOAD_ATTR_CLASS']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: 1, + _names['LOAD_ATTR_INSTANCE_VALUE']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_METHOD_LAZY_DICT']: 2, + _names['LOAD_ATTR_METHOD_NO_DICT']: 2, + _names['LOAD_ATTR_METHOD_WITH_VALUES']: 2, + _names['LOAD_ATTR_MODULE']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: 1, + _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: 1, + _names['LOAD_ATTR_PROPERTY']: 0, + _names['LOAD_ATTR_SLOT']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_WITH_HINT']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_BUILD_CLASS']: 1, + _names['LOAD_CLOSURE']: 1, + _names['LOAD_COMMON_CONSTANT']: 1, + _names['LOAD_CONST']: 1, + _names['LOAD_CONST_IMMORTAL']: 1, + _names['LOAD_CONST_MORTAL']: 1, + _names['LOAD_DEREF']: 1, + _names['LOAD_FAST']: 1, + _names['LOAD_FAST_AND_CLEAR']: 1, + _names['LOAD_FAST_BORROW']: 1, + _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW']: 2, + _names['LOAD_FAST_CHECK']: 1, + _names['LOAD_FAST_LOAD_FAST']: 2, + _names['LOAD_FROM_DICT_OR_DEREF']: 1, + _names['LOAD_FROM_DICT_OR_GLOBALS']: 1, + _names['LOAD_GLOBAL']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_GLOBAL_BUILTIN']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_GLOBAL_MODULE']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_LOCALS']: 1, + _names['LOAD_NAME']: 1, + _names['LOAD_SMALL_INT']: 1, + _names['LOAD_SPECIAL']: 2, + _names['LOAD_SUPER_ATTR']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_SUPER_ATTR_ATTR']: 1, + _names['LOAD_SUPER_ATTR_METHOD']: 2, + _names['MAKE_CELL']: 0, + _names['MAKE_FUNCTION']: 1, + _names['MAP_ADD']: lambda oparg: 1 + (oparg - 1), + _names['MATCH_CLASS']: 1, + _names['MATCH_KEYS']: 3, + _names['MATCH_MAPPING']: 2, + _names['MATCH_SEQUENCE']: 2, + _names['NOP']: 0, + _names['NOT_TAKEN']: 0, + _names['POP_BLOCK']: 0, + _names['POP_EXCEPT']: 0, + _names['POP_ITER']: 0, + _names['POP_JUMP_IF_FALSE']: 0, + _names['POP_JUMP_IF_NONE']: 0, + _names['POP_JUMP_IF_NOT_NONE']: 0, + _names['POP_JUMP_IF_TRUE']: 0, + _names['POP_TOP']: 0, + _names['PUSH_EXC_INFO']: 2, + _names['PUSH_NULL']: 1, + _names['RAISE_VARARGS']: 0, + _names['RERAISE']: lambda oparg: oparg, + _names['RESERVED']: 0, + _names['RESUME']: 0, + _names['RESUME_CHECK']: 0, + _names['RETURN_GENERATOR']: 1, + _names['RETURN_VALUE']: 1, + _names['SEND']: 2, + _names['SEND_GEN']: 1, + _names['SETUP_ANNOTATIONS']: 0, + _names['SETUP_CLEANUP']: 2, + _names['SETUP_FINALLY']: 1, + _names['SETUP_WITH']: 1, + _names['SET_ADD']: lambda oparg: 1 + (oparg-1), + _names['SET_FUNCTION_ATTRIBUTE']: 1, + _names['SET_UPDATE']: lambda oparg: 1 + (oparg-1), + _names['STORE_ATTR']: 0, + _names['STORE_ATTR_INSTANCE_VALUE']: 0, + _names['STORE_ATTR_SLOT']: 0, + _names['STORE_ATTR_WITH_HINT']: 0, + _names['STORE_DEREF']: 0, + _names['STORE_FAST']: 0, + _names['STORE_FAST_LOAD_FAST']: 1, + _names['STORE_FAST_MAYBE_NULL']: 0, + _names['STORE_FAST_STORE_FAST']: 0, + _names['STORE_GLOBAL']: 0, + _names['STORE_NAME']: 0, + _names['STORE_SLICE']: 0, + _names['STORE_SUBSCR']: 0, + _names['STORE_SUBSCR_DICT']: 0, + _names['STORE_SUBSCR_LIST_INT']: 0, + _names['SWAP']: lambda oparg: 2 + (oparg-2), + _names['TO_BOOL']: 1, + _names['TO_BOOL_ALWAYS_TRUE']: 1, + _names['TO_BOOL_BOOL']: 1, + _names['TO_BOOL_INT']: 1, + _names['TO_BOOL_LIST']: 1, + _names['TO_BOOL_NONE']: 1, + _names['TO_BOOL_STR']: 1, + _names['UNARY_INVERT']: 1, + _names['UNARY_NEGATIVE']: 1, + _names['UNARY_NOT']: 1, + _names['UNPACK_EX']: lambda oparg: 1 + (oparg & 0xFF) + (oparg >> 8), + _names['UNPACK_SEQUENCE']: lambda oparg: oparg, + _names['UNPACK_SEQUENCE_LIST']: lambda oparg: oparg, + _names['UNPACK_SEQUENCE_TUPLE']: lambda oparg: oparg, + _names['UNPACK_SEQUENCE_TWO_TUPLE']: 2, + _names['WITH_EXCEPT_START']: 6, + _names['YIELD_VALUE']: 1, + } + _STACK_TABLES = (deopt, popped, pushed) + return _STACK_TABLES + _BLOCK_PUSH = frozenset(_names[n] for n in ['SETUP_FINALLY', 'SETUP_WITH', 'SETUP_CLEANUP']) +# The valid opcodes, and those with each flag `has_*` reports: `opcode` +# (Lib/opcode.py) asks every predicate about every opcode at import, so +# a valid opcode is answered by one set lookup. +_VALID = frozenset(_FLAGS) +_HAS_ARG = frozenset({ + _names['BINARY_OP'], + _names['BUILD_INTERPOLATION'], + _names['BUILD_LIST'], + _names['BUILD_MAP'], + _names['BUILD_SET'], + _names['BUILD_SLICE'], + _names['BUILD_STRING'], + _names['BUILD_TUPLE'], + _names['CALL'], + _names['CALL_ALLOC_AND_ENTER_INIT'], + _names['CALL_BOUND_METHOD_EXACT_ARGS'], + _names['CALL_BOUND_METHOD_GENERAL'], + _names['CALL_BUILTIN_CLASS'], + _names['CALL_BUILTIN_FAST'], + _names['CALL_BUILTIN_FAST_WITH_KEYWORDS'], + _names['CALL_BUILTIN_O'], + _names['CALL_INTRINSIC_1'], + _names['CALL_INTRINSIC_2'], + _names['CALL_ISINSTANCE'], + _names['CALL_KW'], + _names['CALL_KW_BOUND_METHOD'], + _names['CALL_KW_NON_PY'], + _names['CALL_KW_PY'], + _names['CALL_LIST_APPEND'], + _names['CALL_METHOD_DESCRIPTOR_FAST'], + _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS'], + _names['CALL_METHOD_DESCRIPTOR_NOARGS'], + _names['CALL_METHOD_DESCRIPTOR_O'], + _names['CALL_NON_PY_GENERAL'], + _names['CALL_PY_EXACT_ARGS'], + _names['CALL_PY_GENERAL'], + _names['CALL_STR_1'], + _names['CALL_TUPLE_1'], + _names['CALL_TYPE_1'], + _names['COMPARE_OP'], + _names['COMPARE_OP_FLOAT'], + _names['COMPARE_OP_INT'], + _names['COMPARE_OP_STR'], + _names['CONTAINS_OP'], + _names['CONTAINS_OP_DICT'], + _names['CONTAINS_OP_SET'], + _names['CONVERT_VALUE'], + _names['COPY'], + _names['COPY_FREE_VARS'], + _names['DELETE_ATTR'], + _names['DELETE_DEREF'], + _names['DELETE_FAST'], + _names['DELETE_GLOBAL'], + _names['DELETE_NAME'], + _names['DICT_MERGE'], + _names['DICT_UPDATE'], + _names['END_ASYNC_FOR'], + _names['ENTER_EXECUTOR'], + _names['EXTENDED_ARG'], + _names['FOR_ITER'], + _names['FOR_ITER_GEN'], + _names['FOR_ITER_LIST'], + _names['FOR_ITER_RANGE'], + _names['FOR_ITER_TUPLE'], + _names['GET_AWAITABLE'], + _names['IMPORT_FROM'], + _names['IMPORT_NAME'], + _names['INSTRUMENTED_CALL'], + _names['INSTRUMENTED_CALL_KW'], + _names['INSTRUMENTED_END_ASYNC_FOR'], + _names['INSTRUMENTED_FOR_ITER'], + _names['INSTRUMENTED_JUMP_BACKWARD'], + _names['INSTRUMENTED_JUMP_FORWARD'], + _names['INSTRUMENTED_LOAD_SUPER_ATTR'], + _names['INSTRUMENTED_POP_JUMP_IF_FALSE'], + _names['INSTRUMENTED_POP_JUMP_IF_NONE'], + _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE'], + _names['INSTRUMENTED_POP_JUMP_IF_TRUE'], + _names['INSTRUMENTED_RESUME'], + _names['INSTRUMENTED_YIELD_VALUE'], + _names['IS_OP'], + _names['JUMP_BACKWARD'], + _names['JUMP_BACKWARD_JIT'], + _names['JUMP_BACKWARD_NO_INTERRUPT'], + _names['JUMP_BACKWARD_NO_JIT'], + _names['JUMP_FORWARD'], + _names['LIST_APPEND'], + _names['LIST_EXTEND'], + _names['LOAD_ATTR'], + _names['LOAD_ATTR_CLASS'], + _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK'], + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN'], + _names['LOAD_ATTR_INSTANCE_VALUE'], + _names['LOAD_ATTR_METHOD_LAZY_DICT'], + _names['LOAD_ATTR_METHOD_NO_DICT'], + _names['LOAD_ATTR_METHOD_WITH_VALUES'], + _names['LOAD_ATTR_MODULE'], + _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT'], + _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES'], + _names['LOAD_ATTR_PROPERTY'], + _names['LOAD_ATTR_SLOT'], + _names['LOAD_ATTR_WITH_HINT'], + _names['LOAD_COMMON_CONSTANT'], + _names['LOAD_CONST'], + _names['LOAD_CONST_IMMORTAL'], + _names['LOAD_CONST_MORTAL'], + _names['LOAD_DEREF'], + _names['LOAD_FAST'], + _names['LOAD_FAST_AND_CLEAR'], + _names['LOAD_FAST_BORROW'], + _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW'], + _names['LOAD_FAST_CHECK'], + _names['LOAD_FAST_LOAD_FAST'], + _names['LOAD_FROM_DICT_OR_DEREF'], + _names['LOAD_FROM_DICT_OR_GLOBALS'], + _names['LOAD_GLOBAL'], + _names['LOAD_GLOBAL_BUILTIN'], + _names['LOAD_GLOBAL_MODULE'], + _names['LOAD_NAME'], + _names['LOAD_SMALL_INT'], + _names['LOAD_SPECIAL'], + _names['LOAD_SUPER_ATTR'], + _names['LOAD_SUPER_ATTR_ATTR'], + _names['LOAD_SUPER_ATTR_METHOD'], + _names['MAKE_CELL'], + _names['MAP_ADD'], + _names['MATCH_CLASS'], + _names['POP_JUMP_IF_FALSE'], + _names['POP_JUMP_IF_NONE'], + _names['POP_JUMP_IF_NOT_NONE'], + _names['POP_JUMP_IF_TRUE'], + _names['RAISE_VARARGS'], + _names['RERAISE'], + _names['RESUME'], + _names['SEND'], + _names['SEND_GEN'], + _names['SET_ADD'], + _names['SET_FUNCTION_ATTRIBUTE'], + _names['SET_UPDATE'], + _names['STORE_ATTR'], + _names['STORE_ATTR_WITH_HINT'], + _names['STORE_DEREF'], + _names['STORE_FAST'], + _names['STORE_FAST_LOAD_FAST'], + _names['STORE_FAST_STORE_FAST'], + _names['STORE_GLOBAL'], + _names['STORE_NAME'], + _names['SWAP'], + _names['UNPACK_EX'], + _names['UNPACK_SEQUENCE'], + _names['UNPACK_SEQUENCE_LIST'], + _names['UNPACK_SEQUENCE_TUPLE'], + _names['UNPACK_SEQUENCE_TWO_TUPLE'], + _names['YIELD_VALUE'], + _names['JUMP'], + _names['JUMP_IF_FALSE'], + _names['JUMP_IF_TRUE'], + _names['JUMP_NO_INTERRUPT'], + _names['LOAD_CLOSURE'], + _names['SETUP_CLEANUP'], + _names['SETUP_FINALLY'], + _names['SETUP_WITH'], + _names['STORE_FAST_MAYBE_NULL'], +}) +_HAS_CONST = frozenset({ + _names['LOAD_CONST'], + _names['LOAD_CONST_IMMORTAL'], + _names['LOAD_CONST_MORTAL'], +}) +_HAS_NAME = frozenset({ + _names['DELETE_ATTR'], + _names['DELETE_GLOBAL'], + _names['DELETE_NAME'], + _names['IMPORT_FROM'], + _names['IMPORT_NAME'], + _names['INSTRUMENTED_LOAD_SUPER_ATTR'], + _names['LOAD_ATTR'], + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN'], + _names['LOAD_ATTR_WITH_HINT'], + _names['LOAD_FROM_DICT_OR_GLOBALS'], + _names['LOAD_GLOBAL'], + _names['LOAD_NAME'], + _names['LOAD_SUPER_ATTR'], + _names['LOAD_SUPER_ATTR_ATTR'], + _names['LOAD_SUPER_ATTR_METHOD'], + _names['STORE_ATTR'], + _names['STORE_ATTR_WITH_HINT'], + _names['STORE_GLOBAL'], + _names['STORE_NAME'], +}) +_HAS_JUMP = frozenset({ + _names['END_ASYNC_FOR'], + _names['FOR_ITER'], + _names['FOR_ITER_LIST'], + _names['FOR_ITER_RANGE'], + _names['FOR_ITER_TUPLE'], + _names['INSTRUMENTED_END_ASYNC_FOR'], + _names['INSTRUMENTED_FOR_ITER'], + _names['JUMP_BACKWARD'], + _names['JUMP_BACKWARD_JIT'], + _names['JUMP_BACKWARD_NO_INTERRUPT'], + _names['JUMP_BACKWARD_NO_JIT'], + _names['JUMP_FORWARD'], + _names['POP_JUMP_IF_FALSE'], + _names['POP_JUMP_IF_NONE'], + _names['POP_JUMP_IF_NOT_NONE'], + _names['POP_JUMP_IF_TRUE'], + _names['SEND'], + _names['JUMP'], + _names['JUMP_IF_FALSE'], + _names['JUMP_IF_TRUE'], + _names['JUMP_NO_INTERRUPT'], +}) +_HAS_FREE = frozenset({ + _names['DELETE_DEREF'], + _names['LOAD_FROM_DICT_OR_DEREF'], + _names['MAKE_CELL'], + _names['STORE_DEREF'], +}) +_HAS_LOCAL = frozenset({ + _names['BINARY_OP_INPLACE_ADD_UNICODE'], + _names['DELETE_FAST'], + _names['LOAD_DEREF'], + _names['LOAD_FAST'], + _names['LOAD_FAST_AND_CLEAR'], + _names['LOAD_FAST_BORROW'], + _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW'], + _names['LOAD_FAST_CHECK'], + _names['LOAD_FAST_LOAD_FAST'], + _names['STORE_FAST'], + _names['STORE_FAST_LOAD_FAST'], + _names['STORE_FAST_STORE_FAST'], + _names['LOAD_CLOSURE'], + _names['STORE_FAST_MAYBE_NULL'], +}) + def is_valid(opcode): """Return True if opcode is valid, False otherwise.""" - return 0 <= opcode < _NUM_OPCODES and opcode in _FLAGS + return opcode in _VALID or (0 <= opcode < _NUM_OPCODES and opcode in _FLAGS) def _has(opcode, flag): @@ -870,31 +1117,61 @@ def _has(opcode, flag): def has_arg(opcode): """Return True if the opcode uses its oparg, False otherwise.""" + if opcode in _HAS_ARG: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_ARG_FLAG) def has_const(opcode): """Return True if the opcode accesses a constant, False otherwise.""" + if opcode in _HAS_CONST: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_CONST_FLAG) def has_name(opcode): """Return True if the opcode accesses an attribute by name, False otherwise.""" + if opcode in _HAS_NAME: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_NAME_FLAG) def has_jump(opcode): """Return True if the opcode has a jump target, False otherwise.""" + if opcode in _HAS_JUMP: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_JUMP_FLAG) def has_free(opcode): """Return True if the opcode accesses a free variable, False otherwise.""" + if opcode in _HAS_FREE: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_FREE_FLAG) def has_local(opcode): """Return True if the opcode accesses a local variable, False otherwise.""" + if opcode in _HAS_LOCAL: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_LOCAL_FLAG) @@ -914,16 +1191,17 @@ def stack_effect(opcode, oparg=None, /, *, jump=None): jump_int = 0 else: raise ValueError("stack_effect: jump must be False, True or None") + deopt, popped_table, pushed_table = _stack_tables() # `get_stack_effects` (Python/flowgraph.c): specialized forms and # unknown opcodes have no effect entry. - if opcode < 0 or (opcode <= _MAX_REAL_OPCODE and _DEOPT.get(opcode, opcode) != opcode): + if opcode < 0 or (opcode <= _MAX_REAL_OPCODE and deopt.get(opcode, opcode) != opcode): raise ValueError("invalid opcode or oparg") - popped = _POPPED.get(opcode) - pushed = _PUSHED.get(opcode) + popped = popped_table.get(opcode) + pushed = pushed_table.get(opcode) if popped is None or pushed is None: raise ValueError("invalid opcode or oparg") - npop = popped(oparg_int) - npush = pushed(oparg_int) + npop = popped if popped.__class__ is int else popped(oparg_int) + npush = pushed if pushed.__class__ is int else pushed(oparg_int) if npop < 0 or npush < 0: raise ValueError("invalid opcode or oparg") if opcode in _BLOCK_PUSH and not jump_int: diff --git a/crates/weavepy-vm/src/stdlib/python/_weave_flowgraph.py b/crates/weavepy-vm/src/stdlib/python/_weave_flowgraph.py index f2668fe3..8de13ddb 100644 --- a/crates/weavepy-vm/src/stdlib/python/_weave_flowgraph.py +++ b/crates/weavepy-vm/src/stdlib/python/_weave_flowgraph.py @@ -410,22 +410,25 @@ def check_cfg(g): # ---- stack effects --------------------------------------------------------- -_POPPED = _opcode_tables._POPPED -_PUSHED = _opcode_tables._PUSHED -_DEOPT = _opcode_tables._DEOPT +# A table entry is the count, or a function of `oparg` (see `_opcode`). +_DEOPT, _POPPED, _PUSHED = _opcode_tables._stack_tables() _MAX_REAL_OPCODE = _opcode_tables._MAX_REAL_OPCODE def num_popped(opcode, oparg): """``_PyOpcode_num_popped``; ``-1`` for an unknown opcode.""" fn = _POPPED.get(opcode) - return -1 if fn is None else fn(oparg) + if fn is None: + return -1 + return fn if fn.__class__ is int else fn(oparg) def num_pushed(opcode, oparg): """``_PyOpcode_num_pushed``; ``-1`` for an unknown opcode.""" fn = _PUSHED.get(opcode) - return -1 if fn is None else fn(oparg) + if fn is None: + return -1 + return fn if fn.__class__ is int else fn(oparg) def get_stack_effects(opcode, oparg, jump): diff --git a/tools/gen_opcode_shim.py b/tools/gen_opcode_shim.py index 5e3e7763..89ecb0f2 100644 --- a/tools/gen_opcode_shim.py +++ b/tools/gen_opcode_shim.py @@ -28,7 +28,9 @@ The stack-effect expressions in the header are plain C arithmetic over ``oparg`` (``1 + (oparg & 0xFF) + (oparg >> 8)``, ``oparg*2`` ...), which is also valid Python with identical results for the non-negative -``oparg`` values the compiler produces; they are emitted as lambdas. +``oparg`` values the compiler produces; they are emitted as lambdas, and a +count that doesn't depend on ``oparg`` as the integer itself (a module +that defines hundreds of functions is slow to import). """ from __future__ import annotations @@ -172,6 +174,15 @@ def py_expr(expr: str) -> str: return expr +def effect_value(expr: str) -> str: + """A stack-effect table value: the count itself when it doesn't depend + on ``oparg`` (most do not), else a lambda over ``oparg``.""" + expr = py_expr(expr) + if expr.isdigit(): + return expr + return "lambda oparg: %s" % expr + + def render(version: str, size: int, meta, popped, pushed, deopt) -> str: lines: list[str] = [] w = lines.append @@ -211,46 +222,75 @@ def render(version: str, size: int, meta, popped, pushed, deopt) -> str: w(" _names[%r]: %d," % (name, word)) w("}") w("") - w("# `_PyOpcode_Deopt`: specialized form -> base instruction.") - w("_DEOPT = {") - for a, b in deopt: - w(" _names[%r]: _names[%r]," % (a, b)) - w("}") + w("# The tables only `stack_effect` reads, built on its first call (most") + w("# importers of `opcode` and `dis` never compute a stack effect).") + w("_STACK_TABLES = None") w("") - w("# `_PyOpcode_num_popped(opcode, oparg)`.") - w("_POPPED = {") - for name, expr in popped: - w(" _names[%r]: lambda oparg: %s," % (name, py_expr(expr))) - w("}") w("") - w("# `_PyOpcode_num_pushed(opcode, oparg)`.") - w("_PUSHED = {") + w("def _stack_tables():") + w(" global _STACK_TABLES") + w(" if _STACK_TABLES is not None:") + w(" return _STACK_TABLES") + w(" # `_PyOpcode_Deopt`: specialized form -> base instruction.") + w(" deopt = {") + for a, b in deopt: + w(" _names[%r]: _names[%r]," % (a, b)) + w(" }") + w(" # `_PyOpcode_num_popped(opcode, oparg)`: the count, or a function of") + w(" # `oparg` for the opcodes whose count depends on it.") + w(" popped = {") + for name, expr in popped: + w(" _names[%r]: %s," % (name, effect_value(expr))) + w(" }") + w(" # `_PyOpcode_num_pushed(opcode, oparg)`, likewise.") + w(" pushed = {") for name, expr in pushed: - w(" _names[%r]: lambda oparg: %s," % (name, py_expr(expr))) - w("}") + w(" _names[%r]: %s," % (name, effect_value(expr))) + w(" }") + w(" _STACK_TABLES = (deopt, popped, pushed)") + w(" return _STACK_TABLES") + w("") w("") w("_BLOCK_PUSH = frozenset(_names[n] for n in %r)" % (BLOCK_PUSH,)) w("") + w("# The valid opcodes, and those with each flag `has_*` reports: `opcode`") + w("# (Lib/opcode.py) asks every predicate about every opcode at import, so") + w("# a valid opcode is answered by one set lookup.") + w("_VALID = frozenset(_FLAGS)") + predicates = [ + ("has_arg", "HAS_ARG_FLAG", "Return True if the opcode uses its oparg, False otherwise."), + ("has_const", "HAS_CONST_FLAG", "Return True if the opcode accesses a constant, False otherwise."), + ("has_name", "HAS_NAME_FLAG", "Return True if the opcode accesses an attribute by name, False otherwise."), + ("has_jump", "HAS_JUMP_FLAG", "Return True if the opcode has a jump target, False otherwise."), + ("has_free", "HAS_FREE_FLAG", "Return True if the opcode accesses a free variable, False otherwise."), + ("has_local", "HAS_LOCAL_FLAG", "Return True if the opcode accesses a local variable, False otherwise."), + ] + flag_bits = {name: 1 << i for i, name in enumerate(FLAG_NAMES)} + for fn, flag, _doc in predicates: + names = [name for name, word in meta if word & flag_bits[flag]] + w("_%s = frozenset({" % fn.upper()) + for name in names: + w(" _names[%r]," % name) + w("})") + w("") w("") w("def is_valid(opcode):") w(' """Return True if opcode is valid, False otherwise."""') - w(" return 0 <= opcode < _NUM_OPCODES and opcode in _FLAGS") + w(" return opcode in _VALID or (0 <= opcode < _NUM_OPCODES and opcode in _FLAGS)") w("") w("") w("def _has(opcode, flag):") w(" return is_valid(opcode) and bool(_FLAGS[opcode] & flag)") w("") w("") - for fn, flag, doc in [ - ("has_arg", "HAS_ARG_FLAG", "Return True if the opcode uses its oparg, False otherwise."), - ("has_const", "HAS_CONST_FLAG", "Return True if the opcode accesses a constant, False otherwise."), - ("has_name", "HAS_NAME_FLAG", "Return True if the opcode accesses an attribute by name, False otherwise."), - ("has_jump", "HAS_JUMP_FLAG", "Return True if the opcode has a jump target, False otherwise."), - ("has_free", "HAS_FREE_FLAG", "Return True if the opcode accesses a free variable, False otherwise."), - ("has_local", "HAS_LOCAL_FLAG", "Return True if the opcode accesses a local variable, False otherwise."), - ]: + for fn, flag, doc in predicates: w("def %s(opcode):" % fn) w(' """%s"""' % doc) + w(" if opcode in _%s:" % fn.upper()) + w(" return True") + w(" if opcode in _VALID:") + w(" return False") + w(" # An invalid opcode, or not an int (which raises as before).") w(" return _has(opcode, %s)" % flag) w("") w("") @@ -270,16 +310,17 @@ def render(version: str, size: int, meta, popped, pushed, deopt) -> str: w(" jump_int = 0") w(" else:") w(' raise ValueError("stack_effect: jump must be False, True or None")') + w(" deopt, popped_table, pushed_table = _stack_tables()") w(" # `get_stack_effects` (Python/flowgraph.c): specialized forms and") w(" # unknown opcodes have no effect entry.") - w(" if opcode < 0 or (opcode <= _MAX_REAL_OPCODE and _DEOPT.get(opcode, opcode) != opcode):") + w(" if opcode < 0 or (opcode <= _MAX_REAL_OPCODE and deopt.get(opcode, opcode) != opcode):") w(' raise ValueError("invalid opcode or oparg")') - w(" popped = _POPPED.get(opcode)") - w(" pushed = _PUSHED.get(opcode)") + w(" popped = popped_table.get(opcode)") + w(" pushed = pushed_table.get(opcode)") w(" if popped is None or pushed is None:") w(' raise ValueError("invalid opcode or oparg")') - w(" npop = popped(oparg_int)") - w(" npush = pushed(oparg_int)") + w(" npop = popped if popped.__class__ is int else popped(oparg_int)") + w(" npush = pushed if pushed.__class__ is int else pushed(oparg_int)") w(" if npop < 0 or npush < 0:") w(' raise ValueError("invalid opcode or oparg")') w(" if opcode in _BLOCK_PUSH and not jump_int:") From 3f3f03290a3705c72c524b5bfb4c7e964fafd986 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:08:48 -0700 Subject: [PATCH 76/96] perf: run class-body machinery and metaclass attribute reads directly Classes built by a metaclass (`enum`, `abc`, `typing`) spent most of their time outside the class body's own code: - A `__prepare__` namespace with a Python `__setitem__` (enum's `EnumDict`) took a bound method and a fresh, unhashed key string per `STORE_NAME`. The function is now called with the namespace as its first argument and the code's interned name as the key; `LOAD_NAME` probes such a namespace without building a key object. `__set_name__` hooks that are plain functions are called the same way. - Zero-argument `super().name` handed every name it couldn't resolve to a plain function (dunders such as `super().__setitem__` and `super().__new__`, and methods of built-in bases) to the full path, which builds a `super` proxy instance, introspects the frame and runs the generic attribute lookup on the proxy. When nothing observes the call and the receiver passes the basic `supercheck`, `LOAD_SUPER_ATTR` now walks the MRO from the class and receiver already on the stack, sharing the walk with the proxy's own lookup. - Attribute reads on a class whose metaclass is a Python subclass of `type` with the default `__getattribute__` went to the full path. When the metaclass's MRO has no entry for the name, the class's MRO answers exactly as for a plain metaclass, so the leaf paths take them (uncached by site, as the class's version doesn't follow the metaclass). `cls.__dict__` and class-level dicts and lists (`_member_map_`, `_member_names_`) read in the leaf paths too. Creating a three-member `Enum` costs 1.82M -> 1.63M instructions (python3.14 0.37M), an `IntFlag` 2.56M -> 2.27M (0.54M), and a class whose metaclass `__prepare__` returns a dict subclass 216K -> 151K (83K). Import-only instructions, before -> after: typing 67M -> 63M, json 86M -> 84M, dataclasses 186M -> 182M, argparse 80M -> 77M, logging 281M -> 276M, asyncio 447M -> 432M, inspect 183M -> 176M, http.client 228M -> 218M; the ten modules imported together 680M -> 670M (python3.14 447M). --- crates/weavepy-vm/src/lib.rs | 298 ++++++++++++++++++++++++++------- crates/weavepy-vm/src/types.rs | 4 + 2 files changed, 237 insertions(+), 65 deletions(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 5340c256..f3a41ea9 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -15892,8 +15892,12 @@ impl Interpreter { // scalar one is remembered for the core loop's arm. Some(Object::Type(cls)) => { let v = Self::leaf_load_type_attr(code, cls, pc as u32, ins.arg); + // (The stamp is keyed by the class's version alone, + // which a metaclass's changes don't move.) if let Some(v) = &v { - class_attr_fill(code, cls, pc, v); + if Self::plain_metaclass(cls) { + class_attr_fill(code, cls, pc, v); + } } v } @@ -19921,6 +19925,8 @@ impl Interpreter { K::Value(v) => Some(LeafAttr::Value(Self::clone_operand(v))), K::ValueInstance(w) => Some(LeafAttr::Value(Object::Instance(w.upgrade()?))), K::ValueType(w) => Some(LeafAttr::Value(Object::Type(w.upgrade()?))), + K::ValueDict(w) => Some(LeafAttr::Value(Object::Dict(w.upgrade()?))), + K::ValueList(w) => Some(LeafAttr::Value(Object::List(w.upgrade()?))), K::Property(w) => Some(LeafAttr::Property(w.upgrade()?)), K::Other => None, }; @@ -19974,6 +19980,8 @@ impl Interpreter { | Object::Str(_)), ) => K::Value(v), Some(Object::Type(t)) => K::ValueType(Rc::downgrade(&t)), + Some(Object::Dict(d)) => K::ValueDict(Rc::downgrade(&d)), + Some(Object::List(l)) => K::ValueList(Rc::downgrade(&l)), // A plain instance stored on the class (`timezone.utc`): // a value unless its class makes it a descriptor. Some(Object::Instance(i)) => { @@ -19994,6 +20002,8 @@ impl Interpreter { K::Value(v) => Some(LeafAttr::Value(Self::clone_operand(v))), K::ValueInstance(w) => w.upgrade().map(|i| LeafAttr::Value(Object::Instance(i))), K::ValueType(w) => w.upgrade().map(|t| LeafAttr::Value(Object::Type(t))), + K::ValueDict(w) => w.upgrade().map(|d| LeafAttr::Value(Object::Dict(d))), + K::ValueList(w) => w.upgrade().map(|l| LeafAttr::Value(Object::List(l))), K::Property(w) => w.upgrade().map(LeafAttr::Property), K::Other => None, }; @@ -20026,9 +20036,29 @@ impl Interpreter { // (A VM-native builtin class with a faithful C mirror keeps its // own dict authoritative; only an extension-defined type's C // attribute hooks are opaque here.) - if (cls.c_ext_ptr.get() != 0 && !cls.flags.is_builtin) || !Self::plain_metaclass(cls) { + if cls.c_ext_ptr.get() != 0 && !cls.flags.is_builtin { return None; } + // `type.__dict__` is a data descriptor on the metatype that answers + // ahead of everything else, unless the metaclass replaces + // `__getattribute__` (a watched type re-arms on every lookup). + if code.names.get(name_idx as usize).is_some_and(|n| n == "__dict__") { + if crate::capi_watchers::types_active() { + return None; + } + // SAFETY: GIL-serialized raw read, as in `plain_metaclass`. + let meta = unsafe { (*cls.metaclass.as_ptr()).as_ref() }; + if meta.is_some_and(|m| { + !Rc::ptr_eq(m, &builtin_types().type_) + && (m.c_ext_ptr.get() != 0 || !Self::default_getattribute(m)) + }) { + return None; + } + return Some(Object::MappingProxy(cls.dict.clone())); + } + if !Self::plain_metaclass(cls) { + return Self::leaf_load_meta_class_attr(code, cls, name_idx); + } // A natively served class method (`datetime.fromisoformat`; see // `stdlib::datetime_native`), bound as `classmethod.__get__` // binds it. @@ -20064,6 +20094,41 @@ impl Interpreter { } } + /// [`Self::leaf_load_type_attr`] for a class whose metaclass is a + /// Python subclass of `type` (`EnumType`, `ABCMeta`) that keeps the + /// default attribute access. CPython's `type_getattro` lets only a + /// metatype attribute intercept, so when the metaclass's MRO has no + /// entry for the name at all, the class's MRO answers as for a plain + /// metaclass. Nothing is cached per site: the class's attribute + /// version doesn't follow the metaclass. + #[inline(never)] + fn leaf_load_meta_class_attr( + code: &CodeObject, + cls: &Rc, + name_idx: u32, + ) -> Option { + if crate::object::exotic_str_keys_possible() || cls.native_kind.get() != 0 { + return None; + } + // SAFETY: GIL-serialized raw read, as in `plain_metaclass`; the + // reference does not outlive the checks. + let meta = unsafe { (*cls.metaclass.as_ptr()).as_ref() }?; + if meta.c_ext_ptr.get() != 0 || !Self::default_getattribute(meta) { + return None; + } + let Some(Object::Str(name)) = code_name_obj(code, name_idx) else { + return None; + }; + if meta.lookup(name).is_some() { + return None; + } + match Self::leaf_class_attr(code, cls, name_idx)? { + LeafAttr::Method(f) => Some(Object::Function(f)), + LeafAttr::Value(v) => Some(v), + LeafAttr::InstanceOnly | LeafAttr::BuiltinMethod(_) | LeafAttr::Property(_) => None, + } + } + /// Whether a class attribute read of dunder `name` resolves in the /// class's own MRO rather than the metaclass. Deliberately a short /// list: `type` carries data descriptors for most dunders @@ -21155,7 +21220,11 @@ impl Interpreter { // through its `__setitem__` (e.g. `enum._EnumDict`); the old // value is the namespace's to manage, so no reap here. let g = frame.globals.clone(); - self.class_ns_store(&ns_obj, &name, v, &g)?; + let key = match code_name_obj(&frame.code, ins.arg) { + Some(k @ Object::Str(s)) if **s == *name => k.clone(), + _ => Object::from_str(name), + }; + self.class_ns_store_key(&ns_obj, key, v, &g)?; } else { // Dict-subclass exec globals with an overridden // `__setitem__` observe the binding (CPython's @@ -21788,6 +21857,13 @@ impl Interpreter { // `co_names[arg >> 2]` from the super object, pushing a // null self slot after it when the method flag (bit 0) // is set. + if let Some(attr) = self.load_super_attr_direct(frame, ins.arg)? { + frame.push(attr); + if ins.arg & 1 != 0 { + frame.push(Object::Unbound); + } + return Ok(StepOutcome::Continue); + } let self_obj = frame.pop()?; let class_obj = frame.pop()?; // The callable (global_super) is already in position @@ -27202,6 +27278,118 @@ impl Interpreter { } } + /// The first stage of `super_getattro` for a bound proxy over + /// `receiver` (CPython's `su->obj`), walking `obj_type`'s MRO past + /// `thisclass`: the attribute through the descriptor protocol, or the + /// receiver's native payload's when a built-in base keeps it off its + /// type dict. `None` leaves the remaining special cases to the caller. + fn super_mro_attr( + &mut self, + obj_type: &Rc, + receiver: &Object, + thisclass: &Rc, + name: &str, + ) -> Result, RuntimeError> { + // Walk `obj_type`'s MRO *after* `thisclass` (CPython's + // `super_getattro`: start one past `su->type` in + // `su->obj_type->tp_mro`). The proxy's own class (`super` or a + // user subclass) is irrelevant to the lookup target. + let found = { + let mro = obj_type.mro.borrow(); + let start = mro + .iter() + .position(|t| Rc::ptr_eq(t, thisclass)) + .map_or(mro.len(), |i| i + 1); + mro[start..].iter().find_map(|t| { + let v = t.dict.borrow().get(&crate::object::StrKey(name)).cloned()?; + // Introspection-only mirrors in builtin dicts (docs + // surface pass, RFC 0056 WS4) are invisible to super's + // walk too: `super().__repr__()` must fall through to + // the caller's non-virtual `object.__repr__` special case, + // not the virtual `slot_repr` (asyncio.Lock's repr + // chains through super and would re-enter itself). + if t.flags.is_builtin && crate::descr_registry::is_surface_only(&v) { + return None; + } + Some(v) + }) + }; + if let Some(v) = found { + // CPython passes `su->obj_type` as `owner`, and a NULL + // instance when `su->obj == su->obj_type` (the class-bound + // form `super(C, cls)` inside `__new__`/classmethods), so + // plain functions come back unbound while classmethods + // still bind to the class. + let owner = Object::Type(obj_type.clone()); + let instance_for_get = match receiver { + Object::Type(r) if Rc::ptr_eq(r, obj_type) => Object::None, + _ => receiver.clone(), + }; + return self.descriptor_get(&v, &instance_for_get, &owner).map(Some); + } + // The MRO beyond the starting class reaches a built-in base + // (`dict`, `list`, …) whose methods aren't stored on the type + // dict and so don't surface above. Resolve `name` against the + // receiver's native payload so `super().__setitem__`, + // `super().append`, … dispatch to the wrapped built-in and + // operate on the shared payload. (Only names absent from the + // MRO reach here, so this never shadows a user override.) + if let Object::Instance(recv) = receiver { + if let Some(native) = recv.native.get() { + if let Ok(v) = self.load_attr(&native.clone(), name) { + return Ok(Some(v)); + } + } + } + Ok(None) + } + + /// `LOAD_SUPER_ATTR`'s zero-argument form without the proxy: the + /// class and receiver on the stack are what `super()` would find, so + /// when the builtin `super` is the callable, nothing observes the call + /// (CPython's instrumented form reports it), and the receiver passes + /// the basic `supercheck`, the attribute is the MRO walk's. On a hit + /// the three operands leave the stack; `None` leaves them for the + /// general path, which also serves the walk's special cases. + fn load_super_attr_direct( + &mut self, + frame: &mut Frame, + arg: u32, + ) -> Result, RuntimeError> { + if arg & 2 != 0 || crate::trace::any_observers_active() { + return Ok(None); + } + let n = frame.stack.len(); + if n < 3 || !is_super_callable(&frame.stack[n - 3]) { + return Ok(None); + } + let Object::Type(cls) = &frame.stack[n - 2] else { + return Ok(None); + }; + let receiver = &frame.stack[n - 1]; + let obj_type = match receiver { + Object::Type(t) if Rc::ptr_eq(t, cls) || t.is_subclass_of(cls) => t.clone(), + _ => { + let oc = crate::builtins::class_of(receiver); + if !oc.is_subclass_of(cls) { + return Ok(None); + } + oc + } + }; + let (cls, receiver) = (cls.clone(), receiver.clone()); + let code = frame.code.clone(); + let Some(name) = code.names.get((arg >> 2) as usize) else { + return Ok(None); + }; + let Some(attr) = self.super_mro_attr(&obj_type, &receiver, &cls, name)? else { + return Ok(None); + }; + let len = frame.stack.len(); + frame.stack.truncate(len - 3); + Ok(Some(attr)) + } + /// The default `object.__getattribute__` body: data descriptor → instance /// dict → class attr → built-in-subclass payload, ending in /// `AttributeError`. Does **not** consult `__getattr__` (the caller does) @@ -27261,56 +27449,8 @@ impl Interpreter { None }; if let Some((obj_type, receiver, thisclass)) = super_fields { - // Walk `obj_type`'s MRO *after* `thisclass` (CPython's - // `super_getattro`: start one past `su->type` in - // `su->obj_type->tp_mro`). The proxy's own class (`super` or a - // user subclass) is irrelevant to the lookup target. - let found = { - let mro = obj_type.mro.borrow(); - let start = mro - .iter() - .position(|t| Rc::ptr_eq(t, &thisclass)) - .map_or(mro.len(), |i| i + 1); - mro[start..].iter().find_map(|t| { - let v = t.dict.borrow().get(&crate::object::StrKey(name)).cloned()?; - // Introspection-only mirrors in builtin dicts (docs - // surface pass, RFC 0056 WS4) are invisible to super's - // walk too — `super().__repr__()` must fall through to - // the non-virtual `object.__repr__` special case below, - // not the virtual `slot_repr` (asyncio.Lock's repr - // chains through super and would re-enter itself). - if t.flags.is_builtin && crate::descr_registry::is_surface_only(&v) { - return None; - } - Some(v) - }) - }; - if let Some(v) = found { - // CPython passes `su->obj_type` as `owner`, and a NULL - // instance when `su->obj == su->obj_type` — the class-bound - // form `super(C, cls)` inside `__new__`/classmethods — so - // plain functions come back unbound while classmethods - // still bind to the class. - let owner = Object::Type(obj_type.clone()); - let instance_for_get = match &receiver { - Object::Type(r) if Rc::ptr_eq(r, &obj_type) => Object::None, - _ => receiver.clone(), - }; - return self.descriptor_get(&v, &instance_for_get, &owner); - } - // The MRO beyond the starting class reaches a built-in base - // (`dict`, `list`, …) whose methods aren't stored on the type - // dict and so don't surface above. Resolve `name` against the - // receiver's native payload so `super().__setitem__`, - // `super().append`, … dispatch to the wrapped built-in and - // operate on the shared payload. (Only names absent from the - // MRO reach here, so this never shadows a user override.) - if let Object::Instance(recv) = &receiver { - if let Some(native) = recv.native.get() { - if let Ok(v) = self.load_attr(&native.clone(), name) { - return Ok(v); - } - } + if let Some(v) = self.super_mro_attr(&obj_type, &receiver, &thisclass, name)? { + return Ok(v); } // `object.__repr__` / `object.__str__` are identity slots that // WeavePy keeps as native behaviour rather than type-dict @@ -44740,10 +44880,33 @@ impl Interpreter { value: Object, globals: &Rc>, ) -> Result<(), RuntimeError> { - let key = Object::from_str(name); - if let Some(method) = instance_method(ns_obj, "__setitem__") { - self.call(&method, &[key, value], &[], globals)?; - return Ok(()); + self.class_ns_store_key(ns_obj, Object::from_str(name), value, globals) + } + + /// [`Self::class_ns_store`] with the key already an object (the code + /// object's interned name, whose hash is cached, like the `co_names` + /// entry CPython's `STORE_NAME` passes). + fn class_ns_store_key( + &mut self, + ns_obj: &Object, + key: Object, + value: Object, + globals: &Rc>, + ) -> Result<(), RuntimeError> { + if let Object::Instance(inst) = ns_obj { + if let Some(m) = inst.cls().lookup("__setitem__") { + // A plain function takes the namespace as its first + // argument, with no bound method to build; anything else + // binds at call time (a descriptor's `__get__` runs then). + if matches!(m, Object::Function(_)) { + self.call(&m, &[ns_obj.clone(), key, value], &[], globals)?; + } else { + let method = + Object::BoundMethod(Rc::new(BoundMethod::dispatch(ns_obj.clone(), m))); + self.call(&method, &[key, value], &[], globals)?; + } + return Ok(()); + } } self.store_subscr(ns_obj, &key, value, globals) } @@ -44792,7 +44955,7 @@ impl Interpreter { name: &str, globals: &Rc>, ) -> Result, RuntimeError> { - let key = DictKey(Object::from_str(name)); + let key = crate::object::StrKey(name); match ns_obj { Object::Dict(d) => Ok(d.borrow().get(&key).cloned()), Object::MappingProxy(d) => Ok(d.borrow().get(&key).cloned()), @@ -46629,13 +46792,18 @@ impl Interpreter { } if let Object::Instance(inst) = &value { if let Some(hook) = inst.cls().lookup("__set_name__") { - let bound = Object::BoundMethod(Rc::new(BoundMethod::new(value.clone(), hook))); - let res = self.call( - &bound, - &[Object::Type(ty.clone()), Object::from_str(&attr_name)], - &[], - &self.builtins.clone(), - ); + let owner = Object::Type(ty.clone()); + let name = Object::from_str(&attr_name); + let builtins = self.builtins.clone(); + // A plain function takes the descriptor as its first + // argument, with no bound method to build. + let res = if matches!(hook, Object::Function(_)) { + self.call(&hook, &[value.clone(), owner, name], &[], &builtins) + } else { + let bound = + Object::BoundMethod(Rc::new(BoundMethod::new(value.clone(), hook))); + self.call(&bound, &[owner, name], &[], &builtins) + }; // PEP 678 / CPython `type.__new__`: a failing // `__set_name__` is re-raised with a note naming the // descriptor, attribute, and owning class so the diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index 3d7f646a..d0eb0c84 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -447,6 +447,10 @@ pub enum LeafAttrKind { ValueInstance(crate::sync::Weak), /// A class stored on the class, likewise held weakly. ValueType(crate::sync::Weak), + /// A dict stored on the class (`Enum._member_map_`), likewise. + ValueDict(crate::sync::Weak>), + /// A list stored on the class (`Enum._member_names_`), likewise. + ValueList(crate::sync::Weak>>), /// A `property` on the MRO (a data descriptor: it wins over the /// instance dict); its getter is read at access time. Property(crate::sync::Weak), From dc40bc758549c859ae7a3d446931d37f5ebc71d7 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:41:17 -0700 Subject: [PATCH 77/96] style: format, and satisfy Rust 1.99's deprecations and Clippy lints CI builds with `-D warnings` on the current stable toolchain, which now deprecates the `std::u64` module (used by the vendored lzma-sys) and `AtomicU64::fetch_update`, and adds Clippy lints that flagged code on this branch. The vendored crate uses the associated constant, the type version counter its own compare-exchange loop (`try_update` is newer than the MSRV), and the workspace allows the new `assert_is_empty` lint, which fires on tests throughout. The rest is `cargo fmt` and `cargo clippy --fix`. --- Cargo.toml | 2 + crates/weavepy-capi/src/pep741.rs | 4 +- crates/weavepy-capi/src/types.rs | 2 +- crates/weavepy-compiler/src/lib.rs | 3 +- crates/weavepy-jit/src/engine.rs | 12 +- crates/weavepy-jit/src/lib.rs | 39 ++-- crates/weavepy-jit/src/lower.rs | 84 ++++++-- crates/weavepy-vm/src/builtins.rs | 6 +- crates/weavepy-vm/src/frame_jit.rs | 170 ++++++++++------ crates/weavepy-vm/src/gc_trace.rs | 8 +- crates/weavepy-vm/src/gen_fast.rs | 13 +- crates/weavepy-vm/src/leaf_plan.rs | 4 +- crates/weavepy-vm/src/leaf_plan/jit.rs | 29 ++- crates/weavepy-vm/src/lib.rs | 194 ++++++++++++------- crates/weavepy-vm/src/object.rs | 8 +- crates/weavepy-vm/src/stdlib/gc_real.rs | 4 +- crates/weavepy-vm/src/stdlib/pickle_accel.rs | 2 +- crates/weavepy-vm/src/stdlib/sre_mod.rs | 6 +- crates/weavepy-vm/src/stdlib/sys.rs | 2 +- crates/weavepy-vm/src/tier2.rs | 68 +++++-- crates/weavepy-vm/src/types.rs | 21 +- vendor/lzma-sys/src/lib.rs | 1 - 22 files changed, 448 insertions(+), 234 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bce0707c..229aa8bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -288,6 +288,8 @@ manual_map = "allow" comparison_chain = "allow" unnecessary_cast = "allow" len_without_is_empty = "allow" +# New in Clippy 1.99; tests across the workspace assert emptiness this way. +assert_is_empty = "allow" # Interpreter compares IEEE-754 values exactly by design (Python `==`, # rounding, struct packing); an epsilon would be incorrect. float_cmp = "allow" diff --git a/crates/weavepy-capi/src/pep741.rs b/crates/weavepy-capi/src/pep741.rs index 06d90417..2d5b096e 100644 --- a/crates/weavepy-capi/src/pep741.rs +++ b/crates/weavepy-capi/src/pep741.rs @@ -29,8 +29,8 @@ use weavepy_vm::object::Object; use crate::embed::PyInitFn; use crate::initconfig::{ - self, EmbedConfig, PyConfig, PyConfig_InitIsolatedConfig, PyStatus, PyWideStringList, - _PyStatus_TYPE_ERROR, _PyStatus_TYPE_EXIT, + self, _PyStatus_TYPE_ERROR, _PyStatus_TYPE_EXIT, EmbedConfig, PyConfig, + PyConfig_InitIsolatedConfig, PyStatus, PyWideStringList, }; use crate::object::PyObject; diff --git a/crates/weavepy-capi/src/types.rs b/crates/weavepy-capi/src/types.rs index 5570121e..02c0f438 100644 --- a/crates/weavepy-capi/src/types.rs +++ b/crates/weavepy-capi/src/types.rs @@ -2585,7 +2585,7 @@ fn inherit_inline_base_layout( }; let bases = t.bases.borrow(); - let first_ptr = bases.first().and_then(&resolve); + let first_ptr = bases.first().and_then(resolve); // CPython's `best_base()`: the "solid base" is the base with the widest // instance layout (`tp_basicsize`), scanned across *all* bases — not diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index 9d54e798..382128eb 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -284,7 +284,8 @@ impl JitHint { /// back edges (see [`Self::set_backedge_quiet`]). #[inline] pub fn is_backedge_quiet(&self) -> bool { - self.backedge_quiet.load(std::sync::atomic::Ordering::Relaxed) + self.backedge_quiet + .load(std::sync::atomic::Ordering::Relaxed) } pub fn set_backedge_quiet(&self) { diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index 7b0538fc..e2b31984 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -522,7 +522,9 @@ impl JitEngine { .iter() .any(|b| b.stmts.iter().any(|s| matches!(s.op, TOp::IsObj { .. }))) { - return Err(JitVerdict::UnsupportedOpcode("IS_OP (no helper registered)")); + return Err(JitVerdict::UnsupportedOpcode( + "IS_OP (no helper registered)", + )); } let needs_float_divmod = tfunc.blocks.iter().any(|b| { b.stmts @@ -626,7 +628,11 @@ impl JitEngine { } if std::env::var_os("WEAVEPY_JIT_CLIF_STATS").is_some() { let f = &self.ctx.func; - let insts: usize = f.layout.blocks().map(|b| f.layout.block_insts(b).count()).sum(); + let insts: usize = f + .layout + .blocks() + .map(|b| f.layout.block_insts(b).count()) + .sum(); eprintln!( "jit clif {name}: {} blocks, {insts} insts, {} values", f.layout.blocks().count(), @@ -636,7 +642,7 @@ impl JitEngine { let stats = std::env::var_os("WEAVEPY_JIT_CLIF_STATS").is_some(); if stats { - drop(cranelift_codegen::timing::take_current()); + let _ = cranelift_codegen::timing::take_current(); } let t0 = stats.then(std::time::Instant::now); self.module diff --git a/crates/weavepy-jit/src/lib.rs b/crates/weavepy-jit/src/lib.rs index b3eb78eb..227deca9 100644 --- a/crates/weavepy-jit/src/lib.rs +++ b/crates/weavepy-jit/src/lib.rs @@ -42,28 +42,25 @@ pub use ir::{ TOp, TStmt, TTerm, }; pub use runtime::{ - global_obj_list_elem, set_obj_layout, ObjLayout, - register_attr_get_chain_helper, register_attr_helpers, register_build_map_helper, - register_build_set_helper, register_cached_attr_chain_helper, register_call_dyn_helper, - register_call_dyn_int_helper, register_call_method_helper, register_call_native_method_helper, - register_call_py_helper, - register_cell_helpers, register_const_str_helper, register_contains_dyn_helper, - register_dict_del_helper, register_dict_helpers, register_dict_iter_helper, - register_dyn_attr_helpers, register_float_pow_helper, register_global_obj_helper, - register_is_obj_helper, - register_iter_helpers, register_iter_new_helper, register_iter_next_pair_helper, - register_list_extra_helpers, register_list_from_range_helper, register_list_helpers, - register_guard_method_helper, register_list_next_helper, register_math_helpers, - register_obj_getitem_helper, register_poll_helper, - register_self_call_helpers, register_str_format_helpers, register_str_helpers, - register_str_method_helper, register_str_write_helpers, register_truth_helper, - register_tuple_read_helpers, register_unbox_int_helper, AttrGetChainHelper, AttrGetHelper, - AttrSetHelper, BuildListHelper, BuildTupleHelper, BytesGetHelper, CachedAttrChainHelper, - CallDynHelper, CallMethodHelper, CallPyHelper, CallStatus, CellGetHelper, CellSetHelper, - DictAccessHelper, DynAttrHelper, GetIterHelper, GuardMethodHelper, IterNextHelper, - IterNextPairHelper, JitFrame, + global_obj_list_elem, register_attr_get_chain_helper, register_attr_helpers, + register_build_map_helper, register_build_set_helper, register_cached_attr_chain_helper, + register_call_dyn_helper, register_call_dyn_int_helper, register_call_method_helper, + register_call_native_method_helper, register_call_py_helper, register_cell_helpers, + register_const_str_helper, register_contains_dyn_helper, register_dict_del_helper, + register_dict_helpers, register_dict_iter_helper, register_dyn_attr_helpers, + register_float_pow_helper, register_global_obj_helper, register_guard_method_helper, + register_is_obj_helper, register_iter_helpers, register_iter_new_helper, + register_iter_next_pair_helper, register_list_extra_helpers, register_list_from_range_helper, + register_list_helpers, register_list_next_helper, register_math_helpers, + register_obj_getitem_helper, register_poll_helper, register_self_call_helpers, + register_str_format_helpers, register_str_helpers, register_str_method_helper, + register_str_write_helpers, register_truth_helper, register_tuple_read_helpers, + register_unbox_int_helper, set_obj_layout, AttrGetChainHelper, AttrGetHelper, AttrSetHelper, + BuildListHelper, BuildTupleHelper, BytesGetHelper, CachedAttrChainHelper, CallDynHelper, + CallMethodHelper, CallPyHelper, CallStatus, CellGetHelper, CellSetHelper, DictAccessHelper, + DynAttrHelper, GetIterHelper, GuardMethodHelper, IterNextHelper, IterNextPairHelper, JitFrame, JitStatus, ListAppendHelper, ListFromRangeHelper, ListGetHelper, ListLenHelper, ListNextHelper, - ListRepeatHelper, ListSetHelper, ListSliceHelper, MathBinaryHelper, MathUnaryHelper, + ListRepeatHelper, ListSetHelper, ListSliceHelper, MathBinaryHelper, MathUnaryHelper, ObjLayout, PollHelper, SelfEnterHelper, SelfExitHelper, SelfSlowHelper, SlotTag, StrEqHelper, StrLenHelper, StrModHelper, CALL_GAPS, DICT_KEY_INT, DICT_KEY_STR, DICT_VAL_FLOAT, DICT_VAL_INT, DICT_VAL_OBJ, ITER_ELEM_STR, JIT_POLL_STRIDE, MAX_ATTR_CHAIN_LEN, diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index d1fb8eca..60649819 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -11,12 +11,12 @@ //! locals + spilled stack back into the [`JitFrame`] and returns //! [`JitStatus::Deopt`] so the interpreter resumes at the exact pc. +use cranelift_codegen::entity::EntityRef; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::{ types, AbiParam, Block, BlockArg, FuncRef, Function, InstBuilder, MemFlags, SigRef, Signature, StackSlot, StackSlotData, StackSlotKind, Type, Value, }; -use cranelift_codegen::entity::EntityRef; use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; use crate::ir::{ArithKind, CmpKind, MathFunc, MethodRet, SliceOrigin, TFunc, TOp, TStmt, TTerm}; @@ -2366,7 +2366,11 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.b.ins().jump(merge, &[status.into()]); self.b.switch_to_block(merge); let r = self.b.block_params(merge)[0]; - let res = if negate { self.b.ins().bxor_imm(r, 1) } else { r }; + let res = if negate { + self.b.ins().bxor_imm(r, 1) + } else { + r + }; self.vstack.push((res, JitType::Bool)); } @@ -2856,7 +2860,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { .iadd_imm(mbuf, i64::from(token) * i64::from(l.method_size)); let idx = self.b.ins().uload32(t, e, l.method_upd_idx); let from_arg = self.b.ins().uload32(t, e, l.method_upd_from_arg); - let unarmed = self.b.ins().icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); + let unarmed = self + .b + .ins() + .icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); let form = self .b .ins() @@ -2924,7 +2931,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let len = self.b.ins().uload32(t, block, l.split_len); let shadow = self.b.ins().uload32(t, e, l.method_upd_shadow); let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); - let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); let shadowed = self.b.ins().icmp(IntCC::UnsignedGreaterThan, len, shadow); let bad = self.b.ins().bor(foreign, absent); let bad = self.b.ins().bor(bad, shadowed); @@ -2934,7 +2944,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let at = self.b.ins().iadd_imm(at, i64::from(l.split_values)); // An `int` field, plus the increment without overflow. let tag = self.b.ins().uload8(types::I32, t, at, 0); - let not_int = self.b.ins().icmp_imm(IntCC::NotEqual, tag, i64::from(l.tag_int)); + let not_int = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, tag, i64::from(l.tag_int)); self.miss_if(not_int, miss); let old = self.b.ins().load(types::I64, t, at, 8); let inc = match arg { @@ -3125,7 +3138,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let done = self.b.create_block(); let (items, len, list) = self.pinned_list(&l, pin, lanes, miss, false); let cap = self.b.ins().load(types::I64, t, list, l.list_cap); - let full = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, len, cap); + let full = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, len, cap); self.miss_if(full, miss); let off = self.b.ins().ishl_imm(len, 4); let at = self.b.ins().iadd(items, off); @@ -3276,7 +3292,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { .ins() .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); let idx = self.b.ins().uload32(t, g, l.guard_split_idx); - let none = self.b.ins().icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); + let none = self + .b + .ins() + .icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); // The pin. let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); @@ -3287,8 +3306,14 @@ impl<'a, 'b> Lowerer<'a, 'b> { let p = self.b.ins().iadd(buf, off); let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); let otag = self.b.ins().uload8(types::I32, t, p, l.pin_obj); - let not_obj = self.b.ins().icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); - let not_inst = self.b.ins().icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + let not_obj = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); let bad = self.b.ins().bor(not_obj, not_inst); self.miss_if(bad, miss); let inst = self.b.ins().load(ptr, t, p, l.pin_obj + 8); @@ -3319,7 +3344,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let ckeys = self.b.ins().load(ptr, t, cls, l.type_shared_keys); let len = self.b.ins().uload32(t, block, l.split_len); let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); - let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); let bad = self.b.ins().bor(foreign, absent); self.miss_if(bad, miss); let off = self.b.ins().ishl_imm(idx, 4); @@ -3335,7 +3363,13 @@ impl<'a, 'b> Lowerer<'a, 'b> { /// the names before the guard's index and have room for one more, with /// no dict watcher active. Anything else branches to `miss` before /// anything is written. - fn split_append_addr(&mut self, l: &runtime::ObjLayout, pin: Value, site: u32, miss: Block) -> Value { + fn split_append_addr( + &mut self, + l: &runtime::ObjLayout, + pin: Value, + site: u32, + miss: Block, + ) -> Value { let t = MemFlags::trusted(); let ptr = self.ptr_ty; let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); @@ -3346,7 +3380,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { .ins() .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); let idx = self.b.ins().uload32(t, g, l.guard_split_idx); - let none = self.b.ins().icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); + let none = self + .b + .ins() + .icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); let bad = self.b.ins().bor(none, out); @@ -3395,7 +3432,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let cap = self.b.ins().uload32(t, block, l.split_cap); let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); let elsewhere = self.b.ins().icmp(IntCC::NotEqual, idx, len); - let full = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, len, cap); + let full = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, len, cap); let bad = self.b.ins().bor(foreign, elsewhere); let bad = self.b.ins().bor(bad, full); self.miss_if(bad, miss); @@ -3431,7 +3471,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let p = self.b.ins().iadd(buf, off); let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); let lane = self.b.ins().uload8(types::I32, t, p, l.pin_list_elem); - let mut bad = self.b.ins().icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_list_tag)); + let mut bad = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_list_tag)); let mut ok_lane = self.b.ins().iconst(types::I8, 0); for &want in lanes { let hit = self.b.ins().icmp_imm(IntCC::Equal, lane, want as i64); @@ -4218,7 +4261,14 @@ impl<'a, 'b> Lowerer<'a, 'b> { /// only computes, so when the enter helper declines or the leaf deopts /// (an overflow, a zero divisor), the ordinary call helper runs the /// call from the start. - fn emit_call_leaf(&mut self, ix: usize, from: &[LeafArg], call: SiteCall, ret: JitType, pc: u32) { + fn emit_call_leaf( + &mut self, + ix: usize, + from: &[LeafArg], + call: SiteCall, + ret: JitType, + pc: u32, + ) { let trusted = MemFlags::trusted(); let (enter_addr, exit_addr, _) = runtime::self_call_helper_addrs().expect("checked by the engine"); @@ -4310,7 +4360,9 @@ impl<'a, 'b> Lowerer<'a, 'b> { // Declined or deopted: the ordinary call, from the start. self.b.switch_to_block(generic_b); self.vstack.extend(args.iter().copied()); - self.emit_call_py(call.token, call.argc, call.kwc, call.perm, call.gaps, ret, pc); + self.emit_call_py( + call.token, call.argc, call.kwc, call.perm, call.gaps, ret, pc, + ); let (v, _) = self.vstack.pop().expect("the call's result"); self.b.ins().jump(join_b, &[v.into()]); diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index 6fcf5745..a25f9666 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -7175,8 +7175,10 @@ fn simple_key_set(src: &Object) -> Option { if !items.iter().all(simple) { return None; } - let mut out = - crate::object::SetData::with_capacity_and_hasher(items.len(), Default::default()); + let mut out = crate::object::SetData::with_capacity_and_hasher( + items.len(), + crate::fasthash::FxBuildHasher, + ); for v in items { out.insert(DictKey(v.clone())); } diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs index e36f4982..786ee368 100644 --- a/crates/weavepy-vm/src/frame_jit.rs +++ b/crates/weavepy-vm/src/frame_jit.rs @@ -299,13 +299,17 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool && crate::method_site_in_place(ext, pc + 1) { helped[pc + 1] = true; - if let Some(call) = (pc + 2..ins.len().min(pc + 12)).find(|&k| ins[k].op == OpCode::Call) { + if let Some(call) = + (pc + 2..ins.len().min(pc + 12)).find(|&k| ins[k].op == OpCode::Call) + { helped[call] = true; } } } let at_loop = match at { - Heat::BackEdge(pc) => Some(pc).filter(|&pc| ins.get(pc).is_some_and(|i| i.op == OpCode::JumpBackward)), + Heat::BackEdge(pc) => { + Some(pc).filter(|&pc| ins.get(pc).is_some_and(|i| i.op == OpCode::JumpBackward)) + } Heat::Call | Heat::Step => None, }; ins.iter().enumerate().any(|(pc, i)| { @@ -322,7 +326,10 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool } } if stats::enabled() { - eprintln!("frame jit: {} loop at {top}: {native} native, {exits} exits", code.qualname); + eprintln!( + "frame jit: {} loop at {top}: {native} native, {exits} exits", + code.qualname + ); } exits * EXIT_WEIGHT <= native }) @@ -456,7 +463,9 @@ fn tags() -> Option { tag(&Object::new_list(Vec::new())), tag(&Object::new_dict()), tag(&Object::new_tuple_array([Object::None])), - tag(&Object::Type(crate::builtin_types::builtin_types().object_.clone())), + tag(&Object::Type( + crate::builtin_types::builtin_types().object_.clone(), + )), ]; let t = Tags { none: tag(&Object::None), @@ -473,7 +482,9 @@ fn tags() -> Option { if samples.iter().any(|&t| t >= 64) { return None; } - let all = [t.none, t.unbound, t.boolean, t.int, t.float, t.cell, t.instance]; + let all = [ + t.none, t.unbound, t.boolean, t.int, t.float, t.cell, t.instance, + ]; let distinct = all .iter() .enumerate() @@ -893,7 +904,13 @@ unsafe extern "C" fn h_store_attr( let (code, pc) = (&*code, pc as usize); let local = &*st.locals.add(code.instructions[pc].arg as usize); let name = code.instructions[pc + 1].arg; - u32::from(!Interpreter::core_store_local_attr(code, local, pc + 1, name, &*slot)) + u32::from(!Interpreter::core_store_local_attr( + code, + local, + pc + 1, + name, + &*slot, + )) } } @@ -1010,16 +1027,7 @@ fn compile_with( let entries: Vec; let built = { let b = FunctionBuilder::new(&mut engine.ctx.func, &mut engine.fbctx); - let mut lower = Lower::new( - b, - ptr, - engine.tags, - code, - ext, - nlocals, - depths, - field_slots, - ); + let mut lower = Lower::new(b, ptr, engine.tags, code, ext, nlocals, depths, field_slots); let done = lower.lower(); entries = (0..ninstrs).map(|pc| lower.enters_at(pc)).collect(); if done { @@ -1266,11 +1274,7 @@ impl<'a> Lower<'a> { // ---- building blocks ---- fn sig(&mut self, params: &[Type], ret: Option) -> SigRef { - if let Some((_, _, s)) = self - .sigs - .iter() - .find(|(p, r, _)| p == params && *r == ret) - { + if let Some((_, _, s)) = self.sigs.iter().find(|(p, r, _)| p == params && *r == ret) { return *s; } let mut sig = Signature::new(self.b.func.signature.call_conv); @@ -1668,7 +1672,10 @@ impl<'a> Lower<'a> { let t = o.tag.expect("unknown kinds carry a tag"); let byte = o.byte.expect("unknown kinds carry a byte"); let tb = self.tags; - let is_bool = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.boolean)); + let is_bool = self + .b + .ins() + .icmp_imm(IntCC::Equal, t, i64::from(tb.boolean)); let is_int = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.int)); let is_none = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.none)); let mut ok = self.b.ins().bor(is_bool, is_int); @@ -1768,8 +1775,8 @@ impl<'a> Lower<'a> { if ins.op == OpCode::LoadFast && code.instructions.get(next).map(|i| i.op) == Some(OpCode::LoadMethodAttr) { - if let Some(call) = (pc + 2..n.min(pc + 12)) - .find(|&k| code.instructions[k].op == OpCode::Call) + if let Some(call) = + (pc + 2..n.min(pc + 12)).find(|&k| code.instructions[k].op == OpCode::Call) { if call + 1 < n { starts[call + 1] = true; @@ -1878,9 +1885,7 @@ impl<'a> Lower<'a> { self.push(Item::Int(v)); true } - OpCode::StoreFast if (ins.arg as usize) < self.nlocals => { - self.store_fast(pc, ins.arg) - } + OpCode::StoreFast if (ins.arg as usize) < self.nlocals => self.store_fast(pc, ins.arg), OpCode::PopTop => { match self.pop() { Item::Mem(s) => { @@ -1944,7 +1949,10 @@ impl<'a> Lower<'a> { Item::Mem(s) => { let at = self.slot_addr(s); let tag = self.tag_at(at); - let none = self.b.ins().icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)); + let none = + self.b + .ins() + .icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)); let scalar = self.is_scalar(tag); let heap = self.b.create_block(); let done = self.b.create_block(); @@ -1962,14 +1970,18 @@ impl<'a> Lower<'a> { Item::Local(i) => { let at = self.local_addr(i); let tag = self.tag_at(at); - self.b.ins().icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)) + self.b + .ins() + .icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)) } Item::Const(k) => { let none = matches!(self.ext.objects[k as usize], Object::None); self.b.ins().iconst(types::I8, i64::from(none)) } Item::Dyn(tag, ..) => { - self.b.ins().icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)) + self.b + .ins() + .icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)) } Item::Int(_) | Item::Float(_) | Item::Bool(_) => { self.b.ins().iconst(types::I8, 0) @@ -2135,8 +2147,14 @@ impl<'a> Lower<'a> { return false; } self.flush(); - let code = self.b.ins().iconst(self.ptr, std::ptr::from_ref(self.code) as i64); - let ext = self.b.ins().iconst(self.ptr, std::ptr::from_ref(self.ext) as i64); + let code = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(self.code) as i64); + let ext = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(self.ext) as i64); let pcv = self.b.ins().iconst(types::I64, pc as i64); let nl = self.b.ins().iconst(types::I64, self.nlocals as i64); let len = self.b.ins().iconst(types::I64, self.depth as i64); @@ -2194,10 +2212,17 @@ impl<'a> Lower<'a> { } self.flush(); let slot = self.slot_addr(self.depth - 1); - let code = self.b.ins().iconst(self.ptr, std::ptr::from_ref(self.code) as i64); + let code = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(self.code) as i64); let pcv = self.b.ins().iconst(types::I64, pc as i64); let r = self - .call(h_store_attr as *const () as usize, &[self.st, code, pcv, slot], true) + .call( + h_store_attr as *const () as usize, + &[self.st, code, pcv, slot], + true, + ) .expect("returns"); let out = self.exit_with(pc, &[], INTERP); self.branch_out(r, out); @@ -2231,15 +2256,18 @@ impl<'a> Lower<'a> { self.b.ins().iconst(types::I8, 0), )), Item::Bool(v) => Some(( - self.b.ins().iconst(types::I64, i64::from(self.tags.boolean)), + self.b + .ins() + .iconst(types::I64, i64::from(self.tags.boolean)), self.b.ins().iconst(types::I64, 0), v, )), Item::Dyn(t, w, b) => Some((t, w, self.b.ins().ireduce(types::I8, b))), - Item::Const(k) if matches!( - self.ext.objects[k as usize], - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) => + Item::Const(k) + if matches!( + self.ext.objects[k as usize], + Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None + ) => { let o = self.operand(item).expect("a scalar constant"); let tag = match o.kind { @@ -2376,8 +2404,9 @@ impl<'a> Lower<'a> { BinOpKind::Mod, ] .into_iter() - .any(|k| is(k)); - let float_kind = is(BinOpKind::Add) || is(BinOpKind::Sub) || is(BinOpKind::Mult) || is(BinOpKind::Div); + .any(&is); + let float_kind = + is(BinOpKind::Add) || is(BinOpKind::Sub) || is(BinOpKind::Mult) || is(BinOpKind::Div); let ia = self.is_kind(a, Kind::Int, self.tags.int); let ib = self.is_kind(b, Kind::Int, self.tags.int); let fa = self.is_kind(a, Kind::Float, self.tags.float); @@ -2418,7 +2447,12 @@ impl<'a> Lower<'a> { Ok(x) => s.b.ins().iconst(types::I8, i64::from(x)), Err(v) => v, }; - let (iav, ibv, fav, fbv) = (to_v(self, ia), to_v(self, ib), to_v(self, fa), to_v(self, fb)); + let (iav, ibv, fav, fbv) = ( + to_v(self, ia), + to_v(self, ib), + to_v(self, fa), + to_v(self, fb), + ); let done = self.b.create_block(); self.b.append_block_param(done, types::I64); self.b.append_block_param(done, types::I64); @@ -2568,11 +2602,14 @@ impl<'a> Lower<'a> { return false; } self.flush(); - let ins = self + let ins = self.b.ins().iconst( + self.ptr, + std::ptr::from_ref(&self.code.instructions[pc]) as i64, + ); + let code = self .b .ins() - .iconst(self.ptr, std::ptr::from_ref(&self.code.instructions[pc]) as i64); - let code = self.b.ins().iconst(self.ptr, std::ptr::from_ref(self.code) as i64); + .iconst(self.ptr, std::ptr::from_ref(self.code) as i64); let pcv = self.b.ins().iconst(types::I64, pc as i64); let len = self.b.ins().iconst(types::I64, self.depth as i64); let n = self @@ -2652,11 +2689,17 @@ impl<'a> Lower<'a> { } else if is(BinOpKind::BitXor) { self.b.ins().bxor(x, y) } else if is(BinOpKind::RShift) { - let out = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, y, 64); + let out = self + .b + .ins() + .icmp_imm(IntCC::UnsignedGreaterThanOrEqual, y, 64); self.branch_out(out, slow); self.b.ins().sshr(x, y) } else if is(BinOpKind::LShift) { - let out = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThanOrEqual, y, 63); + let out = self + .b + .ins() + .icmp_imm(IntCC::UnsignedGreaterThanOrEqual, y, 63); self.branch_out(out, slow); let r = self.b.ins().ishl(x, y); let back = self.b.ins().sshr(r, y); @@ -2757,8 +2800,12 @@ impl<'a> Lower<'a> { Ok(x) => s.b.ins().iconst(types::I8, i64::from(x)), Err(v) => v, }; - let (iav, ibv, fav, fbv) = - (to_v(self, ia), to_v(self, ib), to_v(self, fa), to_v(self, fb)); + let (iav, ibv, fav, fbv) = ( + to_v(self, ia), + to_v(self, ib), + to_v(self, fa), + to_v(self, fb), + ); let done = self.b.create_block(); self.b.append_block_param(done, types::I8); let both_int = self.b.ins().band(iav, ibv); @@ -2801,9 +2848,7 @@ impl<'a> Lower<'a> { let p = std::ptr::from_ref(obj).cast::(); // SAFETY: a constant's tag, payload byte and word (the // constants live as long as the code). - let (t, b, w) = unsafe { - (*p, *p.add(1), p.add(8).cast::().read_unaligned()) - }; + let (t, b, w) = unsafe { (*p, *p.add(1), p.add(8).cast::().read_unaligned()) }; let (t, w, b) = (i64::from(t), w as i64, i64::from(b)); (c(self, t), c(self, w), c(self, b)) } @@ -2850,7 +2895,10 @@ impl<'a> Lower<'a> { let words_mask = tags.by_pointer | (1i64 << tags.int) | (1i64 << tags.float); let by_word = self.b.ins().band_imm(bit, words_mask); let by_word = self.b.ins().icmp_imm(IntCC::NotEqual, by_word, 0); - let is_bool = self.b.ins().icmp_imm(IntCC::Equal, ta, i64::from(tags.boolean)); + let is_bool = self + .b + .ins() + .icmp_imm(IntCC::Equal, ta, i64::from(tags.boolean)); let unit_mask = (1i64 << tags.none) | (1i64 << tags.unbound); let unit = self.b.ins().band_imm(bit, unit_mask); let unit = self.b.ins().icmp_imm(IntCC::NotEqual, unit, 0); @@ -2883,7 +2931,11 @@ impl<'a> Lower<'a> { let it = self.slot_addr(it_slot); let out_slot = self.slot_addr(self.depth); let r = self - .call(h_for_iter as *const () as usize, &[self.st, it, out_slot], true) + .call( + h_for_iter as *const () as usize, + &[self.st, it, out_slot], + true, + ) .expect("returns"); let int_b = self.b.create_block(); let other = self.b.create_block(); @@ -2971,7 +3023,10 @@ impl<'a> Lower<'a> { let ptr = self.ptr; let inst = self.b.ins().load(ptr, FLAGS, at, 8); let cls = self.b.ins().load(ptr, FLAGS, inst, l.inst_class); - let ver = self.b.ins().load(types::I64, FLAGS, cls, l.type_attr_version); + let ver = self + .b + .ins() + .load(types::I64, FLAGS, cls, l.type_attr_version); let cache = self .b .ins() @@ -2998,7 +3053,10 @@ impl<'a> Lower<'a> { let ckeys = self.b.ins().load(ptr, FLAGS, cls, l.type_shared_keys); let len = self.b.ins().uload32(FLAGS, block, l.split_len); let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); - let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); let bad = self.b.ins().bor(foreign, absent); self.branch_out(bad, miss); let off = self.b.ins().ishl_imm(idx, 4); diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 9e2d795e..9522a49b 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -2664,13 +2664,13 @@ fn container_can_cycle(obj: &Object) -> bool { pub fn track_memoryview_exporter(mv: &Object, exporter: &Object) { debug_assert!(matches!(mv, Object::MemoryView(_))); if !is_atomic(exporter) { - track(&mv); + track(mv); } } pub fn track_if_cyclic(obj: &Object) -> bool { if container_can_cycle(obj) { - track(&obj); + track(obj); true } else { false @@ -2837,7 +2837,7 @@ mod tests { .map(|_| Object::Dict(Rc::new(RefCell::new(DictData::default())))) .collect(); for root in &roots { - state.track_now(&root); + state.track_now(root); } if frozen { state.freeze_all(); @@ -2887,7 +2887,7 @@ mod tests { .collect(); for root in &roots { // See `track_and_untrack` on `track_now`. - state.track_now(&root); + state.track_now(root); } for (collection, expected) in [(0, 1), (1, 2), (2, 2), (2, 2)] { assert_eq!(state.collect(collection), 0); diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs index 0c6779f9..dc22bfbb 100644 --- a/crates/weavepy-vm/src/gen_fast.rs +++ b/crates/weavepy-vm/src/gen_fast.rs @@ -380,7 +380,8 @@ impl Interpreter { let native = { let nlocals = locals.len(); if ext.frame_jit.get(nlocals).is_none() { - ext.frame_jit.warm(code, ext, nlocals, crate::frame_jit::Heat::Step); + ext.frame_jit + .warm(code, ext, nlocals, crate::frame_jit::Heat::Step); } ext.frame_jit.get(nlocals) }; @@ -747,12 +748,10 @@ impl Interpreter { } // (The stack slot keeps `g` alive: the inner step runs no code // that could reach this frame.) - Object::Generator(g) => { - match self.gen_fast_next(g, snap_gen, depth + 1, dead) { - GenNext::Yielded(v) => ForNext::Value(v), - GenNext::Declined | GenNext::Partial => ForNext::Bail, - } - } + Object::Generator(g) => match self.gen_fast_next(g, snap_gen, depth + 1, dead) { + GenNext::Yielded(v) => ForNext::Value(v), + GenNext::Declined | GenNext::Partial => ForNext::Bail, + }, _ => ForNext::Bail, } } diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index 0b46cc61..3ae8ec7f 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -1451,9 +1451,9 @@ fn subscr(owned: &mut Owned, a: V, b: V) -> Option { match (unsafe { &*c }, b) { (Object::List(l), V::I(i)) => { let xs = unsafe { l.peek() }?; - Some(norm(&xs[index(i, xs.len())?])) + Some(norm(&raw const xs[index(i, xs.len())?])) } - (Object::Tuple(t), V::I(i)) => Some(norm(&t[index(i, t.len())?])), + (Object::Tuple(t), V::I(i)) => Some(norm(&raw const t[index(i, t.len())?])), (Object::Str(s), V::I(i)) => { if crate::object::str_char_len(s) != s.len() { return None; diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index 25a266ea..161c543a 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -1333,13 +1333,19 @@ impl Lower<'_> { let bad = self.b.ins().bor(not_scalar, not_ref); self.miss_if(bad, slow); let otag = self.b.ins().uload8(types::I32, f, pr, 0); - let not_inst = self.b.ins().icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); self.miss_if(not_inst, slow); let inst = self.b.ins().load(ptr, f, pr, 8); let cls = self.b.ins().load(ptr, f, inst, l.inst_class); let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); let at = self.b.ins().iconst(ptr, slot); - let want = self.b.ins().load(types::I64, f, at, crate::FIELD_SLOT_VER as i32); + let want = self + .b + .ins() + .load(types::I64, f, at, crate::FIELD_SLOT_VER as i32); let idx = self.b.ins().uload32(f, at, crate::FIELD_SLOT_IDX as i32); let lazy = self.b.ins().load(ptr, f, inst, l.inst_dict_lazy); let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); @@ -1364,7 +1370,10 @@ impl Lower<'_> { let ckeys = self.b.ins().load(ptr, f, cls, l.type_shared_keys); let len = self.b.ins().uload32(f, block, l.split_len); let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); - let absent = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); let bad = self.b.ins().bor(foreign, absent); self.miss_if(bad, slow); let off = self.b.ins().ishl_imm(idx, 4); @@ -2097,11 +2106,15 @@ impl Lower<'_> { /// Whether nothing after op `i` can decline the evaluation: every /// later op is a move, a constant, a return or a forward jump. fn infallible_after(plan: &LeafPlan, i: usize) -> bool { - plan.ops.iter().enumerate().skip(i + 1).all(|(k, op)| match *op { - Op::Move { .. } | Op::Const { .. } | Op::Return { .. } => true, - Op::Jump { target } => usize::from(target) > k, - _ => false, - }) + plan.ops + .iter() + .enumerate() + .skip(i + 1) + .all(|(k, op)| match *op { + Op::Move { .. } | Op::Const { .. } | Op::Return { .. } => true, + Op::Jump { target } => usize::from(target) > k, + _ => false, + }) } /// A `CALL` (op `i`): the callee its site last resolved in line behind diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index f3a41ea9..f4dbcfdf 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -51,6 +51,8 @@ pub mod error; pub mod ext_loader; pub mod fasthash; pub mod foreign; +#[cfg(feature = "jit")] +mod frame_jit; pub mod frozen_code_cache; pub mod frozen_table; pub mod gc_trace; @@ -61,8 +63,6 @@ pub mod hot_gates; pub mod import; pub mod import_time; pub mod inst_dict; -#[cfg(feature = "jit")] -mod frame_jit; mod lazy_arc; mod leaf_plan; pub mod linejump; @@ -5142,9 +5142,8 @@ impl Interpreter { break Err(e); } } - } else { - break Err(err); } + break Err(err); } } }; @@ -7827,7 +7826,8 @@ impl Interpreter { if !code.cellvars.is_empty() { // SAFETY: a bound slot's locals storage is its own. let cells = fresh_cells(f, &code, unsafe { &mut *act.frame.locals.as_ptr() }); - return self.inline_bind_with(frame, shell, pc, act, code, callable, guard, cells, true); + return self + .inline_bind_with(frame, shell, pc, act, code, callable, guard, cells, true); } let cells: *const Rc>>> = f.lean_cells_ref(&code).expect("checked by the shape"); @@ -7843,7 +7843,7 @@ impl Interpreter { frame: &mut Frame, shell: &mut QuietShell<'_>, pc: usize, - mut act: Box, + act: Box, code: Rc, callable: Object, guard: crate::recursion::Guard, @@ -10274,7 +10274,14 @@ impl Interpreter { .is_some() { if let Some(hit) = self.core_native_method( - code, other, pc + 1, next.arg, mslots, lbase, nlocals, consts, + code, + other, + pc + 1, + next.arg, + mslots, + lbase, + nlocals, + consts, ) { return Some(hit); } @@ -10291,11 +10298,28 @@ impl Interpreter { }; if pure_site { let (v, call_pc) = self.core_pure_method( - code, other, pc + 1, next.arg, mslots, lbase, nlocals, consts, depth_cell, + code, + other, + pc + 1, + next.arg, + mslots, + lbase, + nlocals, + consts, + depth_cell, )?; return Some((Ok(v), call_pc)); } - self.core_native_method(code, other, pc + 1, next.arg, mslots, lbase, nlocals, consts) + self.core_native_method( + code, + other, + pc + 1, + next.arg, + mslots, + lbase, + nlocals, + consts, + ) } // A container's site-cached leaf method. Object::List(_) | Object::Dict(_) | Object::Set(_) | Object::Str(_) => { @@ -10446,7 +10470,8 @@ impl Interpreter { #[cfg(feature = "jit")] if pc == 0 && !NATIVE { if let Some(ext) = ext { - ext.frame_jit.warm(code, ext, nlocals, frame_jit::Heat::Call); + ext.frame_jit + .warm(code, ext, nlocals, frame_jit::Heat::Call); } } // The native code's view of the activation, built at its first @@ -11402,7 +11427,12 @@ impl Interpreter { #[cfg(feature = "jit")] if !NATIVE { if let Some(ext) = ext { - ext.frame_jit.warm(code, ext, nlocals, frame_jit::Heat::BackEdge(last)); + ext.frame_jit.warm( + code, + ext, + nlocals, + frame_jit::Heat::BackEdge(last), + ); if ext.frame_jit.get(nlocals).is_some() { break Some(CoreExit::Switch); } @@ -12943,8 +12973,7 @@ impl Interpreter { let Some(mut act) = self.inline_pool.pop() else { return false; }; - let crate::recursion::Enter::Ok(guard) = crate::recursion::enter_with(sw.depth_cell) - else { + let crate::recursion::Enter::Ok(guard) = crate::recursion::enter_with(sw.depth_cell) else { self.inline_pool.push(act); return false; }; @@ -12954,7 +12983,11 @@ impl Interpreter { let nlocals = code.varnames.len(); // SAFETY: a parked slot's locals storage is its own, and empty. let locals = unsafe { &mut *act.frame.locals.as_ptr() }; - let first = if has_self { callee_slot + 1 } else { callee_slot + 2 }; + let first = if has_self { + callee_slot + 1 + } else { + callee_slot + 2 + }; let nargs = n - first; locals.reserve(nlocals.max(nargs)); // SAFETY: the `nargs` operands above `first` move into the locals @@ -13083,7 +13116,8 @@ impl Interpreter { let Some(recv) = frame.stack.last() else { return false; }; - let Some(v) = self.pure_leaf_eval::(code, getter, &[std::ptr::from_ref(recv)]) + let Some(v) = + self.pure_leaf_eval::(code, getter, &[std::ptr::from_ref(recv)]) else { return false; }; @@ -14178,7 +14212,9 @@ impl Interpreter { // `PyFunction::code`); only compared, then cloned below. let code_rc: &Rc = unsafe { &*f.code.as_ptr() }; let (missing, slot_self) = slot.hit(Rc::as_ptr(f), Rc::as_ptr(code_rc))?; - if slot_self != has_self || !Self::lean_code_ok(code_rc) || Self::has_keyword_params(code_rc) + if slot_self != has_self + || !Self::lean_code_ok(code_rc) + || Self::has_keyword_params(code_rc) { return None; } @@ -14254,9 +14290,9 @@ impl Interpreter { unreachable!("checked above") }; let cells = fresh_cells(f, &code, locals); - return Some(self.inline_bind_with( - frame, shell, pc, act, code, callable, guard, cells, true, - )); + return Some( + self.inline_bind_with(frame, shell, pc, act, code, callable, guard, cells, true), + ); } // The cells handle is `f`'s (now `callable`'s) or the shared empty // vector: moving the callable leaves it where it was. @@ -15760,29 +15796,28 @@ impl Interpreter { } } return true; - } else { - let result = if clean { - done.clean = !had_shell; - // SAFETY: as above. - // SAFETY: as above. - unsafe { release_locals(&mut *frame.locals.as_ptr()) }; - drop(done.guard.take()); - Ok(v) - } else { - self.inline_finish( - &mut done, - QuietExit::Outcome { - stepped: Ok(StepOutcome::Return(v)), - cur_pc, - }, - ) - }; - done.clean &= !had_shell; - // SAFETY: the caller is the innermost remaining activation. - (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; - // SAFETY: as above. - entry = unsafe { self.inline_deliver(&mut *cframe, done, result) }; } + let result = if clean { + done.clean = !had_shell; + // SAFETY: as above. + // SAFETY: as above. + unsafe { release_locals(&mut *frame.locals.as_ptr()) }; + drop(done.guard.take()); + Ok(v) + } else { + self.inline_finish( + &mut done, + QuietExit::Outcome { + stepped: Ok(StepOutcome::Return(v)), + cur_pc, + }, + ) + }; + done.clean &= !had_shell; + // SAFETY: the caller is the innermost remaining activation. + (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; + // SAFETY: as above. + entry = unsafe { self.inline_deliver(&mut *cframe, done, result) }; sw.cur = cframe; sw.last = if clast == &raw mut sw.scratch { sw.scratch = usize::MAX; @@ -16464,8 +16499,7 @@ impl Interpreter { return false; }; match slots.get_index_mut(key_idx as usize) { - Some((key, slot)) - if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => + Some((key, slot)) if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => { if !Self::core_droppable(slot) { return false; @@ -16508,9 +16542,7 @@ impl Interpreter { return false; }; match slots.get_index(key_idx as usize) { - Some((key, old)) - if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => - { + Some((key, old)) if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => { Self::core_droppable(old) } _ => slots.get(name).is_none(), @@ -16952,9 +16984,10 @@ impl Interpreter { let mut defaults: Option> = None; let mut closure: Option> = None; let mut k = 0; - while let (Some(next), Some(value)) = - (instrs.get(pc + 1 + k), n.checked_sub(2 + k).map(|i| &stack[i])) - { + while let (Some(next), Some(value)) = ( + instrs.get(pc + 1 + k), + n.checked_sub(2 + k).map(|i| &stack[i]), + ) { let (Object::Tuple(items), OpCode::SetFunctionAttribute) = (value, next.op) else { break; @@ -20042,7 +20075,11 @@ impl Interpreter { // `type.__dict__` is a data descriptor on the metatype that answers // ahead of everything else, unless the metaclass replaces // `__getattribute__` (a watched type re-arms on every lookup). - if code.names.get(name_idx as usize).is_some_and(|n| n == "__dict__") { + if code + .names + .get(name_idx as usize) + .is_some_and(|n| n == "__dict__") + { if crate::capi_watchers::types_active() { return None; } @@ -30942,7 +30979,7 @@ impl Interpreter { // match the `{}` literal and `b_dict` paths so a cycle through // the copy is collectable. if let Some(d) = &out { - gc_trace::track(&d); + gc_trace::track(d); } Ok(out) } @@ -31218,10 +31255,12 @@ impl Interpreter { if self.seqtools.get().is_none() { let module = self.do_import("_seqtools", &Object::None, 0, globals).ok(); let class = |name: &str| match &module { - Some(Object::Module(m)) => match m.dict.borrow().get(&DictKey(Object::from_str(name))) { - Some(Object::Type(t)) => Some(t.clone()), - _ => None, - }, + Some(Object::Module(m)) => { + match m.dict.borrow().get(&DictKey(Object::from_str(name))) { + Some(Object::Type(t)) => Some(t.clone()), + _ => None, + } + } _ => None, }; let tools = class("_MapIter") @@ -31290,9 +31329,9 @@ impl Interpreter { }; let tools = self.seqtools.get()?.as_ref()?; let cls = inst.cls_raw(); - let filter = if std::ptr::eq(cls, &*tools.map) { + let filter = if std::ptr::eq(cls, &raw const *tools.map) { false - } else if std::ptr::eq(cls, &*tools.filter) { + } else if std::ptr::eq(cls, &raw const *tools.filter) { true } else { return None; @@ -31319,7 +31358,10 @@ impl Interpreter { Err(e) => return Some(Err(e)), }; if !filter { - return Some(self.call(&func, std::slice::from_ref(&item), &[], globals).map(Some)); + return Some( + self.call(&func, std::slice::from_ref(&item), &[], globals) + .map(Some), + ); } let keep = match &func { Object::None => self.op_truth(&item), @@ -40935,7 +40977,7 @@ impl Interpreter { // is reclaimable and the drop-time "unclosed file" // `ResourceWarning` still fires // (test_io `test_garbage_collection`). - gc_trace::track(&obj); + gc_trace::track(obj); Ok(()) } }, @@ -57611,8 +57653,7 @@ impl Drop for LeanFrame { /// (what [`fresh_cells`] appends). #[inline] fn closure_cells_ok(f: &PyFunction, code: &CodeObject) -> bool { - f.closure.len() == code.freevars.len() - && f.closure.iter().all(|c| matches!(c, Object::Cell(_))) + f.closure.len() == code.freevars.len() && f.closure.iter().all(|c| matches!(c, Object::Cell(_))) } /// The cells of a call of `f` running `code`, which has cell variables, @@ -57632,7 +57673,9 @@ fn fresh_cells( .iter() .position(|n| n == name) .and_then(|i| locals.get_mut(i)) - .map_or(Object::Unbound, |slot| std::mem::replace(slot, Object::Unbound)); + .map_or(Object::Unbound, |slot| { + std::mem::replace(slot, Object::Unbound) + }); cells.push(Rc::new(RefCell::new(initial))); } if crate::stdlib::tracemalloc_real::is_tracking() { @@ -62969,7 +63012,10 @@ impl LeafSite { let i = poly .entries .iter() - .position(|e| e.as_ref().is_some_and(|e| e.ver == old.ver || e.ver == data.ver)) + .position(|e| { + e.as_ref() + .is_some_and(|e| e.ver == old.ver || e.ver == data.ver) + }) .unwrap_or(poly.next); if i == poly.next { poly.next = (i + 1) % POLY_LEAF_SITES; @@ -64302,10 +64348,8 @@ fn function_slots( let keys = fn_slot_keys(); let module = globals.borrow().get(&DictKey(keys[1].clone())).cloned(); let build = |module: Option| { - let mut map = crate::object::DictMap::with_capacity_and_hasher( - 3, - crate::fasthash::FxBuildHasher, - ); + let mut map = + crate::object::DictMap::with_capacity_and_hasher(3, crate::fasthash::FxBuildHasher); if let Some(m) = module { map.insert(DictKey(keys[0].clone()), m); } @@ -66878,6 +66922,12 @@ fn compare_str(a: &str, b: &str, op: CompareKind) -> bool { pub use object::Object as Value; +/// The `_seqtools` classes behind `map` and `filter`. +struct SeqTools { + map: Rc, + filter: Rc, +} + #[cfg(test)] mod tests { use super::*; @@ -67905,7 +67955,9 @@ assert loop(2000) == 1999000 let native = SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(std::cell::Cell::get) - native; let armed = SCALAR_FIELD_UPDATE_ARMED.with(std::cell::Cell::get) - armed; - eprintln!("Application {name} completed updates: native {native}, armed {armed}"); + eprintln!( + "Application {name} completed updates: native {native}, armed {armed}" + ); if name == "richards" && !crate::tier2::jit_off_for_process() { assert!( native > 100 || (armed > 0 && native > 0), @@ -74133,9 +74185,3 @@ print("native pickle coverage: ok") assert_eq!(run(src), "small\nsmall\nlarge\n"); } } - -/// The `_seqtools` classes behind `map` and `filter`. -struct SeqTools { - map: Rc, - filter: Rc, -} diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 4d672c75..c8860e45 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -12984,6 +12984,10 @@ impl Object { } } +// Every payload is one word or smaller, so a value is a tag and a word. +const _: () = assert!(std::mem::size_of::() == 16); +const _: () = assert!(std::mem::size_of::>() == 16); + #[cfg(test)] mod tests { use super::*; @@ -13153,7 +13157,3 @@ mod tests { assert!(d.is_none()); } } - -// Every payload is one word or smaller, so a value is a tag and a word. -const _: () = assert!(std::mem::size_of::() == 16); -const _: () = assert!(std::mem::size_of::>() == 16); diff --git a/crates/weavepy-vm/src/stdlib/gc_real.rs b/crates/weavepy-vm/src/stdlib/gc_real.rs index 0238fb1f..c65dbfe4 100644 --- a/crates/weavepy-vm/src/stdlib/gc_real.rs +++ b/crates/weavepy-vm/src/stdlib/gc_real.rs @@ -515,7 +515,7 @@ fn object_is_tracked(target: &Object) -> bool { let id = id_of(target); gc_trace::with_state(|s| { if !s.is_tracked(id) { - s.track_now(&target); + s.track_now(target); } s.is_tracked(id) }) @@ -587,7 +587,7 @@ fn get_stats(_args: &[Object]) -> Result { fn track_obj(args: &[Object]) -> Result { if let Some(o) = args.first() { - gc_trace::track(&o); + gc_trace::track(o); } Ok(Object::None) } diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs index f449ad1c..85724186 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs @@ -1113,7 +1113,7 @@ impl<'c> Decoder<'c> { return None; } let mut height = 0; - for pair in stack[begin..].chunks_exact(2) { + for pair in stack[begin..].as_chunks::<2>().0 { // Tuple and float keys retain the full unpickler, including // arbitrary nesting and NaN key-identity behavior. So do // instances, whose hashing can call Python code. diff --git a/crates/weavepy-vm/src/stdlib/sre_mod.rs b/crates/weavepy-vm/src/stdlib/sre_mod.rs index 3992a119..43954ffe 100644 --- a/crates/weavepy-vm/src/stdlib/sre_mod.rs +++ b/crates/weavepy-vm/src/stdlib/sre_mod.rs @@ -610,6 +610,8 @@ struct RepeatCtx { } /// The matcher's capture marks: inline for up to 16 groups. +// (The inline variant's size is the point: no allocation per match.) +#[allow(clippy::large_enum_variant)] enum MarkBuf { Inline { len: usize, buf: [isize; 32] }, Heap(Vec), @@ -1423,7 +1425,7 @@ impl<'a, C: SreChar> Matcher<'a, C> { // eight frames stay far inside the 128 KiB red zone. (The first // check is at the eighth level, so a shallow match, the usual // case, never asks for the stack pointer.) - let r = if self.depth % 8 == 0 { + let r = if self.depth.is_multiple_of(8) { stacker::maybe_grow(128 * 1024, 4 * 1024 * 1024, || { self.do_match_inner(pat, toplevel) }) @@ -2460,7 +2462,7 @@ impl Found<'_> { #[inline] pub(crate) fn mark(&self, i: usize) -> i64 { let (a, b) = self.span(i / 2); - if i % 2 == 0 { + if i.is_multiple_of(2) { a } else { b diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index 86df607a..21f54627 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -1420,7 +1420,7 @@ thread_local! { // (Fx-hashed: interning runs for every name of every code object a // module loads, and SipHash showed in import profiles.) static INTERN_POOL: RefCell> = - RefCell::new(std::collections::HashSet::with_hasher(Default::default())); + const { RefCell::new(std::collections::HashSet::with_hasher(crate::fasthash::FxBuildHasher)) }; } /// Canonicalize `name` through the interpreter's intern pool, seeding it diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index 29a197f6..b9960a7f 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -2974,7 +2974,9 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option usize { @@ -2993,12 +2995,15 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option>::DATA_OFFSET; + let inst_class = + arc_data + std::mem::offset_of!(PyInstance, class) + GilCell::>::DATA_OFFSET; let dict = arc_data + std::mem::offset_of!(PyInstance, dict); let inst_dict_lazy = dict + InstDict::LAZY_OFFSET; let inst_split_borrow = dict + InstDict::SPLIT_OFFSET + GilCell::::BORROW_OFFSET; - let inst_split_block = - dict + InstDict::SPLIT_OFFSET + GilCell::::DATA_OFFSET + SplitValues::BLOCK_OFFSET; + let inst_split_block = dict + + InstDict::SPLIT_OFFSET + + GilCell::::DATA_OFFSET + + SplitValues::BLOCK_OFFSET; let arc_p = arc as *const u8; // The class pointer is its `Arc` allocation too. let cls = inst.cls(); @@ -3031,7 +3036,10 @@ fn obj_layout(obj: &Object, inst: &Rc) -> Option().read_unaligned() }; - let keys = cls.shared_keys.get().map_or(0, |k| std::ptr::from_ref(k) as usize); + let keys = cls + .shared_keys + .get() + .map_or(0, |k| std::ptr::from_ref(k) as usize); let published = inst.dict.published().is_some(); if ver != cls.attr_version.get() || word(cls_p, type_shared_keys) != keys @@ -3916,7 +3924,10 @@ fn guards_hold( } (callees.is_empty() || callee_guards_hold(interp, callees, &guard_snapshot.ctor_vers)) && math.iter().all(MathGuard::holds) - && !guard_snapshot.defaults.iter().any(|f| defaults_overridden(f)) + && !guard_snapshot + .defaults + .iter() + .any(|f| defaults_overridden(f)) } /// [`guards_hold`]'s globals, resolved name by name (and remembered as @@ -3969,7 +3980,8 @@ fn callee_guards_hold( Object::Type(t) => { let ver = t.attr_version.get(); let memo = ctor_vers.get(i); - if memo.is_some_and(|m| m.get() == ver) && ctor_still_inits_with(t, ver, code_snap) { + if memo.is_some_and(|m| m.get() == ver) && ctor_still_inits_with(t, ver, code_snap) + { continue; } let ok = matches!( @@ -4115,7 +4127,11 @@ fn put_pins(mut pins: PinTable) { pins.clear(); SPARE_PINS.with(|spare| { let old = spare.take(); - spare.set(if old.capacity() >= pins.capacity() { old } else { pins }); + spare.set(if old.capacity() >= pins.capacity() { + old + } else { + pins + }); }); } @@ -4727,7 +4743,8 @@ fn arm_update(entry: &MethodEntry, nc: &NativeCallee, receiver: &Object, argc: u u.inc.set(inc); u.code_at.set(func.code.as_ptr() as usize); // SAFETY: `Rc` is one pointer, compared and never dereferenced. - u.code.set(unsafe { std::mem::transmute_copy::, usize>(code) }); + u.code + .set(unsafe { std::mem::transmute_copy::, usize>(code) }); u.idx.set(guard.split_idx); #[cfg(test)] crate::SCALAR_FIELD_UPDATE_ARMED.with(|armed| armed.set(armed.get() + 1)); @@ -5222,8 +5239,10 @@ unsafe fn try_native_call( if status == JitStatus::Returned && !child.dirty { let expect = SlotTag::from_raw(expect_tag); let ret = if njf.ret_tag == expect_tag - && matches!(expect, SlotTag::Int | SlotTag::Float | SlotTag::Bool | SlotTag::None) - { + && matches!( + expect, + SlotTag::Int | SlotTag::Float | SlotTag::Bool | SlotTag::None + ) { Some(njf.ret_bits) } else if njf.ret_tag == SlotTag::ObjPin as u32 && njf.ret_bits == u64::MAX { // `None`. @@ -5522,7 +5541,16 @@ unsafe fn finish_native_call( } else { None }; - match finish_deopted_callee(interp, nc, nctx, exit.locals, exit.spill, exit.tags, njf, pending) { + match finish_deopted_callee( + interp, + nc, + nctx, + exit.locals, + exit.spill, + exit.tags, + njf, + pending, + ) { Ok(v) => Done::Obj(v), Err(e) => Done::Raised(e), } @@ -8509,7 +8537,9 @@ unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i Some(Pin::Obj(o @ (Object::Generator(_) | Object::Iter(_)))) => o.clone(), // An instance iterator whose `__next__` is a registered native // leaf (a `deque` iterator): stepped below without Python. - Some(Pin::Obj(o @ Object::Instance(_))) if !crate::gil::free_threading_enabled() => { + Some(Pin::Obj(o @ Object::Instance(_))) + if !crate::gil::free_threading_enabled() => + { o.clone() } _ => return 2, @@ -8538,7 +8568,9 @@ unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i match interp.leaf_next_step(&it, inst) { None => return 2, Some(Ok(v)) => Ok(Some(v)), - Some(Err(RuntimeError::PyException(e))) if e.type_name() == "StopIteration" => Ok(None), + Some(Err(RuntimeError::PyException(e))) if e.type_name() == "StopIteration" => { + Ok(None) + } Some(Err(e)) => Err(e), } } else { @@ -11573,9 +11605,11 @@ unsafe extern "C" fn wpjit_iter_next_pair( // store-pc deopt and unpack generically). if let Object::Tuple(items) = &v { if items.len() == 2 { - let packed1 = pack_iter_elem(&items[0], tag1, &mut ctx.pins, &mut ctx.pin_memo); - let packed2 = - packed1.and_then(|_| pack_iter_elem(&items[1], tag2, &mut ctx.pins, &mut ctx.pin_memo)); + let packed1 = + pack_iter_elem(&items[0], tag1, &mut ctx.pins, &mut ctx.pin_memo); + let packed2 = packed1.and_then(|_| { + pack_iter_elem(&items[1], tag2, &mut ctx.pins, &mut ctx.pin_memo) + }); if let (Some(b1), Some(b2)) = (packed1, packed2) { jf.ret_bits = b1; // SAFETY: the marshal buffer is at least diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index d0eb0c84..711d78b6 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -155,15 +155,18 @@ fn next_type_version() -> u64 { /// Never publish zero or reuse a token, including at counter exhaustion. fn allocate_type_version(counter: &std::sync::atomic::AtomicU64) -> Option { use std::sync::atomic::Ordering::Relaxed; - counter - .fetch_update(Relaxed, Relaxed, |value| { - if value == 0 { - None - } else { - value.checked_add(1) - } - }) - .ok() + let mut value = counter.load(Relaxed); + loop { + let next = if value == 0 { + None + } else { + value.checked_add(1) + }?; + match counter.compare_exchange_weak(value, next, Relaxed, Relaxed) { + Ok(previous) => return Some(previous), + Err(current) => value = current, + } + } } #[cfg(test)] diff --git a/vendor/lzma-sys/src/lib.rs b/vendor/lzma-sys/src/lib.rs index 38f8307b..86ad932f 100644 --- a/vendor/lzma-sys/src/lib.rs +++ b/vendor/lzma-sys/src/lib.rs @@ -2,7 +2,6 @@ #![doc(html_root_url = "https://docs.rs/lzma-sys/0.1")] use libc::{c_char, c_uchar, c_void, size_t}; -use std::u64; #[cfg(target_env = "msvc")] #[doc(hidden)] From 94fdb3a2af5e3d0fc0c4bd6d3f674244c69f0b24 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Fri, 2 Oct 2026 23:56:27 -0700 Subject: [PATCH 78/96] perf: keep Cranelift's mid-end on for tier 2 Compiling tier-2 code without the mid-end optimizer saved a few percent of compile time but cost numeric loops dearly: the A/B bench gate against main showed `jitloop` 3.7x and `spectral_norm` 1.36x slower. Tier 2 optimizes again by default (`WEAVEPY_JIT_OPT=0` turns it off); leaf plans stay unoptimized unless `WEAVEPY_JIT_OPT=1`. Bench gate vs main (macOS, A/B): geometric mean 0.77x, no fixture regressed; 0.52x CPython. --- crates/weavepy-jit/src/engine.rs | 13 ++++++------- crates/weavepy-vm/src/leaf_plan/jit.rs | 8 ++++---- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index e2b31984..6fd20b74 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -269,14 +269,13 @@ impl JitEngine { // in-process. flag_builder.set("use_colocated_libcalls", "false").ok()?; flag_builder.set("is_pic", "false").ok()?; - // No mid-end optimization: the lowering already emits the code it - // wants, and the e-graph pass found nothing in it worth its share - // of compile time (measured: the fixtures ran no slower without it - // and compiled 3-8% faster). WEAVEPY_JIT_OPT=1 restores it. - let opt = if std::env::var_os("WEAVEPY_JIT_OPT").is_some() { - "speed" - } else { + // Cranelift's mid-end pays for itself in numeric loops: without it + // `jitloop` ran 2x and `spectral_norm` 1.2x slower, for a few + // percent of compile time saved. `WEAVEPY_JIT_OPT=0` turns it off. + let opt = if std::env::var_os("WEAVEPY_JIT_OPT").is_some_and(|v| v == "0") { "none" + } else { + "speed" }; flag_builder.set("opt_level", opt).ok()?; if std::env::var_os("WEAVEPY_JIT_QUICK").is_some() { diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index 161c543a..4e80e1ee 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -827,10 +827,10 @@ impl Engine { let mut flags = settings::builder(); flags.set("use_colocated_libcalls", "false").ok()?; flags.set("is_pic", "false").ok()?; - // No mid-end optimization (as for tier 2): the lowering emits the - // code it wants, and the pass cost more to run than it saved. - // `WEAVEPY_JIT_OPT=1` restores it. - let opt = if std::env::var_os("WEAVEPY_JIT_OPT").is_some() { + // No mid-end optimization: a plan's lowering emits the code it + // wants, and the pass cost more to run than it saved. + // `WEAVEPY_JIT_OPT=1` turns it on. + let opt = if std::env::var_os("WEAVEPY_JIT_OPT").is_some_and(|v| v == "1") { "speed" } else { "none" From 7dd3c531fedf10b1a60868348f910fbfe008f83b Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:33:28 -0700 Subject: [PATCH 79/96] perf: count attribute reads that leave native code against frame compiles The frame compiler's admission rule counted every `LOAD_ATTR` as native, but its code serves only split-layout fields and natively served instances' fields. A loop reading a class attribute through an instance (`o.K`) compiled anyway and left for the core loop on every iteration, which cost about 1,000 instructions per read against the interpreter's 550. A site now counts as native only once the core loop has recorded a field there. --- crates/weavepy-vm/src/frame_jit.rs | 8 +++++++- crates/weavepy-vm/src/lib.rs | 23 +++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs index 786ee368..29f5db27 100644 --- a/crates/weavepy-vm/src/frame_jit.rs +++ b/crates/weavepy-vm/src/frame_jit.rs @@ -319,7 +319,13 @@ fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool let top = (pc + 1).saturating_sub(i.arg as usize); let (mut native, mut exits) = (0usize, 0usize); for k in top..=pc { - if native_op(ins[k].op) || helped[k] { + // An attribute read stays in native code only when the core + // loop has seen it read a split-layout field or a natively + // served instance's field; a class attribute, a module's or a + // method leaves on every run. + let attr_exits = ins[k].op == OpCode::LoadAttr + && !(crate::field_site(ext, k) || crate::native_site(ext, k)); + if (native_op(ins[k].op) && !attr_exits) || helped[k] { native += 1; } else { exits += 1; diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index f4dbcfdf..d132ddc9 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -12744,7 +12744,7 @@ impl Interpreter { // A natively served instance's public field (see // `stdlib::datetime_native`). if inst.cls_raw().native_kind.get() != 0 { - match Self::core_native_field(ext, inst, ins.arg) { + match Self::core_native_field(ext, inst, ins.arg, pc, ninstrs) { Some(v) if Self::core_droppable(unsafe { &*top }) => { // SAFETY: the receiver (droppable) is // replaced in place. @@ -16193,17 +16193,23 @@ impl Interpreter { } /// A natively served instance's public field (`dt.hour`; see - /// [`crate::stdlib::datetime_native::leaf_field`]), or `None`. + /// [`crate::stdlib::datetime_native::leaf_field`]), or `None`. A read + /// marks the site at `pc` native (see [`native_site`]). #[inline(never)] fn core_native_field( ext: Option<&CodeConstObjects>, inst: &PyInstance, name_idx: u32, + pc: usize, + ninstrs: usize, ) -> Option { - let Object::Str(name) = ext?.name_objs.get(name_idx as usize)? else { + let ext = ext?; + let Object::Str(name) = ext.name_objs.get(name_idx as usize)? else { return None; }; - crate::stdlib::datetime_native::leaf_field(inst, name) + let v = crate::stdlib::datetime_native::leaf_field(inst, name)?; + native_site_note(ext, ninstrs, pc); + Some(v) } /// A module attribute through the `LOAD_ATTR` site's cached index: @@ -63156,6 +63162,15 @@ fn native_site_note(ext: &CodeConstObjects, ninstrs: usize, pc: usize) { } } +/// Whether the `LOAD_ATTR` at `pc` has recorded a split-layout field (see +/// [`FieldSlot`]). +pub(crate) fn field_site(ext: &CodeConstObjects, pc: usize) -> bool { + ext.field_slots + .get() + .and_then(|s| s.get(pc)) + .is_some_and(|s| s.get().0 != 0) +} + /// Whether the operator at `pc` has run on a natively served operand. pub(crate) fn native_site(ext: &CodeConstObjects, pc: usize) -> bool { ext.native_sites From e7d5aa33aba4ede8608dce498be851f2ea40e7ae Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:53:50 -0700 Subject: [PATCH 80/96] fix: park a frame that caught an exception through the full path `core_return`'s in-line return parked every clean-stack activation through `inline_park_clean`, including one that caught an exception and so had its handler state set. That path skips resetting the frame's handler, saved-exception, and frame-object fields, and its debug assertion failed in every debug build that imported the standard library. Such a frame now goes through `inline_park`, as `inline_deliver` already does. --- crates/weavepy-vm/src/lib.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index d132ddc9..8a8621a7 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -15756,7 +15756,13 @@ impl Interpreter { // SAFETY: the callable is moved out exactly once; the parked // slot treats the field as stale (see `inline_deliver`). let callable = unsafe { std::ptr::read(&raw const done.callable) }; - self.inline_park_clean(done); + // A frame that caught an exception carries the handler path's + // state, which only the full park resets. + if had_shell { + self.inline_park(done); + } else { + self.inline_park_clean(done); + } // SAFETY: as above. unsafe { push_fast(&mut (*cframe).stack, v) }; match callable { From 78c5a0eea374043dda9f6280ec480a3d278b4517 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 00:53:50 -0700 Subject: [PATCH 81/96] fix: avoid an unsigned underflow in the regex prefix search When the literal prefix is skipped in full, `prefix_len - prefix_skip - 1` is -1. CPython computes it with signed sizes; the Rust port overflowed, which panicked in debug builds (release builds wrapped to the right value). --- crates/weavepy-vm/src/stdlib/sre_mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/weavepy-vm/src/stdlib/sre_mod.rs b/crates/weavepy-vm/src/stdlib/sre_mod.rs index 43954ffe..b7d2d7c9 100644 --- a/crates/weavepy-vm/src/stdlib/sre_mod.rs +++ b/crates/weavepy-vm/src/stdlib/sre_mod.rs @@ -2149,7 +2149,8 @@ impl<'a, C: SreChar> Matcher<'a, C> { // A potential match. let start = ptr - (prefix_len - 1); self.start = start; - self.ptr = ptr - (prefix_len - prefix_skip - 1); + // (`prefix_skip` may be the whole prefix.) + self.ptr = ptr + 1 + prefix_skip - prefix_len; if flags & SRE_INFO_LITERAL != 0 { return Ok(Some(start)); } From 7c937712bd754bcbfc51705a999a77c6a6735423 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:57:11 -0700 Subject: [PATCH 82/96] fix: render OSError as `[Errno N] strerror` in unraisable reports Natively built exceptions no longer store a separate message, so the report's Rust-side rendering fell back to `args[0]` and printed a failed system call's error as `OSError: 9`. `exception_message` now uses `OSError.__str__`'s errno form, shared with `oserror_str`, which fixes `test_signal`'s wakeup-fd write report. --- crates/weavepy-vm/src/builtin_types.rs | 49 ++++++++++++---------- tests/regrtest/test_unraisable_format.py | 52 ++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 tests/regrtest/test_unraisable_format.py diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index e0bb50ba..49e3257f 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -3908,29 +3908,10 @@ fn install_os_error_init(os_error: &Rc) { let Some(Object::Instance(inst)) = args.first() else { return Ok(Object::from_static("")); }; - let get = |name: &'static str| exc_attr(inst, name); - let set = |o: &Option| matches!(o, Some(v) if !matches!(v, Object::None)); - let errno = get("errno"); - let strerror = get("strerror"); - let filename = get("filename"); - let filename2 = get("filename2"); - let errno_s = errno.as_ref().map(Object::to_str).unwrap_or_default(); - let strerror_s = strerror.as_ref().map(Object::to_str).unwrap_or_default(); - if set(&filename) { - let f1 = filename.as_ref().map(Object::repr).unwrap_or_default(); - if set(&filename2) { - let f2 = filename2.as_ref().map(Object::repr).unwrap_or_default(); - return Ok(Object::from_str(format!( - "[Errno {errno_s}] {strerror_s}: {f1} -> {f2}" - ))); - } - return Ok(Object::from_str(format!( - "[Errno {errno_s}] {strerror_s}: {f1}" - ))); - } - if set(&errno) && set(&strerror) { - return Ok(Object::from_str(format!("[Errno {errno_s}] {strerror_s}"))); + if let Some(m) = oserror_fields_message(inst) { + return Ok(Object::from_str(m)); } + let get = |name: &'static str| exc_attr(inst, name); // BaseException.__str__: "" / str(arg) / repr(args). match get("args") { Some(Object::Tuple(items)) => Ok(match &items[..] { @@ -5659,6 +5640,11 @@ pub fn exception_message(obj: &Object) -> Option { if let Some(Object::Str(s)) = exc_attr(inst, "message") { return Some(s.to_string()); } + if inst.cls().is_subclass_of(&builtin_types().os_error) { + if let Some(m) = oserror_fields_message(inst) { + return Some(m); + } + } if let Some(Object::Tuple(items)) = exc_attr(inst, "args") { // `KeyError.__str__` renders a lone key by its repr. if let [single] = &items[..] { @@ -5676,6 +5662,25 @@ pub fn exception_message(obj: &Object) -> Option { } } +/// `OSError.__str__`'s `[Errno N] strerror[: filename[ -> filename2]]` +/// rendering, when the named fields that shape needs are set (see +/// `oserror_str`). +fn oserror_fields_message(inst: &crate::types::PyInstance) -> Option { + let get = |name: &'static str| exc_attr(inst, name).filter(|v| !matches!(v, Object::None)); + let errno = get("errno"); + let strerror = get("strerror"); + let errno_s = errno.as_ref().map(Object::to_str).unwrap_or_default(); + let strerror_s = strerror.as_ref().map(Object::to_str).unwrap_or_default(); + if let Some(f1) = get("filename") { + let f1 = f1.repr(); + return Some(match get("filename2") { + Some(f2) => format!("[Errno {errno_s}] {strerror_s}: {f1} -> {}", f2.repr()), + None => format!("[Errno {errno_s}] {strerror_s}: {f1}"), + }); + } + (errno.is_some() && strerror.is_some()).then(|| format!("[Errno {errno_s}] {strerror_s}")) +} + /// `True` when `obj` is an instance whose class derives from `cls`. pub fn instance_is_subclass(obj: &Object, cls: &TypeObject) -> bool { match obj { diff --git a/tests/regrtest/test_unraisable_format.py b/tests/regrtest/test_unraisable_format.py new file mode 100644 index 00000000..2d679f52 --- /dev/null +++ b/tests/regrtest/test_unraisable_format.py @@ -0,0 +1,52 @@ +"""The default unraisable report renders exceptions as `str()` does. + +Exceptions built natively don't store a separate message, so the report +must derive it the way each class's `__str__` does: an `OSError` from a +failed system call reads `[Errno N] strerror`, not its bare errno. +""" + +import errno +import os +import unittest +from test.support import captured_stderr + + +class UnraisableFormat(unittest.TestCase): + def report(self, exc_source): + class Holder: + def __del__(self): + exc_source() + + with captured_stderr() as err: + Holder() + return err.getvalue() + + def test_os_error_from_a_system_call(self): + def close_bad(): + os.close(-1) + + out = self.report(close_bad) + self.assertIn("Exception ignored", out) + self.assertIn("OSError: [Errno %d] " % errno.EBADF, out) + + def test_os_error_with_a_filename(self): + def open_missing(): + open("/nonexistent/weavepy-unraisable") + + out = self.report(open_missing) + self.assertIn("FileNotFoundError: [Errno %d] " % errno.ENOENT, out) + self.assertIn("'/nonexistent/weavepy-unraisable'", out) + + def test_key_error_and_plain_message(self): + def key(): + {}["missing"] + + def value(): + raise ValueError("bad value") + + self.assertIn("KeyError: 'missing'", self.report(key)) + self.assertIn("ValueError: bad value", self.report(value)) + + +if __name__ == "__main__": + unittest.main() From 550fe923e55614eae1f243205778b7805912c934 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:25:05 -0700 Subject: [PATCH 83/96] fix: let threads and signals in after the quiet loop catches an exception The quiet loop resumed a handler without the per-instruction prologue's GIL countdown and hot-gate check. Code that only calls and catches, such as recursion retried from its `RecursionError` handler, then never gave up the GIL, so `_thread.interrupt_main()` from another thread never landed and `test_threading` timed out. Every catch site now takes the prologue's checks, as the full step's already did. --- crates/weavepy-vm/src/lib.rs | 30 ++++++++++- .../regrtest/test_interrupt_while_catching.py | 50 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/regrtest/test_interrupt_while_catching.py diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 8a8621a7..1dd46cee 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -7463,6 +7463,20 @@ impl Interpreter { } } + /// After the quiet loop caught an exception: whether the activation + /// yields as the outer loop's per-instruction prologue would (the GIL + /// countdown, a hot gate). Code that only calls and catches, such as + /// recursion retried in its `RecursionError` handler, would otherwise + /// never let another thread or a signal in. + #[inline] + fn quiet_caught_yields(&mut self, snap_gen: u64) -> bool { + if self.gil_countdown <= 2 { + return true; + } + self.gil_countdown -= 1; + crate::hot_gates::loop_gen() != snap_gen + } + /// One activation's stretch of the quiet loop (see /// [`Self::quiet_run`]): runs `frame` from `entry` until it makes an /// inline call or exits. @@ -7506,7 +7520,12 @@ impl Interpreter { } } QuietEntry::Raised { err, cur_pc } => match self.quiet_catch(frame, shell, err) { - Ok(()) => *last = cur_pc, + Ok(()) => { + *last = cur_pc; + if self.quiet_caught_yields(snap_gen) { + break 'run QuietExit::Yield; + } + } Err(err) => { // The raise path drops the staged operands wholesale // (see the `CALL` arm of `step`). @@ -7540,6 +7559,9 @@ impl Interpreter { let err = match self.quiet_catch(frame, shell, err) { Ok(()) => { *last = cur_pc; + if self.quiet_caught_yields(snap_gen) { + break 'run QuietExit::Yield; + } continue; } Err(err) => err, @@ -7627,6 +7649,9 @@ impl Interpreter { let err = match self.quiet_catch(frame, shell, err) { Ok(()) => { *last = cur_pc; + if self.quiet_caught_yields(snap_gen) { + break 'run QuietExit::Yield; + } continue; } Err(err) => err, @@ -7698,6 +7723,9 @@ impl Interpreter { let err = match self.quiet_catch(frame, shell, err) { Ok(()) => { *last = cur_pc; + if self.quiet_caught_yields(snap_gen) { + break 'run QuietExit::Yield; + } continue; } Err(err) => err, diff --git a/tests/regrtest/test_interrupt_while_catching.py b/tests/regrtest/test_interrupt_while_catching.py new file mode 100644 index 00000000..1ee4ede2 --- /dev/null +++ b/tests/regrtest/test_interrupt_while_catching.py @@ -0,0 +1,50 @@ +"""An interrupt reaches code that only calls and catches exceptions. + +Recursion retried from its own `RecursionError` handler never returns +normally and has no loop, so the interpreter must still give up the GIL +and look for signals when it catches an exception (gh-102056's shape, +from `test_threading`). +""" + +import subprocess +import sys +import unittest + +SCRIPT = r""" +import time +import threading +import _thread + +def f(): + try: + f() + except RecursionError: + f() + +def h(): + time.sleep(0.5) + _thread.interrupt_main() + +t = threading.Thread(target=h) +t.start() +try: + f() +except KeyboardInterrupt: + print("interrupted") +t.join() +""" + + +class InterruptWhileCatching(unittest.TestCase): + def test_interrupt_main_lands(self): + proc = subprocess.run( + [sys.executable, "-c", SCRIPT], + capture_output=True, + text=True, + timeout=45, + ) + self.assertIn("interrupted", proc.stdout, proc.stderr[-2000:]) + + +if __name__ == "__main__": + unittest.main() From 46a97fc167170e1a67004f96d01326fada0adc24 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:32:03 -0700 Subject: [PATCH 84/96] docs: report the branch's final numbers against CPython and main The parity report now carries the bench gate's A/B results against `main` and CPython 3.14.8, current import and memory costs, per-operation costs for what still trails, and the validation the branch passed. --- docs/PERFORMANCE-CPYTHON-PARITY.md | 191 +++++++++++++++++------------ 1 file changed, 112 insertions(+), 79 deletions(-) diff --git a/docs/PERFORMANCE-CPYTHON-PARITY.md b/docs/PERFORMANCE-CPYTHON-PARITY.md index 3c922af1..cf41bcf5 100644 --- a/docs/PERFORMANCE-CPYTHON-PARITY.md +++ b/docs/PERFORMANCE-CPYTHON-PARITY.md @@ -10,17 +10,13 @@ the reports it links to. ## Method -Timings compare a profile-guided release build of the branch -(`tools/pgo_build.py --skip-regrtest`, JIT on) with CPython 3.14.7 on the -macOS x86-64 development host (Intel Core i9-9980HK). Both executables are -native x86-64. - -Wall-time ratios come from paired, interleaved runs. Each cycle runs every -fixture once under each interpreter at the harness work sizes, alternating -which goes first; one warmup cycle is discarded and five are measured. A -fixture's ratio is the median of its per-cycle WeavePy/CPython ratios, using -each fixture's own timer (`WEAVEPY_BENCH_NS`), which excludes startup and -imports. Values below 1.00 mean WeavePy is faster. +Timings come from the CI bench gate (`weavepy-bench gate`), run on the +macOS x86-64 development host (Intel Core i9-9980HK). It compares the +branch's release build with a release build of `main` (the merge base) and +with CPython 3.14.8, interleaving the three interpreters' runs on the same +machine. Each fixture's time is the median of its samples, measured by the +fixture's own timer (`WEAVEPY_BENCH_NS`), which excludes startup and +imports. Values below 1.00 mean the branch is faster. Instruction counts come from `/usr/bin/time -l` ("instructions retired") on the host, or from callgrind in a Linux container when a per-function or @@ -32,57 +28,76 @@ checkpoint confirms them. ## Benchmark fixtures -Geometric mean over the 23 timed fixtures: **0.592**. - -| Fixture | Work | WeavePy | CPython | Ratio | -| --- | ---: | ---: | ---: | ---: | -| `deltablue` | 50 | 213.8 ms | 67.8 ms | 3.15 | -| `deque_ops` | 200,000 | 97.9 ms | 64.1 ms | 1.53 | -| `pickle_bench` | 40 | 11.6 ms | 8.5 ms | 1.34 | -| `generators` | 300,000 | 59.3 ms | 49.6 ms | 1.19 | -| `datetime_ops` | 60,000 | 50.7 ms | 52.7 ms | 0.98 | -| `str_methods` | 15,000 | 57.0 ms | 59.9 ms | 0.95 | -| `float_math` | 100,000 | 78.0 ms | 81.6 ms | 0.94 | -| `dict_ops` | 100,000 | 55.9 ms | 61.7 ms | 0.90 | -| `call_overhead` | 150,000 | 69.8 ms | 82.2 ms | 0.84 | -| `fannkuch` | 100,000 | 17.1 ms | 20.9 ms | 0.83 | -| `richards` | 50,000 | 16.1 ms | 20.3 ms | 0.80 | -| `nbody` | 20,000 | 33.0 ms | 42.9 ms | 0.77 | -| `attr_access` | 200,000 | 35.4 ms | 47.1 ms | 0.77 | -| `list_ops` | 10,000 | 33.8 ms | 45.1 ms | 0.75 | -| `json_bench` | 150 | 57.4 ms | 81.1 ms | 0.71 | -| `pidigits` | 500,000 | 1,421.0 ms | 2,167.8 ms | 0.67 | -| `pyaes` | 400 | 15.8 ms | 28.9 ms | 0.54 | -| `jitkernels` | 2,000 | 24.3 ms | 49.0 ms | 0.48 | -| `fib` | 27 | 8.0 ms | 21.6 ms | 0.37 | -| `spectral_norm` | 100 | 12.1 ms | 57.7 ms | 0.21 | -| `nested_loops` | 120 | 6.2 ms | 69.3 ms | 0.09 | -| `jitloop` | 1,000 | 7.5 ms | 92.0 ms | 0.08 | -| `sumvm` | 2,000,000 | 3.8 ms | 72.5 ms | 0.05 | - -Times are the medians of each interpreter's samples; ratios are the medians -of the paired ratios, so they needn't equal the quotient of the two times. +Geometric mean over the 24 fixtures: **0.50** of CPython's time and +**0.75** of `main`'s. + +| Fixture | Work | Branch | `main` | CPython | vs `main` | vs CPython | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `deltablue` | 50 | 244.3 ms | 292.5 ms | 80.5 ms | 0.83 | 3.03 | +| `generators` | 300,000 | 64.0 ms | 76.8 ms | 58.0 ms | 0.83 | 1.10 | +| `dict_ops` | 100,000 | 68.1 ms | 82.2 ms | 64.2 ms | 0.83 | 1.06 | +| `pickle_bench` | 40 | 9.7 ms | 15.9 ms | 10.2 ms | 0.61 | 0.95 | +| `float_math` | 100,000 | 86.8 ms | 112.8 ms | 92.5 ms | 0.77 | 0.94 | +| `deque_ops` | 200,000 | 65.7 ms | 134.5 ms | 74.7 ms | 0.49 | 0.88 | +| `json_bench` | 150 | 76.8 ms | 93.3 ms | 96.8 ms | 0.82 | 0.79 | +| `list_ops` | 10,000 | 36.4 ms | 38.7 ms | 46.0 ms | 0.94 | 0.79 | +| `startup` | 1 | 19.4 ms | 20.6 ms | 24.5 ms | 0.94 | 0.79 | +| `fannkuch` | 100,000 | 16.8 ms | 24.4 ms | 22.4 ms | 0.69 | 0.75 | +| `nbody` | 20,000 | 37.9 ms | 50.1 ms | 50.9 ms | 0.76 | 0.74 | +| `str_methods` | 15,000 | 48.7 ms | 83.1 ms | 70.5 ms | 0.59 | 0.69 | +| `call_overhead` | 150,000 | 67.3 ms | 109.3 ms | 97.5 ms | 0.62 | 0.69 | +| `attr_access` | 200,000 | 37.8 ms | 64.2 ms | 57.0 ms | 0.59 | 0.66 | +| `pyaes` | 400 | 19.5 ms | 19.1 ms | 30.4 ms | 1.02 | 0.64 | +| `pidigits` | 500,000 | 1,580 ms | 1,680 ms | 2,570 ms | 0.94 | 0.61 | +| `richards` | 50,000 | 11.7 ms | 20.3 ms | 21.5 ms | 0.58 | 0.55 | +| `datetime_ops` | 60,000 | 30.7 ms | 84.2 ms | 60.8 ms | 0.36 | 0.51 | +| `jitkernels` | 2,000 | 21.6 ms | 34.9 ms | 58.8 ms | 0.62 | 0.37 | +| `fib` | 27 | 8.9 ms | 9.3 ms | 24.6 ms | 0.96 | 0.36 | +| `spectral_norm` | 100 | 12.8 ms | 13.3 ms | 63.9 ms | 0.97 | 0.20 | +| `nested_loops` | 120 | 7.3 ms | 7.4 ms | 86.2 ms | 0.99 | 0.08 | +| `jitloop` | 1,000 | 8.7 ms | 8.9 ms | 111.7 ms | 0.99 | 0.08 | +| `sumvm` | 2,000,000 | 4.3 ms | 4.4 ms | 85.8 ms | 0.99 | 0.05 | + +The largest gains over `main` came from these changes: + +- **A baseline frame compiler** (`frame_jit`) runs loops that tier 2 + declines, such as loops of method calls and container operations, in + native code. It compiles a loop only when most of its instructions stay + native, since every instruction that leaves for the core loop costs a + round trip. +- **Native containers and values.** `collections.deque`, `datetime` values, + `pickle`, compiled regular expressions and their matches, and `map` and + `filter` over native containers run natively instead of through their + pure-Python implementations. +- **Cheaper calls and exceptions.** Small functions that native code calls + (`key=lambda t: t[1]`) run without an activation, tracebacks are built + only when something reads them, and the core loop catches exceptions + without leaving. +- **Class bodies and metaclass reads.** Class creation and metaclass + attribute reads no longer pass through the general call path. ## Startup and imports -Whole-process instructions retired, warm stdlib cache: - -| Command | WeavePy | CPython | -| --- | ---: | ---: | -| `-c pass` | 91M | 127M | -| `import json` | 205M | 153M | -| `import dataclasses` | 349M | 196M | -| `import logging` | 513M | 258M | -| `import asyncio` | 689M | 365M | -| `import unittest` | 496M | 270M | +Whole-process instructions retired and peak memory, warm stdlib cache: -Startup is cheaper than CPython's. Importing large parts of the standard -library still costs about twice as much. Three changes on this branch cut -those costs by 20% to 60%: +| Command | WeavePy | CPython | WeavePy RSS | CPython RSS | +| --- | ---: | ---: | ---: | ---: | +| `-c pass` | 77M | 103M | 11.7 MiB | 10.5 MiB | +| `import json` | 177M | 141M | 24.3 MiB | 12.6 MiB | +| `import dataclasses` | 282M | 183M | 32.9 MiB | 13.9 MiB | +| `import logging` | 421M | 241M | 38.8 MiB | 16.0 MiB | +| `import asyncio` | 577M | 350M | 48.0 MiB | 19.1 MiB | +| `import unittest` | 407M | 253M | 40.4 MiB | 16.0 MiB | + +Startup is cheaper than CPython's, and imports retire 14% to 19% fewer +instructions than when this report was first written. Importing large parts +of the standard library still costs 1.3 to 1.8 times as much as in CPython. +These changes did most of the work: - Compiled stdlib modules are cached in a WeavePy-native code format - (`weavepy_compiler::native_code`), which loads without unmarshalling CPython - bytecode and transcoding it back into WeavePy instructions. + (`weavepy_compiler::native_code`, now version 3), which loads without + unmarshalling CPython bytecode and transcoding it back into WeavePy + instructions. Column tables stay encoded until something reads them. - Slicing a tuple, `bytes`, `bytearray`, or a string with surrogates no longer copies the whole source sequence. `re.compile` with `IGNORECASE` slices a 64K-entry charset map 256 times, which had made `import logging` cost 1.19B @@ -90,29 +105,39 @@ those costs by 20% to 60%: - The collector's per-drop sweep of suspected-dead objects skips entries that have used up their probe budget, so it no longer walks up to 256 entries at every drop during imports. +- The opcode shim builds its stack-effect tables on first use, and the frame + compiler stays within tier 2's start-up and import budget. -Peak memory after these imports is about twice CPython's (for example, -23 MiB against 12 MiB for `import json`). An allocation-site profile -attributes most of the difference to compiled code objects: their per-code -structure, per-instruction line and column tables, and the materialized -constants and names built on first use. +Peak memory is where WeavePy trails most: two to two and a half times +CPython's after a large import, and up to twice CPython's on the fixtures. An +allocation-site profile attributes most of the difference to compiled code +objects: their per-code structure, per-instruction line tables, and the +materialized constants and names built on first use. ## What still trails CPython -The four slower fixtures share a cause: operations that CPython's specialized -bytecodes do in 10 to 30 instructions take WeavePy two to four times as many. -Per operation, with the JIT off: +`deltablue` (3.03) and `generators` (1.10) are still slower than CPython, +and `dict_ops` is at parity (between 0.92 and 1.06 across runs). They +spend their time in operations that CPython's specialized bytecodes do in +a few dozen instructions. Per operation, with +the default settings, measured in a loop that tier 2 compiles: | Operation | WeavePy | CPython | | --- | ---: | ---: | -| Instance attribute read (`o.x`) | 160 | 64 | -| Class attribute read through an instance (`o.K`) | 240 | 60 | -| Global read | 150 | 50 | -| `isinstance(o, C)` | 980 | 240 | -| Call of a small non-leaf function | 1,700 | 440 | +| Instance attribute read (`o.x`) | 64 | 37 | +| Class attribute read through an instance (`o.K`) | 357 | 36 | +| Global read | under 5 | 20 | +| `isinstance(o, C)` | 1,635 | 271 | +| Call of a small non-leaf function | 1,389 | 763 | + +Global reads and the arithmetic around them compile to native code. The +rest leave compiled code for a general helper: a class attribute read +through an instance resolves through the class on every read, and +`isinstance` and calls to Python functions go through the generic call +path. Teaching tier 2 and the frame compiler those three shapes is the +next step for `deltablue`. -The costs come from the interpreter's structure rather than from any single -slow path: +The interpreter's structure adds to these costs where nothing compiles: - **Loop state.** The core loop keeps about 17 values live across its arms, which contain calls; x86-64 has six callee-saved registers, so most of the @@ -131,11 +156,19 @@ branches predict poorly. ## Validation -Every change on the branch passes the VM unit tests (420), the bundled -regression suite (248), the semantics comparison scripts, and the CPython -regression suites most exposed to it. The changes described above were -checked with, among others, `test_bytes`, `test_tuple`, `test_slice`, -`test_str`, `test_re`, `test_marshal`, `test_import`, `test_code`, -`test_compile`, `test_logging`, `test_gc`, `test_weakref`, `test_io`, -`test_ssl`, `test_asyncio`, `test_scope`, `test_global`, `test_module`, -`test_builtin`, `test_sys_settrace`, and `test_monitoring`. +The branch passes the workspace's unit, integration, and doc tests, the +bundled regression suite (272 tests), its free-threaded lane, the tools' +tests, and the bench gate against `main` with no fixture regressing. + +The curated CPython 3.14 gate (616 suites, among them `test_re`, +`test_pickle`, `test_deque`, `test_datetime`, `test_generators`, +`test_exceptions`, `test_threading`, `test_signal`, `test_asyncio`, and +`test_sys_settrace`) passes except where `main` fails the same way on the +development host: `test_capi`, `test_genexps`, `test_tempfile`, and +`test_utf8_mode` fail there too, and the `multiprocessing` spawn and +forkserver suites time out on both. + +New regression tests cover the frame compiler, frameless subscripts, +native `map` and `filter`, packed `datetime` values, deque lanes in +compiled code, unraisable `OSError` reports, and interrupts that arrive +while code only calls and catches. From d0b76ef3b05f7098b291aea4bc1cb8ae28a5d47f Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 10:04:03 -0700 Subject: [PATCH 85/96] ci: cap lints when building the bench gate's merge-base reference The A/B gate builds the merge-base with the workflow's `-D warnings`, so a lint a newer stable toolchain added fails the reference build before anything is timed: Rust 1.99 deprecates `std::u64`, which `main`'s vendored lzma-sys still uses. The reference binary is only timed, so its build now caps lints at warnings. --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e7da213..b0c881ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,7 +231,13 @@ jobs: # rust-cache still primes dependency compilation, and the # merge-base is usually a recent main commit CI has built # before, so this step is mostly cache hits. + # + # The reference binary is only timed, so a lint that a newer + # toolchain added (one the merge-base predates) mustn't fail + # the build: lints are capped at warnings here. shell: bash + env: + RUSTFLAGS: "--cap-lints=warn" run: | BASE_SHA=$(git merge-base "${{ github.event.pull_request.base.sha }}" HEAD) echo "merge-base: $BASE_SHA" From e478a16bd93293a1f36d9e882c05a5312adbc850 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 11:55:24 -0700 Subject: [PATCH 86/96] fix: drop an invalid extension from the plan JIT's in-line method cache `inline_method` widened the split block's length with `uextend.i64`, but `uload32` already yields an `i64`. Release builds skip Cranelift's verifier; x86-64 tolerated the instruction, but AArch64 encoded an invalid one, so compiled plans with in-line method loads died with SIGILL on macOS ARM (`test_jit_plan_caches`, `deltablue`). A new regression test runs the plan-cache tests with both JIT verifiers on. --- crates/weavepy-vm/src/leaf_plan/jit.rs | 2 +- tests/regrtest/test_jit_ir_verifies.py | 39 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/regrtest/test_jit_ir_verifies.py diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs index 4e80e1ee..904d1b54 100644 --- a/crates/weavepy-vm/src/leaf_plan/jit.rs +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -1255,8 +1255,8 @@ impl Lower<'_> { self.b.switch_to_block(fields); let keys = self.b.ins().load(ptr, f, block, l.split_keys); let ckeys = self.b.ins().load(ptr, f, cls, l.type_shared_keys); + // (`uload32` zero-extends to `i64` itself.) let len = self.b.ins().uload32(f, block, l.split_len); - let len = self.b.ins().uextend(types::I64, len); let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); let shadows = self.b.ins().icmp(IntCC::UnsignedGreaterThan, len, limit); let bad = self.b.ins().bor(foreign, shadows); diff --git a/tests/regrtest/test_jit_ir_verifies.py b/tests/regrtest/test_jit_ir_verifies.py new file mode 100644 index 00000000..10397b49 --- /dev/null +++ b/tests/regrtest/test_jit_ir_verifies.py @@ -0,0 +1,39 @@ +"""Compiled plans and frames pass Cranelift's IR verifier. + +Release builds skip the verifier, and some invalid IR (an extension to +the type a value already has) still compiles on x86-64 but encodes an +invalid instruction on AArch64. This runs the plan-cache tests, which +exercise the in-line method, class-attribute and global caches, with +both verifiers on, and fails if any function didn't compile. +""" + +import os +import subprocess +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +class IrVerifies(unittest.TestCase): + def test_plan_caches_compile_verified(self): + script = os.path.join(HERE, "test_jit_plan_caches.py") + if not os.path.exists(script): + self.skipTest("test_jit_plan_caches.py isn't beside this test") + env = dict(os.environ) + env["WEAVEPY_PLAN_VERIFY"] = "1" + env["WEAVEPY_FRAME_JIT_VERIFY"] = "1" + proc = subprocess.run( + [sys.executable, script], + capture_output=True, + text=True, + env=env, + timeout=120, + ) + self.assertEqual(proc.returncode, 0, proc.stderr[-3000:]) + self.assertNotIn("failed to compile", proc.stderr, proc.stderr[-3000:]) + self.assertNotIn("verifier error", proc.stderr, proc.stderr[-3000:]) + + +if __name__ == "__main__": + unittest.main() From 904f4a963dd41e986019a1e96f303387889540b1 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:58:38 -0700 Subject: [PATCH 87/96] ci: diagnose call_overhead when the Windows bench gate fails The Windows gate has measured `call_overhead` at 1.2x the merge-base on every PR run, while the same build, environment, and gate pass at 0.65x on separately dispatched runners. `bench_cold_jit.py` now takes its fixture list as an option, and the failure-only Windows diagnostic adds `call_overhead`, so the next failure records its timings and JIT traces from inside the failing job. --- .github/workflows/ci.yml | 1 + tools/bench_cold_jit.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0c881ed..c9b2b23f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -269,6 +269,7 @@ jobs: python tools/bench_cold_jit.py \ --base "$BASE_WEAVEPY" --new target/release/weavepy.exe \ --samples 7 \ + --fixtures sumvm:2000000,nested_loops:120,jitloop:1000,call_overhead:150000 \ --out "target/performance/cold-jit-ci/${{ github.run_id }}-${{ github.run_attempt }}/report.json" - uses: actions/upload-artifact@v4 if: always() && steps.cold_jit_diagnostic.outcome != 'skipped' && steps.cold_jit_diagnostic.outcome != '' diff --git a/tools/bench_cold_jit.py b/tools/bench_cold_jit.py index 839b2b80..a09b074c 100644 --- a/tools/bench_cold_jit.py +++ b/tools/bench_cold_jit.py @@ -89,6 +89,8 @@ def main(): parser.add_argument("--fixture-root", type=Path, default=Path("crates/weavepy-bench/fixtures")) parser.add_argument("--samples", type=int, default=7) parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--fixtures", default="sumvm:2000000,nested_loops:120,jitloop:1000", + help="comma-separated NAME:WORK pairs to diagnose") args = parser.parse_args() if args.samples < 1: parser.error("samples must be positive") @@ -107,7 +109,9 @@ def main(): "environment": {name: os.environ.get(name) for name in ("PYTHONHASHSEED", "PYTHONDONTWRITEBYTECODE", "PYTHON_JIT", "PYTHON_GIL", "WEAVEPY_STDLIB_CACHE")}} args.out.write_text(json.dumps(report, indent=2) + "\n") - for name, work in [("sumvm", 2000000), ("nested_loops", 120), ("jitloop", 1000)]: + fixtures = [(name, int(work)) for name, work in + (item.split(":") for item in args.fixtures.split(",") if item)] + for name, work in fixtures: path = (args.fixture_root / (name + ".py")).resolve(strict=True) row = {"work": work, "fixture": str(path), "sha256": digest(path), "modes": {}} for warm in (False, True): From bbd274b18a355c88ac8530f7850bfc5ba1c14c9d Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:42:50 -0700 Subject: [PATCH 88/96] ci: time call_overhead directly when the Windows bench gate fails In the last failing run, the gate timed the PR binary's `call_overhead` at 89 ms while the cold-JIT diagnostic, in the same job, timed it at 38 ms (the merge-base and CPython agreed between the two at 60 ms). A failure-only step now times direct runs that each change one thing the two differ in: the environment, the caches, and the binary path. --- .github/workflows/ci.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9b2b23f..6ba1aa71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -260,6 +260,32 @@ jobs: target/release/weavepy-bench gate $FLAGS \ --weavepy=target/release/weavepy \ | tee bench-report.md + - name: Time call_overhead the way the gate runs it + # The gate has timed the PR binary's `call_overhead` at 1.2-1.5x + # the merge-base on every Windows PR run, while direct runs on the + # same runner time it at 0.6x. This separates the differences + # between the two (environment, caches, binary path). + if: failure() && matrix.os == 'windows-latest' && github.event_name == 'pull_request' && steps.bench_gate.outcome == 'failure' + timeout-minutes: 5 + shell: bash + run: | + F=crates/weavepy-bench/fixtures/call_overhead.py + export WEAVEPY_BENCH_WORK=150000 + unset WEAVEPY_JIT + for label in gate-like jit=1 own-caches exe-path base; do + for i in 1 2 3; do + case $label in + gate-like) out=$(target/release/weavepy $F) ;; + jit=1) out=$(WEAVEPY_JIT=1 target/release/weavepy $F) ;; + own-caches) out=$(WEAVEPY_FROZEN_CACHE="$RUNNER_TEMP/fz" WEAVEPY_STDLIB_CACHE="$RUNNER_TEMP/sc" target/release/weavepy $F) ;; + exe-path) out=$(target/release/weavepy.exe $F) ;; + base) out=$("$BASE_WEAVEPY" $F) ;; + esac + echo "$label: $out" + done + done + echo "--- gate-like trace" + WEAVEPY_JIT_TRACE=1 target/release/weavepy $F 2>&1 | tail -30 - name: Diagnose cold Windows JIT costs after a failed gate id: cold_jit_diagnostic if: failure() && matrix.os == 'windows-latest' && github.event_name == 'pull_request' && steps.bench_gate.outcome == 'failure' From aec04c0f77f21cb32d955c6798416820c57cef09 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:35:14 -0700 Subject: [PATCH 89/96] ci: time call_overhead through the bench harness when the Windows gate fails Direct runs in the failing job time the PR binary's `call_overhead` at 0.65x the merge-base while the gate times it at 1.2x, whatever the environment, caches, or binary path. The failure-only step now also times it with an absolute fixture path, launched from Python, and through `weavepy-bench run` with each binary alone. --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ba1aa71..f9e4b7df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -266,7 +266,7 @@ jobs: # same runner time it at 0.6x. This separates the differences # between the two (environment, caches, binary path). if: failure() && matrix.os == 'windows-latest' && github.event_name == 'pull_request' && steps.bench_gate.outcome == 'failure' - timeout-minutes: 5 + timeout-minutes: 15 shell: bash run: | F=crates/weavepy-bench/fixtures/call_overhead.py @@ -284,8 +284,15 @@ jobs: echo "$label: $out" done done + echo "absolute-path: $(target/release/weavepy "$PWD/$F")" + echo "python-launched: $(python -c 'import subprocess,sys; print(subprocess.run([sys.argv[1], sys.argv[2]], capture_output=True, text=True, stdin=subprocess.DEVNULL).stdout.strip())' target/release/weavepy.exe "$PWD/$F")" echo "--- gate-like trace" WEAVEPY_JIT_TRACE=1 target/release/weavepy $F 2>&1 | tail -30 + for B in target/release/weavepy "$BASE_WEAVEPY"; do + echo "--- harness run, $B alone" + target/release/weavepy-bench run --weavepy="$B" --no-cpython --no-interp --samples=3 2>&1 \ + | grep -E "\| (call_overhead|attr_access|sumvm) \|" + done - name: Diagnose cold Windows JIT costs after a failed gate id: cold_jit_diagnostic if: failure() && matrix.os == 'windows-latest' && github.event_name == 'pull_request' && steps.bench_gate.outcome == 'failure' From 5d339980b0eb42e7194be24a9741b8fc7da8c3df Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 18:23:29 -0700 Subject: [PATCH 90/96] ci: compare absolute and relative fixture paths when the Windows gate fails The last failing run pinned the slowdown to the fixture path: the PR binary times `call_overhead` at 75 ms from a relative path and 138 ms from an absolute one, however it's launched, while the merge-base is unaffected. The failure-only step now separates the caches involved and records JIT and burst statistics for both paths. --- .github/workflows/ci.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9e4b7df..27ac7da6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -284,15 +284,19 @@ jobs: echo "$label: $out" done done - echo "absolute-path: $(target/release/weavepy "$PWD/$F")" - echo "python-launched: $(python -c 'import subprocess,sys; print(subprocess.run([sys.argv[1], sys.argv[2]], capture_output=True, text=True, stdin=subprocess.DEVNULL).stdout.strip())' target/release/weavepy.exe "$PWD/$F")" - echo "--- gate-like trace" - WEAVEPY_JIT_TRACE=1 target/release/weavepy $F 2>&1 | tail -30 - for B in target/release/weavepy "$BASE_WEAVEPY"; do - echo "--- harness run, $B alone" - target/release/weavepy-bench run --weavepy="$B" --no-cpython --no-interp --samples=3 2>&1 \ - | grep -E "\| (call_overhead|attr_access|sumvm) \|" + A="$PWD/$F" + echo "absolute-path: $(target/release/weavepy "$A")" + echo "absolute, fresh frozen cache: $(WEAVEPY_FROZEN_CACHE="$RUNNER_TEMP/fz2" target/release/weavepy "$A")" + echo "absolute, fresh stdlib cache: $(WEAVEPY_STDLIB_CACHE="$RUNNER_TEMP/sc2" target/release/weavepy "$A")" + echo "absolute, no frozen cache: $(WEAVEPY_FROZEN_CACHE=0 target/release/weavepy "$A")" + for P in "$F" "$A"; do + echo "--- trace for $P" + WEAVEPY_JIT_TRACE=1 WEAVEPY_FRAME_JIT_STATS=1 target/release/weavepy "$P" 2>&1 | grep -v " loop at " | tail -40 + echo "--- burst stats for $P" + WEAVEPY_BURST_STATS=1 target/release/weavepy "$P" 2>&1 | head -40 done + echo "--- default cache" + ls -la "$LOCALAPPDATA/weavepy" 2>&1 | head -20 - name: Diagnose cold Windows JIT costs after a failed gate id: cold_jit_diagnostic if: failure() && matrix.os == 'windows-latest' && github.event_name == 'pull_request' && steps.bench_gate.outcome == 'failure' From 89987689736330a05d88d25a389513629dab8190 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:12:02 -0700 Subject: [PATCH 91/96] ci: isolate the JIT tier behind the Windows absolute-path slowdown The slow configuration (an absolute fixture path with warm caches) has the same tier-2 compiles as the fast one, and any change to the caches or instrumentation makes it fast again. The failure-only step now runs that configuration with each tier switched off or reconfigured, twice each, to show which one it depends on. --- .github/workflows/ci.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27ac7da6..d295b49d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -286,15 +286,10 @@ jobs: done A="$PWD/$F" echo "absolute-path: $(target/release/weavepy "$A")" - echo "absolute, fresh frozen cache: $(WEAVEPY_FROZEN_CACHE="$RUNNER_TEMP/fz2" target/release/weavepy "$A")" - echo "absolute, fresh stdlib cache: $(WEAVEPY_STDLIB_CACHE="$RUNNER_TEMP/sc2" target/release/weavepy "$A")" - echo "absolute, no frozen cache: $(WEAVEPY_FROZEN_CACHE=0 target/release/weavepy "$A")" - for P in "$F" "$A"; do - echo "--- trace for $P" - WEAVEPY_JIT_TRACE=1 WEAVEPY_FRAME_JIT_STATS=1 target/release/weavepy "$P" 2>&1 | grep -v " loop at " | tail -40 - echo "--- burst stats for $P" - WEAVEPY_BURST_STATS=1 target/release/weavepy "$P" 2>&1 | head -40 + for E in X=1 WEAVEPY_FRAME_JIT=0 WEAVEPY_JIT_QUICK=1 WEAVEPY_JIT_OPT=1 "WEAVEPY_PLAN_VERIFY=1 WEAVEPY_FRAME_JIT_VERIFY=1" WEAVEPY_JIT=0; do + for i in 1 2; do echo "absolute, $E: $(env $E target/release/weavepy "$A" 2>"$RUNNER_TEMP/err.txt") $(head -c 300 "$RUNNER_TEMP/err.txt")"; done done + for i in 1 2; do echo "relative, WEAVEPY_JIT=0: $(WEAVEPY_JIT=0 target/release/weavepy "$F")"; done echo "--- default cache" ls -la "$LOCALAPPDATA/weavepy" 2>&1 | head -20 - name: Diagnose cold Windows JIT costs after a failed gate From 3025dc14321d4ec3c75dfcce9674028be91e44fa Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:58:53 -0700 Subject: [PATCH 92/96] ci: record VM stats for the Windows absolute-path slowdown Every JIT tier switch leaves the absolute-path run slow, and with the JIT off both paths time alike, so the difference is in how compiled code runs rather than what compiles. The failure-only step now records the VM's tier-2 counters (native calls, fallbacks, deopts) for both paths. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d295b49d..e1eb25b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -286,10 +286,10 @@ jobs: done A="$PWD/$F" echo "absolute-path: $(target/release/weavepy "$A")" - for E in X=1 WEAVEPY_FRAME_JIT=0 WEAVEPY_JIT_QUICK=1 WEAVEPY_JIT_OPT=1 "WEAVEPY_PLAN_VERIFY=1 WEAVEPY_FRAME_JIT_VERIFY=1" WEAVEPY_JIT=0; do - for i in 1 2; do echo "absolute, $E: $(env $E target/release/weavepy "$A" 2>"$RUNNER_TEMP/err.txt") $(head -c 300 "$RUNNER_TEMP/err.txt")"; done + for P in "$A" "$F"; do + echo "--- VM stats for $P" + WEAVEPY_VM_STATS=1 target/release/weavepy "$P" 2>&1 | grep -vE "^\| 0x|^\|--|^\| op" | head -70 done - for i in 1 2; do echo "relative, WEAVEPY_JIT=0: $(WEAVEPY_JIT=0 target/release/weavepy "$F")"; done echo "--- default cache" ls -la "$LOCALAPPDATA/weavepy" 2>&1 | head -20 - name: Diagnose cold Windows JIT costs after a failed gate From a0d23e0ebf84f31e2f3d1be89dfb1a5e6a5672b9 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 21:34:19 -0700 Subject: [PATCH 93/96] ci: profile the Windows absolute-path slowdown with samply The slow and fast configurations run the same tier-2 paths, and every in-process statistic tried so far shifts the layout enough to hide the slowdown. The failure-only step now records sampling profiles of both from outside the process and uploads them with the diagnostic artifact. --- .github/workflows/ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1eb25b7..05ab1819 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -266,7 +266,7 @@ jobs: # same runner time it at 0.6x. This separates the differences # between the two (environment, caches, binary path). if: failure() && matrix.os == 'windows-latest' && github.event_name == 'pull_request' && steps.bench_gate.outcome == 'failure' - timeout-minutes: 15 + timeout-minutes: 30 shell: bash run: | F=crates/weavepy-bench/fixtures/call_overhead.py @@ -286,10 +286,18 @@ jobs: done A="$PWD/$F" echo "absolute-path: $(target/release/weavepy "$A")" - for P in "$A" "$F"; do - echo "--- VM stats for $P" - WEAVEPY_VM_STATS=1 target/release/weavepy "$P" 2>&1 | grep -vE "^\| 0x|^\|--|^\| op" | head -70 + # A sampling profile of each run, taken from outside the process + # (in-process statistics shift the layout and hide the slowdown). + cargo install samply --locked --quiet || echo "samply install failed" + OUT="target/performance/cold-jit-ci/${{ github.run_id }}-${{ github.run_attempt }}/profiles" + mkdir -p "$OUT" + for spec in "slow:$A" "fast:$F"; do + name=${spec%%:*}; path=${spec#*:} + samply record --save-only --unstable-presymbolicate --rate 4000 \ + -o "$OUT/$name.json.gz" -- target/release/weavepy.exe "$path" \ + || echo "samply failed for $name" done + ls -la "$OUT" echo "--- default cache" ls -la "$LOCALAPPDATA/weavepy" 2>&1 | head -20 - name: Diagnose cold Windows JIT costs after a failed gate From e00f6625d1193be3919f61ee78341f5eb0d466a5 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:31:34 -0700 Subject: [PATCH 94/96] ci: keep the Windows binaries and PDBs with the call_overhead profiles The slow run's profile spends a fifth of its samples in an unnamed cluster of `python314.dll` functions, which samply couldn't symbolicate on the runner. The failure-only step now uploads the binaries and their PDBs beside the profiles so they can be symbolicated offline. --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05ab1819..4fcd9812 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -297,6 +297,9 @@ jobs: -o "$OUT/$name.json.gz" -- target/release/weavepy.exe "$path" \ || echo "samply failed for $name" done + # The binaries and their PDBs, to symbolicate the profiles. + cp target/release/python314.dll target/release/python314.pdb \ + target/release/weavepy.exe target/release/weavepy.pdb "$OUT/" || true ls -la "$OUT" echo "--- default cache" ls -la "$LOCALAPPDATA/weavepy" 2>&1 | head -20 From e753adf958d663970712db87ddef8ee0f62cb43b Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:41:36 -0700 Subject: [PATCH 95/96] perf: reuse a keyword leaf call's `**kwargs` dictionary Each tier-2 keyword call of a pure leaf that collects `**kwargs` allocated a dictionary and freed it on return. When those blocks were alone on their mimalloc page, the page went back to the arena on every free and was fetched fresh on the next call. A Windows profile of the bench gate's `call_overhead` put a quarter of its samples there, which made it 1.2x the merge-base whenever the heap happened to fall that way. The emptied dictionary of a call that nothing kept now serves the next call; clearing restamps it, so the callee still sees a new dictionary. A regression test checks that returned, stored, and captured dictionaries stay intact and that no keys leak between calls. --- crates/weavepy-vm/src/lib.rs | 59 ++++++++++++++--- tests/regrtest/test_kwargs_dict_reuse.py | 83 ++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 tests/regrtest/test_kwargs_dict_reuse.py diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index 1dd46cee..ed8cba52 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -15167,13 +15167,11 @@ impl Interpreter { args[k] = o; } // The `**kwargs` dictionary is built where it lives: no move of a - // filled table into its cell. - let varkw = code_rc.has_varkeywords.then(|| { - Object::Dict(Rc::new(RefCell::new(DictData::with_capacity_and_hasher( - kwc, - crate::fasthash::FxBuildHasher, - )))) - }); + // filled table into its cell. It reuses the last call's when that + // one didn't escape (see `recycle_kw_dict`). + let varkw = code_rc + .has_varkeywords + .then(|| Object::Dict(spare_kw_dict(kwc))); for (j, o) in vals[eff_argc..].iter().enumerate() { let slot = ((perm >> (4 * j)) & 0xF) as usize; if slot == specialize::KW_TO_VARKW as usize { @@ -15200,7 +15198,11 @@ impl Interpreter { if let Some(d) = &varkw { args[total] = d; } - self.pure_leaf_eval::(code_rc, f, &args[..arity]) + let result = self.pure_leaf_eval::(code_rc, f, &args[..arity]); + if let Some(Object::Dict(d)) = varkw { + recycle_kw_dict(d); + } + result } /// A keyword call's parameters, bound as `kw_names_fill_locals` leaves @@ -57787,6 +57789,47 @@ enum QuietShell<'a> { Lazy(&'a mut LeanAct), } +thread_local! { + /// The `**kwargs` dictionary of the last keyword leaf call, emptied, + /// when nothing kept it (see [`recycle_kw_dict`]). + static SPARE_KW_DICT: std::cell::Cell>>> = + const { std::cell::Cell::new(None) }; +} + +/// An empty `**kwargs` dictionary with room for `n` keys: the spare one, +/// if any, else a fresh one. +fn spare_kw_dict(n: usize) -> Rc> { + SPARE_KW_DICT + .try_with(std::cell::Cell::take) + .ok() + .flatten() + .unwrap_or_else(|| { + Rc::new(RefCell::new(DictData::with_capacity_and_hasher( + n, + crate::fasthash::FxBuildHasher, + ))) + }) +} + +/// Keep a keyword leaf call's `**kwargs` dictionary for the next call +/// when nothing else holds it. A call per iteration otherwise allocates +/// and frees the same blocks, and when they are alone on their allocator +/// page the page is released and fetched again every time (on Windows +/// that doubled `call_overhead`). Clearing restamps the dictionary, so +/// the next call sees a new one. +fn recycle_kw_dict(mut d: Rc>) { + const MAX_KEPT_KEYS: usize = 16; + let Some(cell) = Rc::get_mut(&mut d) else { + return; + }; + let data = cell.get_mut(); + if data.capacity() > MAX_KEPT_KEYS { + return; + } + data.clear(); + let _ = SPARE_KW_DICT.try_with(|spare| spare.set(Some(d))); +} + #[cfg(test)] thread_local! { pub(crate) static NATIVE_FAST_SUBSCRIPTS: std::cell::Cell = const { std::cell::Cell::new(0) }; diff --git a/tests/regrtest/test_kwargs_dict_reuse.py b/tests/regrtest/test_kwargs_dict_reuse.py new file mode 100644 index 00000000..00d51245 --- /dev/null +++ b/tests/regrtest/test_kwargs_dict_reuse.py @@ -0,0 +1,83 @@ +"""A keyword leaf call's `**kwargs` dictionary may be reused across calls. + +The dictionary of a call nothing else kept is emptied and serves the next +call. Each call must still see a fresh, correct dictionary, and one that +escaped (returned, stored, captured) must stay intact and distinct. +""" + +import unittest + +WARM = 3000 + + +def total(a, **kw): + return a + kw.get("delta", 0) + + +def count(**kw): + return len(kw) + + +def echo(**kw): + return kw + + +def add_key(**kw): + kw["extra"] = 1 + return len(kw) + + +kept = [] + + +def keep(**kw): + kept.append(kw) + return 0 + + +class KwargsDictReuse(unittest.TestCase): + def test_values_per_call(self): + acc = 0 + for i in range(WARM): + acc += total(i, delta=2) + self.assertEqual(acc, sum(range(WARM)) + 2 * WARM) + for i in range(WARM): + self.assertEqual(count(a=i, b=i), 2) + self.assertEqual(count(a=i), 1) + + def test_no_keys_leak_between_calls(self): + for i in range(WARM): + self.assertEqual(add_key(a=i), 2) + self.assertEqual(count(b=i), 1) + + def test_returned_dicts_stay_distinct(self): + seen = [echo(n=i) for i in range(WARM)] + self.assertEqual([d["n"] for d in seen], list(range(WARM))) + self.assertEqual(len({id(d) for d in seen}), WARM) + + def test_stored_dicts_stay_intact(self): + kept.clear() + for i in range(WARM): + keep(n=i, m=-i) + self.assertEqual([(d["n"], d["m"]) for d in kept], [(i, -i) for i in range(WARM)]) + + def test_captured_dict(self): + def make(**kw): + return lambda: kw + + getters = [make(v=i) for i in range(WARM)] + self.assertEqual([g()["v"] for g in getters], list(range(WARM))) + + def test_identity_inside_call(self): + ids = [] + + def ident(**kw): + ids.append(id(kw)) + return kw + + held = [ident(x=i) for i in range(50)] + self.assertEqual(len(set(ids)), len(held)) + + +if __name__ == "__main__": + unittest.main() From b8adb6b47643d7517664b1ec9229d139da557dd0 Mon Sep 17 00:00:00 2001 From: Owen Carey <37121709+owenthcarey@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:09:56 -0700 Subject: [PATCH 96/96] ci: drop the temporary call_overhead diagnostic step The Windows bench gate passes now that keyword leaf calls reuse their `**kwargs` dictionary, so the failure-only step that profiled and timed the slowdown has done its job. The cold-JIT diagnostic keeps `call_overhead` in its fixture list. --- .github/workflows/ci.yml | 43 ---------------------------------------- 1 file changed, 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fcd9812..c9b2b23f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -260,49 +260,6 @@ jobs: target/release/weavepy-bench gate $FLAGS \ --weavepy=target/release/weavepy \ | tee bench-report.md - - name: Time call_overhead the way the gate runs it - # The gate has timed the PR binary's `call_overhead` at 1.2-1.5x - # the merge-base on every Windows PR run, while direct runs on the - # same runner time it at 0.6x. This separates the differences - # between the two (environment, caches, binary path). - if: failure() && matrix.os == 'windows-latest' && github.event_name == 'pull_request' && steps.bench_gate.outcome == 'failure' - timeout-minutes: 30 - shell: bash - run: | - F=crates/weavepy-bench/fixtures/call_overhead.py - export WEAVEPY_BENCH_WORK=150000 - unset WEAVEPY_JIT - for label in gate-like jit=1 own-caches exe-path base; do - for i in 1 2 3; do - case $label in - gate-like) out=$(target/release/weavepy $F) ;; - jit=1) out=$(WEAVEPY_JIT=1 target/release/weavepy $F) ;; - own-caches) out=$(WEAVEPY_FROZEN_CACHE="$RUNNER_TEMP/fz" WEAVEPY_STDLIB_CACHE="$RUNNER_TEMP/sc" target/release/weavepy $F) ;; - exe-path) out=$(target/release/weavepy.exe $F) ;; - base) out=$("$BASE_WEAVEPY" $F) ;; - esac - echo "$label: $out" - done - done - A="$PWD/$F" - echo "absolute-path: $(target/release/weavepy "$A")" - # A sampling profile of each run, taken from outside the process - # (in-process statistics shift the layout and hide the slowdown). - cargo install samply --locked --quiet || echo "samply install failed" - OUT="target/performance/cold-jit-ci/${{ github.run_id }}-${{ github.run_attempt }}/profiles" - mkdir -p "$OUT" - for spec in "slow:$A" "fast:$F"; do - name=${spec%%:*}; path=${spec#*:} - samply record --save-only --unstable-presymbolicate --rate 4000 \ - -o "$OUT/$name.json.gz" -- target/release/weavepy.exe "$path" \ - || echo "samply failed for $name" - done - # The binaries and their PDBs, to symbolicate the profiles. - cp target/release/python314.dll target/release/python314.pdb \ - target/release/weavepy.exe target/release/weavepy.pdb "$OUT/" || true - ls -la "$OUT" - echo "--- default cache" - ls -la "$LOCALAPPDATA/weavepy" 2>&1 | head -20 - name: Diagnose cold Windows JIT costs after a failed gate id: cold_jit_diagnostic if: failure() && matrix.os == 'windows-latest' && github.event_name == 'pull_request' && steps.bench_gate.outcome == 'failure'