diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e7da213..c9b2b23f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -231,7 +231,13 @@ jobs: # rust-cache still primes dependency compilation, and the # merge-base is usually a recent main commit CI has built # before, so this step is mostly cache hits. + # + # The reference binary is only timed, so a lint that a newer + # toolchain added (one the merge-base predates) mustn't fail + # the build: lints are capped at warnings here. shell: bash + env: + RUSTFLAGS: "--cap-lints=warn" run: | BASE_SHA=$(git merge-base "${{ github.event.pull_request.base.sha }}" HEAD) echo "merge-base: $BASE_SHA" @@ -263,6 +269,7 @@ jobs: python tools/bench_cold_jit.py \ --base "$BASE_WEAVEPY" --new target/release/weavepy.exe \ --samples 7 \ + --fixtures sumvm:2000000,nested_loops:120,jitloop:1000,call_overhead:150000 \ --out "target/performance/cold-jit-ci/${{ github.run_id }}-${{ github.run_attempt }}/report.json" - uses: actions/upload-artifact@v4 if: always() && steps.cold_jit_diagnostic.outcome != 'skipped' && steps.cold_jit_diagnostic.outcome != '' diff --git a/Cargo.lock b/Cargo.lock index 1036cdee..a3fef29c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2935,6 +2935,11 @@ dependencies = [ "bzip2", "chrono", "corosensei", + "cranelift-codegen", + "cranelift-frontend", + "cranelift-jit", + "cranelift-module", + "cranelift-native", "crc32fast", "crossbeam-channel", "crossbeam-utils", diff --git a/Cargo.toml b/Cargo.toml index bce0707c..229aa8bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -288,6 +288,8 @@ manual_map = "allow" comparison_chain = "allow" unnecessary_cast = "allow" len_without_is_empty = "allow" +# New in Clippy 1.99; tests across the workspace assert emptiness this way. +assert_is_empty = "allow" # Interpreter compares IEEE-754 values exactly by design (Python `==`, # rounding, struct packing); an epsilon would be incorrect. float_cmp = "allow" diff --git a/crates/weavepy-capi/src/gc_bridge.rs b/crates/weavepy-capi/src/gc_bridge.rs index c6b73f9d..6aa96bed 100644 --- a/crates/weavepy-capi/src/gc_bridge.rs +++ b/crates/weavepy-capi/src/gc_bridge.rs @@ -48,8 +48,8 @@ fn instance_type_ptr(obj: &Object) -> Option<*mut PyTypeObject> { // An instance that never crossed into C (no box, no inline body) // has no C-held edges for `tp_traverse`/`tp_clear` to report or // drop: the VM traverses its own fields. This keeps the bridge — - // a mint, a C call and a free per node — off the prompt-reap - // cascade of every native exception instance. + // a mint, a C call and a free per node — off every collection + // that visits a native exception instance. Object::Instance(i) if i.c_body.get() != 0 => crate::types::type_ptr_for_class(&i.cls()), _ => None, } @@ -212,7 +212,7 @@ pub unsafe extern "C" fn PyObject_GC_Track(op: *mut c_void) { } let obj = unsafe { crate::object::clone_object(op as *mut PyObject) }; if matches!(obj, Object::Instance(_)) { - weavepy_vm::gc_trace::track(obj); + weavepy_vm::gc_trace::track(&obj); } } diff --git a/crates/weavepy-capi/src/instance.rs b/crates/weavepy-capi/src/instance.rs index b58818e7..dc74d22c 100644 --- a/crates/weavepy-capi/src/instance.rs +++ b/crates/weavepy-capi/src/instance.rs @@ -329,30 +329,10 @@ pub unsafe fn release_c_ownership(p: *mut PyObject) { .lock() .ok() .and_then(|mut g| g.as_mut().and_then(|m| m.remove(&(p as usize)))); - if let Some(inst) = &pinned { + if pinned.is_some() { // C just dropped what may be this instance's last program-visible - // reference (CPython would run `tp_dealloc` right here). If the VM - // side still pins it through GC-handle / weakref-registry clones, a - // plain `Rc` drop below cannot reap it — a *tracked* extension - // temporary that dies inside a C call (a Cython generator abandoned - // by `BlockManager.iget`, still holding `self` in its closure) would - // otherwise stay pinned by its own handle until the next full - // collection, keeping everything it references alive with it - // (pandas' `_is_view_after_cow_rules` then reads a stale live - // `Block` weakref). Park it for the eval loop's between-bytecodes - // reap; anything still genuinely alive fails the drain's - // refcount-dead test untouched. - // - // Only a *GC-tracked* instance needs the park: an untracked one has - // no handle pinning it, so the plain `Rc` drop below reclaims it - // immediately (running `tp_dealloc` through the free hook) — and a - // queued clone would instead keep it alive until the next eval-loop - // safe point, which never comes for a drop performed outside - // bytecode execution (an embedding host dropping its last handle). - let obj = weavepy_vm::object::Object::Instance(inst.clone()); - if weavepy_vm::gc_trace::is_tracked(weavepy_vm::weakref_registry::id_of(&obj)) { - weavepy_vm::vm_singletons::queue_cext_dropped(&obj); - } + // reference; the plain `Rc` drop below reclaims it then (running + // `tp_dealloc` through the free hook), as CPython's would. } else if unsafe { crate::mirror::is_orphaned_instance_body(p) } && !body_free_in_flight(p as usize) { @@ -618,8 +598,8 @@ unsafe fn dealloc_and_free_body(p: *mut PyObject) { (*ty).tp_basicsize = orig_basicsize.wrapping_add(8); // A `tp_dealloc` is extension code: its decref chains can // re-enter the VM, and any bytecode that runs beneath it - // must see a live C frame (RFC 0047 — the prompt reaper's - // borrowed-pointer window). + // must see a live C frame (RFC 0047 — the borrowed-pointer + // window that defers parked C drops). let _cext_guard = weavepy_vm::vm_singletons::enter_cext_call(); // Consent protocol: the dealloc must reach a `tp_free` // (absorbed by `PyObject_Free`/`PyObject_GC_Del`, which diff --git a/crates/weavepy-capi/src/interp.rs b/crates/weavepy-capi/src/interp.rs index f05b3401..654d20bd 100644 --- a/crates/weavepy-capi/src/interp.rs +++ b/crates/weavepy-capi/src/interp.rs @@ -145,11 +145,10 @@ pub fn ensure_active(body: impl FnOnce() -> R) -> R { // RFC 0047 (wave 5): mark a C-extension call live on this thread for // the duration of `body`. Every bridged C invocation (dunder shims, // `PyCFunction`s, descriptors, tp_call, foreign hooks) funnels through - // here, so the VM's prompt reaper can tell "plain bytecode, no - // extension frame below" (`depth == 0` — reclaiming a dead escaped - // subgraph is as safe as CPython's own `tp_dealloc`) from "inside a C - // call that may hold borrowed body pointers" (`depth > 0` — defer - // memory reclamation to the tracing collector). + // here, so the VM can tell "plain bytecode, no extension frame below" + // (`depth == 0` — releasing objects C dropped is as safe as CPython's + // own `tp_dealloc`) from "inside a C call that may hold borrowed body + // pointers" (`depth > 0` — keep them parked). let _cext_guard = weavepy_vm::vm_singletons::enter_cext_call(); if current_interpreter_mut().is_some() { if ea_trace_enabled() { diff --git a/crates/weavepy-capi/src/object.rs b/crates/weavepy-capi/src/object.rs index dd2b5290..764a1bb1 100644 --- a/crates/weavepy-capi/src/object.rs +++ b/crates/weavepy-capi/src/object.rs @@ -1659,14 +1659,13 @@ pub(crate) unsafe fn free_box(p: *mut PyObject) { evict_property_box(weavepy_vm::sync::Rc::as_ptr(rc) as usize, p); } // RFC 0047 (wave 5): C is dropping what may be the payload's last - // program-visible reference, from inside an extension call the VM's - // prompt reaper cannot see. Park instance/container payloads for a - // refcount-guarded reap at the next eval-loop safe point so anything - // that died here gets its weakrefs cleared with CPython's timing - // (pandas' `BlockValuesRefs.has_reference` prunes dead block - // weakrefs the very next Python-level call after a Cython-internal - // `self.blocks = ...` rebind). A payload that is still alive - // elsewhere fails the reap's deadness test untouched. + // program-visible reference, from inside an extension call that may + // still hold borrowed body pointers. Park instance/container payloads + // and release them at the next eval-loop safe point outside any C + // call, so anything that dies there gets its weakrefs cleared before + // the next Python-level read (pandas' `BlockValuesRefs.has_reference` + // prunes dead block weakrefs right after a Cython-internal + // `self.blocks = ...` rebind). weavepy_vm::vm_singletons::queue_cext_dropped(&bx.payload.obj); if let Some(d) = bx.payload.destructor { let raw = Box::into_raw(bx); diff --git a/crates/weavepy-capi/src/types.rs b/crates/weavepy-capi/src/types.rs index 5570121e..02c0f438 100644 --- a/crates/weavepy-capi/src/types.rs +++ b/crates/weavepy-capi/src/types.rs @@ -2585,7 +2585,7 @@ fn inherit_inline_base_layout( }; let bases = t.bases.borrow(); - let first_ptr = bases.first().and_then(&resolve); + let first_ptr = bases.first().and_then(resolve); // CPython's `best_base()`: the "solid base" is the base with the widest // instance layout (`tp_basicsize`), scanned across *all* bases — not diff --git a/crates/weavepy-compiler/src/cpython_code.rs b/crates/weavepy-compiler/src/cpython_code.rs index ca02b7f6..0a2e1d7b 100644 --- a/crates/weavepy-compiler/src/cpython_code.rs +++ b/crates/weavepy-compiler/src/cpython_code.rs @@ -1480,7 +1480,7 @@ fn encode_exception_table(code: &CodeObject, starts: &[usize]) -> Vec { /// depths for inlined comprehensions; entries the walk never reaches /// stay -1. Handlers whose depth is still the sentinel do not seed /// the walk (their depth is exactly what's being computed). -pub(crate) fn compute_startdepths(code: &CodeObject) -> Vec { +pub fn compute_startdepths(code: &CodeObject) -> Vec { use OpCode as O; let n = code.instructions.len(); let mut startdepth: Vec = vec![-1; n]; diff --git a/crates/weavepy-compiler/src/flowgraph.rs b/crates/weavepy-compiler/src/flowgraph.rs index a4be97f7..68740409 100644 --- a/crates/weavepy-compiler/src/flowgraph.rs +++ b/crates/weavepy-compiler/src/flowgraph.rs @@ -3092,7 +3092,7 @@ impl Cfg { } co.instructions = instructions; co.linetable = linetable; - co.coltable = coltable; + co.coltable = coltable.into(); co.exception_table = table; co.constants = self.consts; no_interrupt.sort_unstable(); diff --git a/crates/weavepy-compiler/src/lib.rs b/crates/weavepy-compiler/src/lib.rs index 6f840348..382128eb 100644 --- a/crates/weavepy-compiler/src/lib.rs +++ b/crates/weavepy-compiler/src/lib.rs @@ -280,6 +280,14 @@ impl JitHint { true } + /// Whether the tier-2 state has no further interest in this code's + /// back edges (see [`Self::set_backedge_quiet`]). + #[inline] + pub fn is_backedge_quiet(&self) -> bool { + self.backedge_quiet + .load(std::sync::atomic::Ordering::Relaxed) + } + pub fn set_backedge_quiet(&self) { self.backedge_quiet .store(true, std::sync::atomic::Ordering::Relaxed); @@ -504,8 +512,9 @@ pub struct CodeObject { /// PEP-657 fine-grained column spans, one per instruction (same length /// as `instructions` once emission finishes). Drives the column fields /// of `co_positions()`. Empty when never populated (e.g. code objects - /// reconstructed from marshal, which doesn't carry columns). - pub coltable: Vec, + /// reconstructed from marshal, which doesn't carry columns). Decoded + /// from the code cache on first use (see [`ColTable`]). + pub coltable: ColTable, /// Number of positional + keyword arguments (excluding `*args`/`**kwargs`). pub arg_count: u32, /// Number of positional-only arguments. @@ -625,6 +634,74 @@ pub struct WireOverrides { /// A per-instruction source-column span (PEP-657). `col`/`end_col` are /// 0-based UTF-8 byte offsets within their respective source lines, and +/// A code object's column spans (see [`CodeObject::coltable`]): a +/// vector, or the code cache's encoding of one, decoded on first use. +/// Most code never reports a column (only tracebacks and +/// `co_positions()` read them), so a module loaded from the cache skips +/// the work until something does. +#[derive(Clone, Default)] +pub struct ColTable { + spans: std::sync::OnceLock>, + encoded: Option>, +} + +impl ColTable { + /// The spans `encoded` holds in the code cache's form (see + /// `native_code`), decoded when first read. + pub(crate) fn encoded(encoded: std::sync::Arc<[u8]>) -> Self { + ColTable { + spans: std::sync::OnceLock::new(), + encoded: Some(encoded), + } + } + + fn spans(&self) -> &Vec { + self.spans.get_or_init(|| { + self.encoded + .as_deref() + .and_then(native_code::decode_coltable) + .unwrap_or_default() + }) + } +} + +impl From> for ColTable { + fn from(spans: Vec) -> Self { + ColTable { + spans: std::sync::OnceLock::from(spans), + encoded: None, + } + } +} + +impl std::ops::Deref for ColTable { + type Target = Vec; + + fn deref(&self) -> &Vec { + self.spans() + } +} + +impl std::ops::DerefMut for ColTable { + fn deref_mut(&mut self) -> &mut Vec { + self.spans(); + self.encoded = None; + self.spans.get_mut().expect("decoded above") + } +} + +impl PartialEq for ColTable { + fn eq(&self, other: &Self) -> bool { + **self == **other + } +} + +impl std::fmt::Debug for ColTable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + (**self).fmt(f) + } +} + /// are `-1` when the column was not tracked. `end_lineno` is `0` when /// unknown (callers fall back to the instruction's start line). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -2623,7 +2700,8 @@ impl Compiler { } }; self.co.linetable.push(line); - self.co.coltable.push(self.resolve_colspan()); + let span = self.resolve_colspan(); + self.co.coltable.push(span); offset } diff --git a/crates/weavepy-compiler/src/native_code.rs b/crates/weavepy-compiler/src/native_code.rs index b430758a..f12aa104 100644 --- a/crates/weavepy-compiler/src/native_code.rs +++ b/crates/weavepy-compiler/src/native_code.rs @@ -16,7 +16,7 @@ use crate::bytecode::{CacheTable, Instruction, OpCode}; use crate::{CodeObject, ColSpan, Constant, ExcHandler}; /// The layout revision; bump it whenever the encoding changes. -pub const VERSION: u8 = 2; +pub const VERSION: u8 = 3; /// Encode `code` (and its nested code objects). `None` for a code object /// the format doesn't carry: one with raw CPython wire overrides, or with @@ -28,6 +28,21 @@ pub fn encode(code: &CodeObject) -> Option> { Some(w.0) } +/// Decode a column-span block [`encode`] wrote (see `ColTable`). +pub(crate) fn decode_coltable(bytes: &[u8]) -> Option> { + let mut r = Reader { bytes, pos: 0 }; + let end_lines = r.deltas()?; + let mut spans = Vec::with_capacity(end_lines.len()); + for end_lineno in end_lines { + spans.push(ColSpan { + end_lineno, + col: r.i32()?, + end_col: r.i32()?, + }); + } + (r.pos == bytes.len()).then_some(spans) +} + /// Decode what [`encode`] wrote, stamping `filename` on every code /// object. `None` for input it didn't write (or a different version's). pub fn decode(bytes: &[u8], filename: &str) -> Option { @@ -116,11 +131,15 @@ impl Writer { // Line numbers as deltas from the previous entry: mostly zero, so // one byte each whatever the line. self.deltas(c.linetable.iter().copied()); - self.deltas(c.coltable.iter().map(|s| s.end_lineno)); - for s in &c.coltable { - self.int(i64::from(s.col)); - self.int(i64::from(s.end_col)); + // The column spans as one length-prefixed block, which a reader + // keeps encoded until something reads a column (see `ColTable`). + let mut cols = Writer(Vec::new()); + cols.deltas(c.coltable.iter().map(|s| s.end_lineno)); + for s in c.coltable.iter() { + cols.int(i64::from(s.col)); + cols.int(i64::from(s.end_col)); } + self.bytes(&cols.0); self.uint(u64::from(c.arg_count)); self.uint(u64::from(c.posonly_count)); self.uint(u64::from(c.kwonly_count)); @@ -360,15 +379,7 @@ impl Reader<'_> { }); } let linetable = self.deltas()?; - let end_lines = self.deltas()?; - let mut coltable = Vec::with_capacity(end_lines.len()); - for end_lineno in end_lines { - coltable.push(ColSpan { - end_lineno, - col: self.i32()?, - end_col: self.i32()?, - }); - } + let coltable = crate::ColTable::encoded(Arc::from(self.raw()?)); let arg_count = self.u32()?; let posonly_count = self.u32()?; let kwonly_count = self.u32()?; @@ -472,7 +483,7 @@ mod tests { constants: vec![Constant::Int(-5)], varnames: vec!["x".to_owned()], linetable: vec![1, 2, 2], - coltable: vec![ColSpan::default(); 3], + coltable: vec![ColSpan::default(); 3].into(), arg_count: 1, is_generator: true, is_nested: true, diff --git a/crates/weavepy-jit/src/analyze.rs b/crates/weavepy-jit/src/analyze.rs index 45ac5419..d99578f8 100644 --- a/crates/weavepy-jit/src/analyze.rs +++ b/crates/weavepy-jit/src/analyze.rs @@ -49,6 +49,10 @@ pub struct MethodResolution { pub arg_count: u32, pub min_args: u32, pub ret: MethodRet, + /// The callee is a native accelerator's leaf method (`deque.pop`): + /// its object-lane result is a container element, so arithmetic on + /// it speculates the integer lane (see [`SE::num_hint`]). + pub native: bool, } /// Maximum receiver path retained by analysis. The final attribute read adds @@ -2781,6 +2785,8 @@ struct MethodMark { ret: MethodRet, /// The method-form `LOAD_ATTR` pc. load_pc: u32, + /// See [`MethodResolution::native`]. + native: bool, } /// RFC 0073 WS3 — a burned-in native `str` method riding on its @@ -2852,6 +2858,12 @@ struct SE { /// native method (the [`Self::method`] discipline, statically /// resolved). str_method: Option, + /// An object-lane element read from a native accelerator's container + /// (a `deque` subscript or end operation). Arithmetic combining two + /// object-lane operands speculates the integer lane when either + /// carries this hint: each operand is guarded exactly like the + /// integral-peer case, so a miss resumes at the operation itself. + num_hint: bool, } impl SE { @@ -2871,6 +2883,7 @@ impl SE { ctor: None, elem_of: None, str_method: None, + num_hint: false, } } @@ -3984,8 +3997,15 @@ fn step_abstract( stack.push(SE::known(a.ty)); return Ok(()); } + let hinted = a.num_hint || b.num_hint; let (a, b) = resolve_pair(a, b, local_types, changed); - let res = bin_result_type(kind, a.ty, b.ty)?; + // Two native container elements (`q[0] + q[-1]`): both + // operands are guarded as integers at emission. + let res = if hinted && a.ty == JitType::Obj && b.ty == JitType::Obj { + bin_result_type(kind, JitType::Int, JitType::Int)? + } else { + bin_result_type(kind, a.ty, b.ty)? + }; stack.push(SE::known(res)); } OpCode::CompareOp => { @@ -4009,24 +4029,30 @@ fn step_abstract( } let b = stack.pop().ok_or(JitVerdict::StackUnderflow)?; let a = stack.pop().ok_or(JitVerdict::StackUnderflow)?; - let val = match (a.none_const, b.none_const) { - (false, true) => a, - (true, false) => b, + // Two object operands (`c is not determining`): both on the + // object lane, compared by identity. + let vals = match (a.none_const, b.none_const) { + (false, true) => vec![a], + (true, false) => vec![b], + (false, false) => vec![a, b], _ => return Err(JitVerdict::UnsupportedOpcode("IS_OP shape")), }; - if !val.is_plain() { - return Err(escape_verdict(code, &[&val], "CALL (callee escapes)")); - } - match val.ty { - JitType::Obj => {} - JitType::Unknown => { - if let Some(slot) = val.src { - set_local(local_types, slot, JitType::Obj, changed)?; + for val in vals { + if !val.is_plain() { + return Err(escape_verdict(code, &[&val], "CALL (callee escapes)")); + } + match val.ty { + JitType::Obj => {} + JitType::Unknown => { + if let Some(slot) = val.src { + set_local(local_types, slot, JitType::Obj, changed)?; + } + // No provenance: transient — a later iteration + // may type it; emission bails if it never + // resolves. } - // No provenance: transient — a later iteration may - // type it; emission bails if it never resolves. + _ => return Err(JitVerdict::UnsupportedOpcode("IS_OP operand lane")), } - _ => return Err(JitVerdict::UnsupportedOpcode("IS_OP operand lane")), } stack.push(SE::known(JitType::Bool)); } @@ -4160,9 +4186,11 @@ fn step_abstract( // RFC 0069 WS1 — a class-resolved method on a pinned // instance receiver: the receiver stays on the abstract // stack (its pin is the native value), re-marked with the - // resolution. `.append` keeps its dedicated list path. - // RFC 0071 WS3 admits attribute-chain receivers. - if name != "append" { + // resolution. `.append` keeps its dedicated list path (a + // list receiver never resolves here: the probe answers + // instances only). RFC 0071 WS3 admits attribute-chain + // receivers. + if name != "append" || !recv.ty.is_list() { if let Some((slot, path)) = obj_recv_ref(recv.ty, recv.src, recv.path, probes.paths) { let names = probes.paths.names(path); @@ -4188,6 +4216,7 @@ fn step_abstract( min_args: res.min_args, ret: res.ret, load_pc: i as u32, + native: res.native, }), ..recv }); @@ -4341,7 +4370,10 @@ fn step_abstract( poison: true, ..SE::known(JitType::Unknown) }), - MethodRet::Scalar(ty) => stack.push(SE::known(ty)), + MethodRet::Scalar(ty) => stack.push(SE { + num_hint: m.native && ty == JitType::Obj, + ..SE::known(ty) + }), } return Ok(()); } @@ -4602,6 +4634,21 @@ fn step_abstract( stack.push(SE::known(JitType::Int)); return Ok(()); } + // A native container element (`q[0]` on a `deque`): the + // container's class resolves `__getitem__` to a native leaf + // method (see `TOp::ObjGetItem`). + if native_getitem(&cont, probes).is_some() { + if cont.path.is_none() && cont.ty == JitType::Unknown { + if let Some(slot) = cont.src { + set_local(local_types, slot, JitType::Obj, changed)?; + } + } + stack.push(SE { + num_hint: true, + ..SE::known(JitType::Obj) + }); + return Ok(()); + } // RFC 0073 WS3 — `s[i]` reads a single-codepoint `str` // (ASCII fast path; the helper deopts on anything else). if cont.ty == JitType::Str { @@ -5508,8 +5555,84 @@ fn bin_result_type(kind: ArithKind, a: JitType, b: JitType) -> Result bool { + for k in (0..stmts.len()).rev() { + match stmts[k].op { + TOp::GuardMethod { .. } if stmts[k].pc == load_pc => { + stmts[k].op = TOp::GuardNotNone; + return true; + } + TOp::LoadLocal(_) + | TOp::PushConstInt(_) + | TOp::PushConstFloat(_) + | TOp::PushConstBool(_) + | TOp::PushNone + | TOp::IntArith(_) + | TOp::FloatArith(_) + | TOp::IntNeg + | TOp::FloatNeg + | TOp::IntInvert + | TOp::IntToFloatTos { .. } + | TOp::IntToFloatSecond { .. } + | TOp::UnboxInt { .. } => {} + _ => return false, + } + } + false +} + +/// The burned `__getitem__` resolution for a subscript container on the +/// object lane (or an untyped local): the root local and the embedder's +/// method probe, when the class resolves the dunder to a native leaf +/// method (`deque.__getitem__`). Python-level `__getitem__` methods stay +/// out: the helper calls the native body directly. +fn native_getitem(cont: &SE, probes: &mut Probes<'_>) -> Option<(u32, MethodResolution)> { + if !matches!(cont.ty, JitType::Obj | JitType::Unknown) { + return None; + } + let (slot, path) = obj_recv_ref(cont.ty, cont.src, cont.path, probes.paths)?; + let names = probes.paths.names(path); + let res = (probes.method)(slot, &names, "__getitem__")?; + res.native.then_some((slot, res)) +} + +/// Fuse an integer consumer at `pc` into the object-lane producer that +/// ran immediately before it (the instruction at `pc - 1`): the producer +/// then hands over an exact `int` unboxed, and anything else leaves +/// through the producer's own completed-result exit (a generic call or a +/// native method parks its result and resumes here; a pure native +/// subscript is simply re-executed by the interpreter). Returns whether +/// the producer took the consumer's guard. `native` admits a burned-in +/// method call, which only a native method's hinted result asks for. +fn fuse_int_result(stmts: &mut [TStmt], pc: u32, native: bool) -> bool { + let Some(previous) = stmts.last_mut() else { + return false; + }; + if previous.pc.checked_add(1) != Some(pc) { + return false; + } + match &mut previous.op { + TOp::CallDyn { int_result, .. } | TOp::ObjGetItem { int_result, .. } => { + *int_result = true; + true + } + TOp::CallMethod { ret, .. } if native && *ret == MethodRet::Scalar(JitType::Obj) => { + *ret = MethodRet::Scalar(JitType::Int); + true + } + _ => false, + } +} + /// An integral peer provides a useful speculative lane for an opaque -/// operand. Two opaque operands have no such evidence and stay generic. +/// operand. Two opaque operands have no such evidence and stay generic +/// unless one is a native container element (see [`SE::num_hint`]). /// The depth is measured from TOS, so only 0 and 1 can be returned. fn integer_operand_guard(a: JitType, b: JitType) -> Option { if a.is_integral() && b == JitType::Obj { @@ -5528,6 +5651,11 @@ fn cmp_check(kind: CmpKind, a: JitType, b: JitType) -> Result<(), JitVerdict> { if !a.is_representable() || !b.is_representable() { return Ok(()); } + // An object compared with an integral operand is guarded to be an + // exact machine-sized `int` (see `integer_operand_guard`). + if integer_operand_guard(a, b).is_some() { + return Ok(()); + } if (a.is_integral() || a == JitType::Float) && (b.is_integral() || b == JitType::Float) { return Ok(()); } @@ -5671,6 +5799,8 @@ struct ESlot { /// RFC 0073 WS3 — a pinned `str` receiver carrying a burned-in /// native method (see [`SE::str_method`]). str_method: Option, + /// See [`SE::num_hint`]. + num_hint: bool, } impl ESlot { @@ -5689,6 +5819,7 @@ impl ESlot { slice: None, const_slice: false, str_method: None, + num_hint: false, } } @@ -6493,21 +6624,31 @@ fn emit_instr( } OpCode::BinaryOp => { let kind = bin_kind(ins.arg)?; + let b_hint = stack.last().is_some_and(|s| s.num_hint); + let hinted = b_hint || (stack.len() >= 2 && stack[stack.len() - 2].num_hint); let mut b = pop_val(stack)?; let mut a = pop_val(stack)?; + // Two native container elements (see `SE::num_hint`): the top + // operand is guarded (or its producer fused) like the + // integral-peer case below, then the lower one at depth 1. + if hinted && a == JitType::Obj && b == JitType::Obj { + if !fuse_int_result(stmts, pc, b_hint) { + stack.push(ESlot::val(a)); + stack.push(ESlot::val(b)); + push(TOp::UnboxInt { depth: 0 }, None, stack, stmts); + stack.pop(); + stack.pop(); + } + b = JitType::Int; + stack.push(ESlot::val(a)); + stack.push(ESlot::val(b)); + push(TOp::UnboxInt { depth: 1 }, None, stack, stmts); + stack.pop(); + stack.pop(); + a = JitType::Int; + } if let Some(depth) = integer_operand_guard(a, b) { - let fused = depth == 0 - && stmts.last_mut().is_some_and(|previous| { - if previous.pc.checked_add(1) != Some(pc) { - return false; - } - if let TOp::CallDyn { int_result, .. } = &mut previous.op { - *int_result = true; - true - } else { - false - } - }); + let fused = depth == 0 && fuse_int_result(stmts, pc, b_hint); // Re-model both original operands for the guard. A miss // restores the boxed value at this BINARY_OP, never its // producer, so calls and callbacks aren't repeated. @@ -6572,8 +6713,24 @@ fn emit_instr( } OpCode::CompareOp => { let kind = cmp_kind(ins.arg)?; - let b = pop_val(stack)?; - let a = pop_val(stack)?; + let mut b = pop_val(stack)?; + let mut a = pop_val(stack)?; + // An object compared with an integral operand (a constant on + // a class, a field read generically) must be an exact + // machine-sized `int`; a miss restores both operands at this + // `COMPARE_OP` and the interpreter compares them. + if let Some(depth) = integer_operand_guard(a, b) { + stack.push(ESlot::val(a)); + stack.push(ESlot::val(b)); + push(TOp::UnboxInt { depth }, None, stack, stmts); + stack.pop(); + stack.pop(); + if depth == 0 { + b = JitType::Int; + } else { + a = JitType::Int; + } + } if a.is_integral() && b.is_integral() { push(TOp::IntCmp(kind), Some(JitType::Bool), stack, stmts); } else if a == JitType::Float && b == JitType::Float { @@ -6621,6 +6778,23 @@ fn emit_instr( let val = match (a.none_const, b.none_const) { (false, true) => a, (true, false) => b, + (false, false) => { + // Two object operands: identity through the pins. + for v in [&a, &b] { + if !v.is_plain() || v.ty != JitType::Obj { + return Err(JitVerdict::UnsupportedOpcode("IS_OP operand lane")); + } + } + push( + TOp::IsObj { + negate: ins.arg == 1, + }, + Some(JitType::Bool), + stack, + stmts, + ); + return Ok(()); + } _ => return Err(JitVerdict::UnsupportedOpcode("IS_OP shape")), }; if !val.is_plain() || val.ty != JitType::Obj { @@ -6892,8 +7066,10 @@ fn emit_instr( let names = probes.paths.names(path); // RFC 0069 WS1 — a class-resolved method: re-mark the // receiver in place (its pin stays the native value) and - // record the site under the probe's token. - if name != "append" { + // record the site under the probe's token (an object-lane + // `.append`, such as `deque.append`, included: the list + // shape returned above). + if name != "append" || !top.ty.is_list() { // RFC 0073 WS1 (element residue) — mirror the // inference fallback: an unbound receiver local // resolves against an exemplar element of its source @@ -6927,7 +7103,17 @@ fn emit_instr( // fence out `None` here (deopt at this pc, receiver // still on the stack) so the interpreter re-executes // the load and raises the exact `AttributeError`. - push(TOp::GuardNotNone, None, stack, stmts); + // A native method binds at this load: its whole guard + // (class version, no instance shadowing, `None` + // fenced) runs here, and the call then uses the + // builtin bound here even when argument evaluation + // rebinds the class attribute, exactly as the + // interpreter's bound method would. + if res.native { + push(TOp::GuardMethod { token: res.token }, None, stack, stmts); + } else { + push(TOp::GuardNotNone, None, stack, stmts); + } let last = stack.len() - 1; stack[last].method = Some(MethodMark { token: res.token, @@ -6935,6 +7121,7 @@ fn emit_instr( min_args: res.min_args, ret: res.ret, load_pc: pc, + native: res.native, }); stack.push(ESlot { null: Some((pc, plan.hidden_at(i) + stack.len() as u32)), @@ -7220,6 +7407,15 @@ fn emit_instr( token: Some(m.token), }); *max_call_args = (*max_call_args).max(argc as u32); + // A native site whose arguments ran no code since the load + // checks its guard at the call (one helper call, not two). + let mut token = m.token; + if m.native { + token |= crate::runtime::METHOD_NATIVE; + if defer_method_guard(stmts, m.load_pc) { + token |= crate::runtime::METHOD_GUARD_AT_CALL; + } + } // Re-model the receiver + args for the statement // (`CallMethod` pops `argc` scalars and the pin). stack.push(ESlot::val(f.ty)); @@ -7228,7 +7424,7 @@ fn emit_instr( } push( TOp::CallMethod { - token: m.token, + token, argc: argc as u8, ret: m.ret, }, @@ -7252,7 +7448,10 @@ fn emit_instr( if !ty.is_representable() || (ty.is_pinned() && ty != JitType::Obj) { return Err(JitVerdict::TypeUnknown); } - stack.push(ESlot::val(ty)); + stack.push(ESlot { + num_hint: m.native && ty == JitType::Obj, + ..ESlot::val(ty) + }); } } *max_stack = (*max_stack).max(stack.len() as u32); @@ -8075,6 +8274,45 @@ fn emit_instr( push(TOp::BytesGetItem, Some(JitType::Int), stack, stmts); return Ok(()); } + // A native container element (see the inference arm): the + // site records the burned `__getitem__` resolution under its + // probe token, like a method site. + if cont == JitType::Obj { + let probe = SE { + src: cont_slot.src, + path: cont_slot.path, + ..SE::known(cont) + }; + if let Some((slot, res)) = native_getitem(&probe, probes) { + let idx = res.token as usize; + if method_sites.len() <= idx { + method_sites.resize(idx + 1, None); + } + method_sites[idx] = Some(MethodSiteMeta { + slot, + name: "__getitem__".to_owned(), + arg_count: res.arg_count, + min_args: res.min_args, + ret: res.ret, + }); + *max_call_args = (*max_call_args).max(1); + push( + TOp::ObjGetItem { + token: res.token, + int_result: false, + }, + None, + stack, + stmts, + ); + stack.push(ESlot { + num_hint: true, + ..ESlot::val(JitType::Obj) + }); + *max_stack = (*max_stack).max(stack.len() as u32); + return Ok(()); + } + } // RFC 0073 WS3 — `s[i]` through the registered helper // (ASCII-only O(1); anything else deopts there). if cont == JitType::Str { diff --git a/crates/weavepy-jit/src/engine.rs b/crates/weavepy-jit/src/engine.rs index e5408456..6fd20b74 100644 --- a/crates/weavepy-jit/src/engine.rs +++ b/crates/weavepy-jit/src/engine.rs @@ -133,6 +133,26 @@ pub struct DirectLeaf { max_stack: u32, params: Vec, ret: JitType, + /// Per parameter, the bits of the default a call that leaves it out + /// binds (in the parameter's lane), when the caller burns it in. + defaults: Vec>, +} + +impl DirectLeaf { + /// The lanes of the leaf's parameters. + #[must_use] + pub fn params(&self) -> &[JitType] { + &self.params + } + + /// Let calls that leave parameters out bind `defaults` (per + /// parameter, the default's bits in its lane): the embedder guards + /// that they stay the function's defaults. + #[must_use] + pub fn with_defaults(mut self, defaults: Vec>) -> Self { + self.defaults = defaults; + self + } } impl CompiledFrame { @@ -168,6 +188,7 @@ impl CompiledFrame { func_id: self.func_id, n_locals: self.n_locals, max_stack: self.max_stack, + defaults: vec![None; params.len()], params, ret, }) @@ -248,8 +269,19 @@ impl JitEngine { // in-process. flag_builder.set("use_colocated_libcalls", "false").ok()?; flag_builder.set("is_pic", "false").ok()?; - // Favour fast compiles over the last few percent of codegen. - flag_builder.set("opt_level", "speed").ok()?; + // Cranelift's mid-end pays for itself in numeric loops: without it + // `jitloop` ran 2x and `spectral_norm` 1.2x slower, for a few + // percent of compile time saved. `WEAVEPY_JIT_OPT=0` turns it off. + let opt = if std::env::var_os("WEAVEPY_JIT_OPT").is_some_and(|v| v == "0") { + "none" + } else { + "speed" + }; + flag_builder.set("opt_level", opt).ok()?; + if std::env::var_os("WEAVEPY_JIT_QUICK").is_some() { + flag_builder.set("opt_level", "none").ok()?; + flag_builder.set("regalloc_algorithm", "single_pass").ok()?; + } // The IR verifier re-checks every function it compiles: a debug // aid whose cost lands on every warm-up in release builds. if !cfg!(debug_assertions) { @@ -483,6 +515,16 @@ impl JitEngine { "frame coverage (no helper registered)", )); } + if runtime::is_obj_helper_addr() == 0 + && tfunc + .blocks + .iter() + .any(|b| b.stmts.iter().any(|s| matches!(s.op, TOp::IsObj { .. }))) + { + return Err(JitVerdict::UnsupportedOpcode( + "IS_OP (no helper registered)", + )); + } let needs_float_divmod = tfunc.blocks.iter().any(|b| { b.stmts .iter() @@ -526,20 +568,29 @@ impl JitEngine { let mut leaves: Vec<(u32, FuncRef, DirectLeaf)> = Vec::new(); if runtime::self_call_helper_addrs().is_some() { for stmt in tfunc.blocks.iter().flat_map(|b| &b.stmts) { - let TOp::CallPy { - token, - argc, - ret, - is_self: false, - } = stmt.op - else { - continue; + // A site that leaves parameters out binds the defaults the + // leaf carries (checked per site by the lowering). + let (token, argc, ret) = match stmt.op { + TOp::CallPy { + token, + argc, + ret, + is_self: false, + } => (token, argc, ret), + TOp::CallPyKw { + token, + argc, + kwc, + ret, + .. + } => (token, argc + kwc, ret), + _ => continue, }; if leaves.iter().any(|(t, ..)| *t == token) { continue; } if let Some(leaf) = direct(token) { - if leaf.params.len() == argc as usize && leaf.ret == ret { + if leaf.params.len() >= argc as usize && leaf.ret == ret { let fref = self .module .declare_func_in_func(leaf.func_id, &mut self.ctx.func); @@ -549,7 +600,7 @@ impl JitEngine { } } - build_function( + let sound = build_function( &mut self.ctx.func, &mut self.fbctx, tfunc, @@ -563,17 +614,52 @@ impl JitEngine { n_locals: leaf.n_locals, max_stack: leaf.max_stack, params: leaf.params, + defaults: leaf.defaults, }) .collect(), ); + if !sound { + self.module.clear_context(&mut self.ctx); + return Err(JitVerdict::NotConverged); + } + if std::env::var_os("WEAVEPY_JIT_CLIF_DUMP").is_some() { + eprintln!("jit clif {name}:\n{}", self.ctx.func.display()); + } + if std::env::var_os("WEAVEPY_JIT_CLIF_STATS").is_some() { + let f = &self.ctx.func; + let insts: usize = f + .layout + .blocks() + .map(|b| f.layout.block_insts(b).count()) + .sum(); + eprintln!( + "jit clif {name}: {} blocks, {insts} insts, {} values", + f.layout.blocks().count(), + f.dfg.num_values() + ); + } + let stats = std::env::var_os("WEAVEPY_JIT_CLIF_STATS").is_some(); + if stats { + let _ = cranelift_codegen::timing::take_current(); + } + let t0 = stats.then(std::time::Instant::now); self.module .define_function(id, &mut self.ctx) .map_err(|_| JitVerdict::NotConverged)?; + let t1 = stats.then(std::time::Instant::now); self.module.clear_context(&mut self.ctx); self.module .finalize_definitions() .map_err(|_| JitVerdict::NotConverged)?; + if let (Some(t0), Some(t1)) = (t0, t1) { + eprintln!( + "jit define {name}: {:?}, finalize {:?}\n{}", + t1 - t0, + t1.elapsed(), + cranelift_codegen::timing::take_current() + ); + } let code_ptr = self.module.get_finalized_function(id); // SAFETY: `code_ptr` is a finalized function with exactly the @@ -696,7 +782,10 @@ fn op_mix(tfunc: &TFunc) -> OpMix { TOp::CallDyn { .. } => mix.dyn_calls += 1, TOp::DynAttrGet { .. } | TOp::DynAttrSet { .. } => mix.dyn_attrs += 1, TOp::ContainsDyn { .. } => mix.dyn_other += 1, - TOp::CallPy { .. } | TOp::CallPyKw { .. } | TOp::CallMethod { .. } => { + TOp::CallPy { .. } + | TOp::CallPyKw { .. } + | TOp::CallMethod { .. } + | TOp::ObjGetItem { .. } => { mix.guarded_calls += 1; } _ => {} diff --git a/crates/weavepy-jit/src/ir.rs b/crates/weavepy-jit/src/ir.rs index 66f7be00..7cfa5f87 100644 --- a/crates/weavepy-jit/src/ir.rs +++ b/crates/weavepy-jit/src/ir.rs @@ -236,7 +236,7 @@ pub enum TOp { /// (staged through `ret_bits`) and write it into closure cell /// `idx` through the registered `wpjit_cell_set` helper. A /// displaced heap value must drop on the interpreter's store path - /// (prompt reap, parked finalizers), so the helper deopts before + /// (which runs any finalizer it queues), so the helper deopts before /// the store in that case and the interpreter re-executes it. CellSet { idx: u32, lane: JitType }, /// RFC 0065 WS5 — `x.append(v)` on a pinned list: pops the value @@ -284,6 +284,12 @@ pub enum TOp { /// `-1` for `None`) and pushes the `bool` result. Purely native — /// no helper call, no deopt. IsNone { negate: bool }, + /// `a is b` / `a is not b` on two object lanes: pops both (pin + /// indices, or `-1` for `None`) and pushes the `bool` identity + /// result. Equal machine values answer in line; otherwise the + /// registered helper compares the pinned objects (a pin miss + /// deopts). + IsObj { negate: bool }, /// RFC 0070 WS1 — push the `None` singleton in the nullable object /// lane (machine value `-1`). Emitted where a `None` constant must /// occupy a *native* stack slot: a `StoreFast` into an `Obj` local, @@ -312,6 +318,27 @@ pub enum TOp { argc: u8, ret: MethodRet, }, + /// `container[index]` (`BINARY_SUBSCR`) on an object-lane container + /// whose class resolved `__getitem__` to a native leaf method at + /// compile time (`deque.__getitem__`): pops the `int` index and the + /// container pin and calls the registered `wpjit_obj_getitem` helper + /// with the method-site `token` (the [`TOp::CallMethod`] protocol, + /// index staged as the one argument). A guard miss or a declined + /// fast half deopts *at* this pc with both operands spilled (nothing + /// ran; the interpreter re-executes the subscript). The result rides + /// the object lane, or the `int` lane when `int_result` fuses the + /// immediately following integer consumer (any other value parks and + /// resumes at that consumer). + ObjGetItem { token: u32, int_result: bool }, + /// The method-form `LOAD_ATTR` of a burned-in native method site + /// (see [`TOp::CallMethod`]): peeks the receiver pin and calls the + /// registered `wpjit_guard_method` helper with the site's `token`, + /// which checks the receiver's class version and that no instance + /// attribute shadows the name. A miss (or `None`) deopts *at* this + /// pc with the receiver spilled, so the interpreter performs the + /// load itself. The `CALL` then needs no guard: the method was bound + /// here. + GuardMethod { token: u32 }, /// RFC 0071 WS6 — `==`/`!=` on two pinned `str` values: pops both /// pins, calls the registered `wpjit_str_eq` helper (identical-pin /// and pointer equality answer before a content compare), and @@ -349,16 +376,16 @@ pub enum TOp { /// interpreted per the trained `val` lane), and calls the /// registered `wpjit_dict_set` helper — the interpreter's own /// `dict_insert` chokepoint, so PEP 509 / watcher discipline is - /// identical. A displaced value that would run the prompt-reap - /// cascade deopts *before* the store (the `wpjit_attr_set` + /// identical. A displaced value whose release could run a + /// finalizer deopts *before* the store (the `wpjit_attr_set` /// discipline), as do active C-API dict watchers and any key-lane /// surprise. DictSet { key: JitType, val: JitType }, /// `del d[k]` on a pinned exact `dict`: pops the key (`key` lane) /// and the dict pin, and calls the registered `wpjit_dict_del` /// helper — the interpreter's own `dict_remove` chokepoint. A - /// missing key, a displaced value that would run the prompt-reap - /// cascade, active C-API dict watchers, or any key-lane surprise + /// missing key, a displaced value whose release could run a + /// finalizer, active C-API dict watchers, or any key-lane surprise /// deopts *before* the delete, and the interpreter re-executes it /// (raising the exact `KeyError`). DictDel { key: JitType }, @@ -1486,6 +1513,8 @@ impl TOp { | TOp::CallPy { .. } | TOp::CallPyKw { .. } | TOp::CallMethod { .. } + | TOp::ObjGetItem { .. } + | TOp::GuardMethod { .. } | TOp::MathIntrinsic(_) | TOp::FloatArith(ArithKind::FloorDiv | ArithKind::Mod | ArithKind::Pow) | TOp::ListGet { .. } @@ -1499,6 +1528,7 @@ impl TOp { | TOp::AttrSet { .. } | TOp::GuardNotNone | TOp::StrEq { .. } + | TOp::IsObj { .. } | TOp::StrLen | TOp::BytesLen | TOp::BytesGetItem diff --git a/crates/weavepy-jit/src/lib.rs b/crates/weavepy-jit/src/lib.rs index 5e04cb84..227deca9 100644 --- a/crates/weavepy-jit/src/lib.rs +++ b/crates/weavepy-jit/src/lib.rs @@ -42,27 +42,29 @@ pub use ir::{ TOp, TStmt, TTerm, }; pub use runtime::{ - register_attr_get_chain_helper, register_attr_helpers, register_build_map_helper, - register_build_set_helper, register_cached_attr_chain_helper, register_call_dyn_helper, - register_call_dyn_int_helper, register_call_method_helper, register_call_py_helper, - register_cell_helpers, register_const_str_helper, register_contains_dyn_helper, - register_dict_del_helper, register_dict_helpers, register_dict_iter_helper, - register_dyn_attr_helpers, register_float_pow_helper, register_global_obj_helper, - register_iter_helpers, register_iter_new_helper, register_iter_next_pair_helper, - register_list_extra_helpers, register_list_from_range_helper, register_list_helpers, - register_list_next_helper, register_math_helpers, register_poll_helper, - register_self_call_helpers, register_str_format_helpers, register_str_helpers, - register_str_method_helper, register_str_write_helpers, register_truth_helper, - register_tuple_read_helpers, register_unbox_int_helper, AttrGetChainHelper, AttrGetHelper, - AttrSetHelper, BuildListHelper, BuildTupleHelper, BytesGetHelper, CachedAttrChainHelper, - CallDynHelper, CallMethodHelper, CallPyHelper, CallStatus, CellGetHelper, CellSetHelper, - DictAccessHelper, DynAttrHelper, GetIterHelper, IterNextHelper, IterNextPairHelper, JitFrame, + global_obj_list_elem, register_attr_get_chain_helper, register_attr_helpers, + register_build_map_helper, register_build_set_helper, register_cached_attr_chain_helper, + register_call_dyn_helper, register_call_dyn_int_helper, register_call_method_helper, + register_call_native_method_helper, register_call_py_helper, register_cell_helpers, + register_const_str_helper, register_contains_dyn_helper, register_dict_del_helper, + register_dict_helpers, register_dict_iter_helper, register_dyn_attr_helpers, + register_float_pow_helper, register_global_obj_helper, register_guard_method_helper, + register_is_obj_helper, register_iter_helpers, register_iter_new_helper, + register_iter_next_pair_helper, register_list_extra_helpers, register_list_from_range_helper, + register_list_helpers, register_list_next_helper, register_math_helpers, + register_obj_getitem_helper, register_poll_helper, register_self_call_helpers, + register_str_format_helpers, register_str_helpers, register_str_method_helper, + register_str_write_helpers, register_truth_helper, register_tuple_read_helpers, + register_unbox_int_helper, set_obj_layout, AttrGetChainHelper, AttrGetHelper, AttrSetHelper, + BuildListHelper, BuildTupleHelper, BytesGetHelper, CachedAttrChainHelper, CallDynHelper, + CallMethodHelper, CallPyHelper, CallStatus, CellGetHelper, CellSetHelper, DictAccessHelper, + DynAttrHelper, GetIterHelper, GuardMethodHelper, IterNextHelper, IterNextPairHelper, JitFrame, JitStatus, ListAppendHelper, ListFromRangeHelper, ListGetHelper, ListLenHelper, ListNextHelper, - ListRepeatHelper, ListSetHelper, ListSliceHelper, MathBinaryHelper, MathUnaryHelper, + ListRepeatHelper, ListSetHelper, ListSliceHelper, MathBinaryHelper, MathUnaryHelper, ObjLayout, PollHelper, SelfEnterHelper, SelfExitHelper, SelfSlowHelper, SlotTag, StrEqHelper, StrLenHelper, StrModHelper, CALL_GAPS, DICT_KEY_INT, DICT_KEY_STR, DICT_VAL_FLOAT, DICT_VAL_INT, DICT_VAL_OBJ, ITER_ELEM_STR, JIT_POLL_STRIDE, MAX_ATTR_CHAIN_LEN, - MAX_CACHED_ATTR_CHAIN_LEN, + MAX_CACHED_ATTR_CHAIN_LEN, METHOD_GUARD_AT_CALL, METHOD_NATIVE, }; pub use value::JitType; diff --git a/crates/weavepy-jit/src/lower.rs b/crates/weavepy-jit/src/lower.rs index 9f327aac..60649819 100644 --- a/crates/weavepy-jit/src/lower.rs +++ b/crates/weavepy-jit/src/lower.rs @@ -11,6 +11,7 @@ //! locals + spilled stack back into the [`JitFrame`] and returns //! [`JitStatus::Deopt`] so the interpreter resumes at the exact pc. +use cranelift_codegen::entity::EntityRef; use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; use cranelift_codegen::ir::{ types, AbiParam, Block, BlockArg, FuncRef, Function, InstBuilder, MemFlags, SigRef, Signature, @@ -45,9 +46,31 @@ pub(crate) struct LeafTarget { pub n_locals: u32, pub max_stack: u32, pub params: Vec, + /// Per parameter, the default a call that leaves it out binds. + pub defaults: Vec>, } -/// Build the Cranelift function body for `tfunc` into `func`. +/// Where a direct leaf call takes one parameter from: the call's `j`th +/// value (positionals, then keyword values), or a burned-in default. +#[derive(Clone, Copy)] +enum LeafArg { + Arg(usize), + Default(u64), +} + +/// A `CallPy`/`CallPyKw` site's operands, for its ordinary call. +#[derive(Clone, Copy)] +struct SiteCall { + token: u32, + argc: u8, + kwc: u8, + perm: u32, + gaps: u32, +} + +/// Build the Cranelift function body for `tfunc` into `func`; `false` when +/// the body assigned a local the write-back plan missed (the build must be +/// discarded). pub(crate) fn build_function( func: &mut Function, fbctx: &mut FunctionBuilderContext, @@ -55,14 +78,16 @@ pub(crate) fn build_function( ptr_ty: Type, self_func: Option, leaves: Vec, -) { +) -> bool { let mut builder = FunctionBuilder::new(func, fbctx); let mut lc = Lowerer::new(&mut builder, tfunc, ptr_ty); lc.self_func = self_func; lc.leaves = leaves; lc.build(); + let sound = !lc.unlisted_assignment; builder.seal_all_blocks(); builder.finalize(); + sound } struct Lowerer<'a, 'b> { @@ -131,6 +156,64 @@ struct Lowerer<'a, 'b> { self_slow_sig: Option, /// Direct scalar-leaf call targets by token. leaves: Vec, + /// Per local slot, whether the body assigns it (see + /// [`assigned_slots`]): a local it never assigns still holds the + /// frame's own value, so exits and calls don't write it back. + assignable: Vec, + /// The slot each local's variable stands for, by variable index. + var_slots: Vec, + /// Set when the body assigned a local [`assigned_slots`] missed: the + /// build is then unsound and `build_function` reports it. + unlisted_assignment: bool, +} + +/// Which local slots `tfunc`'s body assigns after the entry loads them: +/// every operation and loop step that defines a local's variable (see +/// [`Lowerer::def_local`], which checks the list). +fn assigned_slots(tfunc: &TFunc) -> Vec { + let mut out = vec![false; tfunc.n_locals as usize]; + let mut mark = |slot: u32| { + if let Some(a) = out.get_mut(slot as usize) { + *a = true; + } + }; + for b in &tfunc.blocks { + for st in &b.stmts { + match st.op { + TOp::StoreLocal(slot) + | TOp::IterCapture { + iter_slot: slot, .. + } + | TOp::DictIterNew { iter_slot: slot } => mark(slot), + _ => {} + } + } + match b.term { + TTerm::ForRange { + cur_slot, var_slot, .. + } => { + mark(cur_slot); + mark(var_slot); + } + TTerm::ForList { + idx_slot, var_slot, .. + } => { + mark(idx_slot); + mark(var_slot); + } + TTerm::ForIter { var_slot, .. } => mark(var_slot), + TTerm::ForIterPair { + var1_slot, + var2_slot, + .. + } => { + mark(var1_slot); + mark(var2_slot); + } + _ => {} + } + } + out } impl<'a, 'b> Lowerer<'a, 'b> { @@ -166,9 +249,27 @@ impl<'a, 'b> Lowerer<'a, 'b> { self_sig: None, self_slow_sig: None, leaves: Vec::new(), + assignable: Vec::new(), + var_slots: Vec::new(), + unlisted_assignment: false, } } + /// Assign the managed local `var` (anything but its entry load). + fn def_local(&mut self, var: Variable, v: Value) { + let listed = self + .var_slots + .get(var.index()) + .and_then(|&slot| self.assignable.get(slot as usize)) + .copied() + .unwrap_or(false); + if !listed { + debug_assert!(false, "an assignment `assigned_slots` doesn't list"); + self.unlisted_assignment = true; + } + self.b.def_var(var, v); + } + fn cl_ty(ty: JitType) -> Type { match ty { JitType::Float => types::F64, @@ -278,6 +379,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { // Declare + initialise a variable per managed local. self.vars = vec![None; self.tfunc.n_locals as usize]; + self.assignable = assigned_slots(self.tfunc); for slot in 0..self.tfunc.local_types.len() { if let Some(ty) = self.tfunc.local_types[slot] { let cl = Self::cl_ty(ty); @@ -286,6 +388,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let v = self.b.ins().load(cl, trusted, self.locals_base, off); self.b.def_var(var, v); self.vars[slot] = Some(var); + if self.var_slots.len() <= var.index() { + self.var_slots.resize(var.index() + 1, u32::MAX); + } + self.var_slots[var.index()] = slot as u32; } } @@ -561,9 +667,9 @@ impl<'a, 'b> Lowerer<'a, 'b> { let eb = self.cl_blocks[exit]; self.b.ins().brif(cond, body_pre, &[], eb, &[]); self.b.switch_to_block(body_pre); - self.b.def_var(loop_var, cur); + self.def_local(loop_var, cur); let next = self.b.ins().iadd_imm(cur, 1); - self.b.def_var(cur_var, next); + self.def_local(cur_var, next); let bb = self.cl_blocks[body]; self.b.ins().jump(bb, &[]); } @@ -595,6 +701,40 @@ impl<'a, 'b> Lowerer<'a, 'b> { // args, and a header deopt spills it. let snapshot = self.vstack.clone(); let args = self.block_args(); + // A scalar element of a scalar-lane list, or the end of + // the list, in line. + let got_b = self.b.create_block(); + self.b.append_block_param(got_b, Self::cl_ty(elem)); + let mut native_done = None; + if let (Some(l), JitType::Int | JitType::Float | JitType::Bool) = + (runtime::obj_layout(), elem) + { + let l = *l; + let miss = self.b.create_block(); + let (items, len, _) = self.pinned_list(&l, seq, &[elem], miss, true); + let negative = self.b.ins().icmp_imm(IntCC::SignedLessThan, idx, 0); + self.miss_if(negative, miss); + let at_end = self.b.ins().icmp(IntCC::SignedGreaterThanOrEqual, idx, len); + let more = self.b.create_block(); + let end = self.b.create_block(); + self.b.ins().brif(at_end, end, &[], more, &[]); + self.b.switch_to_block(more); + let off = self.b.ins().ishl_imm(idx, 4); + let e = self.b.ins().iadd(items, off); + let (want, voff, ty) = Self::lane_tag(&l, elem).expect("scalar lane"); + let tag = self.b.ins().uload8(types::I32, trusted, e, 0); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, tag, i64::from(want)); + self.miss_if(other, miss); + let v = self.b.ins().load(ty, trusted, e, voff); + let v = if ty == types::I8 { + self.b.ins().uextend(types::I64, v) + } else { + v + }; + self.b.ins().jump(got_b, &[BlockArg::from(v)]); + self.b.switch_to_block(miss); + native_done = Some(end); + } let sig = self.list_helper_sig(); let helper = self .b @@ -605,18 +745,26 @@ impl<'a, 'b> Lowerer<'a, 'b> { .ins() .call_indirect(sig, helper, &[self.frame_ptr, seq, idx]); let status = self.b.inst_results(call)[0]; - let got_b = self.b.create_block(); + let fetched = self.b.create_block(); let rest_b = self.b.create_block(); let is_got = self.b.ins().icmp_imm(IntCC::Equal, status, 0); - self.b.ins().brif(is_got, got_b, &[], rest_b, &[]); - self.b.switch_to_block(got_b); + self.b.ins().brif(is_got, fetched, &[], rest_b, &[]); + self.b.switch_to_block(fetched); let res = self.b .ins() .load(Self::cl_ty(elem), trusted, self.frame_ptr, OFF_RET_BITS); - self.b.def_var(loop_var, res); + self.b.ins().jump(got_b, &[BlockArg::from(res)]); + if let Some(end) = native_done { + self.b.switch_to_block(end); + let eb = self.cl_blocks[exit]; + self.b.ins().jump(eb, &args); + } + self.b.switch_to_block(got_b); + let res = self.b.block_params(got_b)[0]; + self.def_local(loop_var, res); let next = self.b.ins().iadd_imm(idx, 1); - self.b.def_var(idx_var, next); + self.def_local(idx_var, next); let bb = self.cl_blocks[body]; self.b.ins().jump(bb, &args); self.b.switch_to_block(rest_b); @@ -684,7 +832,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.b .ins() .load(Self::cl_ty(elem), trusted, self.frame_ptr, OFF_RET_BITS); - self.b.def_var(loop_var, res); + self.def_local(loop_var, res); let bb = self.cl_blocks[body]; self.b.ins().jump(bb, &args); self.b.switch_to_block(rest_b); @@ -772,12 +920,12 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.b .ins() .load(Self::cl_ty(elem1), trusted, self.frame_ptr, OFF_RET_BITS); - self.b.def_var(var1, e1); + self.def_local(var1, e1); let e2 = self .b .ins() .load(Self::cl_ty(elem2), trusted, self.call_args_base, 0); - self.b.def_var(var2, e2); + self.def_local(var2, e2); let bb = self.cl_blocks[body]; self.b.ins().jump(bb, &args); self.b.switch_to_block(rest_b); @@ -928,7 +1076,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { TOp::StoreLocal(slot) => { let (v, _) = self.pop(); let var = self.vars[slot as usize].expect("managed local"); - self.b.def_var(var, v); + self.def_local(var, v); } TOp::IntArith(kind) => self.emit_int_arith(kind, stmt.pc), TOp::FloatArith(kind) => self.emit_float_arith(kind, stmt.pc), @@ -992,8 +1140,15 @@ impl<'a, 'b> Lowerer<'a, 'b> { } => { if is_self && self.self_func.is_some() { self.emit_call_self(token, argc, ret, stmt.pc); - } else if let Some(ix) = self.leaf_for(token, argc) { - self.emit_call_leaf(ix, token, argc, ret, stmt.pc); + } else if let Some((ix, from)) = self.leaf_for(token, argc, 0, 0) { + let call = SiteCall { + token, + argc, + kwc: 0, + perm: 0, + gaps: 0, + }; + self.emit_call_leaf(ix, &from, call, ret, stmt.pc); } else { self.emit_call_py(token, argc, 0, 0, 0, ret, stmt.pc); } @@ -1005,7 +1160,20 @@ impl<'a, 'b> Lowerer<'a, 'b> { perm, gaps, ret, - } => self.emit_call_py(token, argc, kwc, perm, gaps, ret, stmt.pc), + } => { + if let Some((ix, from)) = self.leaf_for(token, argc, kwc, perm) { + let call = SiteCall { + token, + argc, + kwc, + perm, + gaps, + }; + self.emit_call_leaf(ix, &from, call, ret, stmt.pc); + } else { + self.emit_call_py(token, argc, kwc, perm, gaps, ret, stmt.pc); + } + } TOp::ListGet { elem } => self.emit_list_get(elem, stmt.pc), TOp::ListSet => self.emit_list_set(stmt.pc), TOp::CellGet { idx, lane } => self.emit_cell_get(idx, lane, stmt.pc), @@ -1043,10 +1211,44 @@ impl<'a, 'b> Lowerer<'a, 'b> { TOp::CallMethod { token, argc, ret } => { self.emit_call_method(token, argc, ret, stmt.pc); } + TOp::GuardMethod { token } => { + let snapshot = self.vstack.clone(); + let &(pin, _) = self.vstack.last().expect("guard on empty stack"); + let sig = self.list_helper_sig(); + let helper = self + .b + .ins() + .iconst(self.ptr_ty, runtime::guard_method_helper_addr() as i64); + let tokenv = self.b.ins().iconst(types::I64, i64::from(token)); + let call = self + .b + .ins() + .call_indirect(sig, helper, &[self.frame_ptr, pin, tokenv]); + let status = self.b.inst_results(call)[0]; + let bad = self.b.ins().icmp_imm(IntCC::NotEqual, status, 0); + let cont = self.guard(bad, stmt.pc, &snapshot); + self.b.switch_to_block(cont); + } + TOp::ObjGetItem { token, int_result } => { + let ret = MethodRet::Scalar(if int_result { + JitType::Int + } else { + JitType::Obj + }); + self.emit_method_helper_call( + runtime::obj_getitem_helper_addr(), + token, + 1, + ret, + stmt.pc, + false, + ); + } TOp::CallStrMethod { site, argc, ret } => { self.emit_call_str_method(site, argc, ret, stmt.pc); } TOp::StrEq { negate } => self.emit_str_eq(negate, stmt.pc), + TOp::IsObj { negate } => self.emit_is_obj(negate, stmt.pc), TOp::StrLen => self.emit_pin_len(runtime::str_len_helper_addr(), stmt.pc), TOp::BytesLen => self.emit_pin_len(runtime::bytes_len_helper_addr(), stmt.pc), TOp::BytesGetItem => self.emit_bytes_get(stmt.pc), @@ -1133,7 +1335,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { .b .ins() .iconst(self.ptr_ty, runtime::global_obj_helper_addr() as i64); - let tokenv = self.b.ins().iconst(types::I64, i64::from(token)); + let tokenv = self + .b + .ins() + .iconst(types::I64, runtime::global_obj_list_code(token, lane)); let call = self .b .ins() @@ -1168,7 +1373,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); let iter_var = self.vars[iter_slot as usize].expect("managed iter slot"); - self.b.def_var(iter_var, res); + self.def_local(iter_var, res); } /// RFC 0074 WS2 — the opaque-call lane via `wpjit_call_dyn`: @@ -1666,7 +1871,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); let iter_var = self.vars[iter_slot as usize].expect("managed iter slot"); - self.b.def_var(iter_var, pin); + self.def_local(iter_var, pin); } /// RFC 0073 WS2 — the dict-loop capture behind an erased @@ -1692,7 +1897,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); let iter_var = self.vars[iter_slot as usize].expect("managed iter slot"); - self.b.def_var(iter_var, res); + self.def_local(iter_var, res); } /// RFC 0071 WS4 — `BUILD_LIST k`: the elements are staged through @@ -2131,6 +2336,44 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.vstack.push((res, JitType::Bool)); } + /// `a is b` on two object lanes (see [`TOp::IsObj`]): equal machine + /// values (the same pin, or both `None`) are the same object; any + /// other pair asks the helper, whose status above `1` deopts. + fn emit_is_obj(&mut self, negate: bool, pc: u32) { + let snapshot = self.vstack.clone(); + let (b_pin, _) = self.pop(); + let (a_pin, _) = self.pop(); + let same = self.b.ins().icmp(IntCC::Equal, a_pin, b_pin); + let slow = self.b.create_block(); + let merge = self.b.create_block(); + self.b.append_block_param(merge, types::I64); + let one = self.b.ins().iconst(types::I64, 1); + self.b.ins().brif(same, merge, &[one.into()], slow, &[]); + self.b.switch_to_block(slow); + let sig = self.list_helper_sig(); + let helper = self + .b + .ins() + .iconst(self.ptr_ty, runtime::is_obj_helper_addr() as i64); + let call = self + .b + .ins() + .call_indirect(sig, helper, &[self.frame_ptr, a_pin, b_pin]); + let status = self.b.inst_results(call)[0]; + let bad = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThan, status, 1); + let cont = self.guard(bad, pc, &snapshot); + self.b.switch_to_block(cont); + self.b.ins().jump(merge, &[status.into()]); + self.b.switch_to_block(merge); + let r = self.b.block_params(merge)[0]; + let res = if negate { + self.b.ins().bxor_imm(r, 1) + } else { + r + }; + self.vstack.push((res, JitType::Bool)); + } + /// RFC 0071 WS6 — pinned-`str`/`bytes` length (helper selected by /// address; both share the [`Self::pin_helper_sig`] shape and the /// negative-status deopt of `emit_list_len`). @@ -2452,6 +2695,29 @@ impl<'a, 'b> Lowerer<'a, 'b> { /// the bound method + `Unbound` pair), so the interpreter /// re-executes the call generically. fn emit_call_method(&mut self, token: u32, argc: u8, ret: MethodRet, pc: u32) { + let native = token & runtime::METHOD_NATIVE != 0; + let helper = if native { + runtime::call_native_method_helper_addr() + } else { + runtime::call_method_helper_addr() + }; + self.emit_method_helper_call(helper, token, argc, ret, pc, !native); + } + + /// [`Self::emit_call_method`]'s body over any helper sharing the + /// method helper's shape and status protocol (`wpjit_obj_getitem` + /// stages its index as the one argument). `writeback` stores the + /// locals first, for a helper that may run Python on this + /// activation's behalf. + fn emit_method_helper_call( + &mut self, + helper_addr: usize, + token: u32, + argc: u8, + ret: MethodRet, + pc: u32, + writeback: bool, + ) { let trusted = MemFlags::trusted(); let n = argc as usize; // Snapshot *including* receiver + args: the Reject exit re-runs @@ -2465,11 +2731,42 @@ impl<'a, 'b> Lowerer<'a, 'b> { let tagv = self.b.ins().iconst(types::I32, Self::tag(ty)); self.b.ins().store(trusted, tagv, self.call_tags_base, toff); } + // What a field update would add: the `int` argument, or (none + // passed) the method's literal. + let update_arg = match &self.vstack[base..] { + [] => Some(None), + [(v, JitType::Int)] => Some(Some(*v)), + _ => None, + }; self.vstack.truncate(base); let (pin, _) = self.pop(); let snapshot = self.vstack.clone(); - self.writeback_locals(); + // A method whose body is a field update, run in line once the + // helper has armed it (a Python method's helper only); anything + // else takes the helper below. + let inline = match (runtime::obj_layout(), ret, update_arg) { + (Some(l), MethodRet::Scalar(JitType::Int), Some(arg)) + if helper_addr == runtime::call_method_helper_addr() => + { + Some((*l, arg)) + } + _ => None, + }; + let mut join = None; + if let Some((l, arg)) = inline { + let miss = self.b.create_block(); + let done = self.b.create_block(); + self.b.append_block_param(done, types::I64); + let v = self.inline_method_update(&l, pin, token, arg, miss); + self.b.ins().jump(done, &[BlockArg::from(v)]); + self.b.switch_to_block(miss); + join = Some(done); + } + + if writeback { + self.writeback_locals(); + } self.store_call_site_pc(pc); let expect = match ret { @@ -2477,10 +2774,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { MethodRet::Scalar(t) => Self::tag(t), }; let sig = self.call_method_helper_sig(); - let helper = self - .b - .ins() - .iconst(self.ptr_ty, runtime::call_method_helper_addr() as i64); + let helper = self.b.ins().iconst(self.ptr_ty, helper_addr as i64); let tokenv = self.b.ins().iconst(types::I32, i64::from(token)); let argcv = self.b.ins().iconst(types::I32, i64::from(argc)); let expectv = self.b.ins().iconst(types::I32, expect); @@ -2520,10 +2814,152 @@ impl<'a, 'b> Lowerer<'a, 'b> { .b .ins() .load(Self::cl_ty(t), trusted, self.frame_ptr, OFF_RET_BITS); + let res = match join { + Some(done) => { + self.b.ins().jump(done, &[BlockArg::from(res)]); + self.b.switch_to_block(done); + self.b.block_params(done)[0] + } + None => res, + }; self.vstack.push((res, t)); } } + /// The armed field update of method token `token` on the pinned + /// receiver `pin` (see [`runtime::ObjLayout::method_upd_idx`]): the + /// updated field's new value, with the current block continuing past + /// the store. A receiver of another class version, values not split + /// over the class's names or numerous enough to shadow the method, a + /// swapped `__code__`, a non-`int` field, an overflow, or an active + /// observer, dict watcher or exotic key branches to `miss` (the helper, + /// which runs the call exactly) before anything is stored. + fn inline_method_update( + &mut self, + l: &runtime::ObjLayout, + pin: Value, + token: u32, + arg: Option, + miss: Block, + ) -> Value { + let t = MemFlags::trusted(); + let ptr = self.ptr_ty; + let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); + // The entry: armed, for this call's increment form. + let methods = self.b.ins().load(ptr, t, ctx, l.ctx_methods); + let n = self.b.ins().load(types::I64, t, methods, l.methods_len); + let out = self + .b + .ins() + .icmp_imm(IntCC::UnsignedLessThanOrEqual, n, i64::from(token)); + self.miss_if(out, miss); + let mbuf = self.b.ins().load(ptr, t, methods, l.methods_buf); + let e = self + .b + .ins() + .iadd_imm(mbuf, i64::from(token) * i64::from(l.method_size)); + let idx = self.b.ins().uload32(t, e, l.method_upd_idx); + let from_arg = self.b.ins().uload32(t, e, l.method_upd_from_arg); + let unarmed = self + .b + .ins() + .icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); + let form = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, from_arg, i64::from(arg.is_some())); + // The function's code, and the gates. + let code_at = self.b.ins().load(ptr, t, e, l.method_upd_code_at); + let want = self.b.ins().load(ptr, t, e, l.method_upd_code); + let observers = self.b.ins().iconst(ptr, l.observers as i64); + let observers = self.b.ins().load(types::I64, t, observers, 0); + let watchers = self.b.ins().iconst(ptr, l.dict_watchers as i64); + let watchers = self.b.ins().uload8(types::I64, t, watchers, 0); + let exotic = self.b.ins().iconst(ptr, l.exotic_keys as i64); + let exotic = self.b.ins().load(types::I64, t, exotic, 0); + let gates = self.b.ins().bor(observers, watchers); + let gates = self.b.ins().bor(gates, exotic); + let gated = self.b.ins().icmp_imm(IntCC::NotEqual, gates, 0); + let bad = self.b.ins().bor(unarmed, form); + let bad = self.b.ins().bor(bad, gated); + self.miss_if(bad, miss); + let code = self.b.ins().load(ptr, t, code_at, 0); + let swapped = self.b.ins().icmp(IntCC::NotEqual, code, want); + self.miss_if(swapped, miss); + // The pin: an instance. + let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); + self.miss_if(out, miss); + let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); + let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); + let p = self.b.ins().iadd(buf, off); + let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); + let otag = self.b.ins().uload8(types::I32, t, p, l.pin_obj); + let not_obj = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + let bad = self.b.ins().bor(not_obj, not_inst); + self.miss_if(bad, miss); + let inst = self.b.ins().load(ptr, t, p, l.pin_obj + 8); + // Its class and split values, as `split_field_addr` checks them + // for a write, against the entry's version. + let ver = self.b.ins().load(types::I64, t, e, l.method_ver); + let cls = self.b.ins().load(ptr, t, inst, l.inst_class); + let cver = self.b.ins().load(types::I64, t, cls, l.type_attr_version); + let lazy = self.b.ins().load(ptr, t, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, t, flag, 0); + let borrow = self.b.ins().sload32(t, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, t, inst, l.inst_split_block); + let stale = self.b.ins().icmp(IntCC::NotEqual, cver, ver); + let busy = self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(stale, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, miss); + // The class's names, holding the field but not the method's name. + let keys = self.b.ins().load(ptr, t, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, t, cls, l.type_shared_keys); + let len = self.b.ins().uload32(t, block, l.split_len); + let shadow = self.b.ins().uload32(t, e, l.method_upd_shadow); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let shadowed = self.b.ins().icmp(IntCC::UnsignedGreaterThan, len, shadow); + let bad = self.b.ins().bor(foreign, absent); + let bad = self.b.ins().bor(bad, shadowed); + self.miss_if(bad, miss); + let off = self.b.ins().ishl_imm(idx, 4); + let at = self.b.ins().iadd(block, off); + let at = self.b.ins().iadd_imm(at, i64::from(l.split_values)); + // An `int` field, plus the increment without overflow. + let tag = self.b.ins().uload8(types::I32, t, at, 0); + let not_int = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, tag, i64::from(l.tag_int)); + self.miss_if(not_int, miss); + let old = self.b.ins().load(types::I64, t, at, 8); + let inc = match arg { + Some(a) => a, + None => self.b.ins().load(types::I64, t, e, l.method_upd_inc), + }; + let (new, overflow) = self.checked_add(old, inc); + self.miss_if(overflow, miss); + self.b.ins().store(t, new, at, 8); + new + } + /// RFC 0073 WS3 — burned-in native `str`-method call via the /// registered `wpjit_str_method` helper (the [`Self::emit_call_method`] /// discipline, sharing its signature). The helper receives the @@ -2656,6 +3092,7 @@ impl<'a, 'b> Lowerer<'a, 'b> { let snapshot = self.vstack.clone(); let (val, ty) = self.pop(); let (pin, _) = self.pop(); + let appended = self.inline_list_append(pin, val, ty); self.b .ins() .store(trusted, val, self.frame_ptr, OFF_RET_BITS); @@ -2676,10 +3113,54 @@ impl<'a, 'b> Lowerer<'a, 'b> { let bad = self.b.ins().icmp_imm(IntCC::NotEqual, status, 0); let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); + if let Some(done) = appended { + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } // `append` returns `None`, which the following `POP_TOP` // consumes — neither ever exists on the native stack. } + /// `append` of a scalar to a pinned list with room for it, in line; + /// the block to continue at, with the current block the helper's + /// path (`None` when not in line). + fn inline_list_append(&mut self, pin: Value, val: Value, ty: JitType) -> Option { + let l = *runtime::obj_layout()?; + // The lanes the helper boxes this scalar into. + let lanes: &[JitType] = match ty { + JitType::Int => &[JitType::Int, JitType::Obj], + JitType::Float => &[JitType::Float, JitType::Obj], + JitType::Bool => &[JitType::Obj], + _ => return None, + }; + let t = MemFlags::trusted(); + let miss = self.b.create_block(); + let done = self.b.create_block(); + let (items, len, list) = self.pinned_list(&l, pin, lanes, miss, false); + let cap = self.b.ins().load(types::I64, t, list, l.list_cap); + let full = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, len, cap); + self.miss_if(full, miss); + let off = self.b.ins().ishl_imm(len, 4); + let at = self.b.ins().iadd(items, off); + let (tag, voff, _) = Self::lane_tag(&l, ty).expect("scalar lane"); + let v = if ty == JitType::Bool { + self.b.ins().ireduce(types::I8, val) + } else { + val + }; + let tagv = self.b.ins().iconst(types::I8, i64::from(tag)); + self.b.ins().store(t, tagv, at, 0); + self.b.ins().store(t, v, at, voff); + let len1 = self.b.ins().iadd_imm(len, 1); + self.b.ins().store(t, len1, list, l.list_len); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(miss); + Some(done) + } + /// RFC 0073 WS1 — `LIST_APPEND` inside an inlined comprehension: /// identical to [`Self::emit_list_append`] except the accumulator /// pin stays on the native stack (it is live across the whole @@ -2719,6 +3200,36 @@ impl<'a, 'b> Lowerer<'a, 'b> { let trusted = MemFlags::trusted(); let snapshot = self.vstack.clone(); let (pin, _) = self.pop(); + // A scalar field of a split-layout instance, read in line; any + // other shape takes the helper below. + let inline = match (runtime::obj_layout(), out) { + (Some(l), JitType::Int | JitType::Float | JitType::Bool) => Some(*l), + _ => None, + }; + let mut join = None; + if let Some(l) = inline { + let miss = self.b.create_block(); + let done = self.b.create_block(); + self.b.append_block_param(done, Self::cl_ty(out)); + let addr = self.split_field_addr(&l, pin, site, miss, true); + let (want, off, ty) = match out { + JitType::Int => (l.tag_int, 8, types::I64), + JitType::Float => (l.tag_float, 8, types::F64), + _ => (l.tag_bool, 1, types::I8), + }; + let tag = self.b.ins().uload8(types::I32, trusted, addr, 0); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, tag, i64::from(want)); + self.miss_if(other, miss); + let v = self.b.ins().load(ty, trusted, addr, off); + let v = if ty == types::I8 { + self.b.ins().uextend(types::I64, v) + } else { + v + }; + self.b.ins().jump(done, &[BlockArg::from(v)]); + self.b.switch_to_block(miss); + join = Some(done); + } let sig = self.list_helper_sig(); let helper = self .b @@ -2737,9 +3248,280 @@ impl<'a, 'b> Lowerer<'a, 'b> { .b .ins() .load(Self::cl_ty(out), trusted, self.frame_ptr, OFF_RET_BITS); + let res = match join { + Some(done) => { + self.b.ins().jump(done, &[BlockArg::from(res)]); + self.b.switch_to_block(done); + self.b.block_params(done)[0] + } + None => res, + }; self.vstack.push((res, out)); } + /// Branch to `miss` when `cond`; continue in a fresh block otherwise. + fn miss_if(&mut self, cond: Value, miss: Block) { + let ok = self.b.create_block(); + self.b.ins().brif(cond, miss, &[], ok, &[]); + self.b.switch_to_block(ok); + } + + /// The address of the field the attribute guard of `site` names in the + /// pinned instance `pin`'s split values (see [`runtime::ObjLayout`]): + /// an instance pin whose class still has the guard's version, whose + /// values are split over its class's names, unborrowed (`read`) or + /// unborrowed and exclusive (a write), and hold the guard's index. + /// Anything else branches to `miss`. + fn split_field_addr( + &mut self, + l: &runtime::ObjLayout, + pin: Value, + site: u32, + miss: Block, + read: bool, + ) -> Value { + let t = MemFlags::trusted(); + let ptr = self.ptr_ty; + let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); + // The guard first: a site whose field has no split position (a + // `__slots__` member) leaves straight away. + let guards = self.b.ins().load(ptr, t, ctx, l.ctx_guards); + let gbuf = self.b.ins().load(ptr, t, guards, l.guards_buf); + let g = self + .b + .ins() + .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); + let idx = self.b.ins().uload32(t, g, l.guard_split_idx); + let none = self + .b + .ins() + .icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); + // The pin. + let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); + let bad = self.b.ins().bor(none, out); + self.miss_if(bad, miss); + let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); + let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); + let p = self.b.ins().iadd(buf, off); + let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); + let otag = self.b.ins().uload8(types::I32, t, p, l.pin_obj); + let not_obj = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + let bad = self.b.ins().bor(not_obj, not_inst); + self.miss_if(bad, miss); + let inst = self.b.ins().load(ptr, t, p, l.pin_obj + 8); + let ver = self.b.ins().load(types::I64, t, g, l.guard_ver); + // The class and the split values (grouped by what each group's + // loads need proven: fewer blocks compile faster). + let cls = self.b.ins().load(ptr, t, inst, l.inst_class); + let cver = self.b.ins().load(types::I64, t, cls, l.type_attr_version); + let lazy = self.b.ins().load(ptr, t, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, t, flag, 0); + let borrow = self.b.ins().sload32(t, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, t, inst, l.inst_split_block); + let stale = self.b.ins().icmp(IntCC::NotEqual, cver, ver); + let busy = if read { + self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0) + } else { + self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0) + }; + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(stale, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, miss); + let keys = self.b.ins().load(ptr, t, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, t, cls, l.type_shared_keys); + let len = self.b.ins().uload32(t, block, l.split_len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let bad = self.b.ins().bor(foreign, absent); + self.miss_if(bad, miss); + let off = self.b.ins().ishl_imm(idx, 4); + let at = self.b.ins().iadd(block, off); + self.b.ins().iadd_imm(at, i64::from(l.split_values)) + } + + /// The address the new-key store at `site` appends its field to, in + /// the pinned instance `pin`'s split values (see + /// [`runtime::ObjLayout`]), having counted it in: an ordinary instance + /// pin whose class still has the guard's version, whose values are + /// split over its class's names, unborrowed and exclusive, hold exactly + /// the names before the guard's index and have room for one more, with + /// no dict watcher active. Anything else branches to `miss` before + /// anything is written. + fn split_append_addr( + &mut self, + l: &runtime::ObjLayout, + pin: Value, + site: u32, + miss: Block, + ) -> Value { + let t = MemFlags::trusted(); + let ptr = self.ptr_ty; + let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); + let guards = self.b.ins().load(ptr, t, ctx, l.ctx_guards); + let gbuf = self.b.ins().load(ptr, t, guards, l.guards_buf); + let g = self + .b + .ins() + .iadd_imm(gbuf, i64::from(site) * i64::from(l.guard_size)); + let idx = self.b.ins().uload32(t, g, l.guard_split_idx); + let none = self + .b + .ins() + .icmp_imm(IntCC::Equal, idx, i64::from(u32::MAX)); + let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); + let bad = self.b.ins().bor(none, out); + self.miss_if(bad, miss); + let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); + let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); + let p = self.b.ins().iadd(buf, off); + let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); + let otag = self.b.ins().uload8(types::I32, t, p, l.pin_obj); + let not_obj = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_obj_tag)); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + let bad = self.b.ins().bor(not_obj, not_inst); + self.miss_if(bad, miss); + let inst = self.b.ins().load(ptr, t, p, l.pin_obj + 8); + let ver = self.b.ins().load(types::I64, t, g, l.guard_ver); + let cls = self.b.ins().load(ptr, t, inst, l.inst_class); + let cver = self.b.ins().load(types::I64, t, cls, l.type_attr_version); + let lazy = self.b.ins().load(ptr, t, inst, l.inst_dict_lazy); + let body = self.b.ins().load(ptr, t, inst, l.inst_c_body); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, t, flag, 0); + let watchers = self.b.ins().iconst(ptr, l.dict_watchers as i64); + let watchers = self.b.ins().uload8(types::I64, t, watchers, 0); + let borrow = self.b.ins().sload32(t, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, t, inst, l.inst_split_block); + let stale = self.b.ins().icmp(IntCC::NotEqual, cver, ver); + let busy = self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, body); + let other = self.b.ins().bor(other, shared); + let other = self.b.ins().bor(other, watchers); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(stale, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, miss); + let keys = self.b.ins().load(ptr, t, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, t, cls, l.type_shared_keys); + let len = self.b.ins().uload32(t, block, l.split_len); + let cap = self.b.ins().uload32(t, block, l.split_cap); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let elsewhere = self.b.ins().icmp(IntCC::NotEqual, idx, len); + let full = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, len, cap); + let bad = self.b.ins().bor(foreign, elsewhere); + let bad = self.b.ins().bor(bad, full); + self.miss_if(bad, miss); + let len1 = self.b.ins().iadd_imm(len, 1); + let len1 = self.b.ins().ireduce(types::I32, len1); + self.b.ins().store(t, len1, block, l.split_len); + let off = self.b.ins().ishl_imm(idx, 4); + let at = self.b.ins().iadd(block, off); + self.b.ins().iadd_imm(at, i64::from(l.split_values)) + } + + /// The pinned list `pin`'s items (buffer pointer and length; and its + /// capacity, and where the length lives, for an append): a list pin + /// whose element lane is one of `lanes`, unborrowed (`read`) or + /// unborrowed and exclusive (a write). Anything else branches to + /// `miss`. + fn pinned_list( + &mut self, + l: &runtime::ObjLayout, + pin: Value, + lanes: &[JitType], + miss: Block, + read: bool, + ) -> (Value, Value, Value) { + let t = MemFlags::trusted(); + let ptr = self.ptr_ty; + let ctx = self.b.ins().load(ptr, t, self.frame_ptr, OFF_CTX); + let n = self.b.ins().load(types::I64, t, ctx, l.ctx_pins_len); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, pin, n); + self.miss_if(out, miss); + let buf = self.b.ins().load(ptr, t, ctx, l.ctx_pins_ptr); + let off = self.b.ins().imul_imm(pin, i64::from(l.pin_size)); + let p = self.b.ins().iadd(buf, off); + let ptag = self.b.ins().uload8(types::I32, t, p, l.pin_tag); + let lane = self.b.ins().uload8(types::I32, t, p, l.pin_list_elem); + let mut bad = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, ptag, i64::from(l.pin_list_tag)); + let mut ok_lane = self.b.ins().iconst(types::I8, 0); + for &want in lanes { + let hit = self.b.ins().icmp_imm(IntCC::Equal, lane, want as i64); + ok_lane = self.b.ins().bor(ok_lane, hit); + } + let wrong_lane = self.b.ins().bxor_imm(ok_lane, 1); + bad = self.b.ins().bor(bad, wrong_lane); + self.miss_if(bad, miss); + let list = self.b.ins().load(ptr, t, p, l.pin_list); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I32, t, flag, 0); + let borrow = self.b.ins().sload32(t, list, l.list_borrow); + let busy = if read { + self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0) + } else { + self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0) + }; + let shared = self.b.ins().icmp_imm(IntCC::NotEqual, shared, 0); + let bad = self.b.ins().bor(busy, shared); + self.miss_if(bad, miss); + let items = self.b.ins().load(ptr, t, list, l.list_ptr); + let len = self.b.ins().load(types::I64, t, list, l.list_len); + (items, len, list) + } + + /// The element address for a (possibly negative) in-range `idx` of + /// `len` items at `items`; out of range branches to `miss`. + fn list_elem(&mut self, items: Value, len: Value, idx: Value, miss: Block) -> Value { + let neg = self.b.ins().icmp_imm(IntCC::SignedLessThan, idx, 0); + let wrapped = self.b.ins().iadd(idx, len); + let i = self.b.ins().select(neg, wrapped, idx); + let out = self.b.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, i, len); + self.miss_if(out, miss); + let off = self.b.ins().ishl_imm(i, 4); + self.b.ins().iadd(items, off) + } + + /// The object tag of a scalar lane. + fn lane_tag(l: &runtime::ObjLayout, lane: JitType) -> Option<(u8, i32, Type)> { + match lane { + JitType::Int => Some((l.tag_int, 8, types::I64)), + JitType::Float => Some((l.tag_float, 8, types::F64)), + JitType::Bool => Some((l.tag_bool, 1, types::I8)), + _ => None, + } + } + /// Consecutive callback-free reads can replay from the first read on a /// later miss. Preserve its original receiver in the deopt snapshot. fn emit_attr_get_chain( @@ -2892,7 +3674,62 @@ impl<'a, 'b> Lowerer<'a, 'b> { let trusted = MemFlags::trusted(); let snapshot = self.vstack.clone(); let (pin, _) = self.pop(); - let (val, _) = self.pop(); + let (val, lane) = self.pop(); + // A scalar over a scalar field of a split-layout instance, stored + // in line (no release, no collector bookkeeping); any other shape + // takes the helper below. + let inline = match (runtime::obj_layout(), lane) { + (Some(l), JitType::Int | JitType::Float | JitType::Bool) => Some(*l), + _ => None, + }; + let new_key = self + .tfunc + .attr_sites + .get(site as usize) + .is_some_and(|s| s.new_key); + let mut done = None; + if let (Some(l), true) = (inline, new_key) { + // The constructor pattern: a fresh instance's next field. + let miss = self.b.create_block(); + let stored = self.b.create_block(); + let addr = self.split_append_addr(&l, pin, site, miss); + let (tag, off, v) = match lane { + JitType::Int => (l.tag_int, 8, val), + JitType::Float => (l.tag_float, 8, val), + _ => (l.tag_bool, 1, self.b.ins().ireduce(types::I8, val)), + }; + let tagv = self.b.ins().iconst(types::I8, i64::from(tag)); + self.b.ins().store(trusted, tagv, addr, 0); + self.b.ins().store(trusted, v, addr, off); + self.b.ins().jump(stored, &[]); + self.b.switch_to_block(miss); + done = Some(stored); + } else if let Some(l) = inline { + let miss = self.b.create_block(); + let stored = self.b.create_block(); + let addr = self.split_field_addr(&l, pin, site, miss, false); + let old = self.b.ins().uload8(types::I64, trusted, addr, 0); + let one = self.b.ins().iconst(types::I64, 1); + let bit = self.b.ins().ishl(one, old); + let scalars = (1i64 << l.tag_int) + | (1i64 << l.tag_float) + | (1i64 << l.tag_bool) + | (1i64 << l.tag_none); + let m = self.b.ins().band_imm(bit, scalars); + let heap = self.b.ins().icmp_imm(IntCC::Equal, m, 0); + self.miss_if(heap, miss); + let (tag, off, v) = match lane { + JitType::Int => (l.tag_int, 8, val), + JitType::Float => (l.tag_float, 8, val), + _ => (l.tag_bool, 1, self.b.ins().ireduce(types::I8, val)), + }; + let tagv = self.b.ins().iconst(types::I8, i64::from(tag)); + self.b.ins().store(trusted, tagv, addr, 0); + self.b.ins().store(trusted, v, addr, off); + self.b.ins().jump(stored, &[]); + self.b.switch_to_block(miss); + done = Some(stored); + } self.b .ins() .store(trusted, val, self.frame_ptr, OFF_RET_BITS); @@ -2910,6 +3747,10 @@ impl<'a, 'b> Lowerer<'a, 'b> { let bad = self.b.ins().icmp_imm(IntCC::NotEqual, status, 0); let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); + if let Some(stored) = done { + self.b.ins().jump(stored, &[]); + self.b.switch_to_block(stored); + } } /// RFC 0076 WS6 — closure-cell read via `wpjit_cell_get`. The cell @@ -3055,6 +3896,24 @@ impl<'a, 'b> Lowerer<'a, 'b> { let snapshot = self.vstack.clone(); let (idx, _) = self.pop(); let (pin, _) = self.pop(); + // A scalar element of a scalar-lane list, read in line. + let mut join = None; + if let (Some(l), JitType::Int | JitType::Float) = (runtime::obj_layout(), elem) { + let l = *l; + let miss = self.b.create_block(); + let done = self.b.create_block(); + self.b.append_block_param(done, Self::cl_ty(elem)); + let (items, len, _) = self.pinned_list(&l, pin, &[elem], miss, true); + let e = self.list_elem(items, len, idx, miss); + let (want, off, ty) = Self::lane_tag(&l, elem).expect("scalar lane"); + let tag = self.b.ins().uload8(types::I32, trusted, e, 0); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, tag, i64::from(want)); + self.miss_if(other, miss); + let v = self.b.ins().load(ty, trusted, e, off); + self.b.ins().jump(done, &[BlockArg::from(v)]); + self.b.switch_to_block(miss); + join = Some(done); + } let sig = self.list_helper_sig(); let helper = self .b @@ -3072,6 +3931,14 @@ impl<'a, 'b> Lowerer<'a, 'b> { .b .ins() .load(Self::cl_ty(elem), trusted, self.frame_ptr, OFF_RET_BITS); + let res = match join { + Some(done) => { + self.b.ins().jump(done, &[BlockArg::from(res)]); + self.b.switch_to_block(done); + self.b.block_params(done)[0] + } + None => res, + }; self.vstack.push((res, elem)); } @@ -3083,7 +3950,21 @@ impl<'a, 'b> Lowerer<'a, 'b> { let snapshot = self.vstack.clone(); let (idx, _) = self.pop(); let (pin, _) = self.pop(); - let (val, _) = self.pop(); + let (val, vty) = self.pop(); + // A scalar over a scalar element of a list of that lane, stored in + // line (no release). + let mut stored = None; + if let (Some(l), JitType::Int | JitType::Float) = (runtime::obj_layout(), vty) { + let l = *l; + let miss = self.b.create_block(); + let done = self.b.create_block(); + let (items, len, _) = self.pinned_list(&l, pin, &[vty], miss, false); + let e = self.list_elem(items, len, idx, miss); + self.store_scalar_over_scalar(&l, e, val, vty, miss); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(miss); + stored = Some(done); + } // Typed store: an F64 value lands as its bit pattern. self.b .ins() @@ -3101,6 +3982,43 @@ impl<'a, 'b> Lowerer<'a, 'b> { let bad = self.b.ins().icmp_imm(IntCC::NotEqual, status, 0); let cont = self.guard(bad, pc, &snapshot); self.b.switch_to_block(cont); + if let Some(done) = stored { + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + } + + /// Store the scalar `val` (of `lane`) over the value at `at` when that + /// one is a scalar too (nothing to release); otherwise branch to + /// `miss`, untouched. + fn store_scalar_over_scalar( + &mut self, + l: &runtime::ObjLayout, + at: Value, + val: Value, + lane: JitType, + miss: Block, + ) { + let t = MemFlags::trusted(); + let old = self.b.ins().uload8(types::I64, t, at, 0); + let one = self.b.ins().iconst(types::I64, 1); + let bit = self.b.ins().ishl(one, old); + let scalars = (1i64 << l.tag_int) + | (1i64 << l.tag_float) + | (1i64 << l.tag_bool) + | (1i64 << l.tag_none); + let m = self.b.ins().band_imm(bit, scalars); + let heap = self.b.ins().icmp_imm(IntCC::Equal, m, 0); + self.miss_if(heap, miss); + let (tag, off, _) = Self::lane_tag(l, lane).expect("scalar lane"); + let v = if lane == JitType::Bool { + self.b.ins().ireduce(types::I8, val) + } else { + val + }; + let tagv = self.b.ins().iconst(types::I8, i64::from(tag)); + self.b.ins().store(t, tagv, at, 0); + self.b.ins().store(t, v, at, off); } /// The shared `(frame, pin, idx) -> status` signature of the @@ -3304,28 +4222,57 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.vstack.push((v, ret)); } - /// The direct leaf target of `token` when this site's argument lanes - /// are exactly the leaf's parameter lanes. - fn leaf_for(&self, token: u32, argc: u8) -> Option { + /// The direct leaf target of `token`, and where each of its + /// parameters comes from, when this site's `argc` positional and `kwc` + /// keyword values (keyword `j` binding slot `(perm >> 4j) & 0xF`) have + /// the leaf's parameter lanes and every parameter they leave out has a + /// default the leaf carries. + fn leaf_for(&self, token: u32, argc: u8, kwc: u8, perm: u32) -> Option<(usize, Vec)> { let ix = self.leaves.iter().position(|l| l.token == token)?; - let base = self.vstack.len().checked_sub(argc as usize)?; - let lanes = self.vstack[base..].iter().map(|&(_, ty)| ty); - lanes - .eq(self.leaves[ix].params.iter().copied()) - .then_some(ix) + let leaf = &self.leaves[ix]; + let n = argc as usize + kwc as usize; + let base = self.vstack.len().checked_sub(n)?; + let mut from: Vec> = vec![None; leaf.params.len()]; + for j in 0..n { + let slot = if j < argc as usize { + j + } else { + ((perm >> (4 * (j - argc as usize))) & 0xF) as usize + }; + let at = from.get_mut(slot)?; + if at.is_some() || self.vstack[base + j].1 != leaf.params[slot] { + return None; + } + *at = Some(LeafArg::Arg(j)); + } + let from = from + .iter() + .zip(&leaf.defaults) + .map(|(&f, &d)| f.or(d.map(LeafArg::Default))) + .collect::>>()?; + Some((ix, from)) } /// Lower a direct call of a compiled scalar leaf (see [`LeafTarget`]): /// charge the activation through the self-call enter helper, fill the - /// leaf's `JitFrame` on this function's native stack frame, call it, - /// and release the charge. The leaf only computes, so when the enter - /// helper declines or the leaf deopts (an overflow, a zero divisor), - /// the ordinary call helper runs the call from the start. - fn emit_call_leaf(&mut self, ix: usize, token: u32, argc: u8, ret: JitType, pc: u32) { + /// leaf's `JitFrame` on this function's native stack frame (the + /// parameters from the call's values and the leaf's burned-in + /// defaults, per `from`), call it, and release the charge. The leaf + /// only computes, so when the enter helper declines or the leaf deopts + /// (an overflow, a zero divisor), the ordinary call helper runs the + /// call from the start. + fn emit_call_leaf( + &mut self, + ix: usize, + from: &[LeafArg], + call: SiteCall, + ret: JitType, + pc: u32, + ) { let trusted = MemFlags::trusted(); let (enter_addr, exit_addr, _) = runtime::self_call_helper_addrs().expect("checked by the engine"); - let n = argc as usize; + let n = call.argc as usize + call.kwc as usize; let base = self.vstack.len() - n; let args: Vec<(Value, JitType)> = self.vstack[base..].to_vec(); self.vstack.truncate(base); @@ -3334,8 +4281,8 @@ impl<'a, 'b> Lowerer<'a, 'b> { let sig = self.self_sig(); let enter = self.b.ins().iconst(self.ptr_ty, enter_addr as i64); - let call = self.b.ins().call_indirect(sig, enter, &[self.frame_ptr]); - let declined = self.b.inst_results(call)[0]; + let c = self.b.ins().call_indirect(sig, enter, &[self.frame_ptr]); + let declined = self.b.inst_results(c)[0]; let direct_b = self.b.create_block(); let generic_b = self.b.create_block(); @@ -3386,11 +4333,15 @@ impl<'a, 'b> Lowerer<'a, 'b> { self.b.ins().store(trusted, null, fp, OFF_CALL_ARGS); self.b.ins().store(trusted, null, fp, OFF_CALL_TAGS); for slot_ix in 0..n_locals { - let v = args.get(slot_ix as usize).map_or(zero64, |&(v, _)| v); + let v = match from.get(slot_ix as usize) { + Some(&LeafArg::Arg(j)) => args[j].0, + Some(&LeafArg::Default(bits)) => self.b.ins().iconst(types::I64, bits as i64), + None => zero64, + }; self.b.ins().store(trusted, v, locals, slot_ix * 8); } - let call = self.b.ins().call(func, &[fp]); - let status = self.b.inst_results(call)[0]; + let c = self.b.ins().call(func, &[fp]); + let status = self.b.inst_results(c)[0]; let exit = self.b.ins().iconst(self.ptr_ty, exit_addr as i64); self.b.ins().call_indirect(sig, exit, &[self.frame_ptr]); let returned_b = self.b.create_block(); @@ -3409,7 +4360,9 @@ impl<'a, 'b> Lowerer<'a, 'b> { // Declined or deopted: the ordinary call, from the start. self.b.switch_to_block(generic_b); self.vstack.extend(args.iter().copied()); - self.emit_call_py(token, argc, 0, 0, 0, ret, pc); + self.emit_call_py( + call.token, call.argc, call.kwc, call.perm, call.gaps, ret, pc, + ); let (v, _) = self.vstack.pop().expect("the call's result"); self.b.ins().jump(join_b, &[v.into()]); @@ -3897,10 +4850,16 @@ impl<'a, 'b> Lowerer<'a, 'b> { } /// Write back every managed local into the frame's locals buffer. + /// Store the locals the body assigns back to the frame (the others + /// still hold the values the entry loaded: reading one here would only + /// keep it live to every exit and call). fn writeback_locals(&mut self) { let trusted = MemFlags::trusted(); for slot in 0..self.vars.len() { if let Some(var) = self.vars[slot] { + if !self.assignable.get(slot).copied().unwrap_or(true) { + continue; + } let v = self.b.use_var(var); let off = (slot as i32) * 8; self.b.ins().store(trusted, v, self.locals_base, off); diff --git a/crates/weavepy-jit/src/runtime.rs b/crates/weavepy-jit/src/runtime.rs index bd9e6b57..7e0f0295 100644 --- a/crates/weavepy-jit/src/runtime.rs +++ b/crates/weavepy-jit/src/runtime.rs @@ -132,6 +132,18 @@ pub enum SlotTag { /// are tagged [`SlotTag::Default`]. pub const CALL_GAPS: u32 = 1 << 31; +/// Set in a [`crate::TOp::CallMethod`] token when a native method site's +/// guard runs at the call instead of at its load: nothing between the +/// two can run code (see `TOp::GuardMethod`), so the call helper checks +/// the receiver itself and resolves a miss generically. +pub const METHOD_GUARD_AT_CALL: u32 = 1 << 31; + +/// Set in a [`crate::TOp::CallMethod`] token when the site resolved a +/// native leaf method: its helper never runs Python on this activation's +/// behalf (any call that would leaves through a side exit, which writes +/// the locals back itself), so the call skips the locals write-back. +pub const METHOD_NATIVE: u32 = 1 << 30; + impl SlotTag { /// Decode a raw tag written by native code. #[inline] @@ -150,6 +162,116 @@ impl SlotTag { } } +/// Where the embedder keeps what a pinned instance's field read or write +/// touches, so compiled code can do the common one in line: byte offsets +/// (from the pointer named in each field's doc), the object tags, and the +/// address of the flag that says borrow-free reads are off. The embedder +/// measures and checks them; compiled code falls back to its helpers on +/// anything else. +#[derive(Clone, Copy, Debug, Default)] +pub struct ObjLayout { + /// [`JitFrame::ctx`] → the pin table's buffer pointer and length. + pub ctx_pins_ptr: i32, + pub ctx_pins_len: i32, + /// A pin: its size, its discriminant byte and the object pin's value, + /// and where that object sits in it. + pub pin_size: i32, + pub pin_tag: i32, + pub pin_obj_tag: u8, + pub pin_obj: i32, + /// [`JitFrame::ctx`] → the attribute guards' shared pointer; that + /// pointer → the guard array. + pub ctx_guards: i32, + pub guards_buf: i32, + /// A guard: its size, its class version (`u64`), and the split-values + /// index its name sits at (`u32`, `u32::MAX` for none). + pub guard_size: i32, + pub guard_ver: i32, + pub guard_split_idx: i32, + /// Object tags (the first byte of a value); a word payload is at 8, a + /// `bool`'s at 1. + pub tag_instance: u8, + pub tag_int: u8, + pub tag_float: u8, + pub tag_bool: u8, + pub tag_none: u8, + /// A class value's tag (its payload pointer is the class pointer). + pub tag_type: u8, + /// An instance value's payload pointer → the instance's class pointer, + /// its published dict pointer (null while split), its split values' + /// borrow counter (`i32`) and block pointer (null while empty). + pub inst_class: i32, + pub inst_dict_lazy: i32, + pub inst_split_borrow: i32, + pub inst_split_block: i32, + /// A class pointer → its attribute version (`u64`) and its shared + /// names' pointer. + pub type_attr_version: i32, + pub type_shared_keys: i32, + /// A split block → its names' pointer, its length and capacity + /// (`u32`s) and its first value. + pub split_keys: i32, + pub split_len: i32, + pub split_cap: i32, + pub split_values: i32, + /// An instance value's payload pointer → its native body's pointer + /// (a word; zero for an ordinary instance). + pub inst_c_body: i32, + /// The byte that is nonzero once cells are shared between threads. + pub cells_unguarded: usize, + /// A list pin: its discriminant byte's value, where the list pointer + /// and the element lane (a [`crate::JitType`] byte) sit in it. + pub pin_list_tag: u8, + pub pin_list: i32, + pub pin_list_elem: i32, + /// A list pointer → its cell's borrow counter (`i32`), and its items' + /// buffer pointer, length and capacity. + pub list_borrow: i32, + pub list_ptr: i32, + pub list_len: i32, + pub list_cap: i32, + /// [`JitFrame::ctx`] → the method entries' shared pointer; that + /// pointer → the entry array and its length. + pub ctx_methods: i32, + pub methods_buf: i32, + pub methods_len: i32, + /// A method entry: its size, its class version (`u64`), and its + /// in-line field update: the field's split index (`u32`, `u32::MAX` + /// while unarmed), the most split values a receiver may hold without + /// shadowing the method (`u32`), whether the increment is the call's + /// argument (`u32`: `1`, else `0`), the literal increment (`i64`), + /// where the function keeps its code pointer and the code pointer that + /// must be there (words). + pub method_size: i32, + pub method_ver: i32, + pub method_upd_idx: i32, + pub method_upd_shadow: i32, + pub method_upd_from_arg: i32, + pub method_upd_inc: i32, + pub method_upd_code_at: i32, + pub method_upd_code: i32, + /// Words (or a byte, for `dict_watchers`) that are nonzero while a + /// call can't be skipped: an observer may be active, a dict watcher + /// is, or a class may hold an exotic key. + pub observers: usize, + pub dict_watchers: usize, + pub exotic_keys: usize, +} + +static OBJ_LAYOUT: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Publish the embedder's object layout (once; later calls are ignored). +/// Frames compiled from then on read pinned instances' fields in line. +pub fn set_obj_layout(layout: ObjLayout) { + let _ = OBJ_LAYOUT.set(layout); +} + +/// The published object layout. +#[must_use] +pub(crate) fn obj_layout() -> Option<&'static ObjLayout> { + OBJ_LAYOUT.get() +} + /// The exchange buffer the VM passes to a compiled frame. /// /// The VM owns the backing storage (`Vec` / `Vec`); this @@ -468,6 +590,21 @@ pub type StrLenHelper = unsafe extern "C" fn(frame: *mut JitFrame, pin: i64) -> pub type BytesGetHelper = unsafe extern "C" fn(frame: *mut JitFrame, pin: i64, idx: i64) -> i64; static STR_EQ_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); +static IS_OBJ_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Register the process-wide object identity helper ([`crate::ir::TOp::IsObj`]): +/// `1` when the two object-lane values (pin indices, or `-1` for `None`) +/// name the same object, `0` when not, anything above `1` to deopt (a +/// pin miss). Never runs Python code. Shares the [`StrEqHelper`] shape. +/// Must precede the first compile of a frame containing `IsObj`. +pub fn register_is_obj_helper(f: StrEqHelper) { + IS_OBJ_HELPER.store(f as usize, std::sync::atomic::Ordering::Release); +} + +#[must_use] +pub(crate) fn is_obj_helper_addr() -> usize { + IS_OBJ_HELPER.load(std::sync::atomic::Ordering::Acquire) +} static STR_LEN_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); static BYTES_LEN_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); static BYTES_GET_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); @@ -1039,6 +1176,57 @@ pub(crate) fn call_method_helper_addr() -> usize { CALL_METHOD_HELPER.load(std::sync::atomic::Ordering::Acquire) } +/// The native method-site load guard (see [`crate::TOp::GuardMethod`]): +/// `0` when the receiver pin still matches the site's burned +/// resolution, anything else to deopt at the load. +pub type GuardMethodHelper = + unsafe extern "C" fn(frame: *mut JitFrame, pin: i64, token: i64) -> i64; + +static GUARD_METHOD_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Register the process-wide native method-site load guard. Must precede +/// the first compile of a frame containing `GuardMethod` ops. +pub fn register_guard_method_helper(helper: GuardMethodHelper) { + GUARD_METHOD_HELPER.store(helper as usize, std::sync::atomic::Ordering::Release); +} + +#[must_use] +pub(crate) fn guard_method_helper_addr() -> usize { + GUARD_METHOD_HELPER.load(std::sync::atomic::Ordering::Acquire) +} + +static CALL_NATIVE_METHOD_HELPER: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + +/// Register the process-wide native method-call helper: the +/// [`CallMethodHelper`] for a [`crate::TOp::CallMethod`] whose token +/// carries [`METHOD_NATIVE`]. Must precede the first compile of a frame +/// containing such a call. +pub fn register_call_native_method_helper(helper: CallMethodHelper) { + CALL_NATIVE_METHOD_HELPER.store(helper as usize, std::sync::atomic::Ordering::Release); +} + +#[must_use] +pub(crate) fn call_native_method_helper_addr() -> usize { + CALL_NATIVE_METHOD_HELPER.load(std::sync::atomic::Ordering::Acquire) +} + +static OBJ_GETITEM_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Register the process-wide native-subscript helper (see +/// [`crate::TOp::ObjGetItem`]). Shares [`CallMethodHelper`]'s shape: the +/// `token` names the burned `__getitem__` resolution and the index rides +/// the marshal buffer as the one argument. Must precede the first +/// compile of a frame containing `ObjGetItem` ops. +pub fn register_obj_getitem_helper(helper: CallMethodHelper) { + OBJ_GETITEM_HELPER.store(helper as usize, std::sync::atomic::Ordering::Release); +} + +#[must_use] +pub(crate) fn obj_getitem_helper_addr() -> usize { + OBJ_GETITEM_HELPER.load(std::sync::atomic::Ordering::Acquire) +} + static STR_METHOD_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); /// Register the process-wide native `str`-method helper (RFC 0073 @@ -1300,6 +1488,36 @@ pub(crate) fn global_obj_helper_addr() -> usize { GLOBAL_OBJ_HELPER.load(std::sync::atomic::Ordering::Acquire) } +/// The obj-global helper's argument for `token` on `lane`: a pinned-list +/// lane's element lane rides the high word (`1` int, `2` float, `3` +/// object, `4` float list), so the helper pins the list as a list. +#[must_use] +pub(crate) fn global_obj_list_code(token: u32, lane: crate::JitType) -> i64 { + use crate::JitType; + let elem = match lane.elem_lane() { + Some(JitType::Int) => 1, + Some(JitType::Float) => 2, + Some(JitType::Obj) => 3, + Some(JitType::ListFloat) => 4, + _ => 0, + }; + i64::from(token) | (elem << 32) +} + +/// Inverse of [`global_obj_list_code`]: the element lane a list-lane +/// global pins with, `None` for an object pin. +#[must_use] +pub fn global_obj_list_elem(code: i64) -> Option { + use crate::JitType; + match code >> 32 { + 1 => Some(JitType::Int), + 2 => Some(JitType::Float), + 3 => Some(JitType::Obj), + 4 => Some(JitType::ListFloat), + _ => None, + } +} + static ITER_NEW_HELPER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); /// Register the process-wide *generic* iterator-capture helper (RFC diff --git a/crates/weavepy-vm/Cargo.toml b/crates/weavepy-vm/Cargo.toml index a404cafa..c6aa37de 100644 --- a/crates/weavepy-vm/Cargo.toml +++ b/crates/weavepy-vm/Cargo.toml @@ -76,6 +76,12 @@ crossbeam-utils = { workspace = true } # RFC 0032 — tier-2 Cranelift JIT, behind the (default-off) `jit` feature. weavepy-jit = { workspace = true, optional = true } +# Native code for leaf plans (`leaf_plan::jit`), under the same feature. +cranelift-codegen = { workspace = true, optional = true } +cranelift-frontend = { workspace = true, optional = true } +cranelift-jit = { workspace = true, optional = true } +cranelift-module = { workspace = true, optional = true } +cranelift-native = { workspace = true, optional = true } stacker = "0.1.24" # RFC 0066 WS4: dedicated native stacks for greenlets (aarch64/x86_64 # switching with guard pages — the same contract as upstream greenlet's @@ -100,7 +106,14 @@ rustls-native-certs = { workspace = true } # `--no-default-features` for size-constrained embedders. default = ["jit"] # Compile the tier-2 JIT integration (pulls in Cranelift). -jit = ["dep:weavepy-jit"] +jit = [ + "dep:weavepy-jit", + "dep:cranelift-codegen", + "dep:cranelift-frontend", + "dep:cranelift-jit", + "dep:cranelift-module", + "dep:cranelift-native", +] [lints] workspace = true diff --git a/crates/weavepy-vm/src/builtin_types.rs b/crates/weavepy-vm/src/builtin_types.rs index 46ae4e32..49e3257f 100644 --- a/crates/weavepy-vm/src/builtin_types.rs +++ b/crates/weavepy-vm/src/builtin_types.rs @@ -428,7 +428,7 @@ impl BuiltinTypes { ); pi.dict = dict.into(); let inst = Object::Instance(Rc::new(pi)); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } } @@ -491,7 +491,7 @@ impl BuiltinTypes { let mut pi = crate::types::PyInstance::with_native(cls.clone(), ns); pi.dict = dict.into(); let inst = Object::Instance(Rc::new(pi)); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } } @@ -500,7 +500,7 @@ impl BuiltinTypes { // through a closure cell is a cycle the collector must // see (test_interpreters' captured_thread_exception // left its Thread in `threading._dangling`). - crate::gc_trace::track(ns.clone()); + crate::gc_trace::track(&ns); Ok(ns) }; let mut ns_dict = simple_namespace_.dict.borrow_mut(); @@ -1536,12 +1536,8 @@ fn exc_slot_readonly(name: &str, class_name: &str, default: Object) -> Object { /// applied by an old pickle, or `self.args = …` in a shadowing /// `__init__` that ran before the class descriptor was reachable). pub(crate) fn exc_attr(inst: &crate::types::PyInstance, name: &str) -> Option { - inst.slot_get(name).or_else(|| { - inst.dict_cell() - .borrow() - .get(&crate::object::StrKey(name)) - .cloned() - }) + // Read without publishing a `__dict__` the instance never needed. + inst.slot_get(name).or_else(|| inst.attr_get_str(name)) } /// Construct an exception instance of `class_name` with `message` as @@ -1557,15 +1553,17 @@ pub fn make_exception(class_name: &str, message: impl Into) -> Object { /// Build a built-in exception instance whose single `args[0]` element is the /// *object* `arg`, not a stringified message — `KeyError(key)` where /// `e.args[0] is key`. CPython's `KeyError.__str__` renders `repr(args[0])`, -/// which our `exc_str` already reproduces; we set `message` to that repr so -/// the Rust Display/traceback path matches too. +/// which our `exc_str` already reproduces, and so does +/// [`exception_message`] for the Rust Display/traceback path. pub fn make_exception_with_object(class_name: &str, arg: Object) -> Object { - let exc = make_exception(class_name, ""); - if let Object::Instance(inst) = &exc { - inst.slot_set("args", Object::new_tuple_array([arg.clone()])); - inst.slot_set("message", Object::from_str(arg.repr())); - } - exc + let bt = builtin_types(); + let class = match class_name { + "KeyError" => bt.key_error.clone(), + _ => bt + .by_name(class_name) + .unwrap_or_else(|| bt.exception.clone()), + }; + exception_from_parts(class, Some(arg)) } /// Build a faithful `UnicodeEncodeError` instance carrying the 5-tuple @@ -1834,7 +1832,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { Object::new_tuple(new_args.to_vec()), ); let obj = Object::Instance(inst); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); return Ok(obj); } // `tuple.__new__(tuple, it)` / `int.__new__(int, x)` … on the *built-in @@ -1846,7 +1844,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { // name/doc seeding lives in `module.__init__` only). if cls.is_subclass_of(&builtin_types().module_) { let inst = Object::Instance(Rc::new(PyInstance::new(cls))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } // A `type` subclass reaching the generic allocator is a wrong-arity @@ -1929,7 +1927,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { let interp = unsafe { &mut *ptr }; let s = interp.type_call_default(&bt.str_, &args[1..], &[])?; let inst = Object::Instance(Rc::new(PyInstance::with_native(cls.clone(), s))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } } @@ -1955,7 +1953,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { let interp = unsafe { &mut *ptr }; let v = interp.type_call_default(&bt.int_, &args[1..], &[])?; let inst = Object::Instance(Rc::new(PyInstance::with_native(cls.clone(), v))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } } @@ -1970,7 +1968,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { // `__init__` / `__setstate__` / the `_reconstruct` append-and-update loop. if let Some(native) = native_seed_for_new(&cls, args.get(1)) { let inst = Object::Instance(Rc::new(PyInstance::with_native(cls, native))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); return Ok(inst); } // RFC 0024: explicit `object.__new__(cls)` / `super().__new__(cls)` @@ -1982,7 +1980,7 @@ pub(crate) fn object_new(args: &[Object]) -> Result { // A natively served class's instances are never tracked (see // `stdlib::datetime_native`). if !native { - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); } Ok(inst) } @@ -3910,29 +3908,10 @@ fn install_os_error_init(os_error: &Rc) { let Some(Object::Instance(inst)) = args.first() else { return Ok(Object::from_static("")); }; - let get = |name: &'static str| exc_attr(inst, name); - let set = |o: &Option| matches!(o, Some(v) if !matches!(v, Object::None)); - let errno = get("errno"); - let strerror = get("strerror"); - let filename = get("filename"); - let filename2 = get("filename2"); - let errno_s = errno.as_ref().map(Object::to_str).unwrap_or_default(); - let strerror_s = strerror.as_ref().map(Object::to_str).unwrap_or_default(); - if set(&filename) { - let f1 = filename.as_ref().map(Object::repr).unwrap_or_default(); - if set(&filename2) { - let f2 = filename2.as_ref().map(Object::repr).unwrap_or_default(); - return Ok(Object::from_str(format!( - "[Errno {errno_s}] {strerror_s}: {f1} -> {f2}" - ))); - } - return Ok(Object::from_str(format!( - "[Errno {errno_s}] {strerror_s}: {f1}" - ))); - } - if set(&errno) && set(&strerror) { - return Ok(Object::from_str(format!("[Errno {errno_s}] {strerror_s}"))); + if let Some(m) = oserror_fields_message(inst) { + return Ok(Object::from_str(m)); } + let get = |name: &'static str| exc_attr(inst, name); // BaseException.__str__: "" / str(arg) / repr(args). match get("args") { Some(Object::Tuple(items)) => Ok(match &items[..] { @@ -4858,47 +4837,53 @@ fn install_exception_str_repr(base_exception: &Rc) { } pub fn make_exception_with_class(class: Rc, message: impl Into) -> Object { - use crate::types::PyInstance; - let (mut is_syntax, mut is_stop_iteration, mut is_import) = (false, false, false); - for t in class.mro.borrow().iter() { - match t.name.as_str() { - "SyntaxError" => is_syntax = true, - "StopIteration" => is_stop_iteration = true, - "ImportError" => is_import = true, - _ => {} - } - } - let inst = PyInstance::new(class); - let msg = Object::from_str(message); + let message: String = message.into(); // A messageless raise (`StopIteration()`, `GeneratorExit()`, …) // has *empty* args in CPython, not `("",)`. - let args = if msg.to_str().is_empty() { - Object::new_tuple(Vec::new()) + let msg = if message.is_empty() { + None } else { - Object::new_tuple_array([msg.clone()]) + Some(Object::from_str(message)) }; + exception_from_parts(class, msg) +} + +/// A built-in exception of `class` whose `args` is `(arg,)`, or `()` +/// for `None`: the shared tail of the Rust-side constructors. +/// +/// The pseudo-slots land in one vector sized for the traceback and +/// chaining links a raise adds next, and the class's families come +/// from its memoised flags rather than an MRO walk. CPython keeps no +/// `message` attribute: `str()` and [`exception_message`] derive the +/// text from `args` on demand, so nothing is formatted here. +pub(crate) fn exception_from_parts(class: Rc, arg: Option) -> Object { + use crate::types::{slot_key, PyInstance, SlotStorage}; + let families = crate::exc_family_flags(&class); + let inst = PyInstance::new(class); + let args = match &arg { + None => Object::new_tuple(Vec::new()), + Some(a) => Object::new_tuple_array([a.clone()]), + }; + inst.note_slot_store(&args); + let mut entries: Vec<(crate::object::DictKey, Object)> = Vec::with_capacity(4); // PEP 380: `StopIteration.value` is always present (CPython sets it // in `StopIteration.__init__`, defaulting to None). A Rust-raised - // bare `StopIteration` must answer `.value` too — asyncio's + // bare `StopIteration` must answer `.value` too: asyncio's // `Task.__step` reads `exc.value` on every coroutine return, and a // missing attribute leaves the task wedged (gh: shutdown_asyncgens). - if is_stop_iteration { - let value = if msg.to_str().is_empty() { - Object::None - } else { - msg.clone() - }; - inst.slot_set("value", value); + if families & crate::EXC_FAM_STOP_ITERATION != 0 { + let value = arg.clone().unwrap_or(Object::None); + inst.note_slot_store(&value); + entries.push((slot_key("value"), value)); } - inst.slot_set("args", args); - inst.slot_set("message", msg.clone()); + entries.push((slot_key("args"), args)); // The BaseException pseudo-slots (`__context__`/`__cause__`/ // `__suppress_context__`/`__traceback__`) and OSError's named fields // (`errno`/`strerror`/…) are *not* seeded per-instance: their class // slot descriptors answer the CPython getset defaults while unset, // and a subclass's own class attribute (`class Err(OSError): errno = // EINVAL`, raised bare via `raise Err`) stays visible. - if is_import { + if families & (crate::EXC_FAM_IMPORT_ERROR | crate::EXC_FAM_SYNTAX_ERROR) != 0 { // ImportError/ModuleNotFoundError expose `msg` (the message // string). CPython always defines the slot; a Rust-raised // ImportError must answer `.msg` so consumers (e.g. numpy's @@ -4906,15 +4891,15 @@ pub fn make_exception_with_class(class: Rc, message: impl Into Option { if let Some(Object::Str(s)) = exc_attr(inst, "message") { return Some(s.to_string()); } + if inst.cls().is_subclass_of(&builtin_types().os_error) { + if let Some(m) = oserror_fields_message(inst) { + return Some(m); + } + } if let Some(Object::Tuple(items)) = exc_attr(inst, "args") { + // `KeyError.__str__` renders a lone key by its repr. + if let [single] = &items[..] { + if inst.cls().is_subclass_of(&builtin_types().key_error) { + return Some(single.repr()); + } + } if let Some(first) = items.first() { return Some(first.to_str()); } @@ -5666,6 +5662,25 @@ pub fn exception_message(obj: &Object) -> Option { } } +/// `OSError.__str__`'s `[Errno N] strerror[: filename[ -> filename2]]` +/// rendering, when the named fields that shape needs are set (see +/// `oserror_str`). +fn oserror_fields_message(inst: &crate::types::PyInstance) -> Option { + let get = |name: &'static str| exc_attr(inst, name).filter(|v| !matches!(v, Object::None)); + let errno = get("errno"); + let strerror = get("strerror"); + let errno_s = errno.as_ref().map(Object::to_str).unwrap_or_default(); + let strerror_s = strerror.as_ref().map(Object::to_str).unwrap_or_default(); + if let Some(f1) = get("filename") { + let f1 = f1.repr(); + return Some(match get("filename2") { + Some(f2) => format!("[Errno {errno_s}] {strerror_s}: {f1} -> {}", f2.repr()), + None => format!("[Errno {errno_s}] {strerror_s}: {f1}"), + }); + } + (errno.is_some() && strerror.is_some()).then(|| format!("[Errno {errno_s}] {strerror_s}")) +} + /// `True` when `obj` is an instance whose class derives from `cls`. pub fn instance_is_subclass(obj: &Object, cls: &TypeObject) -> bool { match obj { @@ -5817,7 +5832,7 @@ fn int_new_kw( return Some(Ok(v)); } let inst = Object::Instance(Rc::new(PyInstance::with_native(cls.clone(), v))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); Some(Ok(inst)) } diff --git a/crates/weavepy-vm/src/builtins.rs b/crates/weavepy-vm/src/builtins.rs index a71e7de9..a25f9666 100644 --- a/crates/weavepy-vm/src/builtins.rs +++ b/crates/weavepy-vm/src/builtins.rs @@ -2662,7 +2662,7 @@ fn slot_sizeof(args: &[Object]) -> Result { // is 56 bytes + (len+1) units of the kind width (1 for latin-1, // 2 for BMP, 4 beyond). Object::Str(s) => { - let len = str_char_len(s) as i64; + let len = crate::object::str_char_len(s) as i64; let max_cp = s.chars().map(u32::from).max().unwrap_or(0); match max_cp { 0..=0x7f => 40 + len + 1, @@ -4681,7 +4681,7 @@ fn install_wire_code(nc: &mut weavepy_compiler::CodeObject, bytes: Vec) { // instruction count; line info defaults to the first line. let first = nc.linetable.iter().copied().find(|l| *l > 0).unwrap_or(1); nc.linetable = vec![first; instructions.len()]; - nc.coltable = Vec::new(); + nc.coltable = Vec::new().into(); nc.caches = weavepy_compiler::CacheTable::with_len(instructions.len()); nc.instructions = instructions; nc.exception_table = Vec::new(); @@ -4869,7 +4869,7 @@ pub(crate) fn code_type_call( nc.caches = weavepy_compiler::CacheTable::with_len(decoded.instructions.len()); nc.instructions = decoded.instructions; nc.linetable = decoded.linetable; - nc.coltable = decoded.coltable; + nc.coltable = decoded.coltable.into(); nc.exception_table = decoded.exception_table; nc.no_interrupt_jumps = decoded.no_interrupt_jumps; } @@ -5272,7 +5272,7 @@ fn attr_delete(obj: &Object, name: &str) -> Result<(), RuntimeError> { } Object::Function(f) => { if crate::object::is_function_slot(name) { - f.slots + f.slots() .borrow_mut() .shift_remove(&crate::object::DictKey(Object::from_str(name))); } else { @@ -7013,7 +7013,7 @@ fn b_list(args: &[Object]) -> Result { let obj = Object::new_list(out); // CPython tracks every list; keep `list(...)` consistent with the // `[]` literal path so `gc.is_tracked` and cycle collection agree. - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); // tracemalloc parity with the `[]` literal path // (`test_tracemalloc.test_reset_peak` builds `list(range(100000))` // and expects the peak to reflect it). @@ -7047,7 +7047,7 @@ fn b_tuple(args: &[Object]) -> Result { fn b_dict(args: &[Object]) -> Result { if args.is_empty() { let obj = Object::new_dict(); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); return Ok(obj); } if args.len() > 1 { @@ -7063,7 +7063,7 @@ fn b_dict(args: &[Object]) -> Result { // Clone the stored table so copying never rehashes user keys. let d = src.borrow().clone(); let obj = Object::Dict(Rc::new(RefCell::new(d))); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); return Ok(obj); } // Mapping path for user-defined classes (`__keys__` style) is @@ -7117,7 +7117,7 @@ fn b_dict(args: &[Object]) -> Result { i += 1; } let obj = Object::Dict(Rc::new(RefCell::new(d))); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } @@ -7139,6 +7139,8 @@ fn b_set(args: &[Object]) -> Result { } let out = if args.is_empty() { crate::object::SetData::default() + } else if let Some(out) = simple_key_set(&args[0]) { + out } else { // `set([[]])` raises `TypeError: unhashable type: 'list'` — check // each element as it is admitted, like CPython's `set_init`. A @@ -7154,10 +7156,41 @@ fn b_set(args: &[Object]) -> Result { }; let obj = Object::Set(Rc::new(RefCell::new(out))); // CPython tracks every set (`gc.is_tracked(set())` is True). - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } +/// The set of a list's or tuple's items when every item is an `int`, +/// `str`, `float`, `bool` or `None` (whose hashing and comparison run no +/// Python and can't fail): sized once and filled without the per-insert +/// error scope. `None` leaves the general path to it. +fn simple_key_set(src: &Object) -> Option { + fn build(items: &[Object]) -> Option { + let simple = |v: &Object| { + matches!( + v, + Object::Int(_) | Object::Str(_) | Object::Float(_) | Object::Bool(_) | Object::None + ) + }; + if !items.iter().all(simple) { + return None; + } + let mut out = crate::object::SetData::with_capacity_and_hasher( + items.len(), + crate::fasthash::FxBuildHasher, + ); + for v in items { + out.insert(DictKey(v.clone())); + } + Some(out) + } + match src { + Object::List(l) => build(&l.try_borrow().ok()?), + Object::Tuple(t) => build(t), + _ => None, + } +} + fn b_frozenset(args: &[Object]) -> Result { // `frozenset()` takes at most one positional argument (CPython // `frozenset_new`); `frozenset([], 2)` is a `TypeError`. @@ -8243,7 +8276,7 @@ fn b_sorted(args: &[Object]) -> Result { crate::compare_op(a, b, weavepy_compiler::CompareKind::Lt) })?; let obj = Object::new_list(buf); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } @@ -10424,7 +10457,8 @@ fn b_mark_iterable_coroutine(args: &[Object]) -> Result { // Shared, not copied: `func.__dict__` mutations stay visible on // both, matching CPython where the function object is the same. attrs: RefCell::new(Some(f.attrs())), - slots: RefCell::new(f.slots.borrow().clone()), + slots_raw: RefCell::new(f.slots().borrow().clone()), + slot_seed: RefCell::new(None), closure_cells: std::sync::OnceLock::new(), // The copied slot store carries any override along. defaults_override: crate::object::OverrideFlag::new(f.defaults_maybe_overridden()), @@ -10777,14 +10811,41 @@ fn str_arity(name: &str, args: &[Object], min: usize, max: usize) -> Result<(), Ok(()) } +/// An ASCII `str` receiver's case mapping, written straight into the +/// result, which is ASCII too and born knowing its count; `None` for any +/// other receiver. +fn ascii_case_result(args: &[Object], case: crate::unicode_case::AsciiCase) -> Option { + let Some(Object::Str(s)) = args.first() else { + return None; + }; + if !SharedStr::is_ascii(s) { + return None; + } + // SAFETY: an ASCII case mapping of ASCII text is ASCII of the same + // length, all of it written (see `ascii_case_into`). + Some(Object::Str(unsafe { + SharedStr::build_unchecked(s.len(), Some(s.len()), |out| { + out.push_with(s.len(), |buf| { + crate::unicode_case::ascii_case_into(case, s.as_bytes(), buf); + }); + }) + })) +} + fn str_upper(args: &[Object]) -> Result { str_arity("upper", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Upper) { + return Ok(out); + } let up = crate::unicode_case::upper(&str_self(args)?); Ok(str_result(args, up)) } fn str_lower(args: &[Object]) -> Result { str_arity("lower", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Lower) { + return Ok(out); + } let lo = crate::unicode_case::lower(&str_self(args)?); Ok(str_result(args, lo)) } @@ -10794,6 +10855,9 @@ fn str_lower(args: &[Object]) -> Result { /// and folding is context-free (no Greek final-sigma special-casing). fn str_casefold(args: &[Object]) -> Result { str_arity("casefold", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Lower) { + return Ok(out); + } let out = crate::unicode_case::casefold(&str_self(args)?); Ok(str_result(args, out)) } @@ -10850,33 +10914,38 @@ fn ascii_space_table() -> &'static [bool; 128] { TABLE.get_or_init(|| std::array::from_fn(|i| crate::unicode_case::is_space(i as u8 as char))) } -fn str_split_whitespace(s: &str, maxsplit: i64) -> Vec { +fn str_split_whitespace(s: &str, ascii: bool, maxsplit: i64) -> Vec { use crate::unicode_case::is_space; - // Presized: `collect()` over a filtered split carries no size hint, - // so the vector grew several times for a sentence-length string. - let mut out = Vec::with_capacity((s.len() / 8).min(64) + 1); + // Presized: `collect()` over a filtered split carries no size hint. + // English prose averages under five letters a word, so a quarter of + // the length covers a sentence without growing (each growth copies). + let mut out = Vec::with_capacity((s.len() / 4).min(256) + 1); let limit = if maxsplit < 0 { i64::MAX } else { maxsplit }; let mut splits = 0i64; - if str_is_ascii_cached(s) { + if ascii { + // Fields of ASCII text are ASCII: each is born knowing its count. let table = ascii_space_table(); + // Every ASCII space is at most `' '`, so most bytes are settled + // by one comparison. + let is_space = |c: u8| c <= b' ' && table[usize::from(c)]; let b = s.as_bytes(); let mut i = 0; loop { - while i < b.len() && table[b[i] as usize] { + while i < b.len() && is_space(b[i]) { i += 1; } if i == b.len() { break; } if splits >= limit { - out.push(Object::Str(SharedStr::from(&s[i..]))); + out.push(Object::Str(SharedStr::from_ascii(&s[i..]))); break; } let start = i; - while i < b.len() && !table[b[i] as usize] { + while i < b.len() && !is_space(b[i]) { i += 1; } - out.push(Object::Str(SharedStr::from(&s[start..i]))); + out.push(Object::Str(SharedStr::from_ascii(&s[start..i]))); splits += 1; } return out; @@ -10903,8 +10972,14 @@ fn str_split_whitespace(s: &str, maxsplit: i64) -> Vec { /// initial elements are all strings. Register them before returning /// through either the interpreter or a native string helper. fn new_str_split_list(items: Vec) -> Object { + // Plain `str` fields can't close a cycle yet, however many there are. + let inert = items.iter().all(|o| matches!(o, Object::Str(_))); let result = Object::new_list(items); - crate::gc_trace::track(result.clone()); + if inert { + crate::gc_trace::with_state(|s| s.track_inert(&result)); + } else { + crate::gc_trace::track(&result); + } result } @@ -10928,28 +11003,57 @@ fn str_split(args: &[Object], kwargs: &[(String, Object)]) -> Result = match sep { - None | Some(Object::None) => str_split_whitespace(&s, maxsplit), + let (_, ascii) = recv_char_len(args, &s); + let mut out: Vec = match sep { + None | Some(Object::None) => str_split_whitespace(&s, ascii, maxsplit), Some(sep_obj) => { let sep = str_arg_bridged(sep_obj) .ok_or_else(|| type_error("must be str or None, not other"))?; if sep.is_empty() { return Err(value_error("empty separator")); } - if maxsplit < 0 { - s.split(&*sep) - .map(|s| Object::Str(SharedStr::from(s))) - .collect() - } else { - s.splitn((maxsplit as usize).saturating_add(1), &*sep) - .map(|s| Object::Str(SharedStr::from(s))) - .collect() - } + str_split_sep(&s, ascii, &sep, maxsplit) } }; + // CPython returns an exact `str` that needed no splitting as the + // list's one item itself, not a copy. + if let ([Object::Str(only)], Some(recv @ Object::Str(_))) = (&out[..], args.first()) { + if only.len() == s.len() { + out[0] = recv.clone(); + } + } Ok(new_str_split_list(wrap(out))) } +/// `s.split(sep, maxsplit)` for a non-empty separator. A one-byte +/// separator scans with `memchr` rather than `str::split`'s searcher, +/// whose setup costs more than splitting a short line; fields of ASCII +/// text are born knowing their counts. +fn str_split_sep(s: &str, ascii: bool, sep: &str, maxsplit: i64) -> Vec { + let piece = |text: &str| { + Object::Str(if ascii { + SharedStr::from_ascii(text) + } else { + SharedStr::from(text) + }) + }; + let limit = usize::try_from(maxsplit).unwrap_or(usize::MAX); + if let [byte] = *sep.as_bytes() { + let mut out = Vec::new(); + let mut start = 0; + for at in memchr::memchr_iter(byte, s.as_bytes()) { + if out.len() == limit { + break; + } + out.push(piece(&s[start..at])); + start = at + 1; + } + out.push(piece(&s[start..])); + return out; + } + s.splitn(limit.saturating_add(1), sep).map(piece).collect() +} + fn str_join(args: &[Object]) -> Result { use std::borrow::Cow; @@ -10960,10 +11064,10 @@ fn str_join(args: &[Object]) -> Result { // Exact built-in sequences of exact strings need no iteration hooks, // reference clones, or intermediate copies. Hold a list's read guard // across both passes so concurrent mutation cannot change its size. - if matches!(args.first(), Some(Object::Str(_))) { + if let Some(Object::Str(sep)) = args.first() { let joined = match &args[1] { - Object::List(items) => join_exact_strings(&items.borrow(), &sep), - Object::Tuple(items) => join_exact_strings(items, &sep), + Object::List(items) => join_exact_strings(&items.borrow(), sep), + Object::Tuple(items) => join_exact_strings(items, sep), _ => None, }; if let Some(joined) = joined { @@ -11017,27 +11121,50 @@ fn str_join(args: &[Object]) -> Result { } /// Return `None` for shapes that need the general Unicode/iterator path. -fn join_exact_strings(items: &[Object], sep: &str) -> Option { - let mut size = sep.len().checked_mul(items.len().saturating_sub(1))?; +/// The result is written straight into its final allocation, and knows its +/// code-point count when every part (and the separator) already does. +fn join_exact_strings(items: &[Object], sep: &SharedStr) -> Option { + let gaps = items.len().saturating_sub(1); + let mut size = sep.len().checked_mul(gaps)?; + // The separator is usually a constant: counting it once memoises it. + let mut chars = SharedStr::char_count(sep).checked_mul(gaps); for item in items { let Object::Str(s) = item else { return None; }; size = size.checked_add(s.len())?; + chars = chars.and_then(|n| Some(n + SharedStr::known_char_count(s)?)); } if items.len() == 1 { return Some(items[0].clone()); } - let mut joined = String::with_capacity(size); - for (i, item) in items.iter().enumerate() { - if i != 0 { - joined.push_str(sep); + let fill = |out: &mut crate::shared_value::TextWriter<'_>| { + let mut items = items.iter().filter_map(|item| match item { + Object::Str(s) => Some(s.as_bytes()), + _ => None, + }); + if let Some(first) = items.next() { + out.push(first); } - if let Object::Str(s) = item { - joined.push_str(s); + // The usual one-byte separator (`" "`, `","`) is stored directly. + if let [byte] = *sep.as_bytes() { + for item in items { + out.push_byte(byte); + out.push(item); + } + } else { + for item in items { + out.push(sep.as_bytes()); + out.push(item); + } } - } - Some(Object::from_str(joined)) + }; + // SAFETY: a concatenation of `str`s is UTF-8, `size` bytes long (the + // caller holds the list's guard, so its items cannot change), and + // `chars` sums their counts. + Some(Object::Str(unsafe { + SharedStr::build_unchecked(size, chars, fill) + })) } fn str_startswith(args: &[Object]) -> Result { @@ -11182,6 +11309,94 @@ fn str_match_prefix_suffix( } } +/// `recv.replace(from, to, limit)` for an exact `str` receiver and a +/// non-empty `from`: one scan records the matches, so the result is +/// written straight into an allocation of its exact size, and a receiver +/// with no match is returned itself (CPython's `unicode_result_unchanged`). +/// `None` when the result's size overflows, for the general path to +/// report. +fn str_replace_exact(recv: &SharedStr, from: &str, to: &str, limit: usize) -> Option { + /// Match offsets, on the stack for the usual handful. + struct Hits { + inline: [usize; 32], + len: usize, + spill: Vec, + } + impl Hits { + fn push(&mut self, at: usize) { + if self.len < self.inline.len() { + self.inline[self.len] = at; + } else { + if self.spill.is_empty() { + self.spill.extend_from_slice(&self.inline); + } + self.spill.push(at); + } + self.len += 1; + } + fn as_slice(&self) -> &[usize] { + if self.len <= self.inline.len() { + &self.inline[..self.len] + } else { + &self.spill + } + } + } + let (h, n) = (recv.as_bytes(), from.as_bytes()); + let mut hits = Hits { + inline: [0; 32], + len: 0, + spill: Vec::new(), + }; + if n.len() <= 16 && n[0].is_ascii() { + // A short needle that starts with an ASCII byte (never inside a + // multibyte character, so every hit is a boundary) is found by a + // `memchr` scan for that byte; comparing at most 16 bytes per + // candidate keeps it linear. + if let Some(last_start) = h.len().checked_sub(n.len()) { + let mut i = 0; + while i <= last_start && hits.len < limit { + let Some(off) = memchr::memchr(n[0], &h[i..=last_start]) else { + break; + }; + let at = i + off; + if short_bytes_eq(&h[at + 1..at + n.len()], &n[1..]) { + hits.push(at); + i = at + n.len(); + } else { + i = at + 1; + } + } + } + } else { + for (at, _) in recv.match_indices(from).take(limit) { + hits.push(at); + } + } + let hits = hits.as_slice(); + if hits.is_empty() { + return Some(Object::Str(recv.clone())); + } + let size = (h.len() - hits.len() * n.len()).checked_add(hits.len().checked_mul(to.len())?)?; + // ASCII text with ASCII replacements stays ASCII. + let chars = (SharedStr::is_ascii(recv) && to.is_ascii()).then_some(size); + let fill = |out: &mut crate::shared_value::TextWriter<'_>| { + let mut copied = 0; + for &at in hits { + out.push(&h[copied..at]); + out.push(to.as_bytes()); + copied = at + n.len(); + } + out.push(&h[copied..]); + }; + // SAFETY: the result splices whole `str`s at character boundaries + // (every match is one), `size` bytes in all; `chars` is given only + // when every byte is ASCII. + Some(Object::Str(unsafe { + SharedStr::build_unchecked(size, chars, fill) + })) +} + fn str_replace_kw(args: &[Object], kwargs: &[(String, Object)]) -> Result { let s = str_self(args)?; let positional = args.len().saturating_sub(1); @@ -11255,10 +11470,16 @@ fn str_replace_kw(args: &[Object], kwargs: &[(String, Object)]) -> Result 3631). s.replace(from, to) } else { s.replacen(from, to, count as usize) @@ -11454,15 +11675,10 @@ fn str_find(args: &[Object]) -> Result { Some(p) => p, None => return Err(type_error("find() expected str")), }; - // One ASCII probe for the whole call: each of the char/byte - // conversions below consults the same thread-local cache, and the - // TLS access costs more than the work it guards. - let ascii = str_is_ascii_cached(s); - let total_chars = if ascii { - s.len() as i64 - } else { - str_char_len(s) as i64 - }; + // One ASCII probe for the whole call, O(1) from the receiver's + // memoised count. + let (total_chars, ascii) = recv_char_len(args, s); + let total_chars = total_chars as i64; let Some((start, end)) = str_search_window(args, total_chars) else { return Ok(Object::Int(-1)); }; @@ -11488,6 +11704,17 @@ fn str_find(args: &[Object]) -> Result { } } +/// The method receiver's length in code points, and whether it is pure +/// ASCII (one byte per code point): O(1) from a `str`'s memoised count, a +/// scan of the bridged text for a `WStr`. +fn recv_char_len(args: &[Object], s: &str) -> (usize, bool) { + let n = match args.first() { + Some(Object::Str(recv)) => SharedStr::char_count(recv), + _ => s.chars().count(), + }; + (n, n == s.len()) +} + /// Whether `s` is pure ASCII, memoized by buffer identity. CPython's PEP 393 /// layout makes char↔byte offset mapping O(1); our UTF-8 `SharedStr` needs a /// scan. Callers like `str.find(sub, start)`-in-a-loop (email's @@ -11511,15 +11738,6 @@ pub(crate) fn str_is_ascii_cached(s: &str) -> bool { }) } -/// Total `len()` in code points, O(1) for (cached-)ASCII strings. -pub(crate) fn str_char_len(s: &str) -> usize { - if str_is_ascii_cached(s) { - s.len() - } else { - s.chars().count() - } -} - fn char_offset_to_byte(s: &str, n: usize) -> usize { if n == 0 { return 0; @@ -11539,18 +11757,27 @@ fn byte_offset_to_char(s: &str, byte: usize) -> usize { fn str_title(args: &[Object]) -> Result { str_arity("title", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Title) { + return Ok(out); + } let out = crate::unicode_case::title(&str_self(args)?); Ok(str_result(args, out)) } fn str_capitalize(args: &[Object]) -> Result { str_arity("capitalize", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Capitalize) { + return Ok(out); + } let out = crate::unicode_case::capitalize(&str_self(args)?); Ok(str_result(args, out)) } fn str_swapcase(args: &[Object]) -> Result { str_arity("swapcase", args, 0, 0)?; + if let Some(out) = ascii_case_result(args, crate::unicode_case::AsciiCase::Swap) { + return Ok(out); + } let out = crate::unicode_case::swapcase(&str_self(args)?); Ok(str_result(args, out)) } @@ -11733,17 +11960,27 @@ fn str_rfind(args: &[Object]) -> Result { Some(p) => p, None => return Err(type_error("rfind() expected str")), }; - let total_chars = str_char_len(s) as i64; - let Some((start, end)) = str_search_window(args, total_chars) else { + let (total_chars, ascii) = recv_char_len(args, s); + let Some((start, end)) = str_search_window(args, total_chars as i64) else { return Ok(Object::Int(-1)); }; - let start_byte = char_offset_to_byte(s, start as usize); - let end_byte = char_offset_to_byte(s, end as usize); + let (start_byte, end_byte) = if ascii { + ((start as usize).min(s.len()), (end as usize).min(s.len())) + } else { + ( + char_offset_to_byte(s, start as usize), + char_offset_to_byte(s, end as usize), + ) + }; let hay = &s[start_byte..end_byte]; match substr_rfind(hay, &sub) { Some(byte_idx) => { let abs_byte = byte_idx + start_byte; - Ok(Object::Int(byte_offset_to_char(s, abs_byte) as i64)) + Ok(Object::Int(if ascii { + abs_byte as i64 + } else { + byte_offset_to_char(s, abs_byte) as i64 + })) } None => Ok(Object::Int(-1)), } @@ -11779,12 +12016,8 @@ fn str_count(args: &[Object]) -> Result { None => return Err(type_error("count() expected str")), }; // One ASCII probe for the whole call, as `str.find` does. - let ascii = str_is_ascii_cached(s); - let total_chars = if ascii { - s.len() as i64 - } else { - str_char_len(s) as i64 - }; + let (total_chars, ascii) = recv_char_len(args, s); + let total_chars = total_chars as i64; let Some((start, end)) = str_search_window(args, total_chars) else { return Ok(Object::Int(0)); }; @@ -12470,7 +12703,7 @@ fn list_setitem(args: &[Object]) -> Result { } let replaced = crate::apply_slice_assignment(&mut l.borrow_mut(), s, replacement)?; for old in replaced { - queue_removed(old); + drop(old); } return Ok(Object::None); } @@ -12479,7 +12712,7 @@ fn list_setitem(args: &[Object]) -> Result { let n = list_index_arg(l.len(), key, "__setitem__")?; std::mem::replace(&mut l[n], val.clone()) }; - queue_removed(old); + drop(old); Ok(Object::None) } @@ -12493,7 +12726,7 @@ fn list_delitem(args: &[Object]) -> Result { let mut indices = crate::slice_indices(l.len(), s)?; indices.sort_unstable(); for i in indices.into_iter().rev() { - queue_removed(l.remove(i)); + drop(l.remove(i)); } return Ok(Object::None); } @@ -12502,7 +12735,7 @@ fn list_delitem(args: &[Object]) -> Result { let n = list_index_arg(l.len(), key, "__delitem__")?; l.remove(n) }; - queue_removed(removed); + drop(removed); Ok(Object::None) } @@ -12672,7 +12905,7 @@ fn list_remove(args: &[Object]) -> Result { } }; if let Some(removed) = removed { - queue_removed(removed); + drop(removed); } return Ok(Object::None); } @@ -12931,7 +13164,7 @@ fn list_clear(args: &[Object]) -> Result { } let evicted: Vec = std::mem::take(&mut *list_self(args)?.borrow_mut()); for v in evicted { - queue_removed(v); + drop(v); } Ok(Object::None) } @@ -13141,7 +13374,7 @@ fn dict_setitem(args: &[Object]) -> Result { ensure_dict_key(key)?; let old = dict_insert(&d, key.clone(), val.clone())?; if let Some(old) = old { - queue_removed(old); + drop(old); } Ok(Object::None) } @@ -13160,15 +13393,6 @@ fn dict_getitem(args: &[Object]) -> Result { found.ok_or_else(|| key_error_object(key.clone())) } -/// Route a key/value evicted by a container mutator to the prompt-reap -/// queue (see [`crate::vm_singletons::queue_container_removed`]) before -/// dropping our reference. The queue holds a clone; the eval loop reaps it -/// at the next between-bytecodes safe point if the eviction was its last -/// program-visible reference. -pub(crate) fn queue_removed(v: Object) { - crate::vm_singletons::queue_container_removed(&v); -} - fn dict_delitem(args: &[Object]) -> Result { let d = dict_self(args)?; let key = args @@ -13177,8 +13401,8 @@ fn dict_delitem(args: &[Object]) -> Result { ensure_dict_key(key)?; let removed = dict_remove(&d, key)?; if let Some((k, v)) = removed { - queue_removed(k); - queue_removed(v); + drop(k); + drop(v); Ok(Object::None) } else { Err(key_error_object(key.clone())) @@ -13255,7 +13479,7 @@ fn dict_pop(args: &[Object]) -> Result { if let Some((k, v)) = removed { // The *stored* key (equal to, but possibly distinct from, the // lookup key) is evicted too; the value is returned to the caller. - queue_removed(k); + drop(k); Ok(v) } else if let Some(default) = args.get(2).cloned() { Ok(default) @@ -13294,7 +13518,7 @@ fn dict_update(args: &[Object]) -> Result { let src_len = entries.len(); for (k, v) in entries { if let Some(old) = dict_insert(&d, k.0, v)? { - queue_removed(old); + drop(old); } // CPython's `PyDict_Merge` re-checks the source size // *after* every insert: a key `__eq__` that mutates the @@ -13315,7 +13539,7 @@ fn dict_update(args: &[Object]) -> Result { let mut dst = d.borrow_mut(); for (k, v) in entries { if let Some(old) = dst.insert(k, v) { - queue_removed(old); + drop(old); } } } @@ -13337,8 +13561,8 @@ fn dict_clear(args: &[Object]) -> Result { } } for (k, v) in evicted { - queue_removed(k.0); - queue_removed(v); + drop(k.0); + drop(v); } Ok(Object::None) } @@ -13456,7 +13680,7 @@ fn dict_copy(args: &[Object]) -> Result { // CPython's `PyDict_Copy` preserves GC tracking: the copy is tracked // iff the source is (test_dict `test_copy_maintains_tracking`). if crate::gc_trace::is_tracked(crate::weakref_registry::id_of(&Object::Dict(d))) { - crate::gc_trace::track(out.clone()); + crate::gc_trace::track(&out); } Ok(out) } @@ -13548,7 +13772,7 @@ fn dict_or(args: &[Object]) -> Result { out.insert(k.clone(), v.clone()); } let obj = Object::Dict(Rc::new(RefCell::new(out))); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } @@ -13563,7 +13787,7 @@ fn dict_ror(args: &[Object]) -> Result { out.insert(k.clone(), v.clone()); } let obj = Object::Dict(Rc::new(RefCell::new(out))); - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } @@ -13582,7 +13806,7 @@ fn dict_ior(args: &[Object]) -> Result { .collect(); for (k, v) in entries { if let Some(old) = dict_insert(&d, k.0, v)? { - queue_removed(old); + drop(old); } } return Ok(args[0].clone()); @@ -13630,7 +13854,7 @@ fn dict_ior(args: &[Object]) -> Result { let (k, v) = (kv.next().unwrap(), kv.next().unwrap()); ensure_dict_key(&k)?; if let Some(old) = dict_insert(&d, k, v)? { - queue_removed(old); + drop(old); } i += 1; } @@ -13675,12 +13899,12 @@ fn apply_set_inplace_op(s: &mut crate::object::SetData, op: SetInplaceOp) { } SetInplaceOp::Discard(k) => { if let Some(removed) = s.swap_take(&k) { - queue_removed(removed.0); + drop(removed.0); } } SetInplaceOp::Clear => { for k in s.drain(..) { - queue_removed(k.0); + drop(k.0); } } } @@ -16299,7 +16523,7 @@ pub(crate) fn file_readlines(args: &[Object]) -> Result { loop { let line = file_readline(&[Object::File(f.clone())])?; let len = match &line { - Object::Str(s) => str_char_len(s), + Object::Str(s) => crate::object::str_char_len(s), Object::WStr(cps) => cps.len(), Object::Bytes(b) => b.len(), _ => 0, diff --git a/crates/weavepy-vm/src/capi_watchers.rs b/crates/weavepy-vm/src/capi_watchers.rs index 79d435ab..508d319b 100644 --- a/crates/weavepy-vm/src/capi_watchers.rs +++ b/crates/weavepy-vm/src/capi_watchers.rs @@ -105,6 +105,12 @@ pub fn dicts_active() -> bool { DICTS_ACTIVE.load(Ordering::Relaxed) } +/// The flag [`dicts_active`] reads, for native code that reads it in line +/// (a byte: nonzero while any dict watcher is active). +pub(crate) fn dicts_active_flag() -> *const bool { + DICTS_ACTIVE.as_ptr() +} + pub fn watch_dict(watcher_id: u8, d: &Rc>) { let mut w = WATCHED_DICTS.lock(); let ptr = Rc::as_ptr(d) as usize; @@ -221,11 +227,7 @@ pub fn dict_update_begin(target: &Rc>, target_was_empty: bool) pub fn note_dropped(vm_dropped: &Object) { if let Object::Dict(d) = vm_dropped { if dicts_active() { - let id = crate::weakref_registry::id_of(vm_dropped); - let registry_clones = crate::weakref_registry::strong_clone_count(id); - // The GC registry's `track` handle is a strong clone too. - let gc_clone = usize::from(crate::gc_trace::is_tracked(id)); - if Rc::strong_count(d) <= 1 + registry_clones + gc_clone && dict_mask(d) != 0 { + if Rc::strong_count(d) <= 1 && dict_mask(d) != 0 { dict_event("DEALLOCATED", d, None, None); let ptr = Rc::as_ptr(d) as usize; let mut w = WATCHED_DICTS.lock(); @@ -238,12 +240,7 @@ pub fn note_dropped(vm_dropped: &Object) { } if let Object::Function(f) = vm_dropped { if funcs_active() { - let id = crate::weakref_registry::id_of(vm_dropped); - let registry_clones = crate::weakref_registry::strong_clone_count(id); - // The GC registry holds one strong clone from MAKE_FUNCTION's - // `track`; a dying function is down to our handle + that one. - let gc_clone = usize::from(crate::gc_trace::is_tracked(id)); - if Rc::strong_count(f) <= 1 + registry_clones + gc_clone { + if Rc::strong_count(f) <= 1 { func_event("DESTROY", vm_dropped, &Object::None); } } diff --git a/crates/weavepy-vm/src/frame_jit.rs b/crates/weavepy-vm/src/frame_jit.rs new file mode 100644 index 00000000..29f5db27 --- /dev/null +++ b/crates/weavepy-vm/src/frame_jit.rs @@ -0,0 +1,3126 @@ +//! Native code for hot frames: a baseline compiler for the core loop. +//! +//! The core loop ([`Interpreter::leaf_core`]) decodes and dispatches one +//! instruction at a time, and every operand it handles makes a round trip +//! through the frame's operand stack. A code object that keeps running is +//! compiled here into one function over its instructions, entered at the +//! start of any basic block. The function keeps a *virtual* operand stack +//! while it lowers a block: a local or constant load is just a note of +//! where the value lives, scalar arithmetic and compares produce unboxed +//! values in registers, and a branch tests them directly, so a typical +//! `i = i + 1` or `if a < b:` touches neither the stack nor a reference +//! count. Values reach the frame's stack (copied, with their references +//! taken) only where something else must see them: at the end of a block, +//! and before any instruction the native code leaves to the core loop. +//! +//! An instruction the native code doesn't take, and any shape an in-line +//! one doesn't settle (an overflow, a heap operand, a cache miss), returns +//! to the core loop with the frame exactly as the loop would have it +//! before that instruction: the loop runs it, and enters the native code +//! again at the next block start it reaches. The core loop's fused shapes +//! (a local receiver's method call or attribute store) start at their +//! `LOAD_FAST`, so the native code hands those over at it. +//! +//! Compiling costs far more than interpreting a few iterations, and every +//! hand-over to the core loop costs a round trip, so a code object +//! compiles only once a loop of it is hot and that loop runs mostly in +//! native code (see `worth_compiling`); a loop of calls and global loads +//! stays with the core loop. Code a tier-2 loop may still take waits for +//! tier 2 to settle first. +//! +//! The code reads and writes [`Object`]s in place, through the layout +//! `repr(u8)` fixes: a tag byte at offset 0, a `bool` payload at offset 1, +//! and a word payload at offset 8. Instance fields are read through the +//! object layout tier 2 measures (see `tier2::obj_layout`). + +use std::cell::Cell; +use std::sync::atomic::{AtomicU32, Ordering}; + +use cranelift_codegen::ir::{ + condcodes::{FloatCC, IntCC}, + types, AbiParam, Block, InstBuilder, JumpTableData, MemFlags, SigRef, Signature, Type, Value, +}; +use cranelift_codegen::settings::{self, Configurable}; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; +use cranelift_jit::{JITBuilder, JITModule}; +use cranelift_module::{Linkage, Module}; +use weavepy_compiler::{BinOpKind, CodeObject, CompareKind, OpCode, COMPARE_OP_TO_BOOL_FLAG}; + +use crate::error::RuntimeError; +use crate::object::{Object, PyIterator}; +use crate::sync::Rc; +use crate::{CodeConstObjects, FieldSlot, Interpreter}; + +/// The core loop runs the instruction at `pc` itself. +pub(crate) const INTERP: u32 = 0; +/// A release queued a finalizer: the core loop stops (`LeafStop::Marked`) +/// with `pc` past the releasing instruction. +pub(crate) const MARKED: u32 = 1; +/// A call raised `State::err`, with `pc` past it. +pub(crate) const RAISED: u32 = 2; + +/// Activations and back edges before a code object is compiled +/// (`WEAVEPY_FRAME_JIT_HOT` overrides it, a tuning aid). +fn hot() -> u32 { + static HOT: std::sync::OnceLock = std::sync::OnceLock::new(); + *HOT.get_or_init(|| { + std::env::var("WEAVEPY_FRAME_JIT_HOT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(1000) + }) +} + +/// How many times [`hot`] a code object's heat must reach: compiling +/// costs about the same for every instruction of the code, while what it +/// saves comes from the loop that got hot alone, so a long body (a +/// driver's setup around a short loop) waits longer to earn its compile. +#[inline(always)] +fn size_factor(code: &CodeObject) -> u32 { + (code.instructions.len() / SIZE_UNIT).max(1) as u32 +} + +/// The code length [`hot`] alone covers. +const SIZE_UNIT: usize = 16; + +/// The longest code compiled. +const MAX_INSTRUCTIONS: usize = 4096; + +/// The running activation's state, as the core loop holds it: the native +/// code reads it on entry and writes it back on exit. +#[repr(C)] +pub(crate) struct State { + pub(crate) locals: *mut Object, + pub(crate) stack: *mut Object, + pub(crate) len: usize, + pub(crate) cap: usize, + pub(crate) pc: usize, + pub(crate) last: usize, + /// The interpreter's GIL countdown (the back edges' eval breaker). + pub(crate) countdown: *mut u32, + /// The loop generation the burst started under. + pub(crate) snap_gen: u64, + /// The running thread's queued-finalizer flag. + pub(crate) maybe_dead: *const Cell, + pub(crate) interp: *const Interpreter, + /// A helper's scalar result. + pub(crate) out: u64, + /// The thread's recursion-depth cell. + pub(crate) depth_cell: *const Cell, + /// What a `RAISED` exit raised. + pub(crate) err: Option, +} + +const S_LOCALS: i32 = 0; +const S_STACK: i32 = 8; +const S_LEN: i32 = 16; +const S_CAP: i32 = 24; +const S_PC: i32 = 32; +const S_LAST: i32 = 40; +const S_COUNTDOWN: i32 = 48; +const S_SNAP: i32 = 56; +const S_DEAD: i32 = 64; +const S_OUT: i32 = 80; +const _: () = { + assert!(std::mem::offset_of!(State, locals) == S_LOCALS as usize); + assert!(std::mem::offset_of!(State, stack) == S_STACK as usize); + assert!(std::mem::offset_of!(State, len) == S_LEN as usize); + assert!(std::mem::offset_of!(State, cap) == S_CAP as usize); + assert!(std::mem::offset_of!(State, pc) == S_PC as usize); + assert!(std::mem::offset_of!(State, last) == S_LAST as usize); + assert!(std::mem::offset_of!(State, countdown) == S_COUNTDOWN as usize); + assert!(std::mem::offset_of!(State, snap_gen) == S_SNAP as usize); + assert!(std::mem::offset_of!(State, maybe_dead) == S_DEAD as usize); + assert!(std::mem::offset_of!(State, out) == S_OUT as usize); +}; + +type NativeFn = unsafe extern "C" fn(*mut State) -> u32; + +/// A code object's native code. +pub(crate) struct Native { + func: NativeFn, + /// The locals vector's length the code was compiled for. + nlocals: usize, + /// The pcs worth entering at: block starts whose first instruction + /// the native code runs. + entries: Box<[bool]>, + /// The code's name and opcodes (for `WEAVEPY_FRAME_JIT_STATS`). + name: String, + ops: Box<[OpCode]>, +} + +// SAFETY: the code only touches the state it is handed; it is only +// entered with the GIL held (never in the free-threaded build). +unsafe impl Send for Native {} +// SAFETY: as above. +unsafe impl Sync for Native {} + +impl Native { + /// Whether entering at `pc` gets anywhere. + #[inline(always)] + pub(crate) fn enters_at(&self, pc: usize) -> bool { + self.entries.get(pc).copied().unwrap_or(false) + } + + /// Run from `st.pc` until an instruction the core loop must run. + /// + /// # Safety + /// + /// `st` is the running activation of the code this was compiled + /// from, as the core loop holds it, with `nlocals` locals. + #[inline(always)] + pub(crate) unsafe fn run(&self, st: &mut State) -> u32 { + let from = st.pc; + // SAFETY: the caller's contract. + let status = unsafe { (self.func)(st) }; + if stats::enabled() { + stats::note(&self.name, &self.ops, from, st.pc, status); + } + status + } +} + +/// A code object's compilation state, kept in its extension. +#[derive(Default)] +pub(crate) struct Slot { + native: std::sync::OnceLock>>, + heat: AtomicU32, + /// How many times the code got hot while start-up or an import ran + /// (see `try_compile`). + deferred: AtomicU32, +} + +impl Slot { + /// The native code for an activation with `nlocals` locals, if + /// compiled. + #[inline(always)] + pub(crate) fn get(&self, nlocals: usize) -> Option<&Native> { + match self.native.get() { + Some(Some(n)) if n.nlocals == nlocals => Some(n), + _ => None, + } + } + + /// Count one activation, back edge or generator step of code without + /// native code, compiling it once it is hot. + #[inline(always)] + pub(crate) fn warm(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize, at: Heat) { + if self.native.get().is_some() { + return; + } + // (A racing thread losing a count is harmless.) + let h = self.heat.load(Ordering::Relaxed) + 1; + self.heat.store(h, Ordering::Relaxed); + if h >= hot().saturating_mul(size_factor(code)) { + self.heat.store(0, Ordering::Relaxed); + self.try_compile(code, ext, nlocals, at); + } + } + + #[cold] + #[inline(never)] + fn try_compile(&self, code: &CodeObject, ext: &CodeConstObjects, nlocals: usize, at: Heat) { + if !enabled() { + let _ = self.native.set(None); + return; + } + // Code that start-up or an import runs is mostly run once (a + // regular expression compiler's loops, a module's set-up), and a + // compile costs as much as thousands of interpreted iterations: + // there, code compiles only after sustained work, as tier 2's + // budget has it. + let d = self.deferred.load(Ordering::Relaxed).saturating_add(1); + if !crate::tier2::frame_compile_allowed(d.saturating_mul(hot()), hot()) { + self.deferred.store(d, Ordering::Relaxed); + return; + } + // Loops the tier-2 compiler may still take keep their back edges' + // consultation in the core loop (a generator body the fast steps + // run never reaches it). + let hint = &code.jit_hint; + if !(matches!(at, Heat::Step) + || hint.is_not_jitable() + || crate::tier2::jit_off_for_process() + || hint.loop_free(code) + || hint.is_backedge_quiet()) + { + return; + } + if !name_admitted(&code.qualname) { + let _ = self.native.set(None); + return; + } + if !worth_compiling(code, ext, at) { + // (Another loop may yet get hot.) + return; + } + let t0 = stats::enabled().then(std::time::Instant::now); + let n = self + .native + .get_or_init(|| compile(code, ext, nlocals).map(Box::new)); + if let Some(t0) = t0 { + eprintln!( + "frame jit: {} compiled: {} ({} instructions, {:?})", + code.qualname, + n.is_some(), + code.instructions.len(), + t0.elapsed() + ); + } + } +} + +/// What warmed a code object. +#[derive(Clone, Copy)] +pub(crate) enum Heat { + /// An activation. + Call, + /// The back edge at this pc. + BackEdge(usize), + /// A generator's fast step (see `gen_fast`). + Step, +} + +/// Whether the loop of `code` that got hot (any loop, for a call or a +/// generator step) runs mostly in native code: each +/// instruction the core loop runs costs a round trip out of the native +/// code and back, so a loop of calls and global loads gains nothing from +/// compiling (nor does a loop-free body, whose entry costs as much as the +/// little it would save). +fn worth_compiling(code: &CodeObject, ext: &CodeConstObjects, at: Heat) -> bool { + use weavepy_compiler::OpCode; + let ins = &code.instructions; + // A method call on a local runs through a helper without leaving, when + // its callee runs in place. + let mut helped = vec![false; ins.len()]; + for pc in 0..ins.len().saturating_sub(1) { + if ins[pc].op == OpCode::LoadFast + && ins[pc + 1].op == OpCode::LoadMethodAttr + && crate::method_site_in_place(ext, pc + 1) + { + helped[pc + 1] = true; + if let Some(call) = + (pc + 2..ins.len().min(pc + 12)).find(|&k| ins[k].op == OpCode::Call) + { + helped[call] = true; + } + } + } + let at_loop = match at { + Heat::BackEdge(pc) => { + Some(pc).filter(|&pc| ins.get(pc).is_some_and(|i| i.op == OpCode::JumpBackward)) + } + Heat::Call | Heat::Step => None, + }; + ins.iter().enumerate().any(|(pc, i)| { + if i.op != OpCode::JumpBackward || at_loop.is_some_and(|at| pc != at) { + return false; + } + let top = (pc + 1).saturating_sub(i.arg as usize); + let (mut native, mut exits) = (0usize, 0usize); + for k in top..=pc { + // An attribute read stays in native code only when the core + // loop has seen it read a split-layout field or a natively + // served instance's field; a class attribute, a module's or a + // method leaves on every run. + let attr_exits = ins[k].op == OpCode::LoadAttr + && !(crate::field_site(ext, k) || crate::native_site(ext, k)); + if (native_op(ins[k].op) && !attr_exits) || helped[k] { + native += 1; + } else { + exits += 1; + } + } + if stats::enabled() { + eprintln!( + "frame jit: {} loop at {top}: {native} native, {exits} exits", + code.qualname + ); + } + exits * EXIT_WEIGHT <= native + }) +} + +/// How many natively run instructions a loop needs per instruction that +/// leaves to the core loop before compiling it pays. +const EXIT_WEIGHT: usize = 8; + +/// Whether a code object of this qualified name may compile: +/// `WEAVEPY_FRAME_JIT_ONLY` / `WEAVEPY_FRAME_JIT_SKIP` admit only, or +/// skip, names containing their value (a debugging aid). +fn name_admitted(name: &str) -> bool { + static FILTER: std::sync::OnceLock<(Option, Option)> = + std::sync::OnceLock::new(); + let (only, skip) = FILTER.get_or_init(|| { + let var = |k: &str| std::env::var(k).ok().filter(|v| !v.is_empty()); + (var("WEAVEPY_FRAME_JIT_ONLY"), var("WEAVEPY_FRAME_JIT_SKIP")) + }); + only.as_deref().is_none_or(|o| name.contains(o)) + && skip.as_deref().is_none_or(|s| !name.contains(s)) +} + +/// Whether frames compile at all (`WEAVEPY_FRAME_JIT=0` turns them off). +fn enabled() -> bool { + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + *ON.get_or_init(|| { + std::env::var_os("WEAVEPY_FRAME_JIT").is_none_or(|v| v != "0") + && !crate::gil::free_threading_enabled() + }) +} + +/// `WEAVEPY_FRAME_JIT_STATS=1`: entries and the instructions native code +/// returned at, by code object, printed at exit (a tuning aid). +mod stats { + use std::collections::HashMap; + use std::sync::Mutex; + use weavepy_compiler::OpCode; + + type Counts = HashMap<(String, String), [u64; 2]>; + + static ON: std::sync::OnceLock = std::sync::OnceLock::new(); + static COUNTS: Mutex> = Mutex::new(None); + + #[inline(always)] + pub(super) fn enabled() -> bool { + *ON.get_or_init(|| { + let on = std::env::var_os("WEAVEPY_FRAME_JIT_STATS").is_some(); + if on { + extern "C" fn dump() { + let counts = COUNTS.lock().unwrap_or_else(|e| e.into_inner()); + let mut rows: Vec<_> = counts.iter().flat_map(|c| c.iter()).collect(); + rows.sort_by_key(|(_, n)| std::cmp::Reverse(n[0])); + for ((name, op), n) in rows.iter().take(40) { + eprintln!("{:>10} {:>10} {name}: {op}", n[0], n[1]); + } + } + // SAFETY: registering a plain function. + unsafe { libc::atexit(dump) }; + } + on + }) + } + + #[cold] + pub(super) fn note(name: &str, ops: &[OpCode], from: usize, at: usize, status: u32) { + let op = match (status, ops.get(at)) { + (0, Some(op)) => format!("{op:?}"), + (0, None) => "".to_owned(), + _ => "".to_owned(), + }; + let mut counts = COUNTS.lock().unwrap_or_else(|e| e.into_inner()); + let e = counts + .get_or_insert_with(HashMap::new) + .entry((name.to_owned(), op)) + .or_default(); + e[0] += 1; + e[1] += u64::from(at != from); + } +} + +// ---- the object layout ---------------------------------------------------- + +/// The tags of the variants the code handles in line. +#[derive(Clone, Copy)] +struct Tags { + none: u8, + unbound: u8, + boolean: u8, + int: u8, + float: u8, + cell: u8, + instance: u8, + /// The heap variants whose identity is their payload pointer (and so + /// `is` compares payload words): instances, strings, lists, dicts, + /// tuples, classes. + by_pointer: i64, +} + +impl Tags { + /// `None`, `bool`, `int` and `float`: copied by value, dropped by + /// forgetting. + fn scalar_mask(self) -> i64 { + (1i64 << self.none) | (1 << self.boolean) | (1 << self.int) | (1 << self.float) + } +} + +/// The tags, read off sample values, if the layout is the one the code +/// assumes. +fn tags() -> Option { + let tag = |v: &Object| { + // SAFETY: `Object` is `repr(u8)`: its first byte is the tag. + unsafe { *std::ptr::from_ref(v).cast::() } + }; + let word = |v: &Object| { + // SAFETY: every `Object` is 16 bytes; the second word is + // initialized for the word payloads read here. + unsafe { *std::ptr::from_ref(v).cast::().add(1) } + }; + let byte1 = |v: &Object| { + // SAFETY: as above (a `bool` payload's byte). + unsafe { *std::ptr::from_ref(v).cast::().add(1) } + }; + let cell = Object::Cell(Rc::new(crate::sync::RefCell::new(Object::None))); + let inst = Object::Instance(Rc::new(crate::types::PyInstance::new( + crate::builtin_types::builtin_types().object_.clone(), + ))); + let samples = [ + tag(&inst), + tag(&Object::from_str("")), + tag(&Object::new_list(Vec::new())), + tag(&Object::new_dict()), + tag(&Object::new_tuple_array([Object::None])), + tag(&Object::Type( + crate::builtin_types::builtin_types().object_.clone(), + )), + ]; + let t = Tags { + none: tag(&Object::None), + unbound: tag(&Object::Unbound), + boolean: tag(&Object::Bool(true)), + int: tag(&Object::Int(1)), + float: tag(&Object::Float(1.0)), + cell: tag(&cell), + instance: tag(&inst), + by_pointer: samples + .iter() + .fold(0, |m, &t| if t < 64 { m | (1i64 << t) } else { m }), + }; + if samples.iter().any(|&t| t >= 64) { + return None; + } + let all = [ + t.none, t.unbound, t.boolean, t.int, t.float, t.cell, t.instance, + ]; + let distinct = all + .iter() + .enumerate() + .all(|(i, a)| all[i + 1..].iter().all(|b| a != b)); + (distinct + && all.iter().all(|&t| t < 64) + && tag(&Object::Bool(false)) == t.boolean + && byte1(&Object::Bool(true)) == 1 + && byte1(&Object::Bool(false)) == 0 + && word(&Object::Int(0x0123_4567_89ab_cdef)) == 0x0123_4567_89ab_cdef + && word(&Object::Float(1.5)) == 1.5f64.to_bits() + && std::mem::size_of::() == 16) + .then_some(t) +} + +// ---- helpers (the core loop's arms, for native code) ---------------------- + +#[inline(always)] +fn droppable(v: &Object) -> bool { + Interpreter::core_droppable(v) +} + +/// `*dst = (*src).clone()`. +unsafe extern "C" fn h_clone(dst: *mut Object, src: *const Object) { + // SAFETY: the code passes an initialized value and a free slot. + unsafe { dst.write(crate::clone_hot(&*src)) } +} + +/// Release the heap value at `slot` (it leaves the stack): `0` released, +/// anything else declined with the slot untouched. +unsafe extern "C" fn h_pop(slot: *mut Object) -> u32 { + // SAFETY: the code passes an initialized stack slot. + unsafe { + if !droppable(&*slot) { + return 1; + } + crate::drop_hot(slot.read()); + } + 0 +} + +/// `STORE_FAST i` of the value at stack `slot` over a heap value: `0` +/// stored (the value left the stack), anything else declined untouched. +unsafe extern "C" fn h_store(st: *mut State, slot: *mut Object, i: u64) -> u32 { + // SAFETY: the code passes its live state, an initialized stack slot, + // and a local index in range. + unsafe { + let st = &mut *st; + let local = st.locals.add(i as usize); + if !droppable(&*local) { + return 1; + } + let v = slot.read(); + (*st.interp).release(std::mem::replace(&mut *local, v)); + } + 0 +} + +/// `FOR_ITER` over a range, list or tuple iterator at `it` (the next value +/// goes to `out`, the slot above): `0` an `int` in `st.out`, `1` another +/// value written to `out`, `2` the exhausted iterator retired (it left the +/// stack), anything else declined. +unsafe extern "C" fn h_for_iter(st: *mut State, it: *mut Object, out: *mut Object) -> u32 { + // SAFETY: the code passes its live state and an initialized slot with + // a free one above it. + let (st, top) = unsafe { (&mut *st, &*it) }; + let Object::Iter(rc) = top else { + return 3; + }; + let unique = Rc::strong_count(rc) == 1; + // SAFETY: nothing below runs code while the iterator is borrowed. + let Some(iter) = (unsafe { rc.peek_mut() }) else { + return 3; + }; + let v = match iter { + PyIterator::Range { + current, + stop, + step, + } => { + let live = if *step > 0 { + *current < *stop + } else { + *step < 0 && *current > *stop + }; + if live { + st.out = *current as u64; + *current = current.wrapping_add(*step); + return 0; + } + if !unique { + return 3; + } + // SAFETY: the iterator leaves the stack. + drop(unsafe { it.read() }); + return 2; + } + PyIterator::List { + items, + index, + owner, + } => { + // SAFETY: as above. + let Some(xs) = (unsafe { items.peek() }) else { + return 3; + }; + match xs.get(*index) { + Some(v) => { + let v = crate::clone_hot(v); + *index += 1; + v + } + None if unique && owner.is_none() => { + // SAFETY: as above. + drop(unsafe { it.read() }); + return 2; + } + None => return 3, + } + } + PyIterator::Tuple { items, index } => { + let Some(v) = items.get(*index).cloned() else { + return 3; + }; + *index += 1; + v + } + _ => return 3, + }; + if let Object::Int(i) = v { + st.out = i as u64; + return 0; + } + // SAFETY: the slot above the iterator is free. + unsafe { out.write(v) }; + 1 +} + +/// A container instruction (`BINARY_SUBSCR`, `BINARY_SLICE`, `STORE_SUBSCR`, +/// `LIST_APPEND`, `UNPACK_SEQUENCE`; the one at `ins`, the `pc`th of +/// `code`) on the top of a `len`-deep stack, as the core loop's arm runs +/// it: the new depth, or `u64::MAX` declined untouched. +unsafe extern "C" fn h_container( + st: *mut State, + ins: *const weavepy_compiler::Instruction, + code: *const CodeObject, + pc: u64, + len: u64, +) -> u64 { + // SAFETY: the code passes its live state, one of its own instructions + // and its code, and its stack depth. + unsafe { + let st = &mut *st; + let (ins, len) = (*ins, len as usize); + if let Some(n) = Interpreter::core_container_op(ins, st.stack, len, st.cap) { + return n as u64; + } + // An instance whose class's `__getitem__` is a native fast + // subscript the site cached. + if ins.op != OpCode::BinarySubscr || len < 2 { + return u64::MAX; + } + let ops = std::slice::from_raw_parts(st.stack.add(len - 2), 2); + let Some(fast) = (*st.interp).core_native_subscript(&*code, pc as usize, ops) else { + return u64::MAX; + }; + // (An error leaves the subscript to the core loop, which raises it.) + let Some(Ok(v)) = fast(ops) else { + return u64::MAX; + }; + #[cfg(test)] + crate::NATIVE_FAST_SUBSCRIPTS.with(|calls| calls.set(calls.get() + 1)); + crate::drop_hot(st.stack.add(len - 1).read()); + crate::drop_hot(st.stack.add(len - 2).read()); + st.stack.add(len - 2).write(v); + (len - 1) as u64 + } +} + +/// `CONTAINS_OP` of the item at `at` and the container above it: `0` or +/// `1` (found; both left the stack), anything else declined untouched. +unsafe extern "C" fn h_contains(at: *mut Object) -> u32 { + // SAFETY: the code passes two initialized stack slots. + unsafe { + let (item, container) = (&*at, &*at.add(1)); + if !droppable(item) || !droppable(container) { + return 2; + } + let Some(found) = Interpreter::leaf_contains(container, item) else { + return 2; + }; + crate::drop_hot(at.add(1).read()); + crate::drop_hot(at.read()); + u32::from(found) + } +} + +/// `BINARY_OP` of `kind` over the values at `at` and above it, for the +/// shapes the core loop's arm runs (two numbers; strings' concatenation, +/// repetition and `%` formatting; a natively served instance's operator): +/// `0` the result at `at` (the operands released), anything else declined +/// untouched. +unsafe extern "C" fn h_binop(at: *mut Object, kind: u32) -> u32 { + // SAFETY: the code passes two initialized stack slots, and a kind the + // compiler emitted (`BinOpKind` is `repr(u8)`). + unsafe { + let (a, b) = (&*at, &*at.add(1)); + // Two numbers (the core loop's in-line arm). + let num = |o: &Object| match *o { + Object::Int(i) => Some(crate::leaf_plan::V::I(i)), + Object::Float(x) => Some(crate::leaf_plan::V::F(x)), + _ => None, + }; + if let (Some(x), Some(y)) = (num(a), num(b)) { + let r = match crate::leaf_plan::binary(x, y, kind as u8) { + Some(crate::leaf_plan::V::I(i)) => Object::Int(i), + Some(crate::leaf_plan::V::F(f)) => Object::Float(f), + _ => return 1, + }; + at.write(r); + return 0; + } + let kind: BinOpKind = std::mem::transmute(kind as u8); + let r = match (a, b) { + (Object::Str(_), _) | (Object::Int(_), Object::Str(_)) => { + Interpreter::core_str_binop(a, b, kind) + } + (Object::Instance(i), _) if i.cls_raw().native_kind.get() != 0 => { + match Interpreter::core_native_binop(kind, a, b) { + Some(Ok(r)) if droppable(a) && droppable(b) => Some(r), + _ => None, + } + } + _ => None, + }; + let Some(r) = r else { + return 1; + }; + crate::drop_hot(at.add(1).read()); + crate::drop_hot(at.read()); + at.write(r); + } + 0 +} + +/// `BINARY_OP` of `kind` over the values at `a` and `b` (a local's or a +/// constant's, read in place) for a site that has seen a natively served +/// operand (see `crate::native_site`): `0` the result written to the free +/// slot `dst`, anything else declined untouched (the core loop then runs +/// the instruction, and raises its errors). +unsafe extern "C" fn h_binop_ref( + dst: *mut Object, + a: *const Object, + b: *const Object, + kind: u32, +) -> u32 { + // SAFETY: the code passes a free stack slot, two initialized values + // that outlive the call (nothing it runs can reach them), and a kind + // the compiler emitted (`BinOpKind` is `repr(u8)`). + unsafe { + let (a, b) = (&*a, &*b); + let num = |o: &Object| match *o { + Object::Int(i) => Some(crate::leaf_plan::V::I(i)), + Object::Float(x) => Some(crate::leaf_plan::V::F(x)), + _ => None, + }; + let r = if let (Some(x), Some(y)) = (num(a), num(b)) { + match crate::leaf_plan::binary(x, y, kind as u8) { + Some(crate::leaf_plan::V::I(i)) => Object::Int(i), + Some(crate::leaf_plan::V::F(f)) => Object::Float(f), + _ => return 1, + } + } else { + let kind: BinOpKind = std::mem::transmute(kind as u8); + match a { + Object::Instance(i) if i.cls_raw().native_kind.get() != 0 => { + match Interpreter::core_native_binop(kind, a, b) { + Some(Ok(r)) => r, + _ => return 1, + } + } + _ => return 1, + } + }; + dst.write(r); + } + 0 +} + +/// The field `name` (a `str`) of the natively served instance at `recv` +/// (see `stdlib::datetime_native::leaf_field`): `0` the value written to +/// the free slot `dst`, anything else declined untouched. +unsafe extern "C" fn h_field(dst: *mut Object, recv: *const Object, name: *const Object) -> u32 { + // SAFETY: the code passes a free stack slot, an initialized local and + // one of its code's names; reading the field runs no code. + unsafe { + let (Object::Instance(i), Object::Str(name)) = (&*recv, &*name) else { + return 1; + }; + if i.cls_raw().native_kind.get() == 0 { + return 1; + } + match crate::stdlib::datetime_native::leaf_field(i, name) { + Some(v) => { + dst.write(v); + 0 + } + None => 1, + } + } +} + +/// `COMPARE_OP` with oparg `arg` over the values at `at` and above it, for +/// the shapes the core loop's arm runs (two ints, two floats, two natively +/// served instances): `0` the `bool` at `at` (the operands released), +/// anything else declined untouched. +unsafe extern "C" fn h_compare(at: *mut Object, arg: u32) -> u32 { + // SAFETY: the code passes two initialized stack slots, and the oparg + // the compiler emitted (as `compare_op_step` reads it). + unsafe { + let (a, b) = (&*at, &*at.add(1)); + let kind: CompareKind = std::mem::transmute((arg & !COMPARE_OP_TO_BOOL_FLAG) as u8); + let ord = match (a, b) { + (Object::Int(x), Object::Int(y)) => x.cmp(y), + (Object::Float(x), Object::Float(y)) => match x.partial_cmp(y) { + Some(o) => o, + None => return 1, + }, + (Object::Instance(i), Object::Instance(_)) if i.cls_raw().native_kind.get() != 0 => { + let r = match Interpreter::core_native_compare(kind, a, b) { + Some(Ok(r)) if droppable(a) && droppable(b) => r, + _ => return 1, + }; + crate::drop_hot(at.add(1).read()); + crate::drop_hot(at.read()); + at.write(r); + return 0; + } + _ => return 1, + }; + let r = match kind { + CompareKind::Lt => ord.is_lt(), + CompareKind::LtE => ord.is_le(), + CompareKind::Eq => ord.is_eq(), + CompareKind::NotEq => ord.is_ne(), + CompareKind::Gt => ord.is_gt(), + CompareKind::GtE => ord.is_ge(), + }; + at.write(Object::Bool(r)); + } + 0 +} + +/// `LOAD_FAST x` (the `pc`th instruction of `code`) and the call +/// `x.m()` after it, as the core loop's fused arm runs it +/// (see `Interpreter::core_local_method`), on a stack `len` deep: `0` the +/// result pushed and `st.pc` / `st.last` past the call, `RAISED` the call +/// raised, anything else declined untouched. +unsafe extern "C" fn h_local_method( + st: *mut State, + code: *const CodeObject, + ext: *const CodeConstObjects, + pc: u64, + nlocals: u64, + len: u64, +) -> u32 { + // SAFETY: the code passes its live state, its own code and extension, + // a `LOAD_FAST` of an in-range local, and its stack depth (with room + // for a push). + unsafe { + let st = &mut *st; + let (code, ext, pc) = (&*code, &*ext, pc as usize); + let local = &*st.locals.add(code.instructions[pc].arg as usize); + let Some((r, call_pc)) = (*st.interp).core_local_method( + code, + Some(ext), + local, + pc, + st.locals, + nlocals as usize, + &ext.objects, + st.depth_cell, + ) else { + return 1; + }; + st.last = call_pc; + st.pc = call_pc + 1; + match r { + Ok(v) => { + st.stack.add(len as usize).write(v); + st.len = len as usize + 1; + 0 + } + Err(e) => { + st.len = len as usize; + st.err = Some(e); + RAISED + } + } + } +} + +/// `LOAD_FAST x` (the `pc`th instruction of `code`) and `x.attr = v` +/// after it, with `v` at stack `slot`, as the core loop's fused arm runs +/// it: `0` stored (the value left the stack), anything else declined +/// untouched. +unsafe extern "C" fn h_store_attr( + st: *mut State, + code: *const CodeObject, + pc: u64, + slot: *mut Object, +) -> u32 { + // SAFETY: the code passes its live state, its own code, a `LOAD_FAST` + // of an in-range local followed by `STORE_ATTR`, and the value's slot. + unsafe { + let st = &*st; + let (code, pc) = (&*code, pc as usize); + let local = &*st.locals.add(code.instructions[pc].arg as usize); + let name = code.instructions[pc + 1].arg; + u32::from(!Interpreter::core_store_local_attr( + code, + local, + pc + 1, + name, + &*slot, + )) + } +} + +// ---- the compiler --------------------------------------------------------- + +/// The process's frame compiler. Code it emits lives as long as the +/// process (code objects are shared between threads, and their native +/// code is never freed). +struct Engine { + module: JITModule, + ctx: cranelift_codegen::Context, + fbctx: FunctionBuilderContext, + ptr: types::Type, + tags: Tags, + next: u32, +} + +// SAFETY: the engine is only used under its mutex. +unsafe impl Send for Engine {} + +static ENGINE: std::sync::Mutex>> = std::sync::Mutex::new(None); + +impl Engine { + fn new() -> Option { + let tags = tags()?; + let mut flags = settings::builder(); + flags.set("use_colocated_libcalls", "false").ok()?; + flags.set("is_pic", "false").ok()?; + // The lowering already does the optimizing that pays here (operands + // kept unboxed and in registers): Cranelift's mid-end costs more to + // run than it saves. Its register allocator pays for itself (the + // single-pass one spills around every helper call). + flags.set("opt_level", "none").ok()?; + if !cfg!(debug_assertions) && !verify() { + flags.set("enable_verifier", "false").ok()?; + } + let isa = cranelift_native::builder() + .ok()? + .finish(settings::Flags::new(flags)) + .ok()?; + let module = JITModule::new(JITBuilder::with_isa( + isa, + cranelift_module::default_libcall_names(), + )); + let ptr = module.target_config().pointer_type(); + let ctx = module.make_context(); + Some(Engine { + module, + ctx, + fbctx: FunctionBuilderContext::new(), + ptr, + tags, + next: 0, + }) + } +} + +/// Whether to verify the IR of every compiled frame and report failures +/// (`WEAVEPY_FRAME_JIT_VERIFY`, a debugging aid). +fn verify() -> bool { + static VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); + *VERIFY.get_or_init(|| std::env::var_os("WEAVEPY_FRAME_JIT_VERIFY").is_some()) +} + +/// Compile `code`; `None` leaves it to the core loop. A compiler panic +/// turns frame compilation off for the process rather than taking the +/// interpreter down. +fn compile(code: &CodeObject, ext: &CodeConstObjects, nlocals: usize) -> Option { + if code.instructions.is_empty() || code.instructions.len() > MAX_INSTRUCTIONS { + return None; + } + crate::tier2::ensure_obj_layout(); + let depths = weavepy_compiler::cpython_code::compute_startdepths(code); + let mut guard = ENGINE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let engine = guard.get_or_insert_with(Engine::new).as_mut()?; + let built = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + compile_with(engine, code, ext, nlocals, &depths) + })); + match built { + Ok(native) => native, + Err(_) => { + *guard = Some(None); + None + } + } +} + +fn compile_with( + engine: &mut Engine, + code: &CodeObject, + ext: &CodeConstObjects, + nlocals: usize, + depths: &[i64], +) -> Option { + engine.module.clear_context(&mut engine.ctx); + let ptr = engine.ptr; + let sig = &mut engine.ctx.func.signature; + sig.params.push(AbiParam::new(ptr)); + sig.returns.push(AbiParam::new(types::I32)); + let name = format!("wpframe_{}", engine.next); + engine.next += 1; + let id = engine + .module + .declare_function(&name, Linkage::Local, &engine.ctx.func.signature) + .ok()?; + let ninstrs = code.instructions.len(); + // The attribute sites' field caches (allocated here if no read has + // recorded one yet: the code addresses them). + let field_slots: &[FieldSlot] = ext + .field_slots + .get_or_init(|| (0..ninstrs).map(|_| FieldSlot::empty()).collect()); + let entries: Vec; + let built = { + let b = FunctionBuilder::new(&mut engine.ctx.func, &mut engine.fbctx); + let mut lower = Lower::new(b, ptr, engine.tags, code, ext, nlocals, depths, field_slots); + let done = lower.lower(); + entries = (0..ninstrs).map(|pc| lower.enters_at(pc)).collect(); + if done { + lower.b.seal_all_blocks(); + lower.b.finalize(); + } + done + }; + if !built { + // An unfinished function leaves the builder's state behind. + engine.fbctx = FunctionBuilderContext::new(); + engine.module.clear_context(&mut engine.ctx); + return None; + } + if std::env::var("WEAVEPY_FRAME_JIT_DUMP").is_ok_and(|n| code.qualname.contains(&n)) { + eprintln!("{}", engine.ctx.func.display()); + } + let defined = engine.module.define_function(id, &mut engine.ctx); + if let (Err(e), true) = (&defined, verify()) { + eprintln!( + "weavepy: frame {:?} failed to compile: {e:?}", + code.qualname + ); + } + engine.module.clear_context(&mut engine.ctx); + defined.ok()?; + engine.module.finalize_definitions().ok()?; + let func = engine.module.get_finalized_function(id); + // SAFETY: `func` is a finalized function with exactly the signature + // declared above, and the module (never dropped) keeps it alive. + let func = unsafe { std::mem::transmute::<*const u8, NativeFn>(func) }; + Some(Native { + func, + nlocals, + entries: entries.into_boxed_slice(), + name: code.qualname.clone(), + ops: code.instructions.iter().map(|i| i.op).collect(), + }) +} + +const FLAGS: MemFlags = MemFlags::trusted(); + +/// A value on the virtual operand stack: where it lives until something +/// needs it on the frame's stack. +#[derive(Clone, Copy)] +enum Item { + /// On the frame's stack, at this slot (owned there). + Mem(usize), + /// The value of this local, not copied yet (no reference taken). + Local(u32), + /// This constant, not copied yet. + Const(u32), + /// An `int`, unboxed. + Int(Value), + /// A `float`, unboxed. + Float(Value), + /// A `bool`, unboxed (an `i8`, `0` or `1`). + Bool(Value), + /// A scalar (`None`, `bool`, `int` or `float`) known only at run time: + /// its tag, word payload, and `bool` byte. + Dyn(Value, Value, Value), +} + +/// An operand's type, as far as the code knows it. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + Int, + Float, + Bool, + None, + /// Read at run time. + Unknown, +} + +/// An operand ready for a scalar operation: its kind, and its tag and +/// payload values (the tag only for an unknown kind). +#[derive(Clone, Copy)] +struct Opnd { + kind: Kind, + tag: Option, + /// The `i64` word (an `int`, a `float`'s bits), or a `bool`'s `i8`. + word: Value, + /// A `bool`'s byte (unknown kinds). + byte: Option, +} + +struct Lower<'a> { + b: FunctionBuilder<'a>, + ptr: types::Type, + tags: Tags, + layout: Option<&'static weavepy_jit::ObjLayout>, + code: &'a CodeObject, + ext: &'a CodeConstObjects, + nlocals: usize, + depths: &'a [i64], + field_slots: &'a [FieldSlot], + st: Value, + locals: Value, + stack: Value, + snap: Value, + countdown: Value, + dead: Value, + /// The instruction that last ran, when known here; `None` at a block's + /// start, whose `st.last` the entry or the jump in wrote. + last_pc: Option, + /// Locals that may be unbound when a block starts (parameters no + /// `DELETE_FAST` names never are), and the ones this block has seen + /// bound. + maybe_unbound: Vec, + bound: Vec, + /// Each block start's block (instruction index → block). + blocks: Vec>, + /// Dispatches on the state's pc and depth (the entry, and helpers that + /// move the pc by a run-time amount). + dispatch: Block, + /// Whether copies are on an exit path (one helper call each, not the + /// scalar test in line: exits are rare, and code size costs compile + /// time). + cold: bool, + /// The virtual stack above the frame's, and the logical depth. + vs: Vec, + depth: usize, + sigs: Vec<(Vec, Option, SigRef)>, + /// Side exits waiting for their blocks to be filled (after the body: + /// the builder fills one block at a time). + side_exits: Vec, +} + +/// An exit block of [`Lower::exit_with`], filled after the body from the +/// lowering state at its branch. +struct SideExit { + block: Block, + vs: Vec, + depth: usize, + last_pc: Option, + restore: Vec, + status: u32, + pc: usize, +} + +/// Whether the native code runs `op` itself (the rest are handed to the +/// core loop: their successors start blocks). +fn native_op(op: OpCode) -> bool { + matches!( + op, + OpCode::Nop + | OpCode::NotTaken + | OpCode::Resume + | OpCode::CopyFreeVars + | OpCode::LoadFast + | OpCode::LoadFastBorrow + | OpCode::LoadConst + | OpCode::LoadSmallInt + | OpCode::StoreFast + | OpCode::PopTop + | OpCode::BinaryOp + | OpCode::CompareOp + | OpCode::ToBool + | OpCode::PopJumpIfFalse + | OpCode::PopJumpIfTrue + | OpCode::PopJumpIfNone + | OpCode::PopJumpIfNotNone + | OpCode::JumpForward + | OpCode::JumpBackward + | OpCode::ForIter + | OpCode::LoadAttr + | OpCode::IsOp + | OpCode::CopyTop + | OpCode::Swap + | OpCode::BinarySubscr + | OpCode::BinarySlice + | OpCode::StoreSubscr + | OpCode::ListAppend + | OpCode::UnpackSequence + | OpCode::ContainsOp + ) +} + +impl<'a> Lower<'a> { + #[allow(clippy::too_many_arguments)] + fn new( + mut b: FunctionBuilder<'a>, + ptr: types::Type, + tags: Tags, + code: &'a CodeObject, + ext: &'a CodeConstObjects, + nlocals: usize, + depths: &'a [i64], + field_slots: &'a [FieldSlot], + ) -> Self { + let entry = b.create_block(); + b.append_block_params_for_function_params(entry); + b.switch_to_block(entry); + let st = b.block_params(entry)[0]; + let load = |b: &mut FunctionBuilder<'_>, ty, off| b.ins().load(ty, FLAGS, st, off); + let locals = load(&mut b, ptr, S_LOCALS); + let stack = load(&mut b, ptr, S_STACK); + let snap = load(&mut b, types::I64, S_SNAP); + let countdown = load(&mut b, ptr, S_COUNTDOWN); + let dead = load(&mut b, ptr, S_DEAD); + Lower { + b, + ptr, + tags, + layout: crate::tier2::published_obj_layout(), + code, + ext, + nlocals, + depths, + field_slots, + st, + locals, + stack, + snap, + countdown, + dead, + last_pc: None, + maybe_unbound: { + let params = code.arg_count as usize + + code.kwonly_count as usize + + usize::from(code.has_varargs) + + usize::from(code.has_varkeywords); + let mut m: Vec = (0..nlocals).map(|i| i >= params).collect(); + for ins in &code.instructions { + if ins.op == OpCode::DeleteFast { + if let Some(x) = m.get_mut(ins.arg as usize) { + *x = true; + } + } + } + m + }, + bound: vec![false; nlocals], + blocks: Vec::new(), + dispatch: Block::from_u32(0), + cold: false, + vs: Vec::new(), + depth: 0, + sigs: Vec::new(), + side_exits: Vec::new(), + } + } + + // ---- building blocks ---- + + fn sig(&mut self, params: &[Type], ret: Option) -> SigRef { + if let Some((_, _, s)) = self.sigs.iter().find(|(p, r, _)| p == params && *r == ret) { + return *s; + } + let mut sig = Signature::new(self.b.func.signature.call_conv); + for &t in params { + sig.params.push(AbiParam::new(t)); + } + if let Some(t) = ret { + sig.returns.push(AbiParam::new(t)); + } + let s = self.b.import_signature(sig); + self.sigs.push((params.to_vec(), ret, s)); + s + } + + /// Call the helper at `addr`: its `u32` result, if it has one. + fn call(&mut self, addr: usize, args: &[Value], ret: bool) -> Option { + self.call_typed(addr, args, ret.then_some(types::I32)) + } + + /// Call the helper at `addr`: its result of type `ret`, if any. + fn call_typed(&mut self, addr: usize, args: &[Value], ret: Option) -> Option { + let params: Vec = args + .iter() + .map(|&a| self.b.func.dfg.value_type(a)) + .collect(); + let sig = self.sig(¶ms, ret); + let callee = self.b.ins().iconst(self.ptr, addr as i64); + let inst = self.b.ins().call_indirect(sig, callee, args); + ret.map(|_| self.b.inst_results(inst)[0]) + } + + fn set_last(&mut self, pc: usize) { + self.last_pc = Some(pc); + } + + /// Write the last instruction to the state (unless the state has it). + fn store_last(&mut self) { + if let Some(pc) = self.last_pc { + let v = self.b.ins().iconst(types::I64, pc as i64); + self.b.ins().store(FLAGS, v, self.st, S_LAST); + } + } + + fn slot_addr(&mut self, slot: usize) -> Value { + self.b.ins().iadd_imm(self.stack, 16 * slot as i64) + } + + fn local_addr(&mut self, i: u32) -> Value { + self.b.ins().iadd_imm(self.locals, 16 * i64::from(i)) + } + + fn const_addr(&mut self, k: u32) -> Value { + let at = std::ptr::from_ref(&self.ext.objects[k as usize]) as i64; + self.b.ins().iconst(self.ptr, at) + } + + fn tag_at(&mut self, addr: Value) -> Value { + self.b.ins().uload8(types::I64, FLAGS, addr, 0) + } + + fn is_scalar(&mut self, tag: Value) -> Value { + let one = self.b.ins().iconst(types::I64, 1); + let bit = self.b.ins().ishl(one, tag); + let m = self.b.ins().band_imm(bit, self.tags.scalar_mask()); + self.b.ins().icmp_imm(IntCC::NotEqual, m, 0) + } + + fn write_tag(&mut self, dst: Value, t: u8) { + let t = self.b.ins().iconst(types::I8, i64::from(t)); + self.b.ins().store(FLAGS, t, dst, 0); + } + + fn copy16(&mut self, dst: Value, src: Value) { + let a = self.b.ins().load(types::I64, FLAGS, src, 0); + let w = self.b.ins().load(types::I64, FLAGS, src, 8); + self.b.ins().store(FLAGS, a, dst, 0); + self.b.ins().store(FLAGS, w, dst, 8); + } + + /// Branch to `out` when `cond`; continue in a fresh block otherwise. + fn branch_out(&mut self, cond: Value, out: Block) { + let ok = self.b.create_block(); + self.b.ins().brif(cond, out, &[], ok, &[]); + self.b.switch_to_block(ok); + } + + // ---- the virtual stack ---- + + fn push(&mut self, item: Item) { + self.vs.push(item); + self.depth += 1; + } + + fn pop(&mut self) -> Item { + self.depth -= 1; + self.vs.pop().unwrap_or(Item::Mem(self.depth)) + } + + /// Write `item` to the frame's stack at `slot` (taking a reference for + /// a heap value it copies). + fn materialize(&mut self, item: Item, slot: usize) { + match item { + Item::Mem(s) => debug_assert_eq!(s, slot), + Item::Local(i) => { + let src = self.local_addr(i); + let dst = self.slot_addr(slot); + self.copy_value(dst, src); + } + Item::Const(k) => { + let dst = self.slot_addr(slot); + match self.ext.objects[k as usize] { + Object::Int(v) => { + let v = self.b.ins().iconst(types::I64, v); + self.write_tag(dst, self.tags.int); + self.b.ins().store(FLAGS, v, dst, 8); + } + Object::Float(x) => { + let v = self.b.ins().iconst(types::I64, x.to_bits() as i64); + self.write_tag(dst, self.tags.float); + self.b.ins().store(FLAGS, v, dst, 8); + } + Object::Bool(x) => { + let v = self.b.ins().iconst(types::I8, i64::from(x)); + self.write_tag(dst, self.tags.boolean); + self.b.ins().store(FLAGS, v, dst, 1); + } + Object::None => self.write_tag(dst, self.tags.none), + _ => { + let src = self.const_addr(k); + self.call(h_clone as *const () as usize, &[dst, src], false); + } + } + } + Item::Int(v) => { + let dst = self.slot_addr(slot); + self.write_tag(dst, self.tags.int); + self.b.ins().store(FLAGS, v, dst, 8); + } + Item::Float(v) => { + let dst = self.slot_addr(slot); + self.write_tag(dst, self.tags.float); + let bits = self.b.ins().bitcast(types::I64, MemFlags::new(), v); + self.b.ins().store(FLAGS, bits, dst, 8); + } + Item::Bool(v) => { + let dst = self.slot_addr(slot); + self.write_tag(dst, self.tags.boolean); + self.b.ins().store(FLAGS, v, dst, 1); + } + Item::Dyn(tag, word, byte) => { + let dst = self.slot_addr(slot); + let t = self.b.ins().ireduce(types::I8, tag); + let b8 = self.b.ins().ireduce(types::I8, byte); + self.b.ins().store(FLAGS, t, dst, 0); + self.b.ins().store(FLAGS, b8, dst, 1); + self.b.ins().store(FLAGS, word, dst, 8); + } + } + } + + /// `*dst = (*src).clone()`: a scalar by value, anything else through + /// its clone. + fn copy_value(&mut self, dst: Value, src: Value) { + if self.cold { + self.call(h_clone as *const () as usize, &[dst, src], false); + return; + } + let tag = self.tag_at(src); + let scalar = self.is_scalar(tag); + let by_value = self.b.create_block(); + let by_clone = self.b.create_block(); + let done = self.b.create_block(); + self.b.ins().brif(scalar, by_value, &[], by_clone, &[]); + self.b.switch_to_block(by_value); + self.copy16(dst, src); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(by_clone); + self.call(h_clone as *const () as usize, &[dst, src], false); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + + /// Write the whole virtual stack to the frame's. + fn flush(&mut self) { + let base = self.depth - self.vs.len(); + let items = std::mem::take(&mut self.vs); + for (k, item) in items.into_iter().enumerate() { + self.materialize(item, base + k); + } + } + + /// Write the virtual stack's copies of local `i` to the frame's stack + /// (before the local changes). + fn flush_local(&mut self, i: u32) { + let base = self.depth - self.vs.len(); + for k in 0..self.vs.len() { + if let Item::Local(j) = self.vs[k] { + if j == i { + self.materialize(Item::Local(j), base + k); + self.vs[k] = Item::Mem(base + k); + } + } + } + } + + /// Return `status` with the virtual stack written out and `pc` next. + fn exit(&mut self, status: u32, pc: usize) { + let cold = std::mem::replace(&mut self.cold, true); + self.flush(); + self.cold = cold; + let l = self.b.ins().iconst(types::I64, self.depth as i64); + self.b.ins().store(FLAGS, l, self.st, S_LEN); + self.store_last(); + let pcv = self.b.ins().iconst(types::I64, pc as i64); + self.b.ins().store(FLAGS, pcv, self.st, S_PC); + let s = self.b.ins().iconst(types::I32, i64::from(status)); + self.b.ins().return_(&[s]); + } + + /// Leave for the core loop at `pc` (the instruction about to run), + /// with `restore` (the operands it popped) back on the stack, from a + /// side block; lowering continues in the current block afterwards. + fn exit_with(&mut self, pc: usize, restore: &[Item], status: u32) -> Block { + let block = self.b.create_block(); + self.side_exits.push(SideExit { + block, + vs: self.vs.clone(), + depth: self.depth, + last_pc: self.last_pc, + restore: restore.to_vec(), + status, + pc, + }); + block + } + + /// Fill the side exits' blocks. + fn side_exits(&mut self) { + while let Some(x) = self.side_exits.pop() { + self.b.switch_to_block(x.block); + self.vs = x.vs; + self.depth = x.depth; + self.last_pc = x.last_pc; + for item in x.restore { + self.push(item); + } + self.exit(x.status, x.pc); + } + } + + /// Stop for a queued finalizer (with `pc` next), from the current + /// state. + fn check_released(&mut self, next: usize) { + let f = self.b.ins().uload8(types::I32, FLAGS, self.dead, 0); + let out = self.exit_with(next, &[], MARKED); + self.branch_out(f, out); + } + + /// Jump to the block starting at `pc`, the virtual stack written out. + fn goto(&mut self, pc: usize) { + self.flush(); + match self.blocks.get(pc).copied().flatten() { + Some(b) => { + // The block finds the last instruction in the state. + self.store_last(); + self.b.ins().jump(b, &[]); + } + None => self.exit(INTERP, pc), + } + } + + // ---- operands ---- + + /// `item` as a scalar operand. An unknown-kind operand's tag still has + /// to be checked. + fn operand(&mut self, item: Item) -> Option { + let at = |s: &mut Self, addr: Value| -> Opnd { + let tag = s.tag_at(addr); + let word = s.b.ins().load(types::I64, FLAGS, addr, 8); + let byte = s.b.ins().uload8(types::I64, FLAGS, addr, 1); + Opnd { + kind: Kind::Unknown, + tag: Some(tag), + word, + byte: Some(byte), + } + }; + Some(match item { + Item::Int(v) => Opnd { + kind: Kind::Int, + tag: None, + word: v, + byte: None, + }, + Item::Float(v) => Opnd { + kind: Kind::Float, + tag: None, + word: self.b.ins().bitcast(types::I64, MemFlags::new(), v), + byte: None, + }, + Item::Bool(v) => Opnd { + kind: Kind::Bool, + tag: None, + word: v, + byte: None, + }, + Item::Const(k) => match self.ext.objects[k as usize] { + Object::Int(v) => Opnd { + kind: Kind::Int, + tag: None, + word: self.b.ins().iconst(types::I64, v), + byte: None, + }, + Object::Float(x) => Opnd { + kind: Kind::Float, + tag: None, + word: self.b.ins().iconst(types::I64, x.to_bits() as i64), + byte: None, + }, + Object::Bool(x) => Opnd { + kind: Kind::Bool, + tag: None, + word: self.b.ins().iconst(types::I8, i64::from(x)), + byte: None, + }, + Object::None => Opnd { + kind: Kind::None, + tag: None, + word: self.b.ins().iconst(types::I64, 0), + byte: None, + }, + _ => return None, + }, + Item::Local(i) => { + let a = self.local_addr(i); + at(self, a) + } + Item::Mem(s) => { + let a = self.slot_addr(s); + at(self, a) + } + Item::Dyn(tag, word, byte) => Opnd { + kind: Kind::Unknown, + tag: Some(tag), + word, + byte: Some(byte), + }, + }) + } + + /// Whether `o` is of `kind`: a static answer, or a run-time test. + fn is_kind(&mut self, o: Opnd, kind: Kind, tag: u8) -> Result { + match o.kind { + Kind::Unknown => { + let t = o.tag.expect("unknown kinds carry a tag"); + Err(self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tag))) + } + k => Ok(k == kind), + } + } + + /// `o`'s value as an `f64`, given it is an `int` or a `float` + /// (`is_float` says which, when unknown). + fn as_f64(&mut self, o: Opnd, is_float: Result) -> Value { + match is_float { + Ok(true) => self.b.ins().bitcast(types::F64, MemFlags::new(), o.word), + Ok(false) => self.b.ins().fcvt_from_sint(types::F64, o.word), + Err(f) => { + let x = self.b.ins().bitcast(types::F64, MemFlags::new(), o.word); + let c = self.b.ins().fcvt_from_sint(types::F64, o.word); + self.b.ins().select(f, x, c) + } + } + } + + /// The truth of a scalar operand (`None`, `bool`, `int`; and `float` + /// with `floats`), as an `i8`; anything else branches to `other`. + fn truth(&mut self, o: Opnd, floats: bool, other: Block) -> Value { + match o.kind { + Kind::Bool => o.word, + Kind::Int => self.b.ins().icmp_imm(IntCC::NotEqual, o.word, 0), + Kind::None => self.b.ins().iconst(types::I8, 0), + Kind::Float if floats => { + let x = self.b.ins().bitcast(types::F64, MemFlags::new(), o.word); + let z = self.b.ins().f64const(0.0); + self.b.ins().fcmp(FloatCC::NotEqual, x, z) + } + Kind::Float => { + self.b.ins().jump(other, &[]); + let dead = self.b.create_block(); + self.b.switch_to_block(dead); + self.b.ins().iconst(types::I8, 0) + } + Kind::Unknown => { + let t = o.tag.expect("unknown kinds carry a tag"); + let byte = o.byte.expect("unknown kinds carry a byte"); + let tb = self.tags; + let is_bool = self + .b + .ins() + .icmp_imm(IntCC::Equal, t, i64::from(tb.boolean)); + let is_int = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.int)); + let is_none = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.none)); + let mut ok = self.b.ins().bor(is_bool, is_int); + ok = self.b.ins().bor(ok, is_none); + let fnz = if floats { + let is_float = self.b.ins().icmp_imm(IntCC::Equal, t, i64::from(tb.float)); + ok = self.b.ins().bor(ok, is_float); + let x = self.b.ins().bitcast(types::F64, MemFlags::new(), o.word); + let z = self.b.ins().f64const(0.0); + let nz = self.b.ins().fcmp(FloatCC::NotEqual, x, z); + Some((is_float, nz)) + } else { + None + }; + let bad = self.b.ins().bxor_imm(ok, 1); + self.branch_out(bad, other); + let bnz = self.b.ins().icmp_imm(IntCC::NotEqual, byte, 0); + let wnz = self.b.ins().icmp_imm(IntCC::NotEqual, o.word, 0); + let mut r = self.b.ins().select(is_bool, bnz, wnz); + let not_none = self.b.ins().bxor_imm(is_none, 1); + r = self.b.ins().band(r, not_none); + if let Some((is_float, nz)) = fnz { + r = self.b.ins().select(is_float, nz, r); + } + r + } + } + } + + // ---- the instructions ---- + + /// Whether entering at `pc` gets anywhere: a block start whose first + /// instruction the native code runs (not one it hands straight back). + fn enters_at(&self, pc: usize) -> bool { + let instrs = &self.code.instructions; + if !self.blocks.get(pc).is_some_and(Option::is_some) { + return false; + } + native_op(instrs[pc].op) + } + + /// Lower the whole function; `false` when it can't be compiled. + fn lower(&mut self) -> bool { + let code = self.code; + let n = code.instructions.len(); + if self.depths.len() != n { + return false; + } + // Block starts: the entry, jump targets, and every instruction + // after one that ends a block or that the core loop runs. + let mut starts = vec![false; n]; + starts[0] = true; + for (pc, ins) in code.instructions.iter().enumerate() { + let next = pc + 1; + let target = match ins.op { + OpCode::JumpForward + | OpCode::PopJumpIfFalse + | OpCode::PopJumpIfTrue + | OpCode::PopJumpIfNone + | OpCode::PopJumpIfNotNone + | OpCode::ForIter => Some(next + ins.arg as usize), + OpCode::JumpBackward => Some(next.saturating_sub(ins.arg as usize)), + _ => None, + }; + if let Some(t) = target { + if t < n { + starts[t] = true; + } + if ins.op == OpCode::ForIter { + // The loop exit past `END_FOR` / `POP_ITER`. + let mut to = t; + if code.instructions.get(to).map(|i| i.op) == Some(OpCode::EndFor) { + to += 1; + if matches!( + code.instructions.get(to).map(|i| i.op), + Some(OpCode::PopIter | OpCode::PopTop) + ) { + to += 1; + } + } + if to < n { + starts[to] = true; + } + } + } + if next < n && (target.is_some() || !native_op(ins.op)) { + starts[next] = true; + } + // A fused attribute store continues after its `STORE_ATTR`. + if ins.op == OpCode::LoadFast + && code.instructions.get(next).map(|i| i.op) == Some(OpCode::StoreAttr) + && pc + 2 < n + { + starts[pc + 2] = true; + } + // A fused method call continues after its `CALL`. + if ins.op == OpCode::LoadFast + && code.instructions.get(next).map(|i| i.op) == Some(OpCode::LoadMethodAttr) + { + if let Some(call) = + (pc + 2..n.min(pc + 12)).find(|&k| code.instructions[k].op == OpCode::Call) + { + if call + 1 < n { + starts[call + 1] = true; + } + } + } + } + for (pc, s) in starts.iter_mut().enumerate() { + if *s && self.depths[pc] < 0 { + // Unreachable on the ordinary paths (a handler's): no entry. + *s = false; + } + } + self.blocks = starts + .iter() + .map(|&s| s.then(|| self.b.create_block())) + .collect(); + // The entry: to the block at the state's pc, when the stack has the + // block's depth. + self.dispatch = self.b.create_block(); + self.b.ins().jump(self.dispatch, &[]); + self.b.switch_to_block(self.dispatch); + let pc = self.b.ins().load(types::I64, FLAGS, self.st, S_PC); + let len = self.b.ins().load(types::I64, FLAGS, self.st, S_LEN); + let pc32 = self.b.ins().ireduce(types::I32, pc); + let out = self.b.create_block(); + // The code writes stack slots at fixed offsets: the frame's stack + // must have room for the deepest point plus an instruction's own + // pushes (a pooled stack can be smaller; the core loop grows it). + let need = self.depths.iter().copied().max().unwrap_or(0).max(0) + 4; + let cap = self.b.ins().load(types::I64, FLAGS, self.st, S_CAP); + let short = self.b.ins().icmp_imm(IntCC::SignedLessThan, cap, need); + let roomy = self.b.create_block(); + self.b.ins().brif(short, out, &[], roomy, &[]); + self.b.switch_to_block(roomy); + let mut tramps = Vec::new(); + let mut calls = Vec::with_capacity(n); + for pc in 0..n { + let target = match self.blocks[pc] { + Some(blk) => { + let t = self.b.create_block(); + tramps.push((t, blk, self.depths[pc])); + t + } + None => out, + }; + calls.push(self.b.func.dfg.block_call(target, &[])); + } + let def = self.b.func.dfg.block_call(out, &[]); + let jt = self.b.create_jump_table(JumpTableData::new(def, &calls)); + self.b.ins().br_table(pc32, jt); + self.b.switch_to_block(out); + let s = self.b.ins().iconst(types::I32, i64::from(INTERP)); + self.b.ins().return_(&[s]); + for (t, blk, depth) in tramps { + self.b.switch_to_block(t); + let wrong = self.b.ins().icmp_imm(IntCC::NotEqual, len, depth); + self.b.ins().brif(wrong, out, &[], blk, &[]); + } + // The blocks: each from its start until it ends or the next block + // starts. + for start in 0..n { + let Some(blk) = self.blocks[start] else { + continue; + }; + self.b.switch_to_block(blk); + self.vs.clear(); + self.depth = self.depths[start] as usize; + self.last_pc = None; + self.bound.iter_mut().for_each(|b| *b = false); + let mut pc = start; + loop { + if !self.instruction(pc) { + break; + } + pc += 1; + if pc >= n { + self.exit(INTERP, pc); + break; + } + if self.blocks[pc].is_some() { + self.goto(pc); + break; + } + } + } + self.side_exits(); + true + } + + /// Lower the instruction at `pc` into the current block: `true` when + /// the block continues with the next instruction, `false` when it ended + /// (a jump, an exit). + fn instruction(&mut self, pc: usize) -> bool { + let ins = self.code.instructions[pc]; + let next = pc + 1; + let cont = match ins.op { + OpCode::Nop | OpCode::NotTaken | OpCode::Resume | OpCode::CopyFreeVars => true, + OpCode::LoadFast | OpCode::LoadFastBorrow => self.load_fast(pc, ins.arg), + OpCode::LoadConst if (ins.arg as usize) < self.ext.objects.len() => { + self.push(Item::Const(ins.arg)); + true + } + OpCode::LoadSmallInt => { + let v = self.b.ins().iconst(types::I64, i64::from(ins.arg)); + self.push(Item::Int(v)); + true + } + OpCode::StoreFast if (ins.arg as usize) < self.nlocals => self.store_fast(pc, ins.arg), + OpCode::PopTop => { + match self.pop() { + Item::Mem(s) => { + let at = self.slot_addr(s); + let tag = self.tag_at(at); + let scalar = self.is_scalar(tag); + let heap = self.b.create_block(); + let done = self.b.create_block(); + self.b.ins().brif(scalar, done, &[], heap, &[]); + self.b.switch_to_block(heap); + let r = self + .call(h_pop as *const () as usize, &[at], true) + .expect("returns"); + let out = self.exit_with(pc, &[Item::Mem(s)], INTERP); + self.branch_out(r, out); + self.set_last(pc); + self.check_released(next); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + // A copy never taken has nothing to release. + _ => {} + } + true + } + OpCode::BinaryOp => return self.binary(pc, ins.arg), + OpCode::CompareOp => return self.compare(pc, ins.arg), + OpCode::ToBool => { + let item = self.pop(); + let Some(o) = self.operand(item) else { + self.push(item); + self.exit(INTERP, pc); + return false; + }; + if o.kind == Kind::Bool { + self.push(item); + } else { + let other = self.exit_with(pc, &[item], INTERP); + let t = self.truth(o, true, other); + self.push(Item::Bool(t)); + } + true + } + OpCode::PopJumpIfFalse | OpCode::PopJumpIfTrue => { + let item = self.pop(); + let Some(o) = self.operand(item) else { + self.push(item); + self.exit(INTERP, pc); + return false; + }; + let other = self.exit_with(pc, &[item], INTERP); + let t = self.truth(o, false, other); + self.set_last(pc); + let taken = next + ins.arg as usize; + self.branch(t, ins.op == OpCode::PopJumpIfTrue, taken, next); + return false; + } + OpCode::PopJumpIfNone | OpCode::PopJumpIfNotNone => { + let item = self.pop(); + let is_none = match item { + Item::Mem(s) => { + let at = self.slot_addr(s); + let tag = self.tag_at(at); + let none = + self.b + .ins() + .icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)); + let scalar = self.is_scalar(tag); + let heap = self.b.create_block(); + let done = self.b.create_block(); + self.b.ins().brif(scalar, done, &[], heap, &[]); + self.b.switch_to_block(heap); + let r = self + .call(h_pop as *const () as usize, &[at], true) + .expect("returns"); + let out = self.exit_with(pc, &[Item::Mem(s)], INTERP); + self.branch_out(r, out); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + none + } + Item::Local(i) => { + let at = self.local_addr(i); + let tag = self.tag_at(at); + self.b + .ins() + .icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)) + } + Item::Const(k) => { + let none = matches!(self.ext.objects[k as usize], Object::None); + self.b.ins().iconst(types::I8, i64::from(none)) + } + Item::Dyn(tag, ..) => { + self.b + .ins() + .icmp_imm(IntCC::Equal, tag, i64::from(self.tags.none)) + } + Item::Int(_) | Item::Float(_) | Item::Bool(_) => { + self.b.ins().iconst(types::I8, 0) + } + }; + self.set_last(pc); + let taken = next + ins.arg as usize; + self.branch(is_none, ins.op == OpCode::PopJumpIfNone, taken, next); + return false; + } + OpCode::IsOp => return self.is_op(pc, ins.arg), + OpCode::CopyTop => { + let n = (ins.arg as usize).max(1); + if n > self.depth { + self.exit(INTERP, pc); + return false; + } + let k = self.vs.len(); + let item = if n <= k { + self.vs[k - n] + } else { + Item::Mem(self.depth - n) + }; + match item { + Item::Mem(s) => { + // A second reference to a value on the stack. + let dst = self.slot_addr(self.depth); + let src = self.slot_addr(s); + self.copy_value(dst, src); + let d = self.depth; + self.push(Item::Mem(d)); + } + other => self.push(other), + } + true + } + OpCode::Swap => { + let n = ins.arg as usize; + if n < 2 || n > self.depth { + self.exit(INTERP, pc); + return false; + } + let k = self.vs.len(); + let in_registers = n <= k + && !matches!(self.vs[k - 1], Item::Mem(_)) + && !matches!(self.vs[k - n], Item::Mem(_)); + if in_registers { + self.vs.swap(k - 1, k - n); + } else { + // A stack value's slot can't move on the virtual stack: + // swap on the frame's. + self.flush(); + let a = self.slot_addr(self.depth - 1); + let b = self.slot_addr(self.depth - n); + let (a0, a1) = ( + self.b.ins().load(types::I64, FLAGS, a, 0), + self.b.ins().load(types::I64, FLAGS, a, 8), + ); + self.copy16(a, b); + self.b.ins().store(FLAGS, a0, b, 0); + self.b.ins().store(FLAGS, a1, b, 8); + } + true + } + + OpCode::JumpForward => { + self.set_last(pc); + self.goto(next + ins.arg as usize); + return false; + } + OpCode::JumpBackward => { + self.flush(); + // The back edge is the eval breaker (the core loop's arm, + // which the exits below hand the instruction to). + let c = self.b.ins().load(types::I32, FLAGS, self.countdown, 0); + let low = self.b.ins().icmp_imm(IntCC::UnsignedLessThanOrEqual, c, 1); + let gen_addr = self + .b + .ins() + .iconst(self.ptr, crate::hot_gates::loop_gen_ptr() as i64); + let g = self.b.ins().load(types::I64, FLAGS, gen_addr, 0); + let moved = self.b.ins().icmp(IntCC::NotEqual, g, self.snap); + let trip = self.b.ins().bor(low, moved); + let out = self.exit_with(pc, &[], INTERP); + self.branch_out(trip, out); + let c1 = self.b.ins().iadd_imm(c, -1); + self.b.ins().store(FLAGS, c1, self.countdown, 0); + self.set_last(pc); + self.goto(next.saturating_sub(ins.arg as usize)); + return false; + } + OpCode::ForIter => return self.for_iter(pc, ins.arg), + OpCode::BinarySubscr + | OpCode::BinarySlice + | OpCode::StoreSubscr + | OpCode::ListAppend + | OpCode::UnpackSequence => return self.container(pc), + OpCode::ContainsOp => return self.contains(pc, ins.arg), + OpCode::LoadAttr => return self.load_attr(pc, ins.arg), + _ => { + self.exit(INTERP, pc); + return false; + } + }; + if cont { + self.set_last(pc); + } + cont + } + + /// Branch to `taken` when `cond` (an `i8` truth) is `want`. + fn branch(&mut self, cond: Value, want: bool, taken: usize, fall: usize) { + self.flush(); + let (t, f) = if want { (taken, fall) } else { (fall, taken) }; + let tb = self.b.create_block(); + let fb = self.b.create_block(); + self.b.ins().brif(cond, tb, &[], fb, &[]); + let depth = self.depth; + self.b.switch_to_block(tb); + self.goto(t); + self.depth = depth; + self.b.switch_to_block(fb); + self.goto(f); + } + + fn load_fast(&mut self, pc: usize, i: u32) -> bool { + if i as usize >= self.nlocals { + self.exit(INTERP, pc); + return false; + } + // The core loop's fused receiver shapes (`x.m(...)`, `x.attr = v`) + // start here. + match self.code.instructions.get(pc + 1).map(|i| i.op) { + Some(OpCode::LoadMethodAttr) => return self.method_call(pc), + Some(OpCode::StoreAttr) => return self.store_attr(pc), + _ => {} + } + // An unbound local raises in the core loop (a parameter nothing + // deletes is always bound, as is a local this block bound or tested). + let ix = i as usize; + if self.maybe_unbound[ix] && !self.bound[ix] { + let at = self.local_addr(i); + let tag = self.tag_at(at); + let unbound = self + .b + .ins() + .icmp_imm(IntCC::Equal, tag, i64::from(self.tags.unbound)); + let out = self.exit_with(pc, &[], INTERP); + self.branch_out(unbound, out); + self.bound[ix] = true; + } + self.push(Item::Local(i)); + true + } + + /// `LOAD_FAST x; x.m()` through [`h_local_method`], + /// continuing at whatever follows the call (through the dispatch). A + /// generator body's resume can't raise from here: its calls are the + /// general loop's. + fn method_call(&mut self, pc: usize) -> bool { + if self.code.is_generator || self.code.is_coroutine || self.code.is_async_generator { + self.exit(INTERP, pc); + return false; + } + self.flush(); + let code = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(self.code) as i64); + let ext = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(self.ext) as i64); + let pcv = self.b.ins().iconst(types::I64, pc as i64); + let nl = self.b.ins().iconst(types::I64, self.nlocals as i64); + let len = self.b.ins().iconst(types::I64, self.depth as i64); + self.store_last(); + let r = self + .call( + h_local_method as *const () as usize, + &[self.st, code, ext, pcv, nl, len], + true, + ) + .expect("returns"); + let ran = self.b.create_block(); + let other = self.b.create_block(); + self.b.ins().brif(r, other, &[], ran, &[]); + // Ran: the state holds the pc, depth and last instruction after the + // call. + self.b.switch_to_block(ran); + let saved = self.last_pc.take(); + self.check_released_dynamic(); + self.b.ins().jump(self.dispatch, &[]); + // Raised: the core loop raises it. + self.b.switch_to_block(other); + let raised = self.b.ins().icmp_imm(IntCC::Equal, r, i64::from(RAISED)); + let raise_b = self.b.create_block(); + let declined = self.b.create_block(); + self.b.ins().brif(raised, raise_b, &[], declined, &[]); + self.b.switch_to_block(raise_b); + let s = self.b.ins().iconst(types::I32, i64::from(RAISED)); + self.b.ins().return_(&[s]); + self.b.switch_to_block(declined); + self.last_pc = saved; + self.exit(INTERP, pc); + false + } + + /// Stop for a queued finalizer with the state as a helper left it (pc + /// and depth already stored). + fn check_released_dynamic(&mut self) { + let f = self.b.ins().uload8(types::I32, FLAGS, self.dead, 0); + let out = self.b.create_block(); + let ok = self.b.create_block(); + self.b.ins().brif(f, out, &[], ok, &[]); + self.b.switch_to_block(out); + let s = self.b.ins().iconst(types::I32, i64::from(MARKED)); + self.b.ins().return_(&[s]); + self.b.switch_to_block(ok); + } + + /// `LOAD_FAST x; x.attr = v` through [`h_store_attr`] (the value on + /// the stack's top). + fn store_attr(&mut self, pc: usize) -> bool { + if self.depth == 0 { + self.exit(INTERP, pc); + return false; + } + self.flush(); + let slot = self.slot_addr(self.depth - 1); + let code = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(self.code) as i64); + let pcv = self.b.ins().iconst(types::I64, pc as i64); + let r = self + .call( + h_store_attr as *const () as usize, + &[self.st, code, pcv, slot], + true, + ) + .expect("returns"); + let out = self.exit_with(pc, &[], INTERP); + self.branch_out(r, out); + self.depth -= 1; + self.set_last(pc + 1); + self.check_released(pc + 2); + self.goto(pc + 2); + false + } + + fn store_fast(&mut self, pc: usize, i: u32) -> bool { + self.flush_local(i); + self.bound[i as usize] = true; + let item = self.pop(); + if let Item::Local(j) = item { + if j == i { + return true; + } + } + let local = self.local_addr(i); + // The new value's words, for a store that needs no release. + let words: Option<(Value, Value, Value)> = match item { + Item::Int(v) => Some(( + self.b.ins().iconst(types::I64, i64::from(self.tags.int)), + v, + self.b.ins().iconst(types::I8, 0), + )), + Item::Float(v) => Some(( + self.b.ins().iconst(types::I64, i64::from(self.tags.float)), + self.b.ins().bitcast(types::I64, MemFlags::new(), v), + self.b.ins().iconst(types::I8, 0), + )), + Item::Bool(v) => Some(( + self.b + .ins() + .iconst(types::I64, i64::from(self.tags.boolean)), + self.b.ins().iconst(types::I64, 0), + v, + )), + Item::Dyn(t, w, b) => Some((t, w, self.b.ins().ireduce(types::I8, b))), + Item::Const(k) + if matches!( + self.ext.objects[k as usize], + Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None + ) => + { + let o = self.operand(item).expect("a scalar constant"); + let tag = match o.kind { + Kind::Int => self.tags.int, + Kind::Float => self.tags.float, + Kind::Bool => self.tags.boolean, + _ => self.tags.none, + }; + let (w, b) = if o.kind == Kind::Bool { + (self.b.ins().iconst(types::I64, 0), o.word) + } else { + (o.word, self.b.ins().iconst(types::I8, 0)) + }; + Some((self.b.ins().iconst(types::I64, i64::from(tag)), w, b)) + } + _ => None, + }; + let old = self.tag_at(local); + let plain_old = { + let s = self.is_scalar(old); + let u = self + .b + .ins() + .icmp_imm(IntCC::Equal, old, i64::from(self.tags.unbound)); + self.b.ins().bor(s, u) + }; + // The value's slot (it left the virtual stack). + let slot = self.depth; + match words { + Some((tag, word, byte)) => { + // A scalar over a scalar (or nothing): three stores. + let slow = self.b.create_block(); + let done = self.b.create_block(); + let fast = self.b.create_block(); + self.b.ins().brif(plain_old, fast, &[], slow, &[]); + self.b.switch_to_block(fast); + let t8 = self.b.ins().ireduce(types::I8, tag); + self.b.ins().store(FLAGS, t8, local, 0); + self.b.ins().store(FLAGS, byte, local, 1); + self.b.ins().store(FLAGS, word, local, 8); + self.b.ins().jump(done, &[]); + // Over a heap value: through the stack slot, released by + // the helper. + self.b.switch_to_block(slow); + self.materialize(item, slot); + self.store_from_slot(pc, i, slot); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + None => { + // The value into its slot (a copy taken), then moved in. + self.materialize(item, slot); + let src = self.slot_addr(slot); + let vt = self.tag_at(src); + // The PEP 709 cell restore stays on the full path. + let cell = self + .b + .ins() + .icmp_imm(IntCC::Equal, vt, i64::from(self.tags.cell)); + let out = self.exit_with(pc, &[Item::Mem(slot)], INTERP); + self.branch_out(cell, out); + let slow = self.b.create_block(); + let done = self.b.create_block(); + let fast = self.b.create_block(); + self.b.ins().brif(plain_old, fast, &[], slow, &[]); + self.b.switch_to_block(fast); + self.copy16(local, src); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(slow); + self.store_from_slot(pc, i, slot); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + } + true + } + + /// `STORE_FAST i` of the value at stack `slot` (just above the + /// virtual stack) over a heap value, through the helper. + fn store_from_slot(&mut self, pc: usize, i: u32, slot: usize) { + let src = self.slot_addr(slot); + let iv = self.b.ins().iconst(types::I64, i64::from(i)); + let r = self + .call(h_store as *const () as usize, &[self.st, src, iv], true) + .expect("returns"); + // Declined: the value is on the stack, as the core loop expects. + let out = self.exit_with(pc, &[Item::Mem(slot)], INTERP); + self.branch_out(r, out); + let saved = self.last_pc; + self.set_last(pc); + self.check_released(pc + 1); + self.last_pc = saved; + } + + fn binary(&mut self, pc: usize, arg: u32) -> bool { + let kind = arg as u8; + let is = |k: BinOpKind| kind == k as u8; + let bi = self.pop(); + let ai = self.pop(); + // A value on the stack (a call's result, an element) is as likely + // a heap value as a number: the helper takes every shape. So does + // a site the core loop has run on a natively served operand + // (`datetime`'s; see `crate::native_site`). + let native = crate::native_site(self.ext, pc); + if native + && matches!(ai, Item::Local(_) | Item::Const(_)) + && matches!(bi, Item::Local(_) | Item::Const(_)) + { + return self.binary_borrowed(pc, kind, ai, bi); + } + if matches!(ai, Item::Mem(_)) || matches!(bi, Item::Mem(_)) || native { + return self.binary_helper(pc, kind, ai, bi); + } + let (Some(a), Some(b)) = (self.operand(ai), self.operand(bi)) else { + // A string or other heap constant: the core loop's out-of-line + // shapes, through the helper. + return self.binary_helper(pc, kind, ai, bi); + }; + let slow = self.exit_with(pc, &[ai, bi], INTERP); + if matches!(a.kind, Kind::Bool | Kind::None) || matches!(b.kind, Kind::Bool | Kind::None) { + self.b.ins().jump(slow, &[]); + return false; + } + let int_kind = [ + BinOpKind::Add, + BinOpKind::Sub, + BinOpKind::Mult, + BinOpKind::BitAnd, + BinOpKind::BitOr, + BinOpKind::BitXor, + BinOpKind::RShift, + BinOpKind::LShift, + BinOpKind::FloorDiv, + BinOpKind::Mod, + ] + .into_iter() + .any(&is); + let float_kind = + is(BinOpKind::Add) || is(BinOpKind::Sub) || is(BinOpKind::Mult) || is(BinOpKind::Div); + let ia = self.is_kind(a, Kind::Int, self.tags.int); + let ib = self.is_kind(b, Kind::Int, self.tags.int); + let fa = self.is_kind(a, Kind::Float, self.tags.float); + let fb = self.is_kind(b, Kind::Float, self.tags.float); + // Both statically known: one path. + let static_int = matches!((ia, ib), (Ok(true), Ok(true))); + let static_num = matches!( + (ia, ib, fa, fb), + (Ok(x), Ok(y), Ok(p), Ok(q)) if (x || p) && (y || q) + ); + if static_int { + let Some(r) = self.int_op(kind, a.word, b.word, slow) else { + self.b.ins().jump(slow, &[]); + return false; + }; + self.push(Item::Int(r)); + self.set_last(pc); + return true; + } + if static_num { + if !float_kind { + self.b.ins().jump(slow, &[]); + return false; + } + let x = self.as_f64(a, fa); + let y = self.as_f64(b, fb); + let r = self.float_op(kind, x, y, slow); + self.push(Item::Float(r)); + self.set_last(pc); + return true; + } + if matches!((ia, ib), (Ok(false), _) | (_, Ok(false))) && !float_kind { + self.b.ins().jump(slow, &[]); + return false; + } + // At run time: two ints, or two numbers with a float. + let to_v = |s: &mut Self, r: Result| match r { + Ok(x) => s.b.ins().iconst(types::I8, i64::from(x)), + Err(v) => v, + }; + let (iav, ibv, fav, fbv) = ( + to_v(self, ia), + to_v(self, ib), + to_v(self, fa), + to_v(self, fb), + ); + let done = self.b.create_block(); + self.b.append_block_param(done, types::I64); + self.b.append_block_param(done, types::I64); + let both_int = self.b.ins().band(iav, ibv); + let int_b = self.b.create_block(); + let not_int = self.b.create_block(); + self.b.ins().brif(both_int, int_b, &[], not_int, &[]); + self.b.switch_to_block(int_b); + if int_kind { + match self.int_op(kind, a.word, b.word, slow) { + Some(r) => { + let t = self.b.ins().iconst(types::I64, i64::from(self.tags.int)); + self.b.ins().jump(done, &[t.into(), r.into()]); + } + None => { + self.b.ins().jump(slow, &[]); + } + } + } else { + self.b.ins().jump(slow, &[]); + } + self.b.switch_to_block(not_int); + if float_kind { + let na = self.b.ins().bor(iav, fav); + let nb = self.b.ins().bor(ibv, fbv); + let nums = self.b.ins().band(na, nb); + let bad = self.b.ins().bxor_imm(nums, 1); + self.branch_out(bad, slow); + let x = self.as_f64(a, Err(fav)); + let y = self.as_f64(b, Err(fbv)); + let r = self.float_op(kind, x, y, slow); + let bits = self.b.ins().bitcast(types::I64, MemFlags::new(), r); + let t = self.b.ins().iconst(types::I64, i64::from(self.tags.float)); + self.b.ins().jump(done, &[t.into(), bits.into()]); + } else { + self.b.ins().jump(slow, &[]); + } + self.b.switch_to_block(done); + let (t, w) = (self.b.block_params(done)[0], self.b.block_params(done)[1]); + let zero = self.b.ins().iconst(types::I64, 0); + self.push(Item::Dyn(t, w, zero)); + self.set_last(pc); + true + } + + /// `BINARY_OP` through [`h_binop`]: the operands onto the stack, and + /// the result in the lower one's slot. + fn binary_helper(&mut self, pc: usize, kind: u8, ai: Item, bi: Item) -> bool { + let slot = self.depth; + self.materialize(ai, slot); + self.materialize(bi, slot + 1); + let at = self.slot_addr(slot); + let k = self.b.ins().iconst(types::I32, i64::from(kind)); + let r = self + .call(h_binop as *const () as usize, &[at, k], true) + .expect("returns"); + let out = self.exit_with(pc, &[Item::Mem(slot), Item::Mem(slot + 1)], INTERP); + self.branch_out(r, out); + self.push(Item::Mem(slot)); + self.set_last(pc); + true + } + + /// The address of a local's or a constant's value (`None` for any + /// other item). + fn item_addr(&mut self, item: Item) -> Option { + match item { + Item::Local(i) => Some(self.local_addr(i)), + Item::Const(k) => Some(self.const_addr(k)), + _ => None, + } + } + + /// `BINARY_OP` of two locals or constants through [`h_binop_ref`], + /// which reads them in place (no references taken or released): the + /// result in the next free slot. + fn binary_borrowed(&mut self, pc: usize, kind: u8, ai: Item, bi: Item) -> bool { + let (Some(a), Some(b)) = (self.item_addr(ai), self.item_addr(bi)) else { + return self.binary_helper(pc, kind, ai, bi); + }; + let slot = self.depth; + let dst = self.slot_addr(slot); + let k = self.b.ins().iconst(types::I32, i64::from(kind)); + let r = self + .call(h_binop_ref as *const () as usize, &[dst, a, b, k], true) + .expect("returns"); + let out = self.exit_with(pc, &[ai, bi], INTERP); + self.branch_out(r, out); + self.push(Item::Mem(slot)); + self.set_last(pc); + true + } + + /// `LOAD_ATTR` of a local where no split-layout field was seen: a + /// natively served instance's field through [`h_field`], anything + /// else in the core loop. + fn native_field(&mut self, pc: usize, arg: u32) -> bool { + let item = self.pop(); + let (Item::Local(i), Some(name @ Object::Str(_))) = + (item, self.ext.name_objs.get(arg as usize)) + else { + self.push(item); + self.exit(INTERP, pc); + return false; + }; + let at = self.local_addr(i); + let slot = self.depth; + let dst = self.slot_addr(slot); + let name = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(name) as i64); + let r = self + .call(h_field as *const () as usize, &[dst, at, name], true) + .expect("returns"); + let out = self.exit_with(pc, &[item], INTERP); + self.branch_out(r, out); + self.push(Item::Mem(slot)); + self.set_last(pc); + true + } + + /// `COMPARE_OP` through [`h_compare`]: the operands onto the stack, + /// and the result in the lower one's slot. + fn compare_helper(&mut self, pc: usize, arg: u32, ai: Item, bi: Item) -> bool { + let slot = self.depth; + self.materialize(ai, slot); + self.materialize(bi, slot + 1); + let at = self.slot_addr(slot); + let k = self.b.ins().iconst(types::I32, i64::from(arg)); + let r = self + .call(h_compare as *const () as usize, &[at, k], true) + .expect("returns"); + let out = self.exit_with(pc, &[Item::Mem(slot), Item::Mem(slot + 1)], INTERP); + self.branch_out(r, out); + self.push(Item::Mem(slot)); + self.set_last(pc); + true + } + + /// A container instruction through [`h_container`] (the operands on + /// the stack; the depth after is the static one). + fn container(&mut self, pc: usize) -> bool { + let after = self.depths.get(pc + 1).copied().unwrap_or(-1); + if after < 0 { + self.exit(INTERP, pc); + return false; + } + self.flush(); + let ins = self.b.ins().iconst( + self.ptr, + std::ptr::from_ref(&self.code.instructions[pc]) as i64, + ); + let code = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(self.code) as i64); + let pcv = self.b.ins().iconst(types::I64, pc as i64); + let len = self.b.ins().iconst(types::I64, self.depth as i64); + let n = self + .call_typed( + h_container as *const () as usize, + &[self.st, ins, code, pcv, len], + Some(types::I64), + ) + .expect("returns"); + let declined = self.b.ins().icmp_imm(IntCC::Equal, n, -1); + let out = self.exit_with(pc, &[], INTERP); + self.branch_out(declined, out); + self.depth = after as usize; + self.set_last(pc); + self.check_released(pc + 1); + true + } + + /// `CONTAINS_OP` through [`h_contains`]: an unboxed `bool`. + fn contains(&mut self, pc: usize, arg: u32) -> bool { + let bi = self.pop(); + let ai = self.pop(); + let slot = self.depth; + self.materialize(ai, slot); + self.materialize(bi, slot + 1); + let at = self.slot_addr(slot); + let r = self + .call(h_contains as *const () as usize, &[at], true) + .expect("returns"); + let declined = self.b.ins().icmp_imm(IntCC::UnsignedGreaterThan, r, 1); + let out = self.exit_with(pc, &[Item::Mem(slot), Item::Mem(slot + 1)], INTERP); + self.branch_out(declined, out); + let found = self.b.ins().icmp_imm(IntCC::NotEqual, r, 0); + let found = if arg == 1 { + self.b.ins().bxor_imm(found, 1) + } else { + found + }; + self.push(Item::Bool(found)); + self.set_last(pc); + self.check_released(pc + 1); + true + } + + /// Two ints, by `kind`; a result that doesn't fit (or an operand the + /// core loop's arm declines) branches to `slow`. `None` for a kind + /// with no `int` arm. + fn int_op(&mut self, kind: u8, x: Value, y: Value, slow: Block) -> Option { + let is = |k: BinOpKind| kind == k as u8; + Some(if is(BinOpKind::Add) { + let r = self.b.ins().iadd(x, y); + let axr = self.b.ins().bxor(x, r); + let bxr = self.b.ins().bxor(y, r); + let and = self.b.ins().band(axr, bxr); + let ovf = self.b.ins().icmp_imm(IntCC::SignedLessThan, and, 0); + self.branch_out(ovf, slow); + r + } else if is(BinOpKind::Sub) { + let r = self.b.ins().isub(x, y); + let axb = self.b.ins().bxor(x, y); + let axr = self.b.ins().bxor(x, r); + let and = self.b.ins().band(axb, axr); + let ovf = self.b.ins().icmp_imm(IntCC::SignedLessThan, and, 0); + self.branch_out(ovf, slow); + r + } else if is(BinOpKind::Mult) { + let lo = self.b.ins().imul(x, y); + let hi = self.b.ins().smulhi(x, y); + let sign = self.b.ins().sshr_imm(lo, 63); + let ovf = self.b.ins().icmp(IntCC::NotEqual, hi, sign); + self.branch_out(ovf, slow); + lo + } else if is(BinOpKind::BitAnd) { + self.b.ins().band(x, y) + } else if is(BinOpKind::BitOr) { + self.b.ins().bor(x, y) + } else if is(BinOpKind::BitXor) { + self.b.ins().bxor(x, y) + } else if is(BinOpKind::RShift) { + let out = self + .b + .ins() + .icmp_imm(IntCC::UnsignedGreaterThanOrEqual, y, 64); + self.branch_out(out, slow); + self.b.ins().sshr(x, y) + } else if is(BinOpKind::LShift) { + let out = self + .b + .ins() + .icmp_imm(IntCC::UnsignedGreaterThanOrEqual, y, 63); + self.branch_out(out, slow); + let r = self.b.ins().ishl(x, y); + let back = self.b.ins().sshr(r, y); + let lost = self.b.ins().icmp(IntCC::NotEqual, back, x); + self.branch_out(lost, slow); + r + } else if is(BinOpKind::FloorDiv) || is(BinOpKind::Mod) { + let yneg = self.b.ins().icmp_imm(IntCC::SignedLessThanOrEqual, y, 0); + let xneg = self.b.ins().icmp_imm(IntCC::SignedLessThan, x, 0); + let bad = self.b.ins().bor(yneg, xneg); + self.branch_out(bad, slow); + if is(BinOpKind::FloorDiv) { + self.b.ins().udiv(x, y) + } else { + self.b.ins().urem(x, y) + } + } else { + return None; + }) + } + + /// Two floats, by `kind` (one of add, subtract, multiply, divide); a + /// zero divisor or a NaN result branches to `slow` (the core loop + /// raises, or tags the NaN). + fn float_op(&mut self, kind: u8, x: Value, y: Value, slow: Block) -> Value { + let is = |k: BinOpKind| kind == k as u8; + let r = if is(BinOpKind::Add) { + self.b.ins().fadd(x, y) + } else if is(BinOpKind::Sub) { + self.b.ins().fsub(x, y) + } else if is(BinOpKind::Mult) { + self.b.ins().fmul(x, y) + } else { + let zero = self.b.ins().f64const(0.0); + let z = self.b.ins().fcmp(FloatCC::Equal, y, zero); + self.branch_out(z, slow); + self.b.ins().fdiv(x, y) + }; + let nan = self.b.ins().fcmp(FloatCC::Unordered, r, r); + self.branch_out(nan, slow); + r + } + + fn compare(&mut self, pc: usize, arg: u32) -> bool { + let kind = (arg & !COMPARE_OP_TO_BOOL_FLAG) as u8; + let (icc, fcc) = match kind { + k if k == CompareKind::Lt as u8 => (IntCC::SignedLessThan, FloatCC::LessThan), + k if k == CompareKind::LtE as u8 => { + (IntCC::SignedLessThanOrEqual, FloatCC::LessThanOrEqual) + } + k if k == CompareKind::Eq as u8 => (IntCC::Equal, FloatCC::Equal), + k if k == CompareKind::NotEq as u8 => (IntCC::NotEqual, FloatCC::NotEqual), + k if k == CompareKind::Gt as u8 => (IntCC::SignedGreaterThan, FloatCC::GreaterThan), + k if k == CompareKind::GtE as u8 => { + (IntCC::SignedGreaterThanOrEqual, FloatCC::GreaterThanOrEqual) + } + _ => { + self.exit(INTERP, pc); + return false; + } + }; + let bi = self.pop(); + let ai = self.pop(); + // A site the core loop has run on natively served operands: through + // the helper (see `binary`). + if crate::native_site(self.ext, pc) { + return self.compare_helper(pc, arg, ai, bi); + } + let (Some(a), Some(b)) = (self.operand(ai), self.operand(bi)) else { + self.push(ai); + self.push(bi); + self.exit(INTERP, pc); + return false; + }; + let slow = self.exit_with(pc, &[ai, bi], INTERP); + if matches!(a.kind, Kind::Bool | Kind::None) || matches!(b.kind, Kind::Bool | Kind::None) { + self.b.ins().jump(slow, &[]); + return false; + } + let ia = self.is_kind(a, Kind::Int, self.tags.int); + let ib = self.is_kind(b, Kind::Int, self.tags.int); + let fa = self.is_kind(a, Kind::Float, self.tags.float); + let fb = self.is_kind(b, Kind::Float, self.tags.float); + let int_cmp = |s: &mut Self| s.b.ins().icmp(icc, a.word, b.word); + let float_cmp = |s: &mut Self| { + let x = s.b.ins().bitcast(types::F64, MemFlags::new(), a.word); + let y = s.b.ins().bitcast(types::F64, MemFlags::new(), b.word); + // An unordered pair takes the core loop's arm. + let uno = s.b.ins().fcmp(FloatCC::Unordered, x, y); + s.branch_out(uno, slow); + s.b.ins().fcmp(fcc, x, y) + }; + let r = match (ia, ib, fa, fb) { + (Ok(true), Ok(true), ..) => int_cmp(self), + (_, _, Ok(true), Ok(true)) => float_cmp(self), + _ => { + let to_v = |s: &mut Self, r: Result| match r { + Ok(x) => s.b.ins().iconst(types::I8, i64::from(x)), + Err(v) => v, + }; + let (iav, ibv, fav, fbv) = ( + to_v(self, ia), + to_v(self, ib), + to_v(self, fa), + to_v(self, fb), + ); + let done = self.b.create_block(); + self.b.append_block_param(done, types::I8); + let both_int = self.b.ins().band(iav, ibv); + let int_b = self.b.create_block(); + let not_int = self.b.create_block(); + self.b.ins().brif(both_int, int_b, &[], not_int, &[]); + self.b.switch_to_block(int_b); + let r = int_cmp(self); + self.b.ins().jump(done, &[r.into()]); + self.b.switch_to_block(not_int); + let both_float = self.b.ins().band(fav, fbv); + let bad = self.b.ins().bxor_imm(both_float, 1); + self.branch_out(bad, slow); + let r = float_cmp(self); + self.b.ins().jump(done, &[r.into()]); + self.b.switch_to_block(done); + self.b.block_params(done)[0] + } + }; + self.push(Item::Bool(r)); + self.set_last(pc); + true + } + + /// An item's tag, word and `bool` byte, for an identity test (`None` + /// for a stack value, whose release the test would owe). + fn identity(&mut self, item: Item) -> Option<(Value, Value, Value)> { + let c = |s: &mut Self, v: i64| s.b.ins().iconst(types::I64, v); + Some(match item { + Item::Mem(_) => return None, + Item::Local(i) => { + let at = self.local_addr(i); + let t = self.tag_at(at); + let w = self.b.ins().load(types::I64, FLAGS, at, 8); + let b = self.b.ins().uload8(types::I64, FLAGS, at, 1); + (t, w, b) + } + Item::Const(k) => { + let obj = &self.ext.objects[k as usize]; + let p = std::ptr::from_ref(obj).cast::(); + // SAFETY: a constant's tag, payload byte and word (the + // constants live as long as the code). + let (t, b, w) = unsafe { (*p, *p.add(1), p.add(8).cast::().read_unaligned()) }; + let (t, w, b) = (i64::from(t), w as i64, i64::from(b)); + (c(self, t), c(self, w), c(self, b)) + } + Item::Int(v) => { + let t = c(self, i64::from(self.tags.int)); + let z = c(self, 0); + (t, v, z) + } + Item::Float(v) => { + let t = c(self, i64::from(self.tags.float)); + let w = self.b.ins().bitcast(types::I64, MemFlags::new(), v); + let z = c(self, 0); + (t, w, z) + } + Item::Bool(v) => { + let t = c(self, i64::from(self.tags.boolean)); + let z = c(self, 0); + let b = self.b.ins().uextend(types::I64, v); + (t, z, b) + } + Item::Dyn(t, w, b) => (t, w, b), + }) + } + + /// `IS_OP` of two values the code holds no reference to: the + /// variants' identity rules (`Object::is_same`) in line, for the + /// scalars and the pointer-identity heap variants. + fn is_op(&mut self, pc: usize, arg: u32) -> bool { + let bi = self.pop(); + let ai = self.pop(); + let (Some((ta, wa, ba)), Some((tb, wb, bb))) = (self.identity(ai), self.identity(bi)) + else { + self.push(ai); + self.push(bi); + self.exit(INTERP, pc); + return false; + }; + let slow = self.exit_with(pc, &[ai, bi], INTERP); + let tags = self.tags; + let same_tag = self.b.ins().icmp(IntCC::Equal, ta, tb); + // Which rule the (shared) tag takes. + let one = self.b.ins().iconst(types::I64, 1); + let bit = self.b.ins().ishl(one, ta); + let words_mask = tags.by_pointer | (1i64 << tags.int) | (1i64 << tags.float); + let by_word = self.b.ins().band_imm(bit, words_mask); + let by_word = self.b.ins().icmp_imm(IntCC::NotEqual, by_word, 0); + let is_bool = self + .b + .ins() + .icmp_imm(IntCC::Equal, ta, i64::from(tags.boolean)); + let unit_mask = (1i64 << tags.none) | (1i64 << tags.unbound); + let unit = self.b.ins().band_imm(bit, unit_mask); + let unit = self.b.ins().icmp_imm(IntCC::NotEqual, unit, 0); + // A tag no rule here covers (a long, a complex, …) is the core + // loop's, when the tags match. + let known = self.b.ins().bor(by_word, is_bool); + let known = self.b.ins().bor(known, unit); + let unknown = self.b.ins().bxor_imm(known, 1); + let bad = self.b.ins().band(same_tag, unknown); + self.branch_out(bad, slow); + let weq = self.b.ins().icmp(IntCC::Equal, wa, wb); + let beq = self.b.ins().icmp(IntCC::Equal, ba, bb); + let r = self.b.ins().select(is_bool, beq, weq); + let r = self.b.ins().select(unit, same_tag, r); + let r = self.b.ins().band(r, same_tag); + let r = if arg == 1 { + self.b.ins().bxor_imm(r, 1) + } else { + r + }; + self.push(Item::Bool(r)); + self.set_last(pc); + true + } + + fn for_iter(&mut self, pc: usize, arg: u32) -> bool { + let before = self.last_pc; + self.flush(); + let it_slot = self.depth - 1; + let it = self.slot_addr(it_slot); + let out_slot = self.slot_addr(self.depth); + let r = self + .call( + h_for_iter as *const () as usize, + &[self.st, it, out_slot], + true, + ) + .expect("returns"); + let int_b = self.b.create_block(); + let other = self.b.create_block(); + self.b.ins().brif(r, other, &[], int_b, &[]); + // An int: unboxed. + self.b.switch_to_block(int_b); + let v = self.b.ins().load(types::I64, FLAGS, self.st, S_OUT); + let depth = self.depth; + self.push(Item::Int(v)); + self.set_last(pc); + self.goto(pc + 1); + self.depth = depth; + // Another value, on the stack; or the end; or the core loop's. + self.b.switch_to_block(other); + let pushed = self.b.ins().icmp_imm(IntCC::Equal, r, 1); + let pushed_b = self.b.create_block(); + let rest = self.b.create_block(); + self.b.ins().brif(pushed, pushed_b, &[], rest, &[]); + self.b.switch_to_block(pushed_b); + self.depth += 1; + self.set_last(pc); + self.goto(pc + 1); + self.depth = depth; + self.b.switch_to_block(rest); + let retired = self.b.ins().icmp_imm(IntCC::Equal, r, 2); + let ret_b = self.b.create_block(); + let decl = self.b.create_block(); + self.b.ins().brif(retired, ret_b, &[], decl, &[]); + self.b.switch_to_block(ret_b); + // Past the loop's `END_FOR` / `POP_ITER` pair. + let mut to = pc + 1 + arg as usize; + let op = |s: &Self, pc: usize| s.code.instructions.get(pc).map(|i| i.op); + if op(self, to) == Some(OpCode::EndFor) { + to += 1; + if matches!(op(self, to), Some(OpCode::PopIter | OpCode::PopTop)) { + to += 1; + } + } + self.depth -= 1; + self.set_last(pc); + self.goto(to); + self.depth = depth; + self.b.switch_to_block(decl); + self.last_pc = before; + self.exit(INTERP, pc); + false + } + + /// `x.attr` of an instance (a local's, or one on the stack): the + /// site's field shortcut in line. The value goes onto the stack (a + /// scalar copied, anything else with a reference taken); a stack + /// receiver is released after. Anything else, and a miss, is the core + /// loop's. + fn load_attr(&mut self, pc: usize, arg: u32) -> bool { + let (Some(l), Some(cache_slot)) = (self.layout, self.field_slots.get(pc)) else { + return self.native_field(pc, arg); + }; + let l = *l; + let item = self.pop(); + let at = match item { + Item::Local(i) => self.local_addr(i), + Item::Mem(s) => self.slot_addr(s), + _ => { + self.push(item); + self.exit(INTERP, pc); + return false; + } + }; + let exit = self.exit_with(pc, &[item], INTERP); + // A local natively served instance's field (`dt.hour`; see + // `stdlib::datetime_native::leaf_field`) through [`h_field`] before + // the core loop. + let name = self.ext.name_objs.get(arg as usize); + let native = match (item, name) { + (Item::Local(_), Some(Object::Str(_))) => Some(self.b.create_block()), + _ => None, + }; + let miss = native.unwrap_or(exit); + let otag = self.tag_at(at); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(self.tags.instance)); + self.branch_out(not_inst, miss); + let ptr = self.ptr; + let inst = self.b.ins().load(ptr, FLAGS, at, 8); + let cls = self.b.ins().load(ptr, FLAGS, inst, l.inst_class); + let ver = self + .b + .ins() + .load(types::I64, FLAGS, cls, l.type_attr_version); + let cache = self + .b + .ins() + .iconst(ptr, std::ptr::from_ref(cache_slot).cast::() as i64); + let want = self.b.ins().load(types::I64, FLAGS, cache, FIELD_VER); + let idx = self.b.ins().uload32(FLAGS, cache, FIELD_IDX); + let lazy = self.b.ins().load(ptr, FLAGS, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, FLAGS, flag, 0); + let borrow = self.b.ins().sload32(FLAGS, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, FLAGS, inst, l.inst_split_block); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.branch_out(bad, miss); + let keys = self.b.ins().load(ptr, FLAGS, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, FLAGS, cls, l.type_shared_keys); + let len = self.b.ins().uload32(FLAGS, block, l.split_len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let bad = self.b.ins().bor(foreign, absent); + self.branch_out(bad, miss); + let off = self.b.ins().ishl_imm(idx, 4); + let v = self.b.ins().iadd(block, off); + let v = self.b.ins().iadd_imm(v, i64::from(l.split_values)); + let slot = self.depth; + let dst = self.slot_addr(slot); + match item { + Item::Local(_) => { + // The receiver is the local's: the value just takes a copy. + self.copy_value(dst, v); + if let (Some(native), Some(name)) = (native, name) { + let done = self.b.create_block(); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(native); + let dst = self.slot_addr(slot); + let name = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(name) as i64); + let r = self + .call(h_field as *const () as usize, &[dst, at, name], true) + .expect("returns"); + self.b.ins().brif(r, exit, &[], done, &[]); + self.b.switch_to_block(done); + } + } + _ => { + // The receiver is the stack's: the value goes above it + // (copied) until the receiver is released, then down. + let above = self.slot_addr(slot + 1); + self.copy_value(above, v); + let r = self + .call(h_pop as *const () as usize, &[dst], true) + .expect("returns"); + let undo = self.b.create_block(); + let released = self.b.create_block(); + self.b.ins().brif(r, undo, &[], released, &[]); + // Declined: the copy goes, and the core loop takes the + // instruction from the start. + self.b.switch_to_block(undo); + // (A fresh copy has another owner: its release is a plain + // decrement.) + self.call(h_pop as *const () as usize, &[above], true); + self.b.ins().jump(miss, &[]); + self.b.switch_to_block(released); + self.copy16(dst, above); + } + } + self.push(Item::Mem(slot)); + self.set_last(pc); + if matches!(item, Item::Mem(_)) { + self.check_released(pc + 1); + } + true + } +} + +/// Where a field cache's version and position sit. +const FIELD_VER: i32 = std::mem::offset_of!((u64, u32), 0) as i32; +const FIELD_IDX: i32 = std::mem::offset_of!((u64, u32), 1) as i32; diff --git a/crates/weavepy-vm/src/gc_trace.rs b/crates/weavepy-vm/src/gc_trace.rs index 7dda0326..9522a49b 100644 --- a/crates/weavepy-vm/src/gc_trace.rs +++ b/crates/weavepy-vm/src/gc_trace.rs @@ -81,6 +81,7 @@ use std::sync::atomic::{ }; use crate::object::Object; +use crate::weak_object::WeakObject; use crate::weakref_registry::{id_of, ObjectId}; /// A set of object ids (addresses), hashed with the address mixer. @@ -131,92 +132,46 @@ pub trait Finalize { fn finalize(&self); } -/// Per-tracked-object metadata. Stored as a `Vec>` -/// inside each [`Generation`] — the `Arc` makes the per-handle -/// state cheaply shared across the candidate snapshot during a -/// collection. +/// A tracked object's entry: a non-owning handle on the object plus the +/// bookkeeping that outlives a collection. The collector never keeps an +/// object alive. An entry whose object died stays in its generation until +/// the next collection or prune of that generation drops it; its weak +/// handle keeps the allocation reserved until then, so the address the +/// index keys it by can't be reused by another object meanwhile. #[allow(missing_debug_implementations)] pub struct TrackedHandle { - /// Strong handle to the tracked object. Holding a strong - /// reference is fine because the GC's job is to *break* - /// cycles by clearing fields, not by dropping the Rc. - pub object: Object, - /// Identity, computed from `id_of(object)` at - /// `track`-time. Cached so the GC's mark phase doesn't - /// have to recompute on every visit. + /// The tracked object, held weakly. + pub object: WeakObject, + /// Identity, computed from `id_of(object)` at `track` time. pub id: ObjectId, - /// Working `gc_refs` field. Reset to a fresh value at the - /// start of every collection cycle. - pub gc_refs: AtomicI64, - /// Tri-color state. Reset to White at cycle start. + /// [`color::Frozen`] while `gc.freeze()` holds the entry, otherwise + /// [`color::White`]. Collections keep their marking state in the + /// candidates they build, not here. pub color: AtomicU8, - /// Generation index (0..N_GENERATIONS). Survivors are - /// promoted by incrementing this. + /// Generation index (0..N_GENERATIONS). Survivors are promoted by + /// incrementing this. pub generation: AtomicU8, /// Position of this handle within its owning `Vec` — /// `generations[generation].handles` normally, or the `frozen` /// list when `color == Frozen`. Maintained by every site that /// pushes, drains, or rebuilds those vectors so that - /// [`GcState::untrack_id`] can `swap_remove` in O(1) instead of - /// scanning every generation (which made drop-heavy, - /// large-heap workloads quadratic — RFC 0039 WS4). + /// [`GcState::untrack_id`] can `swap_remove` in O(1). pub slot: CachedSlot, /// Has this object's `__del__` already *run* to completion? CPython /// guarantees a finaliser runs at most once. pub finalized: AtomicBool, /// Has this object's `__del__` been *queued* by a collection but not yet - /// run? While set, the object is kept tracked and excluded from the - /// `collected` count: its finalizer (drained after `gc.collect()` returns) - /// may resurrect it, and CPython only counts objects that are actually - /// reclaimed. Cleared once the finalizer completes (`finalized` is set). + /// run? While set, the object is excluded from the `collected` count: + /// its finalizer (drained after `gc.collect()` returns) may resurrect + /// it, and CPython only counts objects that are actually reclaimed. pub finalize_queued: AtomicBool, - /// Cached count of callback-bearing weakref clones the registry holds for - /// this object — refreshed by [`GcState::note_weakref_finalizable`]. The - /// prompt-finalization scan uses it as a fast-path liveness filter: an - /// object whose `strong_count` exceeds `1 (our handle) + weak_clones` - /// definitely still has a program reference, so the scan can skip it - /// without the (per-id) registry lookup that computes the exact clone - /// count. The cache is only ever an *upper bound* on the live clone count - /// (weakrefs can clear without notifying us), and an over-estimate only - /// makes the filter admit *more* objects to the precise check — never - /// fewer — so it can never cause a dead object to be missed. - pub weak_clones: AtomicUsize, - /// RFC 0061 (WS1b): set (never cleared) when this handle leaves the - /// GC index (`untrack_id`, the collection rebuild's White purge). The - /// prompt-reap suspect probe reads it instead of a per-entry - /// `is_tracked` registry lookup — that lookup (`GcState::handle_for`) - /// was the hottest non-dispatch symbol on drop-heavy profiles. A - /// re-tracked object gets a *fresh* handle, so a set flag - /// definitively means "this handle is dead". - pub untracked: AtomicBool, - /// RFC 0077 (WS2): set while this finalizable-index entry is *cold*: - /// its strong count sat more than [`FIN_COLD_MARGIN`] above the - /// dead line at the last probe, so it can't die at a single drop - /// safe point without a cascade that frees many holders at once. - /// Cold entries are re-probed every [`FIN_COLD_STRIDE`]-th scan - /// instead of every one; the ABC machinery alone keeps dozens of - /// weakref-callback-enrolled classes live for the process lifetime, - /// and probing them at every reference-dropping opcode was a - /// first-order interpreter cost. Only meaningful for handles in the - /// finalizable index. - pub fin_cold: AtomicBool, - /// Position in `GcState::finalizable_hot` while hot (RFC 0077 WS2); - /// only valid under that vector's lock, fixed up on swap-remove - /// exactly like [`Self::slot`]. - pub fin_hot_slot: CachedSlot, - /// Strong count seen at the previous hot probe and the number of - /// consecutive probes it has held (RFC 0077 WS2); a count unchanged - /// for [`FIN_STABLE_PROBES`] probes demotes the entry to cold. - pub fin_last_sc: AtomicUsize, - pub fin_stable: AtomicU8, } /// A compact vector-position hint, with separate absent and uncached states. /// -/// Positions too large to cache use the existing pointer-search fallback. -/// They must not look absent: a hot finalizer with an uncached position still -/// belongs to the hot set. Neither decoded sentinel can index a live -/// `Vec>`, whose allocation is bounded by `isize::MAX`. +/// Positions too large to cache use the pointer-search fallback. Neither +/// decoded sentinel can index a live `Vec>`, whose +/// allocation is bounded by `isize::MAX`. #[derive(Debug)] pub struct CachedSlot(AtomicU32); @@ -261,31 +216,6 @@ impl CachedSlot { } } -/// RFC 0077 (WS2): a finalizable entry whose strong count exceeds the -/// dead line (`1 + weak clones`) by more than this many references -/// goes cold. Sixteen references dying between two consecutive drop -/// safe points takes a cascade, and the cold stride bounds even that -/// case's latency. -const FIN_COLD_MARGIN: usize = 16; -/// Cold finalizable entries are re-probed on every N-th scan. -const FIN_COLD_STRIDE: u64 = 32; -/// A hot entry whose strong count is unchanged across this many -/// consecutive probes goes cold regardless of the margin. The margin -/// alone left deltablue's one finalizable (a callback-weakref'd class, -/// three references above its dead line for the whole run) hot for -/// 2.6M probes; a fixed set of holders is the signature of a long-lived -/// object, and the cold stride still finds its eventual death within -/// `FIN_COLD_STRIDE` drop safe points — the direct paths -/// (`prompt_reap_dropped` and the cascade) catch the common shapes -/// sooner regardless. -const FIN_STABLE_PROBES: u8 = 64; - -/// `WEAVEPY_NO_FIN_COLD=1` pins every finalizable entry hot (bisection). -fn fin_no_cold() -> bool { - static NO_FIN_COLD: std::sync::OnceLock = std::sync::OnceLock::new(); - *NO_FIN_COLD.get_or_init(|| std::env::var_os("WEAVEPY_NO_FIN_COLD").is_some()) -} - #[allow(non_upper_case_globals)] pub mod color { pub const White: u8 = 0; @@ -295,24 +225,19 @@ pub mod color { } impl TrackedHandle { - pub fn new(object: Object, generation: usize) -> Self { + /// A generation-`generation` entry for `object`, or `None` for a value + /// with no heap allocation of its own. + pub fn new(object: &Object, generation: usize) -> Option { assert!(generation < N_GENERATIONS, "invalid collector generation"); - Self { - id: id_of(&object), - object, - gc_refs: AtomicI64::new(0), + Some(Self { + id: id_of(object), + object: WeakObject::new(object)?, color: AtomicU8::new(color::White), generation: AtomicU8::new(generation as u8), slot: CachedSlot::new(0), finalized: AtomicBool::new(false), finalize_queued: AtomicBool::new(false), - weak_clones: AtomicUsize::new(0), - untracked: AtomicBool::new(false), - fin_cold: AtomicBool::new(false), - fin_hot_slot: CachedSlot::new(usize::MAX), - fin_last_sc: AtomicUsize::new(0), - fin_stable: AtomicU8::new(0), - } + }) } } @@ -366,46 +291,75 @@ pub struct GcStats { pub uncollectable: u64, } +/// One object under examination by a collection: a strong reference for +/// the collection's duration, its entry (absent for a temporary candidate +/// discovered through an untracked container), and the marking state. +struct Cand { + obj: Object, + id: ObjectId, + entry: Option>, + gc_refs: std::cell::Cell, + color: std::cell::Cell, +} + +impl Cand { + fn new(obj: Object, id: ObjectId, entry: Option>) -> Self { + Self { + obj, + id, + entry, + gc_refs: std::cell::Cell::new(0), + color: std::cell::Cell::new(color::White), + } + } + + fn is_white(&self) -> bool { + self.color.get() == color::White + } + + /// Whether the object still owes a `__del__` (or a generator close). + fn pending_finalizer(&self) -> bool { + has_finalizer(&self.obj) + && !self + .entry + .as_ref() + .is_some_and(|e| e.finalized.load(Ordering::Acquire)) + } +} + +/// How many entries of the older generations each young collection or +/// prune examines for dead objects, so a dead old object's allocation is +/// released within a bounded number of young collections rather than at +/// the next full one. +const OLD_PRUNE_BUDGET: usize = 512; + /// Public state of the cycle GC. /// /// A single instance lives in a process-global `LazyLock` (see /// [`with_state`]) and is shared by every OS thread, mirroring /// CPython's one-collector-per-interpreter model. This is required -/// for correctness: post-RFC-0025 the heap is `Arc`-rooted and a -/// cycle's links can be allocated on different threads, so only a -/// shared tracked-set can ever observe and break such a cycle. All -/// fields are `Sync` (interior `GilCell`s + atomics), so concurrent -/// access is memory-safe; the GIL additionally serializes mutators. +/// for correctness: the heap is `Arc`-rooted and a cycle's links can +/// be allocated on different threads, so only a shared tracked-set can +/// ever observe and break such a cycle. All fields are `Sync` (interior +/// `GilCell`s + atomics), so concurrent access is memory-safe; the GIL +/// additionally serializes mutators. +/// +/// The collector owns nothing: its entries are weak, so objects die by +/// reference count exactly when CPython's would, and collections only +/// have to find and break the cycles. #[allow(missing_debug_implementations)] pub struct GcState { generations: RefCell<[Generation; N_GENERATIONS]>, - /// RFC 0065 (WS4): insert-only miss-filter over `index`, maintained - /// at [`Self::track_now`] and consulted by the usually-miss - /// `is_tracked`/`handle_for` probes on the drop paths. Rebuilt from - /// `index` once most of its bits name objects long gone. - /// - /// One per state, not one per process: a rebuild takes *this* - /// state's index as the truth, so a shared filter let any state's - /// full collection erase every other state's bits, and their - /// `handle_for` then answered `None` for live objects. Production - /// runs one state, which hid it; the test binary runs many. + /// Insert-only miss-filter over `index`, maintained at + /// [`Self::track_now`] and consulted by the usually-miss `is_tracked` + /// probes. Rebuilt from `index` once most of its bits name objects + /// long gone. tracked_filter: crate::hot_filter::RebuildableBloom, - /// Id → handle index over every tracked object (all generations - /// plus the frozen set). Keeps `track` dedupe, `find_handle`, and - /// `is_tracked` O(1) — the linear scans they replace made - /// allocation-heavy workloads quadratic once the tracked - /// population grew past a few thousand. Keyed by object *address*, - /// so the internal fast hasher applies (consulted on every object - /// drop via the prompt reaper — SipHash here was a top-ten CPU - /// consumer under pandas). + /// Id → handle index over every tracked object (all generations plus + /// the frozen set): keeps `track` dedupe and `is_tracked` O(1). index: RefCell, - /// Re-entrancy guard: a collection can indirectly allocate (e.g. - /// queued finalizers running Python at the next safe point may - /// re-enter `track`), and a nested collection would see torn - /// generation lists. An `AtomicBool` (rather than a `Cell`) so the - /// whole `GcState` is `Sync` and can live in a process-global - /// `LazyLock` — the cycle collector is shared across every OS - /// thread, matching the `Arc`-rooted shared heap (RFC 0039 WS4). + /// Re-entrancy guard: a nested collection would see torn generation + /// lists. Atomic so the whole `GcState` is `Sync`. collecting: AtomicBool, /// Per-generation thresholds. Gen 0's threshold is /// "allocations since last gen 0 collection"; gens 1 and 2 @@ -417,25 +371,29 @@ pub struct GcState { /// generation. counts: RefCell<[usize; N_GENERATIONS]>, /// Lock-free mirror of `counts[0]` and `thresholds[0]`, packed as - /// `count << 32 | threshold`. `auto_collect_due` is polled by the - /// burst loop's allocation opcodes, where two `RefCell` borrows and - /// their guards cost more than the test they guard; both halves are - /// small enough that a `u64` holds them without loss on the - /// allocation counts any real program reaches. + /// `count << 32 | threshold`, for the allocation sites' due check. gen0_gauge: AtomicU64, /// Containers born holding only atomic values (see /// [`container_can_cycle`]). They cannot take part in a cycle while /// that holds, so they are not registered with the collector — only - /// weakly remembered here, so that a collection can re-examine them - /// and promote any that has since acquired a non-atomic element. - /// A weak reference never keeps its container alive, so a deferred - /// container still dies by refcount exactly when CPython's would. + /// remembered here, so that a collection can re-examine them and + /// promote any that has since acquired a non-atomic element. deferred: RefCell>, /// Length at which [`GcState::sweep_deferred`] compacts `deferred`. - /// Re-armed to twice the surviving population after each sweep, so - /// the O(live) scan stays amortized O(1) per deferred birth even for - /// a program holding millions of scalar lists. deferred_limit: AtomicUsize, + /// Instances born since the last collection, held weakly: most die + /// young, and registering one with the collector (an index entry and + /// a handle) costs far more than its life. Their births count toward + /// the gen-0 threshold as eager registration's would, and every + /// collection, reflective API and finalization pass first hands the + /// collector the ones still alive ([`GcState::flush_young`]), so it + /// sees exactly the population eager tracking would have shown it. + young: RefCell>>, + /// Functions born since the last collection, held weakly as + /// [`Self::young`] holds instances (a `def` or `lambda` run in a loop + /// makes one per pass). A flush also hands the collector each live + /// one's globals dict, which [`track`] would have tracked at birth. + young_fns: RefCell>>, /// Frozen handles. `gc.freeze()` moves all tracked objects /// here; they are skipped by future collections until /// `gc.unfreeze()` runs. @@ -454,55 +412,15 @@ pub struct GcState { /// Bumped on every change to the tracked-object set so /// callers can know when to invalidate caches. pub tracked_version: AtomicUsize, - /// Total tracked-object population (live count). Useful - /// for `gc.get_count` and for the threshold check. + /// Total tracked-object population (entries, live or not yet pruned). pub tracked_count: AtomicUsize, /// Ids whose `__del__` has been run (or queued) by a finalizing /// collection or teardown. Persists past the point where the handle /// leaves the tracked set so `gc.is_finalized()` still answers `True` /// for an object its finalizer resurrected (PEP 442 / `test_is_finalized`). finalized_ids: RefCell>, - /// Dedicated index over just the *finalizable* tracked objects — - /// instances whose class defines `__del__` and unfinished - /// generator-family objects. CPython runs `__del__` the instant an - /// object's refcount reaches zero; our tracing handle pins it until a - /// collection, so [`Self::reap_dead_finalizable`] emulates the prompt - /// path by scanning *this* small set (not the whole tracked - /// population) at the interpreter's reference-drop safe points. Keyed - /// by id like `index`; an object is in both while finalizable. - finalizable: RefCell>>, - /// Live population of [`Self::finalizable`]. A relaxed load of this - /// atomic is the gate the interpreter checks before every prompt- - /// finalization sweep: when it is zero (the overwhelmingly common - /// case — most code never defines `__del__`) the sweep is skipped - /// entirely, so the feature costs one atomic load per safe point. - finalizable_count: AtomicUsize, - /// Rotating scan position for [`Self::reap_dead_finalizable_locked`] - /// when the finalizable index outgrows its per-safe-point scan budget - /// (70k callback-weakrefs from a `WeakKeyDictionary` stress test must - /// not turn every reference-dropping opcode into a full index walk — - /// test_weakref's threaded-copy tests went quadratic). Stores the id - /// the next bounded scan resumes from. - fin_scan_cursor: std::sync::atomic::AtomicU64, - /// Safe-point call counter paired with the cursor: when the index is - /// over budget, only every [`FIN_SCAN_STRIDE`]-th safe point pays for - /// a window scan, keeping the steady-state per-opcode cost a counter - /// bump instead of 256 atomic strong-count loads. - fin_scan_tick: std::sync::atomic::AtomicU64, - /// RFC 0077 (WS2): the *hot* (not `fin_cold`) subset of - /// [`Self::finalizable`], so the per-drop scan walks only entries - /// that could plausibly die this safe point. Membership is mirrored - /// by each handle's `fin_cold` flag and `fin_hot_slot` index. - finalizable_hot: RefCell>>, - /// `finalizable_hot.len()`, published for the lock-free gate: when - /// zero, a scan that isn't on the cold stride returns without - /// borrowing anything — the steady state of a program whose - /// finalizables are all long-lived classes. - fin_hot_count: AtomicUsize, - /// Rotating index into `finalizable_hot` for over-budget hot scans. - fin_hot_cursor: AtomicUsize, - /// Scan counter for the cold stride (RFC 0077 WS2). - fin_cold_tick: std::sync::atomic::AtomicU64, + /// Where the next prune of the older generations resumes. + old_prune_cursor: AtomicUsize, } impl Default for GcState { @@ -511,51 +429,6 @@ impl Default for GcState { } } -impl Drop for GcState { - fn drop(&mut self) { - // Thread teardown: the tracked set can hold long generator / - // container chains whose recursive field-drops overflow the - // native stack (each `Arc` link is one `drop_in_place` frame). - // Clear every tracked object's container fields *iteratively* - // first so the chains are already severed when the handle - // vectors drop. Safe at this point: the thread is exiting, no - // Python code will observe the cleared objects. - let mut handles: Vec> = Vec::new(); - if let Ok(gens) = self.generations.try_borrow() { - for g in gens.iter() { - handles.extend(g.handles.iter().cloned()); - } - } - if let Ok(frozen) = self.frozen.try_borrow() { - handles.extend(frozen.iter().cloned()); - } - // Only clear objects whose sole remaining strong reference is the - // registry handle itself. Anything with extra references escaped - // into shared state that outlives this thread — e.g. Flag - // pseudo-members a worker published into the enum class's - // `_value2member_map_` — and other threads *will* observe it, so - // wiping its fields would corrupt live objects. Iterate to a - // fixpoint: each cleared object releases its referents, which can - // drop a chained object's count to 1 and make it clearable on the - // next pass — severing long chains without recursive drops. - loop { - let mut progress = false; - handles.retain(|h| { - if strong_count_for(&h.object) <= 1 { - clear_object_fields(&h.object); - progress = true; - false - } else { - true - } - }); - if !progress { - break; - } - } - } -} - impl GcState { pub fn new() -> Self { Self { @@ -568,6 +441,8 @@ impl GcState { gen0_gauge: AtomicU64::new(DEFAULT_THRESHOLDS[0] as u64), deferred: RefCell::new(Vec::new()), deferred_limit: AtomicUsize::new(DEFERRED_FLOOR), + young: RefCell::new(Vec::new()), + young_fns: RefCell::new(Vec::new()), frozen: RefCell::new(Vec::new()), garbage: RefCell::new(Vec::new()), callbacks: RefCell::new(Vec::new()), @@ -577,117 +452,19 @@ impl GcState { tracked_version: AtomicUsize::new(0), tracked_count: AtomicUsize::new(0), finalized_ids: RefCell::new(crate::fasthash::FxHashSet::default()), - finalizable: RefCell::new(std::collections::BTreeMap::new()), - finalizable_count: AtomicUsize::new(0), - fin_scan_cursor: std::sync::atomic::AtomicU64::new(0), - fin_scan_tick: std::sync::atomic::AtomicU64::new(0), - finalizable_hot: RefCell::new(Vec::new()), - fin_hot_count: AtomicUsize::new(0), - fin_hot_cursor: AtomicUsize::new(0), - fin_cold_tick: std::sync::atomic::AtomicU64::new(0), + old_prune_cursor: AtomicUsize::new(0), } } - /// Insert `h` into the finalizable index (RFC 0077 WS2: one place - /// for the population and hot-set bookkeeping). New entries start - /// hot; the next scan grades them. Returns whether it was new. - fn fin_insert(&self, id: ObjectId, h: HandleRc) -> bool { - let mut fin = self.finalizable.borrow_mut(); - if fin.contains_key(&id) { - return false; - } - fin.insert(id, h.clone()); - drop(fin); - self.fin_make_hot(&h); - if self.finalizable_count.fetch_add(1, Ordering::AcqRel) == 0 { - // RFC 0065 (WS1): 0→1 population transition — kick - // dispatch loops off the quiet path. - crate::hot_gates::bump_loop_gen(); - } - true - } - - /// Remove `id` from the finalizable index, if present, keeping the - /// population and hot set exact. - fn fin_remove(&self, id: ObjectId) -> bool { - // The usual case: nothing finalizable is tracked at all. (Inserts - // count after they land, and the same thread removes an id it - // inserted.) - if self.finalizable_count.load(Ordering::Acquire) == 0 { - return false; - } - let removed = self.finalizable.borrow_mut().remove(&id); - let Some(h) = removed else { - return false; - }; - self.fin_remove_bookkeeping(&h); - true - } - - /// The hot-set and population half of [`Self::fin_remove`], for the - /// one caller that already holds the finalizable lock. - fn fin_remove_bookkeeping(&self, h: &HandleRc) { - self.fin_make_cold(h); - if self.finalizable_count.fetch_sub(1, Ordering::AcqRel) == 1 { - // RFC 0065 (WS1): population reached zero — dispatch loops - // may re-enter the quiet path. - crate::hot_gates::bump_loop_gen(); - } - } - - /// Move `h` into the hot set (no-op if already hot). - fn fin_make_hot(&self, h: &HandleRc) { - let mut hot = self.finalizable_hot.borrow_mut(); - if !h.fin_cold.swap(false, Ordering::AcqRel) - && h.fin_hot_slot.load(Ordering::Relaxed) != usize::MAX - { - return; - } - h.fin_hot_slot.store(hot.len(), Ordering::Relaxed); - h.fin_stable.store(0, Ordering::Relaxed); - hot.push(h.clone()); - self.fin_hot_count.store(hot.len(), Ordering::Release); - } - - /// Move `h` out of the hot set (no-op if already cold). O(1) via - /// the handle's cached slot, with the swap-remove fixup. - fn fin_make_cold(&self, h: &HandleRc) { - let mut hot = self.finalizable_hot.borrow_mut(); - h.fin_cold.store(true, Ordering::Release); - let slot = h.fin_hot_slot.swap(usize::MAX, Ordering::AcqRel); - if slot == usize::MAX { - return; - } - if slot < hot.len() && HandleRc::ptr_eq(&hot[slot], h) { - hot.swap_remove(slot); - if let Some(moved) = hot.get(slot) { - moved.fin_hot_slot.store(slot, Ordering::Relaxed); - } - } else if let Some(pos) = hot.iter().position(|x| HandleRc::ptr_eq(x, h)) { - hot.swap_remove(pos); - if let Some(moved) = hot.get(pos) { - moved.fin_hot_slot.store(pos, Ordering::Relaxed); - } - } - self.fin_hot_count.store(hot.len(), Ordering::Release); - } - /// Reinitialise the collector's locks in a `fork(2)` child, preserving the - /// inherited tracked-object state. The collector is process-global and its - /// `GilCell` fields are normally serialised by the GIL, but an `Object` - /// whose last `Arc` is released on a peer thread can drop — and run through - /// the collector's prompt-reap bookkeeping — without that thread holding - /// the GIL. If such a peer vanishes mid-`borrow` in the fork, the inherited + /// inherited tracked-object state. An `Object` whose last `Arc` is + /// released on a peer thread can drop without that thread holding the + /// GIL; if such a peer vanishes mid-`borrow` in the fork, the inherited /// `parking_lot` lock would wedge the child's very first allocation - /// (`test_threading.test_reinit_tls_after_fork` forks from 16 threads; the - /// child deadlocks in `threading._after_fork`'s `set(_enumerate())`). - /// Rebuild every field's lock in place and clear the re-entrancy guard — - /// CPython's `PyOS_AfterFork_Child` reinitialises the runtime's locks for - /// the same reason. - /// - /// Takes a raw `*mut Self` so each field's lock rebuild is driven from a - /// laundered raw pointer rather than a `&self` cast (see - /// [`crate::sync::GilCell::reinit_lock_after_fork`]). + /// (`test_threading.test_reinit_tls_after_fork`). Rebuild every field's + /// lock in place and clear the re-entrancy guard — CPython's + /// `PyOS_AfterFork_Child` reinitialises the runtime's locks for the same + /// reason. /// /// # Safety /// @@ -701,13 +478,14 @@ impl GcState { RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).index)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).thresholds)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).counts)); + RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).deferred)); + RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).young)); + RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).young_fns)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).frozen)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).garbage)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).callbacks)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).stats)); RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).finalized_ids)); - RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).finalizable)); - RefCell::reinit_lock_after_fork(std::ptr::addr_of_mut!((*this).finalizable_hot)); // A peer may have vanished mid-collection with this set. (*this).collecting.store(false, Ordering::Release); } @@ -741,29 +519,145 @@ impl GcState { /// Track `obj` for cycle detection. Idempotent — if `obj` /// is already tracked, this is a no-op. - pub fn track(&self, obj: Object) { + pub fn track(&self, obj: &Object) { // A container holding only atomic values cannot anchor a cycle - // yet, so it stays off the GC's books (and out of the prompt-reap - // cascade that every tracked death pays) until it holds something - // that could close one. `defer_container` keeps a weak reference - // so the next collection can promote it first. - if !container_can_cycle(&obj) && self.defer_container(&obj) { + // yet, so it stays off the GC's books until it holds something + // that could close one. `defer_container` remembers it so the + // next collection can promote it first. + if !container_can_cycle(obj) && self.defer_container(obj) { return; } + if let Object::Instance(inst) = obj { + if self.nurse(&self.young, inst) { + return; + } + } self.track_now(obj); } + /// [`Self::track`] for a function: young (see [`Self::young_fns`]), or + /// registered at once together with its globals dict. + fn track_function(&self, obj: &Object, f: &crate::Rc) { + if self.nurse(&self.young_fns, f) { + return; + } + self.track(&Object::Dict(f.globals.clone())); + self.track_now(obj); + } + + /// Register a newborn instance (or function) in its young set (see + /// [`Self::young`]). `false` leaves it to eager registration. + fn nurse( + &self, + set: &RefCell>>, + obj: &crate::Rc, + ) -> bool { + let over = { + // SAFETY: nothing below runs code while the set is borrowed + // (a dead predecessor's release only frees memory). + let Some(young) = (unsafe { set.peek_mut() }) else { + // A flush is walking the set, or cells are shared. + return false; + }; + // Churn (a loop that builds an object and drops the previous + // one) leaves its dead predecessors at the tail: reclaim them + // here, so the set neither grows nor parks dead allocations. + let n = young.len(); + for i in (n.saturating_sub(2)..n).rev() { + if young[i].strong_count() == 0 { + young.swap_remove(i); + } + } + young.push(crate::Rc::downgrade(obj)); + young.len() >= YOUNG_CAP + }; + // As in `track_now`: an id recycled from a finalized object starts + // un-finalized (a finalizer that runs from here on is recorded). + // SAFETY: a read that runs no code. + if unsafe { self.finalized_ids.peek() }.is_none_or(|f| !f.is_empty()) { + let id = crate::Rc::as_ptr(obj) as usize as ObjectId; + self.finalized_ids.borrow_mut().remove(&id); + } + self.note_gen0_alloc(); + if over { + self.flush_young(); + } + true + } + + /// Register the young instances and functions still alive with the + /// collector. + fn flush_young(&self) { + if let Ok(mut fns) = self.young_fns.try_borrow_mut() { + if !fns.is_empty() { + let fns = std::mem::take(&mut *fns); + for w in fns { + if let Some(f) = w.upgrade() { + self.track(&Object::Dict(f.globals.clone())); + self.register(&Object::Function(f), false); + } + } + } + } + let young = match self.young.try_borrow_mut() { + Ok(mut young) if !young.is_empty() => std::mem::take(&mut *young), + _ => return, + }; + for w in young { + if let Some(inst) = w.upgrade() { + self.register(&Object::Instance(inst), false); + } + } + } + + /// The young instances and functions still alive (the dead ones are + /// dropped). + fn young_live(&self) -> usize { + let fns = self.young_fns.try_borrow_mut().map_or(0, |mut fns| { + fns.retain(|w| w.strong_count() > 0); + fns.len() + }); + let Ok(mut young) = self.young.try_borrow_mut() else { + return fns; + }; + young.retain(|w| w.strong_count() > 0); + young.len() + fns + } + + /// Whether the instance or function `id` is in a young set. + fn is_young(&self, id: ObjectId) -> bool { + fn holds(set: &RefCell>>, id: ObjectId) -> bool { + set.try_borrow().is_ok_and(|young| { + young + .iter() + .any(|w| w.as_ptr() as usize as ObjectId == id && w.strong_count() > 0) + }) + } + holds(&self.young, id) || holds(&self.young_fns, id) + } + + /// [`Self::track`] for a container its builder knows holds only + /// atomic values (a `str.split` result): deferred whatever its size, + /// without the element scan, which `track` caps at `SCAN_CAP` + /// elements and so registers every longer list eagerly. + pub fn track_inert(&self, obj: &Object) { + debug_assert!(match obj { + Object::List(l) => l.borrow().iter().all(element_is_inert), + _ => false, + }); + if !self.defer_container(obj) { + self.track_now(obj); + } + } + /// Remember `obj` weakly instead of tracking it. Returns `false` when - /// the kind has no weak form (the caller tracks it as before). + /// the kind has no deferred form (the caller tracks it as before). /// /// A deferred birth does not advance the gen-0 allocation counter. /// That counter paces automatic collections against the rate cyclic /// garbage can appear, and nothing deferred here can be part of a - /// cycle. Counting the births and repaying them at the next sweep - /// instead made the counter sawtooth: a `[i, i]` loop under the JIT, - /// whose drops the native driver releases in batches, went from 1 - /// automatic collection to 59. A population that genuinely - /// accumulates is paced by `DEFERRED_CAP` instead. + /// cycle. A population that genuinely accumulates is paced by + /// `DEFERRED_CAP` instead. fn defer_container(&self, obj: &Object) -> bool { let Some(weak) = DeferredContainer::new(obj) else { return false; @@ -806,6 +700,7 @@ impl GcState { /// enough for the collector to see every cycle it would have seen /// with eager tracking. fn sweep_deferred(&self, promote_all: bool) { + self.flush_young(); let mut promote: Vec = Vec::new(); { let Ok(mut deferred) = self.deferred.try_borrow_mut() else { @@ -833,7 +728,7 @@ impl GcState { ); } for obj in promote { - self.track_now(obj); + self.track_now(&obj); } } @@ -846,109 +741,95 @@ impl GcState { /// [`Self::track`] without the deferral filter: register `obj` with the /// collector unconditionally. For objects the caller has already decided /// must be tracked (a promoted deferral, `gc.is_tracked`, `gc.get_objects`). - pub fn track_now(&self, obj: Object) { - let new_id = id_of(&obj); + pub fn track_now(&self, obj: &Object) { + if self.register(obj, true) { + self.note_gen0_alloc(); + } + } + + /// Enter `obj` in the index and generation 0 (`false`: it was already + /// there, or has no heap allocation of its own). A `fresh` object's id + /// may be a finalized one's, recycled; a young instance's own + /// finalizer may already have run, and its entry says so. + fn register(&self, obj: &Object, fresh: bool) -> bool { + let new_id = id_of(obj); { let mut index = self.index.borrow_mut(); - // One probe decides the dedupe and the insert (this runs for - // every function, list and dict born). + // One probe decides the dedupe and the insert. An entry whose + // object died can't sit at this address: its weak handle keeps + // the allocation reserved until the entry is pruned. let entry = match index.entry(new_id) { - std::collections::hash_map::Entry::Occupied(_) => return, + std::collections::hash_map::Entry::Occupied(_) => return false, std::collections::hash_map::Entry::Vacant(e) => e, }; - // RFC 0065 (WS4): publish to the miss-filter *before* the - // insert becomes observable (we hold the index borrow, so - // no prober can race past a fresh registration). + let Some(handle) = TrackedHandle::new(obj, 0) else { + return false; + }; + let handle = HandleRc::new(handle); + // Publish to the miss-filter *before* the insert becomes + // observable (we hold the index borrow). self.tracked_filter.insert(new_id); - let handle = HandleRc::new(TrackedHandle::new(obj, 0)); entry.insert(handle.clone()); - // Enroll finalizable objects in the dedicated prompt-finalization - // index so the per-safe-point sweep scans only them, not the whole - // tracked population. - if has_finalizer(&handle.object) { - // Seed the clone-count cache from the registry: weakrefs - // created *before* tracking (asyncio's WeakValueDictionary - // registers a transport before its first cycle-suspect - // mutation tracks it) would otherwise make the fast-path - // filter read the object as permanently live. - handle.weak_clones.store( - crate::weakref_registry::strong_clone_count(new_id), - Ordering::Release, - ); - self.fin_insert(new_id, handle.clone()); + // `finalized_ids` is keyed by object id (a pointer), which the + // allocator recycles. A freshly tracked object at a recycled + // address must start *un*-finalized (`test_is_finalized`). + // Almost always empty (only `__del__`-bearing objects ever land + // there). + if !self.finalized_ids.borrow().is_empty() { + if fresh { + self.finalized_ids.borrow_mut().remove(&new_id); + } else if self.finalized_ids.borrow().contains(&new_id) { + handle.finalized.store(true, Ordering::Release); + } } let mut gens = self.generations.borrow_mut(); handle.slot.store(gens[0].handles.len(), Ordering::Release); gens[0].handles.push(handle); } - // `finalized_ids` is keyed by object id (a pointer), which the - // allocator recycles. A freshly tracked object at a recycled address - // must start *un*-finalized, so drop any stale entry — otherwise - // `gc.is_finalized(new_obj)` would inherit the previous tenant's - // finalized flag (`test_is_finalized`). - // Almost always empty (only `__del__`-bearing objects ever land - // there), and the borrow plus hash would otherwise be paid by - // every tracked birth. - if !self.finalized_ids.borrow().is_empty() { - self.finalized_ids.borrow_mut().remove(&new_id); - } serial_add(&self.tracked_count, 1); serial_add(&self.tracked_version, 1); - self.note_gen0_alloc(); + true } - /// Stop tracking `obj`. Used by the cycle-clearing path - /// after an object is reclaimed, and by the explicit - /// `gc._untrack(obj)` extension. + /// Stop tracking `obj`. Backs the explicit `gc._untrack(obj)` extension + /// and the C-API `PyObject_GC_UnTrack`. pub fn untrack_id(&self, id: ObjectId) { + // (From the newest: an untrack usually follows its birth closely.) + if let Ok(mut fns) = self.young_fns.try_borrow_mut() { + if let Some(i) = fns + .iter() + .rposition(|w| w.as_ptr() as usize as ObjectId == id) + { + fns.swap_remove(i); + let mut counts = self.counts.borrow_mut(); + counts[0] = counts[0].saturating_sub(1); + self.sync_gen0_gauge(counts[0], None); + return; + } + } + if let Ok(mut young) = self.young.try_borrow_mut() { + if let Some(i) = young + .iter() + .position(|w| w.as_ptr() as usize as ObjectId == id) + { + young.swap_remove(i); + let mut counts = self.counts.borrow_mut(); + counts[0] = counts[0].saturating_sub(1); + self.sync_gen0_gauge(counts[0], None); + return; + } + } let Some(handle) = self.index.borrow_mut().remove(&id) else { return; }; - // RFC 0061 (WS1b): let suspect probes see the removal without a - // registry lookup. - handle.untracked.store(true, Ordering::Release); - // Purge any suspect-list clone of this handle in lock-step: the - // suspect entry shares the same `Arc`, so dropping - // the index's Arc alone would leave the handle's strong `object` - // reference alive in the suspect list (see `remove_suspect`). - remove_suspect(id); - // CPython's `PyObject_GC_Del` decrements the gen-0 allocation - // counter for every GC-tracked object freed, whatever its - // generation — so churn workloads whose young objects die by - // refcount (asyncio's per-connection task/future/handle webs) - // barely advance toward `threshold0` and automatic collections - // stay rare. Without the decrement, our prompt-reap untracks kept - // the gross allocation count, firing young collections an order - // of magnitude more often than CPython — frequently enough to - // land inside the window where a pending asyncio task is - // reachable only through its own await cycle. Collecting there is - // *correct* per CPython semantics (the docs tell users to hold - // task references; test_log_destroyed_pending_task relies on it), - // but CPython's cadence means its suite never observes it in e.g. - // test_streams.test_start_server — match the cadence, not just - // the semantics (RFC 0054). { let mut counts = self.counts.borrow_mut(); counts[0] = counts[0].saturating_sub(1); self.sync_gen0_gauge(counts[0], None); } - // Drop the finalizable-index entry in lock-step with the main index so - // the cheap prompt-finalization scan never sees a reclaimed object. - self.fin_remove(id); - // O(1) removal via the handle's cached `slot`. The index is the - // dedupe authority, so exactly one handle existed for `id`, and - // its `slot`/`generation`/`color` pinpoint its position without a - // per-generation scan (which made drop-heavy large heaps - // quadratic — RFC 0039 WS4). - // - // The cached `slot`/`generation` are *only* valid while the owning - // generation/frozen lock is held: a `swap_remove` elsewhere updates a - // moved handle's `slot` under that same lock. So we must acquire the - // vector lock *before* reading the cached position, and — because the - // GC is process-global and shared across OS threads — fall back to a - // pointer search if the cached slot is stale, rather than corrupting - // the vector with a wrong `swap_remove` (the bug behind the - // "generation slot index out of sync" panic under threaded GC). + // O(1) removal via the handle's cached `slot`. The cached position is + // only valid under the owning vector's lock; fall back to a pointer + // search if it's stale. if handle.color.load(Ordering::Acquire) == color::Frozen { let mut frozen = self.frozen.borrow_mut(); let slot = handle.slot.load(Ordering::Acquire); @@ -976,9 +857,6 @@ impl GcState { { swap_remove_handle(&mut gens[g].handles, slot); } else if !remove_handle_by_ptr(&mut gens[g].handles, &handle) { - // Declared generation was wrong too (e.g. a concurrent - // promotion landed between the `generation` and `slot` - // reads). Search the rest before giving up. for gg in 0..N_GENERATIONS { if gg != g && remove_handle_by_ptr(&mut gens[gg].handles, &handle) { break; @@ -990,511 +868,125 @@ impl GcState { serial_add(&self.tracked_version, 1); } - /// Reclaim every tracked object on this thread whose only remaining - /// strong reference is the cycle collector's own handle — dead - /// *acyclic* garbage that CPython's refcounting frees the instant - /// its last binding drops, but which our per-thread strong handle - /// pins until a collection. Skips finalizable objects (their - /// `__del__` must be ordered by a finalizing collection) and - /// weakref-watched objects (clearing their weakrefs runs user - /// callbacks). Because every survivor of these filters runs no - /// Python on the way out, this is safe to call from any GIL-holding - /// safe point — notably a `Thread.join` return, where a worker has - /// just dropped the last *program* reference to objects this thread - /// allocated (RFC 0039 WS4: cross-thread prompt reclamation across - /// the per-thread-heap boundary). Iterates to a fixpoint so freeing - /// one object reclaims any acyclic chain it anchored. Returns the - /// number of objects reclaimed. - pub fn reap_dead_acyclic(&self) -> usize { - // A collection already walks the same set; never re-enter it. - // Atomic claim (see `collect_impl`): overlapping reaps/collections - // from two threads corrupt each other's refcount math. - if self - .collecting - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_err() - { - return 0; - } - let n = self.reap_dead_acyclic_locked(); - self.collecting.store(false, Ordering::Release); - n - } - - /// [`Self::reap_dead_acyclic`] body, called with the `collecting` claim - /// already held (by `reap_dead_acyclic` itself or by `collect_impl`). - fn reap_dead_acyclic_locked(&self) -> usize { - let mut reclaimed = 0usize; - // A worklist of ids that *may* have just died. Freeing an object - // drops its fields, so the only way a tracked object can become - // dead during this pass is by being a (transitive) child of one we - // freed; seeding the worklist from those children makes a long - // acyclic chain (`UserList([UserList([...])])` 200k deep, - // `test_userlist.test_repr_deep`) O(n) instead of one full index - // rescan per link. The outer loop's full scan remains the fixpoint - // authority: it seeds the first batch and confirms the last. - let mut work: Vec = Vec::new(); - loop { - let dead: Vec = { - let index = self.index.borrow(); - index - .iter() - .filter(|(id, h)| { - strong_count_for(&h.object) <= 1 - && !has_finalizer(&h.object) - && crate::weakref_registry::count_for(**id) == 0 - }) - .map(|(id, _)| *id) - .collect() - }; - if dead.is_empty() { - break; - } - work.extend(dead); - while let Some(id) = work.pop() { - // Re-validate under a fresh borrow: a free earlier in this - // batch may have already reclaimed `id` as a child, and a - // worklist candidate may simply still be alive (counts - // only fall here, so nothing is ever revived). - let handle = self.index.borrow().get(&id).cloned(); - let Some(h) = handle else { continue }; - let still_dead = strong_count_for(&h.object) <= 1 - && !has_finalizer(&h.object) - && crate::weakref_registry::count_for(id) == 0; - if !still_dead { - continue; - } - if crate::hot_gates::env_flags::reap_trace() { - eprintln!("[ACYCLIC-REAP] {}", h.object.type_name_owned()); + /// Drop the entries of objects that have died from `handles`, and from + /// the index; `budget` caps how many entries are examined, starting at + /// `start` (wrapping). Returns how many were dropped and where the + /// examination stopped. + fn prune_dead_in( + index: &mut GcIndex, + handles: &mut Vec>, + start: usize, + budget: usize, + ) -> (usize, usize) { + let mut removed = 0; + let mut i = if start < handles.len() { start } else { 0 }; + let mut examined = 0; + while examined < budget && i < handles.len() { + examined += 1; + if handles[i].object.is_dead() { + let h = handles.swap_remove(i); + index.remove(&h.id); + if let Some(moved) = handles.get(i) { + moved.slot.store(i, Ordering::Release); } - // Enqueue the tracked objects this one holds, looking - // through untracked intermediaries (a tuple or frozenset - // holding the next link) which die with their parent. - self.collect_child_candidates(&h.object, &mut work); - self.untrack_id(id); - // `h` was cloned out of the index above; drop that clone so - // the untrack's field drops actually happen now (and the - // children's counts fall before they are re-examined). - drop(h); - reclaimed += 1; - } - } - reclaimed - } - - /// Push the ids of the tracked objects directly held by `obj` onto - /// `work`, descending through *untracked* children (immutable - /// containers and iterators are not persistently tracked) so a tracked - /// grandchild behind a tuple is still found. Bounded by an explicit - /// stack and a visited set so deep or shared untracked webs neither - /// recurse on the C stack nor blow up the candidate list. - fn collect_child_candidates(&self, obj: &Object, work: &mut Vec) { - let mut pending: Vec = Vec::new(); - traverse_object(obj, &mut |child| pending.push(child.clone())); - let mut seen: IdSet = IdSet::default(); - while let Some(child) = pending.pop() { - let id = id_of(&child); - if !seen.insert(id) { - continue; - } - if self.index.borrow().contains_key(&id) { - work.push(id); - } else if strong_count_for(&child) <= 2 { - // Untracked and about to die with its parent (our `child` - // clone plus the parent's own slot are the only counts): its - // fields fall too, so look through it. - traverse_object(&child, &mut |grandchild| pending.push(grandchild.clone())); - } - } - } - - /// True iff at least one finalizable object is currently tracked. A single - /// relaxed atomic load — the gate the interpreter checks at every - /// reference-drop safe point before deciding whether a prompt-finalization - /// sweep is even worth attempting. - #[inline] - pub fn has_any_finalizable(&self) -> bool { - self.finalizable_count.load(Ordering::Relaxed) > 0 - } - - /// Enroll a tracked object in the prompt-finalization index because a - /// weakref *with a callback* now watches it (`weakref.ref(obj, cb)`, - /// `weakref.finalize`, `multiprocessing.util.Finalize`). CPython fires - /// such a callback the instant the referent's last strong reference drops; - /// without this the callback would wait for the next cyclic collection. - /// No-op when the object isn't tracked (untracked weakref targets — - /// plain functions, bound methods — are handled by the collection-time - /// [`sweep_weakref_only_targets`] sweep instead). - pub fn note_weakref_finalizable(&self, id: ObjectId) { - let handle = self.index.borrow().get(&id).cloned(); - if let Some(h) = handle { - // Refresh the cached clone count used by the prompt-finalization - // fast-path filter (see `TrackedHandle::weak_clones`). - let clones = crate::weakref_registry::strong_clone_count(id); - h.weak_clones.store(clones, Ordering::Release); - // A fresh callback-weakref means somebody expects prompt - // notification: re-grade an already-enrolled cold entry. - if !self.fin_insert(id, h.clone()) { - self.fin_make_hot(&h); + removed += 1; + } else { + i += 1; } } + (removed, i) } - /// Drive one prompt-finalization pass over the dedicated finalizable index: - /// for every tracked finalizable object whose last *program* reference just - /// dropped, run its `__del__` and/or fire its weakref callbacks and reclaim - /// it. CPython does this by refcount the instant the count hits zero; our - /// tracing handle pins the object until a collection, so this emulates the - /// prompt path between bytecodes. - /// - /// "Dead" means the effective program refcount is zero: - /// `strong_count - 1 (our own GC handle) - (registry weakref clones)`. - /// A weakref slot keeps one strong clone of its target alive - /// ([`crate::weakref_registry::WeakRefSlot::target`]); discounting those is - /// what lets a `util.Finalize`-watched object — reachable now only through - /// the GC handle and the finalizer's own weakref — collapse to zero. - /// - /// Per dead object: - /// * If a `__del__` is still pending, queue it (set `finalize_queued`) and - /// leave the object tracked — its finalizer might resurrect it, and any - /// weakref callbacks must fire *after* `__del__` (CPython order). The - /// driver's next pass re-checks death and, if it stuck, fires the - /// weakrefs and reclaims. - /// * Otherwise (no `__del__`, or it already ran) fire the weakref callbacks - /// (`notify_clear` → queued for the interpreter) and untrack the object — - /// dropping the handle frees it and cascades the refcount into referents, - /// exposing the next layer of dead finalizables on the following pass. - /// - /// Returns the number of objects that made progress (queued a finalizer or - /// were reclaimed); the driver loops until this is zero. Cheap: scans only - /// the finalizable index, which holds just the `__del__`/callback-weakref - /// objects (typically a handful). - pub fn reap_dead_finalizable(&self) -> usize { - // RFC 0077 (WS2): the hot/cold gate runs *before* the claim below. - // With every finalizable cold (the steady state: the ABC - // machinery's callback-weakref'd type, a long-lived `__del__` - // object) and no cold tick due, the whole scan is two relaxed - // loads and a counter bump, not a CAS on the shared claim word. - // `WEAVEPY_NO_FIN_COLD=1` pins every entry hot for bisection. - let cold_tick = fin_no_cold() - || take_force_cold_tick() - || self - .fin_cold_tick - .fetch_add(1, Ordering::Relaxed) - .is_multiple_of(FIN_COLD_STRIDE); - if !cold_tick && self.fin_hot_count.load(Ordering::Relaxed) == 0 { - return 0; - } - // Atomic claim (see `collect_impl`): the untrack path below mutates - // the shared index/generations, which must not overlap a - // collection's mark walk on another thread. - if self - .collecting - .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) - .is_err() - { - return 0; - } - let n = self.reap_dead_finalizable_locked(cold_tick); - self.collecting.store(false, Ordering::Release); - n - } - - /// [`Self::reap_dead_finalizable`] body, called with the `collecting` - /// claim held. - fn reap_dead_finalizable_locked(&self, cold_tick: bool) -> usize { - // Borrow-only scan: collect just the dead handles. The common case — - // all finalizables still reachable — allocates nothing (an empty - // `Vec::new()` doesn't heap-allocate) and pays only a cheap - // `strong_count` atomic load per object, skipping the per-id registry - // lookup via the `weak_clones` fast-path filter. - static FIN_TRACE: std::sync::OnceLock = std::sync::OnceLock::new(); - let fin_trace = *FIN_TRACE.get_or_init(|| std::env::var_os("WEAVEPY_FIN_TRACE").is_some()); - // Per-safe-point scan budget. Below this size the whole index is - // scanned (full CPython-like promptness — the overwhelmingly common - // shape); above it, a rotating window bounds the cost so a huge - // population of callback-weakrefs (70k `WeakKeyDictionary` keys in - // test_weakref's threaded-copy stress) doesn't make every - // reference-dropping opcode O(index) — quadratic over the run. - // Deaths are still detected within index_len/budget safe points. - const FIN_SCAN_BUDGET: usize = 256; - /// When over budget, additionally scan only every N-th safe point: - /// with tens of thousands of live callback-weakrefs even a bounded - /// window per drop opcode dominates the run; deaths are batched, - /// so probing less often loses nothing but a little latency. - const FIN_SCAN_STRIDE: u64 = 8; - // RFC 0077 (WS2): the hot/cold gate (decided by the caller, see - // `reap_dead_finalizable`). Cold entries — strong count far above - // the dead line, or unchanged across `FIN_STABLE_PROBES` - // consecutive probes — are re-probed only on the cold stride. - let no_cold = fin_no_cold(); - // The over-budget stride applies to whichever population this - // scan walks: the hot set on an ordinary scan, the whole index - // on a cold tick. - let walk_len = if cold_tick { - self.finalizable.borrow().len() - } else { - self.fin_hot_count.load(Ordering::Relaxed) - }; - if walk_len > FIN_SCAN_BUDGET { - let tick = self.fin_scan_tick.fetch_add(1, Ordering::Relaxed); - if !tick.is_multiple_of(FIN_SCAN_STRIDE) { - return 0; - } - } - floor_stats::bump(&floor_stats::FIN_SCANS, 1); - let mut probed = 0u64; - /// Outcome of one probe: keep hot, go cold, or dead. - enum Grade { - Hot, - Cold, - } - let mut out: Vec> = Vec::new(); - let mut probe = |h: &HandleRc, - out: &mut Vec>| - -> Grade { - probed += 1; - let sc = strong_count_for(&h.object); - let cached = h.weak_clones.load(Ordering::Acquire); - if fin_trace { - let tn = h.object.type_name_owned(); - if tn.contains("Transport") || tn.contains("SSLContext") || tn == "SSLProtocol" { - eprintln!( - "[FIN-SCAN] {tn} sc={sc} cached={cached} clones={}", - crate::weakref_registry::strong_clone_count(h.id) - ); - } - } - // Fast reject: more strong refs than our handle plus all of its - // (cached, upper-bound) weakref clones ⇒ a program reference is - // still live. Skip without touching the registry. Far above the - // dead line grades cold; so does a count that hasn't moved in - // `FIN_STABLE_PROBES` consecutive probes (a long-lived object - // with a fixed set of holders — the shape that otherwise pays - // a probe at every drop safe point for the process lifetime). - // Near the dead line and changing stays hot. - if sc > 1 + cached { - if no_cold { - return Grade::Hot; - } - if sc > 1 + cached + FIN_COLD_MARGIN { - return Grade::Cold; - } - // Already cold and still above the dead line: stay cold. - // Only reaching the borderline (handled below) re-arms it, - // so a long-lived object whose count merely jitters - // doesn't oscillate between the sets. - if h.fin_cold.load(Ordering::Relaxed) { - return Grade::Cold; - } - if h.fin_last_sc.swap(sc, Ordering::Relaxed) == sc { - let stable = h.fin_stable.fetch_add(1, Ordering::Relaxed) + 1; - if stable >= FIN_STABLE_PROBES { - h.fin_stable.store(0, Ordering::Relaxed); - return Grade::Cold; - } - } else { - h.fin_stable.store(0, Ordering::Relaxed); - } - return Grade::Hot; - } - // Borderline: compute the exact live clone count and test for - // an effective program refcount of zero. - let clones = crate::weakref_registry::strong_clone_count(h.id); - // Refresh the cached bound with the exact value. A cleared or - // died weakref leaves the cache stale-high, and a stale-high - // bound keeps a live object "borderline" — paying this - // registry lookup again at *every* reference-dropping safe - // point. This scan runs per drop opcode while any finalizable - // is live, so a single stale entry costs a whole test run - // (statistics.kde's hot sum-loop spent ~40% of its time - // here). New weakrefs refresh the cache upward via - // `note_weakref_finalizable`, so tightening it is safe. - h.weak_clones.store(clones, Ordering::Release); - if sc.saturating_sub(1).saturating_sub(clones) == 0 { - out.push(h.clone()); - } - Grade::Hot - }; - if cold_tick { - // Walk the whole index (or a rotating window of it) and - // re-grade every entry; collect the transitions and apply - // them after the index borrow ends. - let mut to_cold: Vec> = Vec::new(); - let mut to_hot: Vec> = Vec::new(); - { - let fin = self.finalizable.borrow(); - let mut grade = |h: &HandleRc| { - let was_cold = h.fin_cold.load(Ordering::Relaxed); - match probe(h, &mut out) { - Grade::Cold if !was_cold => to_cold.push(h.clone()), - Grade::Hot if was_cold => to_hot.push(h.clone()), - _ => {} - } - }; - if fin.len() <= FIN_SCAN_BUDGET { - for h in fin.values() { - grade(h); - } - } else { - let start = self.fin_scan_cursor.load(Ordering::Relaxed); - let mut next_cursor = start; - for (scanned, (id, h)) in - fin.range(start..).chain(fin.range(..start)).enumerate() - { - if scanned == FIN_SCAN_BUDGET { - next_cursor = *id; - break; - } - grade(h); - } - // budget < len guarantees the break above ran and set the - // resume point to the first unscanned id. - self.fin_scan_cursor.store(next_cursor, Ordering::Relaxed); - } - } - for h in &to_cold { - self.fin_make_cold(h); - } - for h in &to_hot { - self.fin_make_hot(h); - } - } else { - // Ordinary scan: only the hot set, demoting in place. Over - // budget, a rotating window (the cursor is an index here) - // keeps every hot entry visited within len/budget scans. - let mut hot = self.finalizable_hot.borrow_mut(); - let len = hot.len(); - let budget = len.min(FIN_SCAN_BUDGET); - let mut i = if len > FIN_SCAN_BUDGET { - self.fin_hot_cursor.load(Ordering::Relaxed) % len + /// Drop the dead young entries, plus a bounded slice of the older + /// generations'. Returns the young generation's live population. + fn prune(&self) -> usize { + let (removed, live) = { + let mut index = self.index.borrow_mut(); + let mut gens = self.generations.borrow_mut(); + let (mut removed, _) = + Self::prune_dead_in(&mut index, &mut gens[0].handles, 0, usize::MAX); + // The older generations, a slice at a time: the cursor runs + // over gen 1 then gen 2 as one sequence. + let cursor = self.old_prune_cursor.load(Ordering::Relaxed); + let n1 = gens[1].handles.len(); + let (g, start) = if cursor < n1 { + (1, cursor) } else { - 0 + (2, cursor - n1) }; - let mut scanned = 0usize; - while scanned < budget && !hot.is_empty() { - if i >= hot.len() { - i = 0; - } - scanned += 1; - let h = hot[i].clone(); - match probe(&h, &mut out) { - Grade::Cold => { - h.fin_cold.store(true, Ordering::Release); - h.fin_hot_slot.store(usize::MAX, Ordering::Relaxed); - hot.swap_remove(i); - if let Some(moved) = hot.get(i) { - moved.fin_hot_slot.store(i, Ordering::Relaxed); - } - } - Grade::Hot => i += 1, - } - } - if len > FIN_SCAN_BUDGET { - self.fin_hot_cursor.store(i, Ordering::Relaxed); - } - self.fin_hot_count.store(hot.len(), Ordering::Release); - } - let dead = out; - floor_stats::bump(&floor_stats::FIN_PROBED, probed); - if dead.is_empty() { - return 0; - } - floor_stats::bump(&floor_stats::FIN_DEAD, dead.len() as u64); - let mut progressed = 0; - for h in dead { - // Re-validate under fresh counts: an earlier finalizer in this batch - // may have resurrected `h`. - let clones = crate::weakref_registry::strong_clone_count(h.id); - let effective = strong_count_for(&h.object) - .saturating_sub(1) - .saturating_sub(clones); - if effective != 0 { - continue; // resurrected between scan and now - } - let del_pending = has_finalizer(&h.object) && !h.finalized.load(Ordering::Acquire); - if del_pending { - // Queue `__del__`; defer weakref callbacks + reclamation to a - // later pass (after the finalizer has run and either resurrected - // the object or left it dead). - if !h.finalize_queued.swap(true, Ordering::AcqRel) { - run_finalizer(&h.object); - progressed += 1; + let (r, stop) = + Self::prune_dead_in(&mut index, &mut gens[g].handles, start, OLD_PRUNE_BUDGET); + removed += r; + let next = if stop >= gens[g].handles.len() { + if g == 1 { + gens[1].handles.len() + } else { + 0 } + } else if g == 1 { + stop } else { - // No pending `__del__`: fire the (callback) weakrefs and - // reclaim. Weakref callbacks receive the weakref wrapper, not - // the target, so they cannot resurrect it — the object is - // definitively dead once they're queued. - crate::weakref_registry::queue_callbacks(crate::weakref_registry::notify_clear( - h.id, - )); - self.untrack_id(h.id); - progressed += 1; - } + gens[1].handles.len() + stop + }; + self.old_prune_cursor.store(next, Ordering::Relaxed); + (removed, gens[0].handles.len()) + }; + if removed > 0 { + self.tracked_count.fetch_sub( + removed.min(self.tracked_count.load(Ordering::Acquire)), + Ordering::AcqRel, + ); + serial_add(&self.tracked_version, 1); } - progressed + live } pub fn is_tracked(&self, id: ObjectId) -> bool { - // RFC 0065 (WS4): usually-miss probe — two relaxed loads - // instead of the index borrow. A stale filter bit (untracked - // id) just takes the precise path. + // Usually-miss probe: two relaxed loads instead of the index + // borrow. A stale filter bit just takes the precise path. if !self.tracked_filter.may_contain(id) { return false; } self.index.borrow().contains_key(&id) } - /// Copy only weakref targets not already owned by the cycle collector. - /// Hold the index before the registry, matching `track_now`'s lock order. - /// Both borrows end before the sweep clears slots or queues callbacks. - fn untracked_weakref_targets( - &self, - registry: &crate::weakref_registry::WeakRefRegistry, - ) -> Vec<(ObjectId, Object)> { - let index = self.index.borrow(); - registry.targets_matching(|id| !index.contains_key(&id)) + /// [`Self::is_tracked`] for an instance or function, which may be + /// young. + pub fn is_tracked_instance(&self, id: ObjectId) -> bool { + self.is_tracked(id) || self.is_young(id) } /// O(1) handle lookup by object id (any generation or frozen). pub fn handle_for(&self, id: ObjectId) -> Option> { - // RFC 0065 (WS4): see `is_tracked`. if !self.tracked_filter.may_contain(id) { return None; } self.index.borrow().get(&id).cloned() } - /// Snapshot every tracked object that still carries an unrun + /// Snapshot every live tracked object that still carries an unrun /// `__del__`. The interpreter's shutdown pass walks this list to - /// finalize objects that are still alive at exit — CPython runs - /// finalizers for everything during interpreter teardown, not just - /// for cyclic garbage. The per-handle `finalized` flag (shared with - /// the cycle collector) guarantees each `__del__` runs at most once. - pub fn finalization_candidates(&self) -> Vec> { + /// finalize objects that are still alive at exit. The per-handle + /// `finalized` flag guarantees each `__del__` runs at most once. + pub fn finalization_candidates(&self) -> Vec<(HandleRc, Object)> { + self.flush_young(); let mut out = Vec::new(); - let pending = |h: &HandleRc| { - !h.finalized.load(Ordering::Acquire) - // A finalizer already queued by a collection (but not yet - // drained) must not be listed again — the pending queue owns - // it, and running it twice would double-fire `__del__`. - && !h.finalize_queued.load(Ordering::Acquire) - && has_finalizer(&h.object) - }; - let gens = self.generations.borrow(); - for gen in gens.iter() { - for h in &gen.handles { - if pending(h) { - out.push(h.clone()); - } + let mut consider = |h: &HandleRc| { + // A finalizer already queued by a collection (but not yet + // drained) must not be listed again. + if h.finalized.load(Ordering::Acquire) || h.finalize_queued.load(Ordering::Acquire) { + return; } - } - for h in self.frozen.borrow().iter() { - if pending(h) { - out.push(h.clone()); + if let Some(obj) = h.object.upgrade() { + if has_finalizer(&obj) { + out.push((h.clone(), obj)); + } } + }; + for gen in self.generations.borrow().iter() { + gen.handles.iter().for_each(&mut consider); } + self.frozen.borrow().iter().for_each(&mut consider); out } @@ -1545,7 +1037,14 @@ impl GcState { /// every allocation site polls `maybe_auto_collect` itself /// (`MAKE_FUNCTION`, `list(it)`, …). fn note_gen0_alloc(&self) { - self.bump_count(0); + // SAFETY: nothing runs while the counts are updated. + match unsafe { self.counts.peek_mut() } { + Some(counts) => { + counts[0] = counts[0].saturating_add(1); + self.sync_gen0_gauge(counts[0], None); + } + None => self.bump_count(0), + } let gauge = self.gen0_gauge.load(Ordering::Relaxed); let threshold = gauge & 0xffff_ffff; if threshold != 0 && (gauge >> 32) == threshold && self.is_enabled() { @@ -1565,14 +1064,7 @@ impl GcState { .store(((count as u64) << 32) | th, Ordering::Relaxed); } - /// Threshold-driven automatic collection (CPython's `gc_alloc` - /// path): when the gen-0 allocation counter passes `threshold0`, - /// collect the *oldest* generation whose own counter has also - /// passed its threshold. Returns the number of objects reclaimed. - /// Callers must be at a safe point (no outstanding container - /// borrows); the interpreter invokes this from its allocation - /// sites. - /// Whether the next [`Self::maybe_auto_collect`] would run a + /// Whether the next [`Self::maybe_auto_collect`] would consider a /// collection (the young-generation counter has reached its /// threshold). Lets an allocation site that cannot run finalizers /// hand the allocation to one that can. @@ -1585,11 +1077,23 @@ impl GcState { threshold != 0 && (gauge >> 32) >= threshold } + /// Threshold-driven automatic collection (CPython's `gc_alloc` + /// path): when the gen-0 counter passes `threshold0`, collect the + /// *oldest* generation whose own counter has also passed its + /// threshold. Returns whether a collection ran. Callers must be at a + /// safe point (no outstanding container borrows). + /// + /// CPython's counter is allocations *minus deallocations* of tracked + /// objects. Deaths aren't counted as they happen here, so the counter + /// is corrected when it trips: the dead young entries are pruned and + /// the count restarts from the survivors, unless they alone make up + /// half the threshold (which also bounds the pruning work at two + /// entries per allocation). pub fn maybe_auto_collect(&self) -> bool { if !self.is_enabled() || self.collecting.load(Ordering::Acquire) { return false; } - let (count0, eligible) = { + let (due, eligible, threshold0) = { let counts = self.counts.borrow(); let thresholds = self.thresholds.borrow(); if thresholds[0] == 0 { @@ -1602,22 +1106,27 @@ impl GcState { gen = 2; } } - (counts[0] >= thresholds[0], gen) + (counts[0] >= thresholds[0], gen, thresholds[0]) }; - if !count0 { + if !due { return false; } - // Automatic young collection: single incremental pass, no whole-index - // acyclic reap (see `collect_impl`'s `exact` discussion). Report that a - // collection ran (regardless of how many objects it reclaimed) so the - // caller drains any `__del__` finalizers it deferred — without paying a - // pending-queue probe on every allocation. + let live = self.prune() + self.young_live(); + if live < threshold0 / 2 { + let mut counts = self.counts.borrow_mut(); + counts[0] = live; + self.sync_gen0_gauge(live, None); + return false; + } + // Automatic young collection: a single pass (see `collect_impl`'s + // `exact` discussion). self.collect_impl(eligible, false); true } /// Total population (across all generations + frozen). pub fn population(&self) -> usize { + self.flush_young(); let gens = self.generations.borrow(); let mut n = 0; for g in gens.iter() { @@ -1626,7 +1135,7 @@ impl GcState { n + self.frozen.borrow().len() } - /// Snapshot all tracked objects. Used by + /// Snapshot all live tracked objects. Used by /// `gc.get_objects(generation=...)`. pub fn snapshot(&self, generation: Option) -> Vec { // `gc.get_objects()` enumerates every container CPython tracks, @@ -1634,24 +1143,15 @@ impl GcState { self.promote_all_deferred(); let gens = self.generations.borrow(); let mut out = Vec::new(); + let mut push = |h: &HandleRc| out.extend(h.object.upgrade()); match generation { - Some(g) if g < N_GENERATIONS => { - for h in &gens[g].handles { - out.push(h.object.clone()); - } - } - _ => { - for g in gens.iter() { - for h in &g.handles { - out.push(h.object.clone()); - } - } - } + Some(g) if g < N_GENERATIONS => gens[g].handles.iter().for_each(&mut push), + _ => gens + .iter() + .for_each(|g| g.handles.iter().for_each(&mut push)), } if generation.is_none() { - for h in self.frozen.borrow().iter() { - out.push(h.object.clone()); - } + self.frozen.borrow().iter().for_each(&mut push); } out } @@ -1659,6 +1159,7 @@ impl GcState { /// `gc.freeze()` — mark every currently-tracked object as /// frozen so it is ignored by future collections. pub fn freeze_all(&self) { + self.flush_young(); let mut gens = self.generations.borrow_mut(); let mut frozen = self.frozen.borrow_mut(); for g in gens.iter_mut() { @@ -1674,9 +1175,7 @@ impl GcState { /// `gc.unfreeze()` — move every frozen object back to /// generation 0. pub fn unfreeze_all(&self) { - // Lock order: generations before frozen, matching `freeze_all` - // (consistent ordering avoids a cross-cell deadlock now that the - // GC is process-global — RFC 0039 WS4). + // Lock order: generations before frozen, matching `freeze_all`. let mut gens = self.generations.borrow_mut(); let mut frozen = self.frozen.borrow_mut(); for h in frozen.drain(..) { @@ -1698,30 +1197,18 @@ impl GcState { /// Runs regardless of `gc.isenabled()`: CPython's `gc.disable()` only /// suppresses the *automatic*, threshold-driven collections (see /// [`Self::maybe_auto_collect`]); an explicit `gc.collect()` always runs a - /// full sweep. (`test_gc` disables the collector module-wide via - /// `setUpModule` and then asserts that explicit collections still reclaim - /// cycles.) The re-entrancy guard still applies — a collection triggered - /// from inside a collection (e.g. an allocating finalizer) is a no-op. + /// full sweep. The re-entrancy guard still applies — a collection + /// triggered from inside a collection is a no-op. pub fn collect(&self, upto: usize) -> usize { - // An explicit `gc.collect()` is "exact": it reaps acyclic dead and - // iterates to a fixpoint so the returned count matches CPython. self.collect_impl(upto, true) } /// Run the cycle collector's mark phase across all generations and fire /// the weakref callbacks of every unreachable, non-finalizable object, /// *without* the destructive teardown of a real collection. See - /// [`Self::collect_generation`]'s `weakref_only` discussion. The - /// re-entrancy guard applies, so this is a no-op inside a collection. + /// [`Self::collect_generation`]'s `weakref_only` discussion. pub fn fire_dead_weakrefs(&self) { - // Promote any deferred container that can now anchor a cycle, so - // the mark phase below sees the whole candidate population - // (see `sweep_deferred`). Before the re-entrancy claim: promotion - // calls `track_now`, which a collection in progress must not see. self.sweep_deferred(false); - // Atomic claim (see `collect_impl`): even a mark-only pass mutates - // the shared `gc_refs` counters, so it must not overlap a real - // collection on another thread. if self .collecting .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -1736,31 +1223,20 @@ impl GcState { /// Shared collection body. `exact` selects between the two cost/precision /// profiles: /// - /// * `true` — an explicit `gc.collect()`. Reap acyclic dead up front (so - /// they stay out of the cyclic count and `DEBUG_SAVEALL`) and iterate the - /// mark-sweep to a fixpoint, reproducing CPython's "one call reclaims all - /// current cyclic garbage" guarantee that `test_gc`'s exact-count - /// assertions depend on. - /// * `false` — a threshold-driven *automatic* young collection. CPython's - /// auto path is a single incremental pass (leftover garbage waits for the - /// next trigger or an explicit collect), so we skip both the whole-index - /// acyclic reap and the fixpoint loop. That keeps the per-allocation cost - /// flat: with the reap+fixpoint on every auto-collect, an allocation-heavy - /// suite (`test_set`'s mutation stress) re-scanned the entire accumulated - /// tracked set several times per trigger and blew the time budget. + /// * `true` — an explicit `gc.collect()`: iterate the mark-sweep to a + /// fixpoint, reproducing CPython's "one call reclaims all current + /// cyclic garbage" guarantee that `test_gc`'s exact-count assertions + /// depend on. + /// * `false` — a threshold-driven automatic young collection: a single + /// pass (leftover garbage waits for the next trigger), keeping the + /// per-allocation cost flat. fn collect_impl(&self, upto: usize, exact: bool) -> usize { // Promote any deferred container that can now anchor a cycle, so - // the mark phase below sees the whole candidate population - // (see `sweep_deferred`). Before the re-entrancy claim: promotion - // calls `track_now`, which a collection in progress must not see. + // the mark phase sees the whole candidate population. Before the + // re-entrancy claim: promotion calls `track_now`. self.sweep_deferred(false); - // Atomic claim — a plain load-then-store gate let two threads both - // observe `false` and run *overlapping* collections over the shared - // heap. Each phase-3 walk then subtracted the same internal edges - // from the same `gc_refs` counters, so a live object with one - // external root went negative and was swept as garbage — observed - // as a peer thread's suspended generator being closed mid-`for` - // loop (test_threading.test_foreign_thread wait_threads_exit). + // Atomic claim: overlapping collections over the shared heap would + // subtract the same internal edges twice. if self .collecting .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -1768,53 +1244,20 @@ impl GcState { { return 0; } - // Drop this thread's parked C-dropped clones (RFC 0047, wave 5) - // before seeding reachability. Each queue entry is a strong - // `Object` clone awaiting the eval loop's prompt-reap safe point — - // but when C code runs without returning to the eval loop (a - // C-driven embedding, or the collector's own traverse/clear boxes - // freed mid-pass) the queue never drains, and every entry inflates - // its object's `Rc::strong_count`. The mark phase seeds `gc_refs` - // from exactly that count, so a queued clone makes its object — - // and everything reachable from it — look externally rooted, - // pinning dead cycles forever. A collection is itself a safe - // point: anything that dies when these clones drop is either - // reaped below (acyclic) or found unreachable by the mark phase, - // with weakrefs/finalizers handled by the normal collection path. - // - // Guarded on no extension frame being live on this thread: while - // one is (`gc.collect()` invoked *from* C), a queued clone may be - // the last count backing a body pointer that C still borrows - // across its call, and dropping it here would free the body under - // C's feet — the exact UAF the queue exists to prevent. The clones - // then simply wait for the eval-loop drain, as designed. + // Drop this thread's parked C-dropped clones before seeding + // reachability: each inflates its object's strong count, which the + // mark phase reads as an external root. Not while an extension frame + // is live on this thread: a queued clone may back a body pointer C + // still borrows. if !crate::vm_singletons::cext_call_active() { drop(crate::vm_singletons::drain_pending_cext_drops()); } - if exact { - // Reap dead *acyclic* garbage first. CPython frees these by refcount - // the instant their last binding drops, so they never reach the - // cycle collector; we pin them on the registry handle until now. - // Doing it up front keeps them out of the cyclic `collected` count - // *and* out of `gc.garbage` under `DEBUG_SAVEALL` (`test_saveall` - // asserts only the genuine cycle is saved, not an incidental dead - // `[]` temporary). - self.reap_dead_acyclic_locked(); - } let gen = upto.min(N_GENERATIONS - 1); - // Iterate the mark-sweep to a fixpoint (exact only). Reachability is - // seeded from an *approximate* outer refcount (`Rc::strong_count`), so a - // transient Rust-side reference (an operand-stack slot not yet - // overwritten, an in-flight clone) can make a dead object — and - // everything reachable only through it — look live for a single pass. - // CPython's collector is refcount-exact and reclaims *all* current - // cyclic garbage in one `gc.collect()`; the count tests in `test_gc` - // (`gc.collect()` returns exactly the cycle size) depend on that - // completeness. Repeating until a pass collects nothing reproduces it: - // each pass re-seeds from a fresh refcount snapshot, so a reference that - // was transient last pass no longer pins its subgraph. Passes collect a - // strictly shrinking set, so this converges quickly; the cap is a guard - // against pathological churn. + // Reachability is seeded from strong counts, so a transient + // Rust-side reference can make a dead object — and everything + // reachable only through it — look live for a single pass. + // Repeating until a pass collects nothing (exact only) reclaims + // all current cyclic garbage, as CPython's one call does. let passes = if exact { MAX_COLLECT_PASSES } else { 1 }; let mut collected = 0usize; for _ in 0..passes { @@ -1830,10 +1273,9 @@ impl GcState { stats[gen].collected = stats[gen].collected.saturating_add(collected as u64); } { - // CPython resets the counters of every collected - // generation and credits one "tick" to the next older - // one — that tick is what eventually promotes a gen-1 / - // gen-2 collection in `maybe_auto_collect`. + // CPython resets the counters of every collected generation and + // credits one "tick" to the next older one — that tick is what + // eventually promotes a gen-1 / gen-2 collection. let mut counts = self.counts.borrow_mut(); for c in counts.iter_mut().take(gen + 1) { *c = 0; @@ -1844,20 +1286,21 @@ impl GcState { self.sync_gen0_gauge(counts[0], None); } // The tracked-id filter only ever gains bits; once most of them - // name objects long gone, rebuild it from the live index (a full - // collection has just walked everything anyway). + // name objects long gone, rebuild it from the live index. let live = self.index.borrow().len(); let stale = self.tracked_filter.inserts_since_rebuild(); if gen == N_GENERATIONS - 1 || stale > 4096.max(live.saturating_mul(4)) { self.rebuild_tracked_filter(); } self.collecting.store(false, Ordering::Release); + // Referents without a death hook (see `weakref_registry`) are + // noticed here. + crate::weakref_registry::sweep_dead_targets(); collected } - /// Rebuild the tracked-id miss filter from the live index (see - /// [`crate::hot_filter::RebuildableBloom`]). Holding the index borrow - /// serializes it against every `track`. + /// Rebuild the tracked-id miss filter from the live index. Holding the + /// index borrow serializes it against every `track`. pub fn rebuild_tracked_filter(&self) { let index = self.index.borrow(); self.tracked_filter.rebuild(index.keys().copied()); @@ -1867,763 +1310,300 @@ impl GcState { /// /// `weakref_only` runs the identical mark phase but stops once the /// unreachable set is known: it fires the weakref callbacks of the dead, - /// non-finalizable objects (flipping `weakref.ref(obj)()` to `None`) and - /// returns *without* running finalizers, clearing fields, untracking, or - /// rebuilding generations. It is used from a blocking `Thread.join` to - /// fire a reference-count-dead `ThreadPoolExecutor`'s `weakref_cb` (which - /// signals its idle workers to exit) without the destructive teardown of a - /// full collection — which, run while a worker holds an in-flight - /// `_WorkItem` in a frame the collector can't see as a root, would clear - /// that live work item mid-use (RFC 0040: `test_shutdown`). Because it - /// never mutates object contents, such a misclassification is harmless - /// here (a `_WorkItem` has no weakref, so its `notify_clear` is a no-op). + /// non-finalizable objects and returns *without* running finalizers, + /// clearing fields or rebuilding generations. It is used from a + /// blocking `Thread.join` to fire a dead `ThreadPoolExecutor`'s + /// `weakref_cb` without the destructive teardown of a full collection + /// (RFC 0040: `test_shutdown`). fn collect_generation(&self, gen: usize, weakref_only: bool) -> usize { - // Phase 1: snapshot the handles in this generation, plus - // any younger ones (collecting gen N also collects all - // gens 0..N). We treat gens 0..=gen as the candidate set. - let candidate_set = self.snapshot_for_collection(gen); - let cs_len = candidate_set.len(); - if cs_len == 0 { + // Phase 1: the live objects of this generation and the younger ones, + // each held strongly for the collection's duration. Entries whose + // object died are dropped on the way. + let mut cands = self.snapshot_for_collection(gen); + let n_real = cands.len(); + if n_real == 0 { return 0; } - - // Phase 2: initialise gc_refs from the *outer* refcount. - // For Rc-wrapped objects we approximate by - // `Rc::strong_count - 1` (the candidate set holds one - // reference itself, in `TrackedHandle::object`). - for handle in &candidate_set { - // A weak reference must not keep its referent reachable, but - // each live slot holds a strong `Object` clone of the target - // (the registry's drop-driven clear model). Discount those - // clones here so an object reachable *only* through weakrefs - // collapses to `gc_refs == 0` and is collected — which fires - // `notify_clear` and flips `weakref.ref(obj)()` to `None`. - let weak_clones = crate::weakref_registry::strong_clone_count(handle.id) as i64; - let outer = strong_count_for(&handle.object) - .saturating_sub(1) - .saturating_sub(weak_clones as usize) as i64; - handle.gc_refs.store(outer, Ordering::Release); - handle.color.store(color::White, Ordering::Release); - } - - // Index the candidate set by id so the per-child lookups in - // phases 3 and 4 are O(1) — a linear `find` here makes the - // whole collection quadratic, which generator-heavy programs - // (itertools pipelines) hit hard. - // Use the same address-mixing hasher as the persistent index. This - // temporary map only serves identity lookups, not collection order. - let mut by_id: GcIndex = candidate_set.iter().map(|h| (h.id, h.clone())).collect(); - - // Phase 2b: promote untracked iterators reachable from the candidate - // set to *temporary* candidates for this pass only. CPython GC-tracks - // its `*_iterator` objects, so an iterator-mediated cycle (bug #3680: - // `obj.x = iter(set_containing_obj)`) is collectible: the iterator's - // single internal ref to the container has to be subtracted off the - // container's `gc_refs`, otherwise the container looks externally - // reachable and pins the whole cycle. We keep transient *loop* - // iterators untracked for speed (enrolling every `for`-loop iterator - // in a generation regressed allocation-heavy suites by triggering far - // more young collections); instead we discover only the iterators that - // are actually reachable from already-tracked objects, here, while a - // collection is already in flight. The temporary handles take part in - // the subtract/mark walk (so their edges are accounted) but never enter - // a generation, are never cleared/finalized, never touch the index, and - // are not counted as collected — they're dropped when this pass ends, - // and the underlying iterator is freed by refcount once the real - // objects in its (dead) cycle are cleared. - let mut temp_handles: Vec> = Vec::new(); - { - // Scan the existing candidate list, then the growing temporary - // list, preserving discovery order without copying either list. - // Borrow real candidates; only a current temporary needs an - // owning handle while discovery extends its list. These are - // `Arc` handles, not extra `Object` - // clones: a discovered object gains only the strong reference - // inside its temporary handle. Newly discovered objects join the - // scan, including private buffers and their iterators. - // We promote untracked iterators and untracked - // `list` buffers: a snapshot iterator (`frozenset`/`dict.values()`/ - // file) hands back a fresh, untracked `Object::List` for its - // buffer, whose `-> elements` edges have to be accounted for the - // cycle to collapse. An `iter(list)` shares the live list's buffer, - // which is already a real candidate and is found by id below. - let mut scanned = 0usize; - while scanned < cs_len + temp_handles.len() { - let current_temporary = if scanned >= cs_len { - Some(temp_handles[scanned - cs_len].clone()) - } else { - None - }; - let h = match ¤t_temporary { - Some(handle) => handle, - None => &candidate_set[scanned], - }; - scanned += 1; - // Immutable containers (tuple/frozenset) and iterators are not - // persistently GC-tracked — pinning them in a generation would - // hold transient `(type, value, tb)` triples and loop iterators - // alive past the point CPython frees them by refcount - // (`test_traceback`'s `getrefcount` asserts, the loop-iterator - // churn that regressed allocation-heavy suites). But a cycle can - // still *route through* one (`l=[]; t=(l,); l.append(t)`; - // `obj.x = iter(set_containing_obj)`), so we discover the ones - // reachable from the (mutable, tracked) candidate set here and - // promote them to temporary candidates: their internal edges are - // accounted, the dead ones are counted, and the handles are - // dropped when the pass ends (no persistent pinning). - // - // Lists are tracked at creation, so an *untracked* list is only - // ever an iterator's private snapshot buffer (`frozenset`/ - // `dict.values()`/file iterators); promote those only when - // reached directly through an iterator, so we never re-scan the - // whole (already tracked) list population. - let parent_is_iter = matches!(&h.object, Object::Iter(_)); - let parent_is_frame = matches!(&h.object, Object::Frame(_)); - traverse_object(&h.object, &mut |child| { - let promote = match child { - // Dict views join iterators here: they're not - // persistently tracked, but a cycle can route - // through one (`obj.v = container.keys()` with - // `container = {obj: 1}` — test_container_iterator). - // Bound methods likewise (CPython GC-tracks - // `method`): a stored `self.cb` closes the classic - // callback cycle — asyncio's `future._callbacks → - // task.__wakeup → task → future` (RFC 0054, - // test_tasks.test_log_destroyed_pending_task). - // Closure cells likewise (CPython GC-tracks `cell`): - // a nested function that calls itself - // (`def inner(): ... inner()`) closes the cycle - // `function → cell → function`, and the cell's edge - // must be subtracted or the function always looks - // externally reachable (RFC 0054: asyncio's - // `iter_one`-style recursive callbacks pin the async - // generator they iterate, test_base_events' - // asyncgen-finalization-by-gc tests). - // Tracebacks and frames likewise (CPython GC-tracks - // both): an exception object owns `__traceback__ → - // frame → f_locals`, and a local that references the - // exception again (`except* E as excs` materialises - // the handled group in the frame's locals mirror) - // closes a cycle whose edges live entirely in these - // untracked node types (RFC 0054, - // test_taskgroups.test_exception_refcycles_*). - // Descriptor wrappers likewise (CPython GC-tracks - // staticmethod/classmethod/property): a user - // `__new__` is stored in the class dict behind a - // staticmethod wrapper, so the wrapper's edge to - // the function must be subtracted or a dead - // `namespace -> class -> __new__ -> __globals__` - // exec cycle keeps the function externally - // reachable forever - // (test_module.test_clear_dict_in_ref_cycle). - Object::Iter(_) - | Object::Tuple(_) - | Object::FrozenSet(_) - | Object::DictView(_) - | Object::Slice(_) - | Object::Cell(_) - | Object::Traceback(_) - | Object::Frame(_) - | Object::BoundMethod(_) - | Object::StaticMethod(_) - | Object::ClassMethod(_) - | Object::Property(_) => true, - Object::List(_) => parent_is_iter, - // An *exception* instance is untracked until a - // mutation marks it a cycle suspect, yet `raise X - // from …` inside an `except` builds the classic - // `group → __context__ exc → __traceback__ → frame → - // f_locals → group` loop where the chained exception - // is the only instance node. Promote untracked - // exceptions so their `__context__`/`__cause__`/ - // traceback edges are subtracted (RFC 0054, - // test_taskgroups.test_exception_refcycles_*). - // An instance whose tracking is still deferred - // (`track_deferred_owner`) holds only atomic values, - // but its edge to its class can still close a cycle - // (`A.a = A(); del A`): promote it so that edge is - // subtracted, like CPython's always-tracked instance. - Object::Instance(i) => i.cls().flags.is_exception || i.is_gc_deferred(), - // A frame's `f_locals` cache is an internal, - // untracked dict; it carries the frame's only - // object-graph edges to the locals (the `eg` in the - // cycle above), so it joins the walk when reached - // through its frame. - Object::Dict(_) => parent_is_frame, - _ => false, - }; - if !promote { - return; - } - let cid = id_of(child); - if by_id.contains_key(&cid) { - return; - } - let handle = HandleRc::new(TrackedHandle::new(child.clone(), 0)); - by_id.insert(cid, handle.clone()); - temp_handles.push(handle); - }); - } - // Seed `gc_refs` *after* discovery: an iterator synthesises a fresh - // `Object::List`/`Object::Set` wrapper for its buffer on each - // traverse, and that wrapper is alive only for the duration of the - // `visit` call above. Computing the outer refcount here — once - // every such transient clone has been dropped — keeps the seed - // exact (referrers + the one clone the handle itself holds). - for handle in &temp_handles { - let weak_clones = crate::weakref_registry::strong_clone_count(handle.id) as i64; - let outer = strong_count_for(&handle.object) - .saturating_sub(1) - .saturating_sub(weak_clones as usize) as i64; - handle.gc_refs.store(outer, Ordering::Release); - handle.color.store(color::White, Ordering::Release); + let mut by_id: GcIdMap = cands.iter().enumerate().map(|(i, c)| (c.id, i)).collect(); + + // Phase 2: promote untracked nodes reachable from the candidates to + // temporary candidates for this pass only. CPython GC-tracks + // iterators, tuples, frames, tracebacks, cells, bound methods and + // descriptor wrappers; we keep them off the books for speed and + // discover the ones a cycle actually routes through here, so their + // internal edges are accounted. Temporaries take part in the + // subtract/mark walk but are never reclaimed or tracked. + let mut scanned = 0usize; + while scanned < cands.len() { + let parent_is_iter = matches!(cands[scanned].obj, Object::Iter(_)); + let parent_is_frame = matches!(cands[scanned].obj, Object::Frame(_)); + let mut found: Vec = Vec::new(); + traverse_object(&cands[scanned].obj, &mut |child| { + if promotes_temporarily(child, parent_is_iter, parent_is_frame) + && !by_id.contains_key(&id_of(child)) + { + found.push(child.clone()); + } + }); + for child in found { + let cid = id_of(&child); + if let std::collections::hash_map::Entry::Vacant(e) = by_id.entry(cid) { + e.insert(cands.len()); + cands.push(Cand::new(child, cid, None)); + } } + scanned += 1; } - // Real candidates plus the temporary iterator candidates take part in - // the subtract/mark walk; only the real ones are reclaimed below. - // Cloning this iterator copies positions, not handles. The two lists - // retain ownership through finalizers and the resurrection re-mark. - let scan_all = candidate_set.iter().chain(temp_handles.iter()); - - // Phase 3: subtract internal refs by walking each - // tracked object's children. Self-references count too — - // a `self.self = self` instance has one internal ref to - // itself which must be subtracted off so a pure self-cycle - // collapses to gc_refs == 0. - for handle in scan_all.clone() { - traverse_object(&handle.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - target.gc_refs.fetch_sub(1, Ordering::AcqRel); + // Phase 3: seed gc_refs from the outer refcount, after discovery (an + // iterator synthesises a fresh wrapper for its buffer on each + // traverse, alive only during the visit). Each candidate holds one + // reference of its own. + for c in &cands { + c.gc_refs.set(strong_count_for(&c.obj) as i64 - 1); + } + // Subtract internal references, self-references included. + for c in &cands { + traverse_object(&c.obj, &mut |child| { + if let Some(&i) = by_id.get(&id_of(child)) { + let t = &cands[i]; + t.gc_refs.set(t.gc_refs.get() - 1); } }); } - // Phase 4: anything with gc_refs > 0 is reachable from - // outside; mark it black and propagate. - // The candidate lists and immutable identity index retain ownership - // throughout marking, so work stacks can borrow their handles. - let mut grey: Vec<&TrackedHandle> = Vec::new(); - for handle in scan_all.clone() { - if handle.gc_refs.load(Ordering::Acquire) > 0 { - handle.color.store(color::Grey, Ordering::Release); - grey.push(handle.as_ref()); + // Phase 4: anything with gc_refs > 0 is reachable from outside; mark + // it black and propagate. + let mut grey: Vec = Vec::new(); + for (i, c) in cands.iter().enumerate() { + if c.gc_refs.get() > 0 { + c.color.set(color::Grey); + grey.push(i); } } if std::env::var_os("WP_ROOT_DBG").is_some() { - for h in &grey { - let d = match &h.object { - Object::Module(m) => format!("module {}", m.name), - Object::Function(f) => format!("function {}", f.name), - Object::Type(t) => format!("class {}", t.name), - Object::Instance(i) => format!("instance {}", i.cls().name), - Object::Dict(d) => format!( - "dict len={} first={:?}", - d.borrow().len(), - d.borrow().iter().next().map(|(k, _)| k.0.repr()) - ), - Object::List(l) => format!("list len={}", l.borrow().len()), - o => o.type_name_owned(), - }; + for &i in &grey { + let c = &cands[i]; eprintln!( "[root] id={} gc_refs={} sc={} {}", - h.id, - h.gc_refs.load(Ordering::Acquire), - strong_count_for(&h.object), - d + c.id, + c.gc_refs.get(), + strong_count_for(&c.obj), + c.obj.type_name_owned() ); } } - while let Some(h) = grey.pop() { - h.color.store(color::Black, Ordering::Release); - traverse_object(&h.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - if target.color.load(Ordering::Acquire) == color::White { - target.color.store(color::Grey, Ordering::Release); - grey.push(target.as_ref()); - } - } - }); - } + Self::propagate_black(&cands, &by_id, &mut grey); - // Phase 5: white objects are unreachable cyclic garbage. - let unreachable: Vec> = candidate_set - .iter() - .filter(|h| h.color.load(Ordering::Acquire) == color::White) - .cloned() - .collect(); + // Phase 5: white real candidates are unreachable cyclic garbage. + let unreachable: Vec = (0..n_real).filter(|&i| cands[i].is_white()).collect(); - if crate::hot_gates::env_flags::reap_dbg() { - let dbg_class = std::env::var("WP_REAP_DBG_CLASS").unwrap_or("Executor".into()); - for h in &candidate_set { - let matches_dbg = match &h.object { - Object::Instance(i) => i.cls().name.contains(&dbg_class), - Object::Type(t) => t.name.contains(&dbg_class), - _ => false, - }; - { - if matches_dbg { - let exec_id = h.id; - let mut referrers: Vec = Vec::new(); - for c in scan_all.clone() { - if c.id == exec_id { - continue; - } - let mut hit = false; - traverse_object(&c.object, &mut |child| { - if id_of(child) == exec_id { - hit = true; - } - }); - if hit { - let nm = match &c.object { - Object::Instance(ci) => format!("Instance({})", ci.cls().name), - other => other.type_name().to_string(), - }; - referrers.push(nm); - } - } - eprintln!( - "[mark wronly={}] {} sc={} clones={} gc_refs={} white={} tracked_referrers={:?}", - weakref_only, - h.object.type_name_owned(), - strong_count_for(&h.object), - crate::weakref_registry::strong_clone_count(h.id), - h.gc_refs.load(Ordering::Acquire), - h.color.load(Ordering::Acquire) == color::White, - referrers, - ); - } - } - } - } - - // Weakref-only pass: fire the dead objects' weakref callbacks and - // stop. We deliberately skip everything destructive below (finalizer - // execution, field clearing, untracking, generation rebuild) so a - // frame-rooted live object the mark mis-coloured White is left fully - // intact — only its (absent) weakrefs would be touched. A genuinely - // dead, weakref-watched object (the `del`'d `ThreadPoolExecutor`) gets - // its `weakref_cb` queued, which is all a blocking `join` needs to - // unblock its idle workers. Finalizable objects are left for a real - // collection so `tp_finalize` ordering is preserved. // CPython's `handle_weakrefs`: a weakref that is *itself* part of the // cyclic trash has its callback cleared without invocation — only // weakrefs rooted outside the dying subgraph observe the deaths - // (test_callbacks_on_callback: `c.wr`/`d.wr` stay silent while the - // external `safe_callback` fires). Snapshot the trash ids so the - // queue loops below can drop callbacks belonging to trash wrappers. - let mut trash_ids: IdSet = unreachable.iter().map(|h| h.id).collect(); - let wrapper_is_trash = |slot: &crate::sync::Rc, - trash: &IdSet| { + // (test_callbacks_on_callback). + let trash_ids: IdSet = unreachable.iter().map(|&i| cands[i].id).collect(); + let wrapper_is_trash = |slot: &crate::sync::Rc| { slot.py_ref .borrow() .as_ref() .and_then(crate::sync::Weak::upgrade) .is_none_or(|inst| { - trash.contains(&(crate::sync::Rc::as_ptr(&inst) as usize as u64)) + trash_ids.contains(&(crate::sync::Rc::as_ptr(&inst) as usize as u64)) }) }; if weakref_only { let mut weakref_callbacks = Vec::new(); - for h in &unreachable { - if has_finalizer(&h.object) && !h.finalized.load(Ordering::Acquire) { + for &i in &unreachable { + if cands[i].pending_finalizer() { continue; } - for (slot, cb) in crate::weakref_registry::notify_clear(h.id) { + for (slot, cb) in crate::weakref_registry::notify_clear(cands[i].id) { if let Some(cb) = cb { - if wrapper_is_trash(&slot, &trash_ids) { - continue; + if !wrapper_is_trash(&slot) { + weakref_callbacks.push((slot, cb)); } - weakref_callbacks.push((slot, cb)); } } } - for (slot, cb) in weakref_callbacks { - let wr = slot - .py_ref - .borrow() - .as_ref() - .and_then(crate::sync::Weak::upgrade) - .map(crate::object::Object::Instance); - if let Some(wr) = wr { - crate::vm_singletons::push_pending_weakref_callback(cb, wr); - } - } + queue_weakref_callbacks(weakref_callbacks, |_| false); return 0; } // CPython clears weakrefs to the *entire* unreachable set - // (`handle_weakrefs`) BEFORE running any finalizer (`finalize_garbage`) - // and before resurrection handling. So a weakref watching an object a - // finalizer later revives stays cleared even though the object itself - // survives — e.g. `_pyio.FileIO.__del__` recording a - // `ResourceWarning(source=self)` into a live `catch_warnings` log - // resurrects the file, yet `weakref.ref(f)()` must read `None` - // (`test_io.test_garbage_collection`). Mirror that ordering: clear and - // queue callbacks for every unreachable object now, regardless of - // whether the resurrection re-mark or a finalizer keeps it alive below. - // (Counting still tracks only objects actually reclaimed — `dead` — so - // a resurrected object is uncounted but its weakref is gone, exactly as - // in CPython.) + // (`handle_weakrefs`) BEFORE running any finalizer, so a weakref + // watching an object a finalizer later revives stays cleared + // (`test_io.test_garbage_collection`). let mut weakref_callbacks = Vec::new(); - for h in &unreachable { - for (slot, cb) in crate::weakref_registry::notify_clear(h.id) { + for &i in &unreachable { + for (slot, cb) in crate::weakref_registry::notify_clear(cands[i].id) { if let Some(cb) = cb { - if wrapper_is_trash(&slot, &trash_ids) { - continue; + if !wrapper_is_trash(&slot) { + weakref_callbacks.push((slot, cb)); } - weakref_callbacks.push((slot, cb)); } } } - // Split the unreachable set into objects whose `__del__` hasn't run - // yet ("deferred") and the rest. A deferred object is queued for - // finalization and kept tracked: its finalizer (drained right after - // `gc.collect()` returns control to the interpreter) might resurrect - // it, and CPython only counts objects it actually reclaims - // (`test_resurrection_*`). The interpreter then collects again — by - // which point the finalizer has set `finalized`, so a survivor that - // wasn't resurrected falls into `dead` and is reclaimed (its weakrefs - // cleared in that second pass, so single-`collect()` weakref tests - // still observe `ref() is None`). - let mut deferred: Vec> = Vec::new(); - let mut maybe_dead: Vec> = Vec::new(); - for h in &unreachable { - let pending_finalizer = - has_finalizer(&h.object) && !h.finalized.load(Ordering::Acquire); - if pending_finalizer { - deferred.push(h.clone()); - } else { - maybe_dead.push(h.clone()); - } - } - - // Run each deferred object's finalizer (once). A finalizer is arbitrary - // Python: it can execute bytecode, hit a `periodic_gil_checkpoint`, and - // hand the GIL to another OS thread — which may then *resurrect* an - // object the mark phase just classified unreachable (store it somewhere - // reachable, or, in the threaded queue reproducers, pull it off a buffer - // into a live frame local). Every mark color computed above predates - // these finalizers, so it is stale the instant any finalizer runs. - for h in &deferred { - if !h.finalize_queued.swap(true, Ordering::AcqRel) { - run_finalizer(&h.object); - } - } - - // CPython's `handle_resurrected_objects`: after `finalize_garbage` runs - // every `tp_finalize`, it re-derives reachability and moves any object - // that came back to life out of the to-be-cleared set. Mirror that — but - // only when a finalizer actually ran, since that is the sole point in - // this routine where the GIL can be released and the object graph can - // change underneath us. Re-seed `gc_refs` from a *fresh* strong-count - // snapshot (so a reference a concurrent thread or a finalizer added is - // counted), re-subtract internal edges, and re-propagate reachability. - // Without this, a live object reachable only through an untraversed root - // (a running thread's frame locals) that a finalizer's GIL hand-off - // revived is cleared mid-use — emptying its `__dict__` while another - // thread pickles it (RFC 0040: `ProcessPoolExecutor` / multiprocessing - // `Queue` feeder dropping a `_CallItem` into a worker's pipe). - if !deferred.is_empty() { - for handle in scan_all.clone() { - let weak_clones = crate::weakref_registry::strong_clone_count(handle.id) as i64; - let outer = strong_count_for(&handle.object) - .saturating_sub(1) - .saturating_sub(weak_clones as usize) as i64; - handle.gc_refs.store(outer, Ordering::Release); - handle.color.store(color::White, Ordering::Release); - } - for handle in scan_all.clone() { - traverse_object(&handle.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - target.gc_refs.fetch_sub(1, Ordering::AcqRel); - } - }); - } - let mut grey: Vec<&TrackedHandle> = Vec::new(); - for handle in scan_all.clone() { - if handle.gc_refs.load(Ordering::Acquire) > 0 { - handle.color.store(color::Grey, Ordering::Release); - grey.push(handle.as_ref()); + // Objects whose `__del__` hasn't run yet are queued for + // finalization and survive this pass: the finalizer (drained after + // the collection) may resurrect them, and CPython only counts + // objects it actually reclaims. The next collection, by which point + // the finalizer has set `finalized`, reclaims the ones it didn't. + let (deferred, maybe_dead): (Vec, Vec) = unreachable + .iter() + .partition(|&&i| cands[i].pending_finalizer()); + for &i in &deferred { + if let Some(e) = &cands[i].entry { + if !e.finalize_queued.swap(true, Ordering::AcqRel) { + run_finalizer(&cands[i].obj); } } - while let Some(h) = grey.pop() { - h.color.store(color::Black, Ordering::Release); - traverse_object(&h.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - if target.color.load(Ordering::Acquire) == color::White { - target.color.store(color::Grey, Ordering::Release); - grey.push(target.as_ref()); - } - } - }); - } } - - // Recolor the deferred roots Black and protect their whole reachable - // subgraph. CPython runs `finalize_garbage` *before* `delete_garbage`, so - // a pending finalizer always sees its own class, closure cells, and - // referents intact — even when those are themselves unreachable cyclic - // garbage (a locally-defined class whose only instance is dying, the - // `__del__` function closing over the cycle, …). Those objects are - // reclaimed by a later pass once the owning finalizer has run and they, - // too, are plain garbage. Re-applied here so it survives the resurrection - // re-mark above (which reset every color from the fresh refcounts). - let mut protect_stack: Vec<&TrackedHandle> = Vec::new(); - for h in &deferred { - h.color.store(color::Black, Ordering::Release); - protect_stack.push(h.as_ref()); - } - while let Some(h) = protect_stack.pop() { - traverse_object(&h.object, &mut |child| { - if let Some(target) = by_id.get(&id_of(child)) { - if target.color.load(Ordering::Acquire) == color::White { - target.color.store(color::Black, Ordering::Release); - protect_stack.push(target.as_ref()); - } - } - }); + // CPython runs `finalize_garbage` before `delete_garbage`, so a + // pending finalizer always sees its own class, closure cells, and + // referents intact: protect the deferred objects' whole subgraphs. + let mut protect: Vec = Vec::new(); + for &i in &deferred { + cands[i].color.set(color::Grey); + protect.push(i); } + Self::propagate_black(&cands, &by_id, &mut protect); - // Whatever stayed White after the resurrection re-mark and the finalizer - // subgraph protection is genuinely dead this pass. - let dead: Vec> = maybe_dead + let dead: Vec = maybe_dead .into_iter() - .filter(|h| h.color.load(Ordering::Acquire) == color::White) + .filter(|&i| cands[i].is_white()) .collect(); let collected = dead.len(); + // Temporarily promoted iterators and immutable containers that ended + // up white are cyclic garbage too, freed by refcount once the cycle's + // mutable anchor is cleared below. CPython counts each (`test_tuple` + // asserts the closing tuple is counted alongside its list). + let reported = collected + + cands[n_real..] + .iter() + .filter(|c| { + c.is_white() + && matches!( + c.obj, + Object::Iter(_) | Object::Tuple(_) | Object::FrozenSet(_) + ) + }) + .count(); - // Temporarily-promoted iterators / immutable containers (tuple, - // frozenset) that ended up White are genuine cyclic garbage: they'll be - // freed by refcount the moment the mutable anchor in their cycle is - // cleared just below. CPython counts each in the `gc.collect()` total - // (`test_tuple` asserts the closing tuple is counted alongside its - // list), so fold the dead real-object temporaries into the *reported* - // count. The private list buffers an iterator snapshots have no CPython - // counterpart, so they don't count; and none of these were ever in - // `tracked_count`, so that bookkeeping uses `collected` (real) below. - let mut reported = collected; - for h in &temp_handles { - if h.color.load(Ordering::Acquire) == color::White - && matches!( - h.object, - Object::Iter(_) | Object::Tuple(_) | Object::FrozenSet(_) - ) - { - reported += 1; - } - } - - // 5a: weakrefs for the unreachable set were already cleared above (the - // CPython `handle_weakrefs`-before-`finalize_garbage` ordering), so the - // `dead` objects' weakrefs are gone and their callbacks are already - // queued in `weakref_callbacks` for invocation in 5d. - - // 5b (RFC 0039 WS5): before tearing the dead objects down, record the - // children they referenced *outside* this collection's candidate set. - // These seed the older-generation refcount cascade in 5c2; they must - // be captured here, while the dead objects' fields are still intact. - let saveall = self.debug.load(Ordering::Acquire) & DEBUG_SAVEALL != 0; - let mut cascade_seed: Vec = Vec::new(); - if !saveall { - for h in &dead { - traverse_object(&h.object, &mut |child| { - cascade_seed.push(id_of(child)); - }); - } - } - - // 5c: break cycles by clearing the reclaimed objects' fields — or, - // under `gc.DEBUG_SAVEALL`, park them in `gc.garbage` intact for - // inspection instead of tearing them down. - if saveall { + // Break the cycles by clearing the reclaimed objects' fields — or, + // under `gc.DEBUG_SAVEALL`, park them in `gc.garbage` intact. + if self.debug.load(Ordering::Acquire) & DEBUG_SAVEALL != 0 { let mut garbage = self.garbage.borrow_mut(); - for h in &dead { - garbage.push(h.object.clone()); + for &i in &dead { + garbage.push(cands[i].obj.clone()); } } else { - // Instances whose `__dict__` is shared are deferred; once - // every other dead object has released its references, a - // dict held only by dead holders is down to one owner and - // the retry clears it (a live holder keeps it intact). - let mut shared_dict_holders: Vec<&HandleRc> = Vec::new(); - for h in &dead { - if !clear_object_fields(&h.object) { - shared_dict_holders.push(h); + // Instances whose `__dict__` is shared are deferred; once every + // other dead object has released its references, a dict held + // only by dead holders is down to one owner and the retry + // clears it (a live holder keeps it intact). + let mut shared_dict_holders: Vec = Vec::new(); + for &i in &dead { + if !clear_object_fields(&cands[i].obj) { + shared_dict_holders.push(i); } } - for h in shared_dict_holders { - clear_object_fields(&h.object); - } - } - - // 5c2 (RFC 0039 WS5): cascade refcount-reclamation into *older* - // generations the current pass didn't scan. CPython frees an object - // the instant its refcount hits zero, regardless of generation: - // clearing a young cyclic-garbage object (`c1`) drops the last - // reference to an old object (`c0`) it pointed at, which frees `c0` - // and fires `c0`'s weakref callback — even during a young-only - // collection (`test_gc` `test_bug1055820c`). Our tracked handle pins - // such an object, so the refcount never reaches zero on its own; - // emulate the cascade explicitly. Starting from the children the now - // cleared dead objects referenced (captured in 5b), reap any tracked - // object that (a) isn't part of this collection's candidate set (those - // are handled by the normal mark/rebuild) and (b) is now reachable only - // through its own tracked handle and weakref slots, firing its weakref - // callbacks and recursing into its children. Finalizable orphans are - // left for a finalizing collection so `__del__` ordering is preserved. - if !saveall { - let dead_ids: IdSet = dead.iter().map(|h| h.id).collect(); - let mut worklist = cascade_seed; - let mut seen: IdSet = IdSet::default(); - while let Some(cid) = worklist.pop() { - if dead_ids.contains(&cid) || by_id.contains_key(&cid) || !seen.insert(cid) { - // Dead (already reaped), a candidate this collection owns, - // or already visited — skip. - continue; - } - let Some(h) = self.index.borrow().get(&cid).cloned() else { - continue; - }; - // Leave finalizable objects to a finalizing collection. - if has_finalizer(&h.object) { - continue; - } - let weak_clones = crate::weakref_registry::strong_clone_count(cid); - let effective = strong_count_for(&h.object) - .saturating_sub(1) - .saturating_sub(weak_clones); - if effective != 0 { - // Still reachable from a survivor — keep it. But the - // extra references may be Rust-side transients rather - // than heap edges: an *automatic* collection can land - // mid-teardown (asyncio cancellation), while the task - // machinery still holds in-flight clones of a dying - // child. The collection consumes the dead subgraph, so - // the eval-loop cascade never walks it and the child is - // pinned by its own handle forever once the transients - // drop. Mirror that cascade's slack: enroll borderline - // survivors for the suspect re-probe so they still die - // at CPython's refcount timing (test_ssl - // test_handshake_timeout_handler_leak with a mid-run - // gen-0 collection). - if effective <= 3 { - note_suspect(h.clone()); - } - continue; - } - // Orphaned: fire its weakref callbacks (queued in 5d below), - // capture its children for the cascade, tear it down, and drop - // it from the tracked set. The orphan joins the trash set - // first so a weakref *wrapper* dying in this cascade never - // fires its own callback (CPython `handle_weakrefs` parity). - trash_ids.insert(cid); - for (slot, cb) in crate::weakref_registry::notify_clear(cid) { - if let Some(cb) = cb { - weakref_callbacks.push((slot, cb)); - } - } - traverse_object(&h.object, &mut |child| { - worklist.push(id_of(child)); - }); - clear_object_fields(&h.object); - self.untrack_id(cid); + for i in shared_dict_holders { + clear_object_fields(&cands[i].obj); } } - // 5d: queue weakref callbacks (after finalisers and cyclic - // clears, matching CPython's order). The interpreter drains - // the queue at its next safe point — the GC layer can't call - // Python itself. Wrappers that turned out to be trash (including - // cascade orphans discovered after their callbacks were queued) - // are dropped here. - for (slot, cb) in weakref_callbacks { - if wrapper_is_trash(&slot, &trash_ids) { - continue; - } - let wr = slot - .py_ref + // Queue the weakref callbacks (after finalisers and cyclic clears, + // matching CPython's order) for the interpreter's next safe point. + queue_weakref_callbacks(weakref_callbacks, |slot| { + slot.py_ref .borrow() .as_ref() .and_then(crate::sync::Weak::upgrade) - .map(crate::object::Object::Instance); - if let Some(wr) = wr { - crate::vm_singletons::push_pending_weakref_callback(cb, wr); - } - } - - // Phase 6: rebuild the generation lists. Survivors of - // generation `g` (color != White) move to generation - // min(g+1, N_GENERATIONS-1). - self.rebuild_generations(gen, &candidate_set); - - // The rebuild dropped the dead objects' index handles *directly* - // (not via `untrack_id`), so purge any suspect-list clones in - // lock-step here — after the index borrow is released, matching - // `untrack_id`'s ordering. A stale suspect entry shares the dead - // handle's `Arc`, whose strong `object` reference would otherwise - // keep the just-collected object alive until the suspect probe - // ages it out: `test_descr.test_remove_subclass` observed a - // collected class still listed in `Parent.__subclasses__()` - // right after an explicit `gc.collect()`. - for h in &dead { - remove_suspect(h.id); - } + .is_none_or(|inst| { + trash_ids.contains(&(crate::sync::Rc::as_ptr(&inst) as usize as u64)) + }) + }); - // Adjust the population counter. + // Phase 6: rebuild the generation lists. Survivors of generation + // `g` move to generation min(g+1, N_GENERATIONS-1); the dead leave + // the index. Dropping `cands` then frees the dead by refcount. + self.rebuild_generations(gen, &cands[..n_real]); self.tracked_count.fetch_sub( collected.min(self.tracked_count.load(Ordering::Acquire)), Ordering::AcqRel, ); self.tracked_version.fetch_add(1, Ordering::AcqRel); - + drop(cands); reported } - fn snapshot_for_collection(&self, upto: usize) -> Vec> { - let gens = self.generations.borrow(); - let selected = &gens[..=upto.min(N_GENERATIONS - 1)]; - let mut out = Vec::with_capacity(selected.iter().map(|g| g.handles.len()).sum()); - for g in selected { - out.extend_from_slice(&g.handles); + /// Blacken everything reachable from the grey candidates in `grey`. + fn propagate_black(cands: &[Cand], by_id: &GcIdMap, grey: &mut Vec) { + while let Some(i) = grey.pop() { + cands[i].color.set(color::Black); + traverse_object(&cands[i].obj, &mut |child| { + if let Some(&j) = by_id.get(&id_of(child)) { + if cands[j].is_white() { + cands[j].color.set(color::Grey); + grey.push(j); + } + } + }); + } + } + + /// The live objects of generations `0..=upto` as collection candidates; + /// entries whose object died are dropped from those generations and the + /// index. + fn snapshot_for_collection(&self, upto: usize) -> Vec { + let mut index = self.index.borrow_mut(); + let mut gens = self.generations.borrow_mut(); + let mut out = Vec::new(); + let mut pruned = 0usize; + for g in gens.iter_mut().take(upto.min(N_GENERATIONS - 1) + 1) { + g.handles.retain(|h| match h.object.upgrade() { + Some(obj) => { + out.push(Cand::new(obj, h.id, Some(h.clone()))); + true + } + None => { + index.remove(&h.id); + pruned += 1; + false + } + }); + for (i, h) in g.handles.iter().enumerate() { + h.slot.store(i, Ordering::Release); + } + } + if pruned > 0 { + self.tracked_count.fetch_sub( + pruned.min(self.tracked_count.load(Ordering::Acquire)), + Ordering::AcqRel, + ); } out } - fn rebuild_generations(&self, upto: usize, candidates: &[HandleRc]) { - // Lock order MUST match `track` (index before generations): the - // collector and a mutator thread can both reach the GC under the - // shared, process-global state, and acquiring these two cells in - // opposite orders is a textbook deadlock (observed under - // `test_weakref`'s background-collector loop — RFC 0039 WS4). + fn rebuild_generations(&self, upto: usize, cands: &[Cand]) { + // Lock order MUST match `track` (index before generations). let mut index = self.index.borrow_mut(); - let mut fin = self.finalizable.borrow_mut(); let mut gens = self.generations.borrow_mut(); for g in 0..=upto.min(N_GENERATIONS - 1) { gens[g].handles.clear(); } - for h in candidates { - let color = h.color.load(Ordering::Acquire); - if color == color::White { + for c in cands { + let Some(h) = &c.entry else { continue }; + if c.is_white() { index.remove(&h.id); - // RFC 0061 (WS1b): mirror `untrack_id`'s flag so a stale - // suspect entry self-identifies as reclaimed. - h.untracked.store(true, Ordering::Release); - // Same bookkeeping as `fin_remove`, under the locks - // already held (lock order: index, finalizable, - // generations — as in `track`). - if let Some(fh) = fin.remove(&h.id) { - self.fin_remove_bookkeeping(&fh); - } continue; } let g = h.generation.load(Ordering::Acquire); let new_g = g.saturating_add(1).min(MAX_GENERATION); h.generation.store(new_g, Ordering::Release); - h.color.store(color::White, Ordering::Release); let new_g = usize::from(new_g); h.slot.store(gens[new_g].handles.len(), Ordering::Release); gens[new_g].handles.push(h.clone()); @@ -2631,6 +1611,71 @@ impl GcState { } } +/// Id → candidate position, for one collection. +type GcIdMap = std::collections::HashMap>; + +/// Whether a collection promotes `child`, an untracked node reached from a +/// candidate, to a temporary candidate (see `collect_generation`'s phase 2). +fn promotes_temporarily(child: &Object, parent_is_iter: bool, parent_is_frame: bool) -> bool { + match child { + // Iterators, immutable containers, dict views, slices, closure + // cells, tracebacks, frames, bound methods and descriptor wrappers + // are all GC-tracked by CPython, and a cycle can route through any + // of them (`obj.x = iter(set_containing_obj)`, `l = []; t = (l,); + // l.append(t)`, a recursive closure's `function -> cell -> + // function`, an exception's `__traceback__ -> frame -> f_locals`). + Object::Iter(_) + | Object::Tuple(_) + | Object::FrozenSet(_) + | Object::DictView(_) + | Object::Slice(_) + | Object::Cell(_) + | Object::Traceback(_) + | Object::Frame(_) + | Object::BoundMethod(_) + | Object::StaticMethod(_) + | Object::ClassMethod(_) + | Object::Property(_) => true, + // Lists are tracked at creation, so an *untracked* list is only an + // iterator's private snapshot buffer. + Object::List(_) => parent_is_iter, + // An untracked exception (the `group -> __context__ -> __traceback__ + // -> frame -> f_locals -> group` loop), or an instance whose + // tracking is still deferred, whose edge to its class can close a + // cycle (`A.a = A(); del A`). + Object::Instance(i) => i.cls().flags.is_exception || i.is_gc_deferred(), + // A frame's `f_locals` cache carries the frame's only edges to its + // locals. + Object::Dict(_) => parent_is_frame, + _ => false, + } +} + +/// Queue the collected weakref callbacks for the interpreter's next safe +/// point, skipping those whose wrapper `is_trash`. +fn queue_weakref_callbacks( + callbacks: Vec<( + crate::sync::Rc, + Object, + )>, + is_trash: impl Fn(&crate::sync::Rc) -> bool, +) { + for (slot, cb) in callbacks { + if is_trash(&slot) { + continue; + } + let wr = slot + .py_ref + .borrow() + .as_ref() + .and_then(crate::sync::Weak::upgrade) + .map(crate::object::Object::Instance); + if let Some(wr) = wr { + crate::vm_singletons::push_pending_weakref_callback(cb, wr); + } + } +} + /// Process-global hook counting `Object` clones held *only* by the C-API /// layer's pin caches (parked argument-pinned identity boxes, dead /// scalar/tuple pins) — infrastructure references `sys.getrefcount` must @@ -2711,6 +1756,11 @@ pub fn strong_count_for(obj: &Object) -> usize { Object::File(f) => Rc::strong_count(f), // Promoted transiently when a cycle routes through one. Object::Slice(s) => Rc::strong_count(s), + Object::Property(p) => Rc::strong_count(p), + Object::StaticMethod(m) | Object::ClassMethod(m) => Rc::strong_count(m), + Object::SlotDescriptor(d) => Rc::strong_count(d), + Object::LazyIter(l) => Rc::strong_count(l), + Object::AsyncGenAwait(a) => Rc::strong_count(a), // Leaf types — no internal refs to trace. _ => 1, } @@ -2998,7 +2048,8 @@ pub fn traverse_object(obj: &Object, visit: &mut dyn FnMut(&Object)) { } } } - if let Ok(slots) = f.slots.try_borrow() { + // (An unplanted seed holds only the shared name strings.) + if let Ok(slots) = f.slots_raw.try_borrow() { for (k, v) in slots.iter() { visit(&k.0); visit(v); @@ -3044,8 +2095,8 @@ fn run_external_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) { /// A small append-only registry of `(matches, hook)` function-pointer /// pairs with lock-free reads: hooks are registered at interpreter -/// init and polled on every prompt-reap cascade and collection walk, -/// so readers must not take a lock (nor allocate) per object. +/// init and polled on every collection walk, so readers must not take a +/// lock (nor allocate) per object. const HOOK_TABLE_CAP: usize = 16; struct HookTable { @@ -3209,9 +2260,12 @@ pub fn clear_object_fields(obj: &Object) -> bool { attrs.clear(); } } - if let Ok(mut slots) = f.slots.try_borrow_mut() { + if let Ok(mut slots) = f.slots_raw.try_borrow_mut() { slots.clear(); } + if let Ok(mut seed) = f.slot_seed.try_borrow_mut() { + seed.take(); + } } Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { // Dropping the suspended frame box breaks the cycle @@ -3262,25 +2316,10 @@ fn has_finalizer(obj: &Object) -> bool { // The class's cached `__del__` verdict (reset whenever `__del__` // or the MRO changes), not an MRO walk per tracked instance. Object::Instance(inst) => inst.cls().instances_need_finalize(), - // RFC 0065 (WS4, item 3) tried gating this on "close can run - // user code" (empty exception table ⇒ skip enrollment) so - // `yield`-loop workloads could reach the fully-quiet dispatch - // path. Measured: a decisive regression — long-lived suspended - // generators that stop enrolling *here* enroll as **active - // suspects** instead, and an active suspect vetoes the quiet - // path outright (finalizable population only demotes it to the - // still-fast fin-quiet mode). jitloop lost its entire quiet- - // path win (236ms → 314ms). Left as-is per the RFC's fallback - // clause; the split would need an active-suspect exemption - // co-designed with the suspects machinery first. // A generator that has never been started has nothing to clean - // up: `close()` on it runs no Python, and neither does its drop. - // Enrolling it cost the prompt-finalization index a pair of - // inserts and removals per generator born, which is most of what - // a short-lived generator costs. A *coroutine* still enrolls + // up: `close()` on it runs no Python. A *coroutine* still counts // unstarted — finalizing one that was never awaited emits the - // RuntimeWarning, which is exactly the observable cleanup this - // index exists to run promptly. + // RuntimeWarning. Object::Generator(g) => !g.is_finished() && !g.is_unstarted(), Object::Coroutine(g) | Object::AsyncGenerator(g) => !g.is_finished(), _ => false, @@ -3319,96 +2358,9 @@ pub fn with_state(f: impl FnOnce(&GcState) -> R) -> R { f(&GC_STATE) } -/// References to `target` held by *zombie* tracked memoryviews — views -/// whose only remaining strong reference is the registry's own handle -/// (plus weakref-slot clones). Under CPython refcounting such a view is -/// already freed, so `sys.getrefcount` must not let its exporter edge -/// inflate the exporter's count (test_memoryview's getitem/setitem tests -/// assert `getrefcount(b)` returns to baseline after a short-lived view -/// over `b` is dropped). Chains (a zombie view of a zombie view) resolve -/// iteratively. Restricted to memoryviews to keep the scan O(#views), -/// well away from `getrefcount`-hot paths like pandas'. -pub fn zombie_memoryview_refs_to(target: ObjectId) -> usize { - with_state(|s| { - let mut handles: Vec> = Vec::new(); - { - let Ok(gens) = s.generations.try_borrow() else { - return 0; - }; - for gen in gens.iter() { - for h in &gen.handles { - if matches!(h.object, Object::MemoryView(_)) { - handles.push(h.clone()); - } - } - } - } - if let Ok(frozen) = s.frozen.try_borrow() { - for h in frozen.iter() { - if matches!(h.object, Object::MemoryView(_)) { - handles.push(h.clone()); - } - } - } - if handles.is_empty() { - return 0; - } - let mut zombies: IdSet = IdSet::default(); - loop { - // Inbound references each candidate receives from the current - // zombie set (a dropped chain of sub-views keeps inner views' - // counts up via exporter edges). - let mut inbound: std::collections::HashMap< - ObjectId, - usize, - BuildHasherDefault, - > = std::collections::HashMap::default(); - for h in &handles { - if zombies.contains(&h.id) { - traverse_object(&h.object, &mut |c| { - *inbound.entry(id_of(c)).or_insert(0) += 1; - }); - } - } - let mut changed = false; - for h in &handles { - if zombies.contains(&h.id) { - continue; - } - let strong = strong_count_for(&h.object); - let weak = crate::weakref_registry::strong_clone_count(h.id); - let from_zombies = inbound.get(&h.id).copied().unwrap_or(0); - if strong - .saturating_sub(1) // the registry handle itself - .saturating_sub(weak) - .saturating_sub(from_zombies) - == 0 - { - zombies.insert(h.id); - changed = true; - } - } - if !changed { - break; - } - } - let mut n = 0usize; - for h in &handles { - if zombies.contains(&h.id) { - traverse_object(&h.object, &mut |c| { - if id_of(c) == target { - n += 1; - } - }); - } - } - n - }) -} - /// Convenience: track `obj` in the shared, process-global GC. -pub fn track(obj: Object) { - if let Object::Instance(inst) = &obj { +pub fn track(obj: &Object) { + if let Object::Instance(inst) = obj { // The instance may have been born with deferred tracking; its // `__dict__` no longer needs to guard it. inst.clear_deferred_tracking(); @@ -3427,16 +2379,40 @@ pub fn track(obj: Object) { // that never went through BuildMap) would otherwise never be a // candidate. CPython tracks every dict; we pair the tracking with // the objects that make the dict cycle-capable. - if let Object::Module(m) = &obj { + // (A young function's dict is tracked when the function is flushed.) + if let Object::Module(m) = obj { let dict = Object::Dict(m.dict.clone()); - with_state(|s| s.track(dict)); - } else if let Object::Function(f) = &obj { - let dict = Object::Dict(f.globals.clone()); - with_state(|s| s.track(dict)); + with_state(|s| s.track(&dict)); + } else if let Object::Function(f) = obj { + return with_state(|s| s.track_function(obj, f)); } with_state(|s| s.track(obj)); } +/// Track a list or dict that native code filled before publishing it (the +/// unpickler's results) as if it had been tracked at birth, while still +/// empty: it joins the deferred containers. Every collection promotes the +/// deferred containers that can anchor a cycle before its mark phase, so +/// the collector still sees every cycle. +pub fn track_built(obj: &Object) { + if !with_state(|s| s.defer_container(obj)) { + track(obj); + } +} + +/// [`track`] for a fresh list, dict or set its builder knows holds only +/// atomic values (a `list(range(..))`): such a container is deferred +/// whatever it holds, so the element scan [`container_can_cycle`] would +/// make first is skipped. +pub fn track_inert(obj: &Object) { + debug_assert!(!container_can_cycle(obj)); + with_state(|s| { + if !s.defer_container(obj) { + s.track_now(obj); + } + }); +} + /// `counter += n` (wrapping) for a collector counter: a plain load and /// store while the GIL serializes every writer, a locked read-modify-write /// only in free-threaded mode. @@ -3455,19 +2431,17 @@ fn serial_add(counter: &AtomicUsize, n: usize) { /// Deferred instance tracking. /// /// CPython tracks every instance of a Python-defined class at -/// allocation. A tracked object here costs an index entry, a strong -/// handle and, on its death, the prompt-reap cascade — several hundred -/// nanoseconds for a temporary whose dict only ever holds ints. An -/// instance that holds only atomic values ([`Object::is_gc_atomic`]) +/// allocation. A tracked object here costs an index entry and a weak +/// handle, which a temporary whose dict only ever holds ints doesn't +/// need. An instance that holds only atomic values ([`Object::is_gc_atomic`]) /// cannot take part in a cycle, so a fresh plain instance starts /// *untracked* and its `__dict__` records the instance address as its /// deferred owner. The first store that could put a non-atomic value /// into the instance — any generic dict mutation (`DictData`'s /// `DerefMut`), a slot store of a non-atomic value, a `__dict__` /// replacement — tracks it through this function; so do `gc.is_tracked` -/// and weakref creation (the registry's strong clone would otherwise -/// keep it alive with no collector to reap it). An instance whose owner -/// record is still set when it dies is freed by its last `Arc` drop. +/// and weakref creation. An instance whose owner record is still set when +/// it dies is freed by its last `Arc` drop. /// /// `owner` is the address recorded by [`crate::object::DictData`]; it is /// 0 when the record was already taken. The instance is alive: it clears @@ -3489,27 +2463,7 @@ pub fn track_deferred_owner(owner: usize) { // instance's so `is_gc_deferred` stops claiming the collector has // never seen it. inst.deferred.set(false); - track(Object::Instance(inst)); -} - -/// Track `obj` *and* enroll it in the prompt-finalization index even though -/// it has no `__del__`/weakref callback. For handle-pinned glue objects whose -/// CPython counterpart dies by refcount with observable timing — e.g. -/// `_asyncio.FutureIter`, whose strong ref on its Future would otherwise keep -/// a finished Task (and its coroutine frame, and every local in it) alive -/// until the next cyclic collection (test_ssl's weakref-based leak tests). -/// The prompt sweep sees the object with no pending finalizer and simply -/// untracks it the moment its last program reference drops, cascading the -/// frees exactly like CPython's refcounting. -pub fn track_prompt_reclaim(obj: Object) { - let id = crate::weakref_registry::id_of(&obj); - with_state(|s| { - s.track(obj); - let handle = s.index.borrow().get(&id).cloned(); - if let Some(h) = handle { - s.fin_insert(id, h); - } - }); + track(&Object::Instance(inst)); } /// True while a collection (mark/sweep or weakref-only pass) is in @@ -3600,6 +2554,10 @@ pub fn is_atomic(obj: &Object) -> bool { /// sweep's fixed cost outweighs what it reclaims. const DEFERRED_FLOOR: usize = 4096; +/// How many young instances wait before the set is handed to the +/// collector regardless (collections normally flush it well before). +const YOUNG_CAP: usize = 8192; + /// How many containers may stay deferred at once. Past this the sweep /// hands the whole set to the collector, which both bounds the weak /// references parked here and restores the allocation pacing a program @@ -3706,23 +2664,19 @@ fn container_can_cycle(obj: &Object) -> bool { pub fn track_memoryview_exporter(mv: &Object, exporter: &Object) { debug_assert!(matches!(mv, Object::MemoryView(_))); if !is_atomic(exporter) { - track(mv.clone()); + track(mv); } } pub fn track_if_cyclic(obj: &Object) -> bool { if container_can_cycle(obj) { - track(obj.clone()); + track(obj); true } else { false } } -/// Convenience: threshold-driven automatic collection on the current -/// thread's GC (see [`GcState::maybe_auto_collect`]). Returns the -/// number of objects reclaimed; the caller should drain pending -/// finalizers when this is non-zero. /// Whether `id` may be tracked: the miss-filter probe (no false /// negatives — every tracked object's bits are set at track time; a bit /// left by an untracked object is a harmless false positive). @@ -3737,12 +2691,11 @@ pub fn auto_collect_due() -> bool { with_state(GcState::auto_collect_due) } +/// Convenience: threshold-driven automatic collection on the shared GC +/// (see [`GcState::maybe_auto_collect`]). Returns whether a collection +/// ran; the caller should then drain pending finalizers. pub fn maybe_auto_collect() -> bool { - let ran = with_state(GcState::maybe_auto_collect); - if ran { - sweep_weakref_only_targets(); - } - ran + with_state(GcState::maybe_auto_collect) } /// Convenience: find a tracked handle by object id (O(1) via the @@ -3751,9 +2704,7 @@ pub fn find_handle(id: ObjectId) -> Option> { with_state(|s| s.handle_for(id)) } -/// Convenience: is `id` currently tracked by the cycle GC? Used -/// by refcount-emulation paths to discount the registry's own -/// strong handle. +/// Convenience: is `id` currently tracked by the cycle GC? pub fn is_tracked(id: ObjectId) -> bool { with_state(|s| s.is_tracked(id)) } @@ -3781,555 +2732,29 @@ pub fn complete_finalizer(id: ObjectId) { with_state(|s| s.complete_finalizer(id)); } -/// Convenience: snapshot all tracked objects with an unrun `__del__` -/// in the shared GC (see [`GcState::finalization_candidates`]). -pub fn finalization_candidates() -> Vec> { +/// Convenience: the live tracked objects with an unrun `__del__` (see +/// [`GcState::finalization_candidates`]). +pub fn finalization_candidates() -> Vec<(HandleRc, Object)> { with_state(|s| s.finalization_candidates()) } -/// Convenience: refcount-reclaim dead acyclic garbage in the shared -/// GC (see [`GcState::reap_dead_acyclic`]). -pub fn reap_dead_acyclic() -> usize { - with_state(|s| s.reap_dead_acyclic()) -} - -/// [`reap_dead_acyclic`] for the prompt-finalization drain's hot path: -/// the full-index cascade scan is O(tracked) and the drain runs it once -/// per pass that freed a finalizable. With a huge tracked population -/// shedding finalizables continuously (70k `WeakKeyDictionary` keys -/// dying one pop at a time in test_weakref's threaded-copy stress) that -/// multiplies into minutes, so over a size threshold only every N-th -/// drain pays for the scan — the skipped cascades are plain containers -/// whose reclamation the next scan (or any collection) picks up. Small -/// heaps keep CPython-like promptness (asyncio's SSL leak chains). -pub fn reap_dead_acyclic_amortized() -> usize { - const TRACKED_THRESHOLD: usize = 8192; - static STRIDE: std::sync::OnceLock = std::sync::OnceLock::new(); - let stride = *STRIDE.get_or_init(|| { - std::env::var("WEAVEPY_ACYCLIC_STRIDE") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(64) - }); - static TICK: AtomicU64 = AtomicU64::new(0); - with_state(|s| { - if s.tracked_count.load(Ordering::Relaxed) > TRACKED_THRESHOLD { - let tick = TICK.fetch_add(1, Ordering::Relaxed); - if !tick.is_multiple_of(stride) { - return 0; - } - } - s.reap_dead_acyclic() - }) -} - -/// Convenience: is any finalizable object currently tracked in the shared GC? -/// The interpreter's prompt-finalization gate (see -/// [`GcState::has_any_finalizable`]). -#[inline] -pub fn has_any_finalizable() -> bool { - with_state(GcState::has_any_finalizable) -} - /// Rebuild the tracked-id miss filter from the live tracked set (see /// [`GcState::rebuild_tracked_filter`]). pub fn rebuild_tracked_filter() { with_state(GcState::rebuild_tracked_filter); } -/// Prompt-reap *suspects* (RFC 0054): tracked, non-finalizable objects a -/// [`Interpreter::reap_dead_subgraph`] cascade visited but had to skip -/// because something still referenced them — typically a Rust-side -/// transient (an in-flight `PyException` clone, a native call's argument) -/// that dies a few opcodes later, *between* safe points, leaving the object -/// pinned by its own collector handle until the next full collection. -/// CPython's refcounting frees such objects the instant the transient dies; -/// re-probing the (tiny) suspect list at drop safe points recovers that -/// timing. Each entry carries a probe budget so a genuinely long-lived -/// skipped object (the event loop itself) stops costing anything after a -/// bounded number of checks. -/// -/// The canonical chain: asyncio's `wait_for` timeout leaves a -/// `CancelledError` (skipped mid-cascade while the task machinery still -/// held a clone) whose traceback pins the cancelled `create_connection` -/// frames — and through their locals the SSL transport, protocol, and -/// `SSLContext` that test_ssl's leak tests watch via weakref. -const SUSPECT_CAP: usize = 256; -/// RFC 0077 (WS2): active probes an entry gets before it goes dormant. -/// Was 64; the census showed the per-drop `retain` walk over the -/// active population as a first-order interpreter cost on object-heavy -/// fixtures. A suspect pinned by a Rust-side transient that "dies a few -/// opcodes later" is caught within 16 drop safe points; one that -/// outlives them (a live object the cascade merely couldn't prove -/// dead, or a transient held across a long native call) moves to the -/// dormant stride as before, so nothing is forgotten — it just stops -/// being re-probed at every drop. (A "reset the budget when the count -/// moves" variant was measured and rejected: live objects churn their -/// counts constantly, which kept ~65 entries permanently active on -/// `deltablue`.) -const SUSPECT_BUDGET: u8 = 16; -/// Aged-out (budget-exhausted) suspects turn *dormant* rather than being -/// forgotten: they are re-probed only every `DORMANT_STRIDE`-th sweep, so -/// a long-lived skipped object costs two atomic loads per stride instead -/// of per safe point — but an object whose last external reference dies -/// *after* its budget ran out is still recovered. (test_ssl's -/// handshake-timeout leak tests: the CancelledError web enrolls at -/// cancellation, the event loop burns >64 drop safe points before the -/// Task machinery lets go, and with eager eviction the web stayed pinned -/// by its collector handle until a full `gc.collect()`.) -const DORMANT_STRIDE: u64 = 64; -/// RFC 0077 (WS2): a *dormant* entry probed at more than this many strong -/// references above its dead line is forgotten outright. The dormant -/// stride exists for the transient-pinned shape — an object one or two -/// Rust-side clones away from death that no Python-visible drop will -/// re-notice. An entry that has outlived its active budget *and* still -/// carries several extra references is simply a live object the cascade -/// couldn't prove dead; its eventual death is re-noticed the ordinary way -/// (the holder's own `prompt_reap_dropped` cascade walks to it), so -/// keeping it enrolled only made every stride sweep pay for the whole -/// live population. `deltablue` filled the map with ~200 such entries -/// and re-probed all of them 40K times per run (the census's ~9.4M -/// suspect probes). -const SUSPECT_LIVE_MARGIN: usize = 3; -/// RFC 0077 (WS2): stride probes a dormant entry survives before it is -/// forgotten regardless of its count. The margin above can't tell "one -/// live holder forever" (deltablue's `Variable`s: one list reference -/// each, 191 of them filling the map to its cap) from "one transient -/// about to die", so dormancy is bounded in time instead: 16 active -/// probes then 16 stride probes gives every suspect ~1000 drop safe -/// points to have its transient released — an order of magnitude past -/// the test_ssl cancellation chain that motivated dormancy — after which -/// its death is left to the cascade through its holders (or a full -/// collection). Cost per enrollment is now bounded at 32 probes. -const SUSPECT_DORMANT_PROBES: u8 = 16; -/// RFC 0061 (WS1b): keyed by [`ObjectId`] so `remove_suspect` (called in -/// lock-step with every `untrack_id`) and enrollment dedup are O(1) -/// map hits instead of linear scans of the list — `remove_suspect`'s -/// `retain` was a measurable share of drop-heavy profiles (`list_ops`). -/// One enrolled suspect: its handle, remaining active probe budget, and -/// (once dormant) the number of stride probes it has survived. -struct Suspect { - handle: HandleRc, - budget: u8, - dormant_probes: u8, -} -type SuspectMap = indexmap::IndexMap>; -/// The enrolled suspects, split by phase: entries with probe budget left -/// (re-probed at every sweep) and aged-out dormant ones (re-probed only on -/// the stride), so an ordinary sweep never walks the dormant population. -#[derive(Default)] -struct Suspects { - /// Every entry has budget remaining. - active: SuspectMap, - /// Every entry's budget is spent. - dormant: SuspectMap, -} -impl Suspects { - fn len(&self) -> usize { - self.active.len() + self.dormant.len() - } - fn contains_key(&self, id: ObjectId) -> bool { - self.active.contains_key(&id) || self.dormant.contains_key(&id) - } - fn values(&self) -> impl Iterator { - self.active.values().chain(self.dormant.values()) - } -} -static SUSPECTS: std::sync::LazyLock> = - std::sync::LazyLock::new(|| parking_lot::Mutex::new(Suspects::default())); -static SUSPECT_COUNT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); -/// Entries with probe budget remaining. When only dormant entries are -/// left, [`has_suspects`] admits a sweep every [`DORMANT_STRIDE`]-th -/// safe point instead of every one. -static SUSPECT_ACTIVE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); -static SUSPECT_TICK: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); -/// RFC 0065 (WS4): insert-only miss-filter over ever-enrolled suspect -/// ids, so the per-`untrack_id` [`remove_suspect`] probe skips the map -/// lock for the overwhelming never-enrolled majority. -static SUSPECT_FILTER: crate::hot_filter::AtomicBloom = crate::hot_filter::AtomicBloom::new(); - -/// RFC 0077 (WS1): floor counters for the `WEAVEPY_VM_STATS` shutdown -/// print — how often the drop-path probes ran and what they found. -pub mod floor_stats { - use std::sync::atomic::{AtomicU64, Ordering}; - pub static SUSPECT_ENROLLED: AtomicU64 = AtomicU64::new(0); - pub static SUSPECT_SWEEPS: AtomicU64 = AtomicU64::new(0); - pub static SUSPECT_PROBED: AtomicU64 = AtomicU64::new(0); - pub static SUSPECT_DEAD: AtomicU64 = AtomicU64::new(0); - pub static SUSPECT_FORGOTTEN: AtomicU64 = AtomicU64::new(0); - pub static FIN_SCANS: AtomicU64 = AtomicU64::new(0); - pub static FIN_PROBED: AtomicU64 = AtomicU64::new(0); - pub static FIN_DEAD: AtomicU64 = AtomicU64::new(0); - pub static DROP_NOTES: AtomicU64 = AtomicU64::new(0); - pub static DROP_NOTES_MARKED: AtomicU64 = AtomicU64::new(0); - - #[inline] - pub fn bump(c: &AtomicU64, n: u64) { - if crate::specialize::stats_enabled() { - c.fetch_add(n, Ordering::Relaxed); - } - } - - /// Coarse "operand stack shrank ⇒ maybe dead" marks by opcode name - /// (stats-only; the eval loop calls this under `stats_enabled`). - static COARSE_MARKS: std::sync::LazyLock< - parking_lot::Mutex>, - > = std::sync::LazyLock::new(Default::default); - - #[cold] - pub fn coarse_mark(op: &'static str) { - *COARSE_MARKS.lock().entry(op).or_insert(0) += 1; - } - - /// Markdown block for the shutdown stats print; `None` when nothing - /// was counted. - pub fn markdown() -> Option { - let rows = [ - ( - "drop notes (audited handlers)", - DROP_NOTES.load(Ordering::Relaxed), - ), - ( - "drop notes that marked maybe-dead", - DROP_NOTES_MARKED.load(Ordering::Relaxed), - ), - ( - "suspects enrolled", - SUSPECT_ENROLLED.load(Ordering::Relaxed), - ), - ("suspect sweeps", SUSPECT_SWEEPS.load(Ordering::Relaxed)), - ( - "suspect entries probed", - SUSPECT_PROBED.load(Ordering::Relaxed), - ), - ("suspects found dead", SUSPECT_DEAD.load(Ordering::Relaxed)), - ( - "dormant suspects forgotten (live)", - SUSPECT_FORGOTTEN.load(Ordering::Relaxed), - ), - ("finalizable scans", FIN_SCANS.load(Ordering::Relaxed)), - ( - "finalizable entries probed", - FIN_PROBED.load(Ordering::Relaxed), - ), - ("finalizables found dead", FIN_DEAD.load(Ordering::Relaxed)), - ]; - if rows.iter().all(|(_, v)| *v == 0) { - return None; - } - let mut out = - String::from("\n## Drop-path floor (RFC 0077 WS2)\n\n| counter | value |\n|---|---|\n"); - for (name, v) in rows { - out.push_str(&format!("| {name} | {v} |\n")); - } - // Residual populations at exit: what the per-drop scans were - // actually walking. Type name, strong count above the dead line. - let mut census = |title: &str, rows: Vec<(String, usize)>| { - if rows.is_empty() { - return; - } - out.push_str(&format!("\n{title} at exit ({}):\n", rows.len())); - let mut grouped: std::collections::BTreeMap> = - std::collections::BTreeMap::new(); - for (tn, excess) in rows { - grouped.entry(tn).or_default().push(excess); - } - for (tn, mut ex) in grouped { - ex.sort_unstable(); - let shown: Vec = ex.iter().take(8).map(|e| e.to_string()).collect(); - out.push_str(&format!( - "- {tn} x{} (excess refs: {}{})\n", - ex.len(), - shown.join(","), - if ex.len() > 8 { ",…" } else { "" } - )); - } - }; - census("hot finalizables", super::residual_hot_finalizables()); - census("enrolled suspects", super::residual_suspects()); - let marks = COARSE_MARKS.lock(); - if !marks.is_empty() { - let mut by_count: Vec<(&&str, &u64)> = marks.iter().collect(); - by_count.sort_by(|a, b| b.1.cmp(a.1)); - out.push_str("\ncoarse maybe-dead marks by opcode:\n"); - for (op, n) in by_count.into_iter().take(12) { - out.push_str(&format!("- {op}: {n}\n")); - } - } - Some(out) - } -} - -/// Stats-only snapshot of the hot finalizable set: `(type name, strong -/// refs above the dead line)` per entry. -fn residual_hot_finalizables() -> Vec<(String, usize)> { - with_state(|gc| { - gc.finalizable_hot - .borrow() - .iter() - .map(|h| { - let sc = strong_count_for(&h.object); - let cached = h.weak_clones.load(Ordering::Relaxed); - (h.object.type_name_owned(), sc.saturating_sub(1 + cached)) - }) - .collect() - }) -} - -/// Stats-only snapshot of the suspect map, same shape. -fn residual_suspects() -> Vec<(String, usize)> { - SUSPECTS - .lock() - .values() - .map(|e| { - let h = &e.handle; - let sc = strong_count_for(&h.object); - let cached = h.weak_clones.load(Ordering::Relaxed); - (h.object.type_name_owned(), sc.saturating_sub(1 + cached)) - }) - .collect() -} - -/// Publish the count gates after the locked suspect maps changed. -fn publish_suspect_counts(s: &Suspects) { - let active = s.active.len(); - let was_active = SUSPECT_ACTIVE.swap(active, Ordering::AcqRel) > 0; - let was_present = SUSPECT_COUNT.swap(s.len(), Ordering::AcqRel) > 0; - // RFC 0065 (WS1): the dispatch loops' quiet-path snapshots consult - // `active_suspects_present` / the population gates, so a - // transition of either invalidates them. Same-state churn (one - // active suspect replacing another) doesn't. - if was_active != (active > 0) || was_present != (s.len() > 0) { - crate::hot_gates::bump_loop_gen(); - } -} - -/// Evict the entry that has had the most chances to die: the oldest -/// dormant one, or with none, the first lowest-budget active one. The -/// caller holds the maps' lock and republishes the counts. -fn evict_lowest_budget_suspect(s: &mut Suspects) -> bool { - if !s.dormant.is_empty() { - s.dormant.shift_remove_index(0); - return true; - } - let Some(victim) = s - .active - .values() - .enumerate() - .min_by_key(|(_, entry)| entry.budget) - .map(|(index, _)| index) - else { - return false; - }; - s.active.swap_remove_index(victim); - true -} - -/// RFC 0065 (WS1): *active*-population probe for the dispatch loop's -/// quiet predicate, with no dormant-stride tick side effect. While an -/// active (budget-remaining) suspect is enrolled the loop must stay -/// on the full prologue so the per-drop re-probe cadence holds; -/// dormant (aged-out) entries are instead re-probed from the quiet -/// loop's GIL-checkpoint cadence via [`has_suspects`] + -/// [`take_dead_suspects`]. -#[inline] -pub fn active_suspects_present() -> bool { - SUSPECT_ACTIVE.load(Ordering::Relaxed) > 0 -} - -/// Enroll a cascade-skipped tracked object for later deadness re-probes. -/// Deduplicated; silently dropped when the list is full (the next full -/// collection reclaims it instead). -pub fn note_suspect(h: HandleRc) { - static NO_SUSPECTS: std::sync::OnceLock = std::sync::OnceLock::new(); - if *NO_SUSPECTS.get_or_init(|| std::env::var_os("WEAVEPY_NO_SUSPECTS").is_some()) { - return; - } - let mut s = SUSPECTS.lock(); - if s.contains_key(h.id) { - return; - } - // RFC 0065 (WS4): publish to the miss-filter before the enrollment - // becomes observable (we hold the map lock). - SUSPECT_FILTER.insert(h.id); - // Refresh the handle's cached weakref-clone upper bound once at - // enrollment so the re-probe loop can fast-reject live suspects with - // two atomic loads instead of a registry lookup per safe point (see - // `take_dead_suspects` — the probe runs at *every* reference-dropping - // opcode, and a hot loop that keeps re-enrolling a long-lived object - // like `statistics.kde`'s sample list would otherwise spend ~40% of - // its time in `strong_clone_count`). - h.weak_clones.store( - crate::weakref_registry::strong_clone_count(h.id), - Ordering::Release, - ); - if s.len() >= SUSPECT_CAP { - // Full: evict the most-probed entry (lowest remaining budget, - // dormant first) — it has had the most chances to die and is - // the closest to aging out anyway. Silently dropping the *new* - // suspect instead loses the one object whose last real - // reference just died (asyncio's wait_for cancellation chain - // arrived after ~200 module-teardown stragglers and was never - // re-probed, pinning the Timeout→Task→frame web the test_ssl - // leak tests watch). - if !evict_lowest_budget_suspect(&mut s) { - return; - } - } - floor_stats::bump(&floor_stats::SUSPECT_ENROLLED, 1); - s.active.insert( - h.id, - Suspect { - handle: h, - budget: SUSPECT_BUDGET, - dormant_probes: 0, - }, - ); - publish_suspect_counts(&s); -} - -/// Cheap gate for the eval loop's safe point: always sweep while an -/// *active* suspect is enrolled; with only dormant (aged-out) entries -/// left, admit every [`DORMANT_STRIDE`]-th safe point so long-lived -/// suspects cost two atomic loads per stride, not per drop. -#[inline] -pub fn has_suspects() -> bool { - if SUSPECT_ACTIVE.load(Ordering::Relaxed) > 0 { - return true; - } - if SUSPECT_COUNT.load(Ordering::Relaxed) == 0 { - return false; - } - SUSPECT_TICK - .fetch_add(1, Ordering::Relaxed) - .is_multiple_of(DORMANT_STRIDE) -} - -/// Drop the suspect entry for `id`, if any. Called in lock-step with -/// `untrack_id`: a suspect's `Arc` is a *clone of the -/// index's handle*, so removing the object from the index alone leaves -/// the handle — and its strong `object` reference — alive in the suspect -/// list until the next re-probe. That stale strong clone pins an object -/// the prompt-reap cascade just untracked and expected to free by `Rc` -/// drop, deferring everything it anchors (`unittest`'s -/// `_AssertRaisesContext` → stored exception → `AttributeError.obj` io -/// temporary whose `close()` must fire at `with`-exit — -/// `test_io.test_error_through_destructor`) to a later safe point. -pub fn remove_suspect(id: ObjectId) { - if SUSPECT_COUNT.load(Ordering::Relaxed) == 0 { - return; - } - // RFC 0065 (WS4): with a resident (typically dormant) suspect - // population this is called for *every* `untrack_id` — probe the - // insert-only filter before taking the map lock; a never-enrolled - // id answers with two relaxed loads. - if !SUSPECT_FILTER.may_contain(id) { - return; - } - let mut s = SUSPECTS.lock(); - let removed = s.active.swap_remove(&id).is_some() || s.dormant.swap_remove(&id).is_some(); - if removed { - publish_suspect_counts(&s); - } -} - -/// Re-probe the suspect list: return the objects that are now dead in -/// refcount terms (nothing beyond the GC handle and weakref strong clones -/// holds them) for the caller to run through the prompt-reap cascade, and -/// decay the probe budget of the rest. -pub fn take_dead_suspects() -> Vec { - let mut out = Vec::new(); - let mut s = SUSPECTS.lock(); - // With only dormant entries left, `has_suspects` already stride-gated - // this sweep; with actives present the stride ticks here instead. - let probe_dormant = s.active.is_empty() - || SUSPECT_TICK - .fetch_add(1, Ordering::Relaxed) - .is_multiple_of(DORMANT_STRIDE); - floor_stats::bump(&floor_stats::SUSPECT_SWEEPS, 1); - let mut probed = 0u64; - // A suspect whose last reference is gone (beyond the handle and any - // weakref strong clones) goes to `out`. Returns whether it stays. - let mut dead = |e: &Suspect, out: &mut Vec| -> Option { - probed += 1; - let h = &e.handle; - // RFC 0061 (WS1b): the handle self-identifies as reclaimed (set - // in lock-step with every index removal) — no registry lookup. - if h.untracked.load(Ordering::Acquire) { - return None; // already reclaimed elsewhere - } - // Fast reject via the cached weakref-clone upper bound (refreshed - // at enrollment): more strong refs than the handle plus every - // possible weakref clone ⇒ a program reference is still live, no - // registry lookup needed. Only borderline counts pay for the - // exact `strong_clone_count`. - let sc = strong_count_for(&h.object); - let cached = h.weak_clones.load(Ordering::Acquire); - if sc <= 1 + cached { - let weak = crate::weakref_registry::strong_clone_count(h.id); - if sc <= 1 + weak { - out.push(h.object.clone()); - return None; - } - } - Some(sc.saturating_sub(1 + cached)) - }; - // Active entries spend a probe; one whose budget runs out turns - // dormant after this sweep (it isn't stride-probed in the same one). - let mut aged = Vec::new(); - s.active.retain(|&id, e| { - if dead(e, &mut out).is_none() { - return false; - } - e.budget -= 1; - if e.budget == 0 { - aged.push(( - id, - Suspect { - handle: e.handle.clone(), - budget: 0, - dormant_probes: e.dormant_probes, - }, - )); - return false; - } - true - }); - if probe_dormant { - // A dormant entry still well above its dead line, or one that has - // sat through its dormant allowance, is plainly alive: drop it - // from the map (see `SUSPECT_LIVE_MARGIN` / `SUSPECT_DORMANT_PROBES`). - s.dormant.retain(|_, e| { - let Some(excess) = dead(e, &mut out) else { - return false; - }; - e.dormant_probes = e.dormant_probes.saturating_add(1); - if excess > SUSPECT_LIVE_MARGIN || e.dormant_probes > SUSPECT_DORMANT_PROBES { - floor_stats::bump(&floor_stats::SUSPECT_FORGOTTEN, 1); - return false; - } - true - }); - } - s.dormant.extend(aged); - floor_stats::bump(&floor_stats::SUSPECT_PROBED, probed); - floor_stats::bump(&floor_stats::SUSPECT_DEAD, out.len() as u64); - publish_suspect_counts(&s); - out -} - thread_local! { - /// Set whenever the interpreter executes an opcode that may have dropped - /// the last reference to an object (a `POP_*`/`STORE_*`/`DELETE_*`, a - /// frame teardown, …). The eval loop only runs a prompt-finalization sweep - /// when this is set *and* a finalizable object is live, so a hot loop that - /// never drops a reference pays nothing beyond the gate's atomic load. - /// Thread-local because each thread reclaims the objects whose last - /// reference *it* dropped, matching CPython's per-thread `tp_dealloc`. + /// Set when work for the interpreter's next safe point was queued on + /// this thread: a finalizer resurrected by a dying object's `Drop`, a + /// weakref callback, an unclosed resource's warning. The dispatch + /// loops poll it between instructions, so the work runs before the + /// instruction after the one that dropped the object — CPython runs + /// it inside that instruction's `Py_DECREF`. static MAYBE_DEAD: std::cell::Cell = const { std::cell::Cell::new(false) }; } -/// A pointer to the calling thread's maybe-dead flag, for a dispatch +/// A pointer to the calling thread's pending-work flag, for a dispatch /// loop that polls it after every instruction: one thread-local lookup /// per loop entry instead of one per poll. Valid for the life of the /// calling thread; only ever dereferenced on that thread. @@ -4338,341 +2763,38 @@ pub(crate) fn maybe_dead_flag() -> *const std::cell::Cell { MAYBE_DEAD.with(std::ptr::from_ref) } -/// Note that the current thread just executed a reference-dropping opcode, so a -/// finalizable object may now be dead. Cheap (a thread-local `Cell` store); the -/// actual sweep is deferred to the next eval-loop safe point. +/// Note that the current thread queued work for its next safe point. +/// Teardown-safe: a thread whose locals are gone has no next safe point. #[inline] pub fn mark_maybe_dead() { - MAYBE_DEAD.with(|c| c.set(true)); -} - -/// RFC 0077 (WS2): set by [`mark_bulk_drop`], consumed by the next -/// finalizable scan, which then re-grades the *whole* index (a forced -/// cold tick) instead of only the hot set. -static FORCE_COLD_TICK: AtomicBool = AtomicBool::new(false); - -/// Note that the current thread just released many references at once -/// without grading them individually: a frame torn down by a -/// propagating exception. Its locals and operands drop wholesale, and -/// any of them may have been the last holder of a finalizable that the -/// hot/cold gate had already demoted to cold (a coroutine created, -/// passed to a call that raised, and never awaited: its count was -/// stable across the exception's construction). The ordinary hot-set -/// scan would find it only on the cold stride, up to `FIN_COLD_STRIDE` -/// safe points later, which is past the `with assertWarns(...)` block -/// that expects the "never awaited" warning (test_asyncio's -/// `test_run_until_complete_nesting`). Exceptional unwinds are rare -/// relative to drop safe points, so forcing a full re-grade here keeps -/// promptness without giving back the steady-state gate. -#[inline] -pub fn mark_bulk_drop() { - MAYBE_DEAD.with(|c| c.set(true)); - FORCE_COLD_TICK.store(true, Ordering::Relaxed); -} - -/// Consume the forced-cold-tick request (see [`mark_bulk_drop`]). -#[inline] -pub(crate) fn take_force_cold_tick() -> bool { - FORCE_COLD_TICK.load(Ordering::Relaxed) && FORCE_COLD_TICK.swap(false, Ordering::Relaxed) -} - -/// Precise per-value drop note (RFC 0059 WS1b). Called by the audited -/// opcode handlers (`POP_TOP`, `STORE_FAST`, the generic `BINARY_OP` / -/// `COMPARE_OP` paths) with each value they discard; the eval loop -/// exempts those opcodes from its coarse "operand stack shrank ⇒ maybe -/// dead" heuristic in return. -/// -/// A **pure leaf** — a variant that owns no [`Object`]s and whose -/// instances can never carry a finalizer (subclass instances live in -/// `Object::Instance`, never in the scalar variants) — cannot make -/// anything finalizable unreachable when dropped, so it schedules no -/// sweep. This is what keeps an int-accumulating hot loop -/// (`total = total + i`, the RFC 0059 `nested_loops` profile) from -/// paying a `reap_dead_finalizable` scan per iteration whenever *some* -/// `__del__`-bearing object is alive anywhere in the process. -/// Everything else is graded by its strong count (RFC 0077 WS2). The -/// drop about to happen decrements `obj`'s count by one and, only if -/// that was the last reference, frees `obj` and cascades into whatever -/// it held. So a tracked finalizable or suspect can become dead here in -/// exactly two ways: -/// -/// * `obj` is about to be freed (`strong_count == 1`): the cascade may -/// release the last program reference to anything — mark. -/// * `obj` itself is tracked and this drop takes it to handle-only -/// (`strong_count == 2`), or `obj` is tracked and weakref clones make -/// the dead line unknowable cheaply — mark. -/// -/// A drop of an untracked object that survives it (`strong_count >= 2`, -/// e.g. `self` rebound in a local, a shared list popped off the stack) -/// changes nothing observable and schedules no sweep. The filters are -/// insert-only Bloom bits (false positives take the conservative -/// path), and the sweep itself refcount-checks, so a false positive -/// costs one scan, never correctness. -#[inline] -pub fn note_dropped(obj: &crate::object::Object) { - if note_dropped_marks(obj) { - mark_maybe_dead(); - } -} - -/// The grading half of [`note_dropped`]: would dropping `obj` right now -/// schedule a prompt-finalization sweep? Split out so the leaf burst -/// (`Interpreter::leaf_burst`) can end its run at exactly the -/// instruction that marks, handing the very next safe point back to the -/// full eval-loop prologue. -#[inline] -pub fn note_dropped_marks(obj: &crate::object::Object) -> bool { - use crate::object::Object as O; - match obj { - O::None - | O::Unbound - | O::Bool(_) - | O::Int(_) - | O::Long(_) - | O::Float(_) - | O::Complex(_) - | O::Str(_) - | O::WStr(_) - | O::Bytes(_) - | O::Range(_) - | O::Code(_) => false, - // The common heap operands, with their count and identity read - // inline (no variant dispatch). - // A natively served class's instance (never tracked, no - // finalizer, only ints and a `tzinfo` inside) cannot start a - // finalization when it dies. - O::Instance(i) if i.cls_raw().native_kind.get() != 0 => false, - O::Instance(i) => note_dropped_counted( - crate::sync::Rc::strong_count(i), - crate::sync::Rc::as_ptr(i) as usize as u64, - ), - O::List(l) => { - let (sc, id) = ( - crate::sync::Rc::strong_count(l), - crate::sync::Rc::as_ptr(l) as usize as u64, - ); - if inert_death(sc, id, || { - l.try_borrow() - .is_ok_and(|v| v.len() <= INERT_SCAN_CAP && v.iter().all(is_atomic)) - }) { - return false; - } - note_dropped_counted(sc, id) - } - O::Dict(d) => { - let (sc, id) = ( - crate::sync::Rc::strong_count(d), - crate::sync::Rc::as_ptr(d) as usize as u64, - ); - if inert_death(sc, id, || { - d.try_borrow().is_ok_and(|m| { - m.len() <= INERT_SCAN_CAP - && m.iter().all(|(k, v)| is_atomic(&k.0) && is_atomic(v)) - }) - }) { - return false; - } - note_dropped_counted(sc, id) - } - O::Function(f) => note_dropped_counted( - crate::sync::Rc::strong_count(f), - crate::sync::Rc::as_ptr(f) as usize as u64, - ), - O::Type(t) => note_dropped_counted( - crate::sync::Rc::strong_count(t), - crate::sync::Rc::as_ptr(t) as usize as u64, - ), - O::Tuple(t) => note_dropped_counted( - ThinArc::strong_count(t), - ThinArc::as_ptr(t).cast::<()>() as usize as u64, - ), - _ => note_dropped_marks_other(obj), - } -} - -/// Does dropping this reference to `obj` leave an instance that others -/// still hold, with no weakref watching it? Past two owners only a -/// weakref-watched object can be at its dead line (see -/// [`note_dropped_counted`]), so such a drop needs neither a prompt reap -/// nor a mark: the cheap test ahead of the full grades. -#[inline(always)] -pub fn drop_survives_plainly(obj: &crate::object::Object) -> bool { - match obj { - crate::object::Object::Instance(i) => { - crate::sync::Rc::strong_count(i) > 2 - && !crate::weakref_registry::may_have_weakrefs( - crate::sync::Rc::as_ptr(i) as usize as u64 - ) - } - _ => false, - } -} - -/// How many elements a dying container may hold and still be graded by -/// inspection (see [`inert_death`]). Capped so the grade stays O(1) on a -/// path that runs for every discarded heap value. -const INERT_SCAN_CAP: usize = 32; - -/// Does this death need no prompt handling at all? A container dying at -/// its last reference, with no collector handle and no weakref, whose -/// every element is atomic, frees nothing that could finalize — so the -/// burst need not end to run a sweep that would find nothing. Grading it -/// `true` is what keeps a `x = [i, i]` loop inside the burst: the store -/// that displaces the previous list ended it on every iteration. -#[inline] -fn inert_death(sc: usize, id: ObjectId, all_atomic: impl FnOnce() -> bool) -> bool { - sc <= 1 - && !process_tracked_filter().may_contain(id) - && !crate::weakref_registry::may_have_weakrefs(id) - && all_atomic() -} - -/// [`inert_death`] for a value the caller still holds the last reference -/// to: dropping it frees nothing that could finalize, clear a weakref or -/// need untracking, so the plain `Rc` drop is its whole teardown and no -/// prompt-reap cascade has to run. -pub fn dies_inert(obj: &crate::object::Object) -> bool { - use crate::object::Object as O; - match obj { - O::List(l) => inert_death( - crate::sync::Rc::strong_count(l), - crate::sync::Rc::as_ptr(l) as usize as u64, - || { - l.try_borrow() - .is_ok_and(|v| v.len() <= INERT_SCAN_CAP && v.iter().all(is_atomic)) - }, - ), - O::Dict(d) => inert_death( - crate::sync::Rc::strong_count(d), - crate::sync::Rc::as_ptr(d) as usize as u64, - || { - d.try_borrow().is_ok_and(|m| { - m.len() <= INERT_SCAN_CAP - && m.iter().all(|(k, v)| is_atomic(&k.0) && is_atomic(v)) - }) - }, - ), - _ => false, - } -} - -/// The grade for an object with strong count `sc` and identity `id` -/// (the identity is the one [`crate::weakref_registry::id_of`] yields). -#[inline(always)] -fn note_dropped_counted(sc: usize, id: ObjectId) -> bool { - floor_stats::bump(&floor_stats::DROP_NOTES, 1); - // A tracked object dies here when only its collector handle (and any - // registry-held weakref clones) remain beside the dropped reference. - // Probed cheapest-first: past two references only a weakref-watched - // object can be at its dead line, and almost none are. - let marked = if sc <= 1 { - true - } else if sc == 2 { - process_tracked_filter().may_contain(id) - } else { - crate::weakref_registry::may_have_weakrefs(id) - && process_tracked_filter().may_contain(id) - && sc <= 2 + crate::weakref_registry::strong_clone_count(id) - }; - if marked { - floor_stats::bump(&floor_stats::DROP_NOTES_MARKED, 1); - } - marked -} - -#[inline(never)] -fn note_dropped_marks_other(obj: &crate::object::Object) -> bool { - use crate::object::Object as O; - match obj { - // A dying bound method (the `obj.m(...)` call temporary, freed at - // every method call) owns exactly one object: its receiver. Grade - // that instead of marking on the method's own count of one. - O::BoundMethod(bm) if crate::sync::Rc::strong_count(bm) <= 1 => { - note_dropped_marks(&bm.receiver) - } - _ => { - let sc = strong_count_for(obj); - let id = crate::weakref_registry::id_of(obj); - note_dropped_counted(sc, id) - } - } + let _ = MAYBE_DEAD.try_with(|c| c.set(true)); } -/// Consume the "a reference may have dropped" flag, returning whether it was -/// set. The eval loop calls this to decide whether a prompt-finalization sweep -/// is warranted this instruction. +/// Consume the pending-work flag, returning whether it was set. #[inline] pub fn take_maybe_dead() -> bool { MAYBE_DEAD.with(|c| c.replace(false)) } -/// Convenience: drive one prompt-finalization pass over refcount-dead -/// finalizable objects in the shared GC (see -/// [`GcState::reap_dead_finalizable`]). -pub fn reap_dead_finalizable() -> usize { - with_state(|s| s.reap_dead_finalizable()) -} - -/// Convenience: enroll a tracked object in the prompt-finalization index -/// because a callback-bearing weakref now watches it (see -/// [`GcState::note_weakref_finalizable`]). -pub fn note_weakref_finalizable(id: ObjectId) { - with_state(|s| s.note_weakref_finalizable(id)); -} - /// Convenience: run a full collection on the shared GC. Returns the /// number of objects collected. pub fn collect_all() -> usize { - let n = with_state(|s| s.collect(N_GENERATIONS - 1)); - sweep_weakref_only_targets(); - n + with_state(|s| s.collect(N_GENERATIONS - 1)) } /// Convenience: run a partial collection of generations /// `0..=upto`. pub fn collect_upto(upto: usize) -> usize { - let n = with_state(|s| s.collect(upto)); - sweep_weakref_only_targets(); - n + with_state(|s| s.collect(upto)) } /// Convenience: fire dead objects' weakref callbacks via a non-destructive -/// mark pass on the shared GC (see [`GcState::fire_dead_weakrefs`]), then -/// sweep the untracked weakref-only targets. Used from a blocking -/// `Thread.join` to unblock idle `ThreadPoolExecutor` workers without the -/// teardown risk of a full collection. +/// mark pass on the shared GC (see [`GcState::fire_dead_weakrefs`]). Used +/// from a blocking `Thread.join` to unblock idle `ThreadPoolExecutor` +/// workers without the teardown risk of a full collection. pub fn fire_dead_weakrefs() { with_state(|s| s.fire_dead_weakrefs()); - sweep_weakref_only_targets(); -} - -/// Clear weakrefs whose referent isn't in the tracked set and whose -/// only remaining strong references are the weakref slots' own -/// clones. Covers weakref-able objects the cycle collector never -/// sees — plain functions, bound methods, types — so -/// `del f; gc.collect()` flips `weakref.ref(f)()` to `None` exactly -/// like CPython's refcount-driven `tp_dealloc` would. -pub fn sweep_weakref_only_targets() -> usize { - let targets = - with_state(|s| crate::weakref_registry::with_registry(|r| s.untracked_weakref_targets(r))); - let mut cleared = 0; - for (id, target) in targets { - if is_tracked(id) { - // Tracked objects belong to the cycle pass (their handle - // holds an extra strong ref this arithmetic doesn't model). - continue; - } - let clones = crate::weakref_registry::strong_clone_count(id); - // `target` itself is one clone we hold for the probe. - if strong_count_for(&target) <= clones + 1 { - crate::weakref_registry::queue_callbacks(crate::weakref_registry::notify_clear(id)); - cleared += 1; - } - } - if cleared > 0 { - crate::weakref_registry::with_registry(|r| r.shrink()); - } - cleared + crate::weakref_registry::sweep_dead_targets(); } #[cfg(test)] @@ -4683,104 +2805,6 @@ mod tests { use crate::object::DictData; - #[test] - fn suspect_eviction_takes_dormant_then_lowest_budget_and_releases() { - let mut suspects = Suspects::default(); - let mut handles = Vec::new(); - for index in 0..8usize { - let object = Object::List(Rc::new(RefCell::new(Vec::new()))); - let handle = HandleRc::new(TrackedHandle::new(object, 0)); - handles.push((handle.id, HandleRc::downgrade(&handle))); - let budget = [0, 5, 0, 3, 9, 3, 0, 1][index]; - let entry = Suspect { - handle, - budget, - dormant_probes: 0, - }; - if budget == 0 { - suspects.dormant.insert(handles[index].0, entry); - } else { - suspects.active.insert(handles[index].0, entry); - } - } - // Dormant entries go first, oldest first. - for expected in [0, 2, 6] { - assert!(evict_lowest_budget_suspect(&mut suspects)); - assert!(!suspects.contains_key(handles[expected].0)); - assert!(handles[expected].1.upgrade().is_none()); - } - // Then active ones by budget: 1, the first 3, the other 3, 5, 9. - for expected in [7, 3, 5, 1, 4] { - assert!(evict_lowest_budget_suspect(&mut suspects)); - assert!(!suspects.contains_key(handles[expected].0)); - assert!(handles[expected].1.upgrade().is_none()); - } - assert_eq!(suspects.len(), 0); - assert!(!evict_lowest_budget_suspect(&mut suspects)); - } - - #[test] - fn weakref_snapshot_skips_tracked_payloads_and_observes_transitions() { - use crate::weakref_registry::{kind, WeakRefRegistry, WeakRefSlot}; - - let state = GcState::new(); - let registry = WeakRefRegistry::new(); - let roots: Vec<_> = (0..3) - .map(|_| Object::List(Rc::new(RefCell::new(Vec::new())))) - .collect(); - let slots: Vec<_> = roots - .iter() - .map(|target| { - let slot = Rc::new(WeakRefSlot::new( - id_of(target), - target.clone(), - false, - kind::REF, - )); - registry.register(slot.clone()); - slot - }) - .collect(); - state.track_now(roots[0].clone()); - // The excluded payload cannot even be borrowed. This catches a - // regression to cloning every target and filtering afterwards. - let excluded = slots[0].target.borrow_mut(); - let before = strong_count_for(&roots[0]); - let snapshot = state.untracked_weakref_targets(®istry); - assert_eq!(snapshot.len(), 2); - assert_eq!(strong_count_for(&roots[0]), before); - for (id, target) in &snapshot { - assert_eq!(*id, id_of(target)); - assert!(!target.is_same(&roots[0])); - } - drop(snapshot); - drop(excluded); - - state.track_now(roots[1].clone()); - let snapshot = state.untracked_weakref_targets(®istry); - assert_eq!(snapshot.len(), 1); - assert!(snapshot[0].1.is_same(&roots[2])); - drop(snapshot); - state.untrack_id(id_of(&roots[0])); - state.untrack_id(id_of(&roots[1])); - assert_eq!(state.untracked_weakref_targets(®istry).len(), 3); - - slots[1].clear(); - let snapshot = state.untracked_weakref_targets(®istry); - assert_eq!(snapshot.len(), 2); - assert!(snapshot - .iter() - .all(|(_, target)| !target.is_same(&roots[1]))); - // A returned snapshot owns its targets independently of the slots. - for slot in &slots { - slot.clear(); - } - assert!(state.untracked_weakref_targets(®istry).is_empty()); - assert!(snapshot.iter().all(|(id, target)| *id == id_of(target))); - drop(slots); - assert!(registry.targets().is_empty()); - } - #[test] fn compact_positions_keep_absent_and_uncached_states_distinct() { for value in [0, 1, (u32::MAX - 2) as usize, usize::MAX] { @@ -4801,7 +2825,7 @@ mod tests { { let hint = CachedSlot::new(u32::MAX as usize + 100); assert_eq!(hint.load(Ordering::Acquire), usize::MAX - 1); - assert_eq!(std::mem::size_of::(), 64); + assert_eq!(std::mem::size_of::(), 40); } } @@ -4813,7 +2837,7 @@ mod tests { .map(|_| Object::Dict(Rc::new(RefCell::new(DictData::default())))) .collect(); for root in &roots { - state.track_now(root.clone()); + state.track_now(root); } if frozen { state.freeze_all(); @@ -4837,105 +2861,6 @@ mod tests { } } - #[test] - fn uncacheable_hot_positions_preserve_membership_and_move_fixups() { - let state = GcState::new(); - let roots: Vec<_> = (0..3) - .map(|_| Object::Dict(Rc::new(RefCell::new(DictData::default())))) - .collect(); - let handles: Vec<_> = roots - .iter() - .map(|root| { - state.track_now(root.clone()); - state.handle_for(id_of(root)).unwrap() - }) - .collect(); - for handle in &handles { - assert!(state.fin_insert(handle.id, handle.clone())); - } - let middle = &handles[1]; - middle - .fin_hot_slot - .store((u32::MAX - 1) as usize, Ordering::Release); - state.fin_make_hot(middle); - assert_eq!(state.finalizable_hot.borrow().len(), 3); - state.fin_make_cold(middle); - assert_eq!(state.finalizable_hot.borrow().len(), 2); - assert_eq!(handles[2].fin_hot_slot.load(Ordering::Acquire), 1); - assert!(middle.fin_cold.load(Ordering::Acquire)); - assert_eq!(middle.fin_hot_slot.load(Ordering::Acquire), usize::MAX); - state.fin_make_hot(middle); - assert_eq!(middle.fin_hot_slot.load(Ordering::Acquire), 2); - handles[0] - .fin_hot_slot - .store((u32::MAX - 1) as usize, Ordering::Release); - assert!(state.fin_remove(handles[0].id)); - assert!(!state.fin_remove(handles[0].id)); - assert_eq!(middle.fin_hot_slot.load(Ordering::Acquire), 0); - for handle in &handles { - state.untrack_id(handle.id); - } - assert!(state.finalizable_hot.borrow().is_empty()); - assert_eq!(state.finalizable_count.load(Ordering::Acquire), 0); - assert_eq!(state.fin_hot_count.load(Ordering::Acquire), 0); - } - - #[test] - fn deferred_instance_drop_ignores_stale_gc_filter() { - const CHILD: &str = "WEAVEPY_DEFERRED_DROP_TEST_CHILD"; - if std::env::var_os(CHILD).is_none() { - let status = std::process::Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "gc_trace::tests::deferred_instance_drop_ignores_stale_gc_filter", - "--nocapture", - ]) - .env(CHILD, "1") - .status() - .expect("spawn deferred-drop test"); - assert!(status.success(), "deferred-drop child: {status}"); - return; - } - let _gil = crate::gil::global_gil().acquire(); - let class = - crate::types::TypeObject::new_user("DeferredDrop", Vec::new(), DictData::default()) - .unwrap(); - let owner = crate::types::PyInstance::new_deferred(class); - let operand = Object::Instance(owner.clone()); - let id = id_of(&operand); - assert!(owner.is_gc_deferred()); - assert!(!is_tracked(id)); - assert_eq!(Rc::strong_count(&owner), 2); - - // Model stale bits from an earlier allocation at this address. - // This makes the reported CI failure deterministic without - // relying on allocator placement or weakening path coverage. - process_tracked_filter().insert(id); - assert!(maybe_tracked(id)); - assert!(!is_tracked(id)); - assert!(note_dropped_marks(&operand)); - assert!(crate::Interpreter::core_droppable(&operand)); - - // A final program owner still requires ordinary teardown. - drop(owner); - assert!(!crate::Interpreter::core_droppable(&operand)); - - // A non-atomic field revokes the deferral proof. After releasing - // the extra program owner, the collector and operand are the - // only owners, so this release must mark prompt cleanup. - let Object::Instance(owner) = &operand else { - unreachable!() - }; - owner.note_slot_store(&Object::List(Rc::new(RefCell::new(Vec::new())))); - assert!(!owner.is_gc_deferred()); - assert!(is_tracked(id)); - assert_eq!(Rc::strong_count(owner), 2); - assert!(note_dropped_marks(&operand)); - assert!(!crate::Interpreter::core_droppable(&operand)); - with_state(|state| state.untrack_id(id)); - assert!(!is_tracked(id)); - } - #[test] fn gc_index_distributes_aligned_object_addresses() { use std::hash::BuildHasher; @@ -4962,7 +2887,7 @@ mod tests { .collect(); for root in &roots { // See `track_and_untrack` on `track_now`. - state.track_now(root.clone()); + state.track_now(root); } for (collection, expected) in [(0, 1), (1, 2), (2, 2), (2, 2)] { assert_eq!(state.collect(collection), 0); @@ -5002,7 +2927,7 @@ mod tests { #[test] fn generation_constructor_rejects_out_of_range_indices() { for generation in [N_GENERATIONS, u8::MAX as usize + 1, usize::MAX] { - let result = std::panic::catch_unwind(|| TrackedHandle::new(Object::None, generation)); + let result = std::panic::catch_unwind(|| TrackedHandle::new(&Object::None, generation)); assert!(result.is_err()); } } @@ -5014,7 +2939,7 @@ mod tests { // `track_now`, not `track`: an empty dict holds nothing that could // close a cycle, so `track` would defer it (see `defer_container`). // This test is about the index's bookkeeping, not that policy. - s.track_now(d.clone()); + s.track_now(&d); assert!(s.is_tracked(id_of(&d))); s.untrack_id(id_of(&d)); assert!(!s.is_tracked(id_of(&d))); @@ -5025,22 +2950,36 @@ mod tests { let s = GcState::new(); let dict = Rc::new(RefCell::new(DictData::default())); let outer = Object::Dict(dict.clone()); - s.track(outer.clone()); + s.track_now(&outer); // The dict references itself: a 1-cycle. dict.borrow_mut().insert( crate::object::DictKey(Object::from_static("self")), outer.clone(), ); - // Drop the local strong ref; only the cycle + the GC's - // tracked handle keep it alive. + let weak = Rc::downgrade(&dict); drop(outer); - let collected = s.collect(2); - // We expect the cyclic dict to be discovered (the cycle's - // gc_refs is balanced by the self-pointer). The actual - // assertion is loose — the GC may or may not collect on - // the first pass depending on Rust-side stash counts. - let _ = collected; - // What we *do* assert: the GC didn't crash. + drop(dict); + // Only the cycle keeps the dict alive; the collector holds nothing. + assert!(weak.upgrade().is_some()); + assert_eq!(s.collect(2), 1); + assert!(weak.upgrade().is_none()); + assert_eq!(s.population(), 0); + } + + #[test] + fn tracked_objects_die_by_refcount() { + let s = GcState::new(); + let dict = Rc::new(RefCell::new(DictData::default())); + let obj = Object::Dict(dict.clone()); + s.track_now(&obj); + let weak = Rc::downgrade(&dict); + drop(obj); + drop(dict); + // No collection ran: the entry is stale but owns nothing. + assert!(weak.upgrade().is_none()); + assert_eq!(s.population(), 1); + assert_eq!(s.collect(0), 0); + assert_eq!(s.population(), 0); } #[test] @@ -5048,7 +2987,7 @@ mod tests { let s = GcState::new(); let d = Object::Dict(Rc::new(RefCell::new(DictData::default()))); // See `track_and_untrack` on `track_now`. - s.track_now(d.clone()); + s.track_now(&d); s.freeze_all(); assert_eq!(s.freeze_count(), 1); s.unfreeze_all(); diff --git a/crates/weavepy-vm/src/gen_fast.rs b/crates/weavepy-vm/src/gen_fast.rs index d2902740..dc22bfbb 100644 --- a/crates/weavepy-vm/src/gen_fast.rs +++ b/crates/weavepy-vm/src/gen_fast.rs @@ -96,8 +96,11 @@ fn op_supported(op: OpCode) -> bool { ) } -/// Whether `code` is a plain generator whose every instruction is one a -/// fast step runs (cached in the code's extension table). +/// Whether `code` is a plain generator whose steady state, everything a +/// resume can reach from a `yield`, is instructions a fast step runs +/// (cached in the code's extension table). The prologue may hold others +/// (`range(n)` built before the loop): the first resume's step stops at +/// them, and the general loop runs them once. fn code_ok(code: &CodeObject) -> bool { use std::sync::atomic::Ordering; let Some(ext) = crate::code_vm_ext(code) else { @@ -113,13 +116,46 @@ fn code_ok(code: &CodeObject) -> bool { && !code.is_async_generator && code.cellvars.is_empty() && code.freevars.is_empty() - && code.instructions.iter().all(|i| op_supported(i.op)) - && in_bounds(code, ext.objects.len()); + && in_bounds(code, ext.objects.len()) + && steady_state_supported(code); ext.gen_fast .store(if ok { 2 } else { 1 }, Ordering::Relaxed); ok } +/// Whether every instruction a resume reaches from one of `code`'s yields +/// is one a fast step runs (jump targets are in range: see [`in_bounds`]). +fn steady_state_supported(code: &CodeObject) -> bool { + let instrs = &code.instructions; + let n = instrs.len(); + let mut seen = vec![false; n]; + let mut work: Vec = (0..n) + .filter(|&p| instrs[p].op == OpCode::YieldValue) + .map(|p| p + 1) + .collect(); + while let Some(p) = work.pop() { + if p >= n || std::mem::replace(&mut seen[p], true) { + continue; + } + let ins = instrs[p]; + if !op_supported(ins.op) { + return false; + } + let arg = ins.arg as usize; + match ins.op { + OpCode::ReturnValue | OpCode::Reraise => {} + OpCode::JumpForward => work.push(p + 1 + arg), + OpCode::JumpBackward => work.push((p + 1).saturating_sub(arg)), + OpCode::PopJumpIfFalse | OpCode::PopJumpIfTrue | OpCode::ForIter => { + work.push(p + 1); + work.push(p + 1 + arg); + } + _ => work.push(p + 1), + } + } + true +} + /// Whether every jump, local and constant `code`'s instructions name is /// in range, and the last instruction can't fall through: the step then /// reads instructions, locals and constants without per-use checks. @@ -299,13 +335,15 @@ impl Interpreter { /// back when it ends. /// /// With `fold` (a draining consumer's sink, top level only), a yield - /// the sink takes resumes the body at once with `None` sent. + /// the sink takes resumes the body at once with `None` sent. `dead` is + /// the thread's queued-finalizer flag (`gc_trace::maybe_dead_flag`). pub(crate) fn gen_fast_step( &mut self, frame: &mut Frame, snap_gen: u64, depth: u8, fold: Option, + dead: *const std::cell::Cell, ) -> GenStep { // SAFETY: the frame's code is immutable and outlives the step (the // frame holds it, and nothing here replaces it). @@ -336,12 +374,58 @@ impl Interpreter { if pc >= ninstrs { return GenStep::Bail; } + // The body's native form (see `frame_jit`), run from every pc the + // arms below leave it at; a hot body compiles. + #[cfg(feature = "jit")] + let native = { + let nlocals = locals.len(); + if ext.frame_jit.get(nlocals).is_none() { + ext.frame_jit + .warm(code, ext, nlocals, crate::frame_jit::Heat::Step); + } + ext.frame_jit.get(nlocals) + }; + #[cfg(feature = "jit")] + let mut nst = crate::frame_jit::State { + locals: lbase, + stack: base, + len, + cap, + pc, + last: pc, + countdown: std::ptr::addr_of_mut!(self.gil_countdown), + snap_gen, + maybe_dead: dead, + interp: std::ptr::from_ref(self), + out: 0, + depth_cell: std::ptr::null(), + err: None, + }; + #[cfg(feature = "jit")] + let mut handed = usize::MAX; // SAFETY (throughout): `base` indexes only below `len` (initialized) // or `cap` as checked; `lbase`, `cbase` and `instrs` only at the // indices and pcs the eligibility scan proved in range (`in_bounds`: // every jump lands on an instruction and the last can't fall // through, so `pc < ninstrs` whenever an instruction is read). let yielded = loop { + #[cfg(feature = "jit")] + if pc != handed { + if let Some(native) = native.filter(|n| n.enters_at(pc)) { + nst.len = len; + nst.pc = pc; + // SAFETY: the body's activation state, as this loop + // holds it (its locals count checked above). A queued + // finalizer waits for the general loop's next check. + let _ = unsafe { native.run(&mut nst) }; + len = nst.len; + pc = nst.pc; + handed = pc; + if pc >= ninstrs { + break None; + } + } + } let ins = unsafe { *instrs.add(pc) }; match ins.op { OpCode::Nop | OpCode::NotTaken | OpCode::Resume => pc += 1, @@ -534,7 +618,7 @@ impl Interpreter { } } } - match self.gen_fast_for_iter(top, snap_gen, depth) { + match self.gen_fast_for_iter(top, snap_gen, depth, dead) { ForNext::Value(v) => { unsafe { base.add(len).write(v) }; len += 1; @@ -605,7 +689,13 @@ impl Interpreter { /// value, a range's end (when the loop alone holds it), or a fast /// generator's next yield. #[inline(never)] - fn gen_fast_for_iter(&mut self, top: &Object, snap_gen: u64, depth: u8) -> ForNext { + fn gen_fast_for_iter( + &mut self, + top: &Object, + snap_gen: u64, + depth: u8, + dead: *const std::cell::Cell, + ) -> ForNext { match top { Object::Iter(it) => { let unique = Rc::strong_count(it) == 1; @@ -656,13 +746,12 @@ impl Interpreter { _ => ForNext::Bail, } } - Object::Generator(g) => { - let g = g.clone(); - match self.gen_fast_next(&g, snap_gen, depth + 1) { - GenNext::Yielded(v) => ForNext::Value(v), - GenNext::Declined | GenNext::Partial => ForNext::Bail, - } - } + // (The stack slot keeps `g` alive: the inner step runs no code + // that could reach this frame.) + Object::Generator(g) => match self.gen_fast_next(g, snap_gen, depth + 1, dead) { + GenNext::Yielded(v) => ForNext::Value(v), + GenNext::Declined | GenNext::Partial => ForNext::Bail, + }, _ => ForNext::Bail, } } @@ -680,7 +769,7 @@ impl Interpreter { if !Self::gen_fast_frame_ok(frame) { return None; } - match self.gen_fast_step(frame, snap_gen, 0, fold) { + match self.gen_fast_step(frame, snap_gen, 0, fold, crate::gc_trace::maybe_dead_flag()) { GenStep::Yielded(v) => Some(v), GenStep::Bail => None, } @@ -694,6 +783,7 @@ impl Interpreter { g: &Rc, snap_gen: u64, depth: u8, + dead: *const std::cell::Cell, ) -> GenNext { if depth > MAX_DEPTH || crate::recursion::current_depth() + usize::from(depth) + 2 @@ -732,7 +822,7 @@ impl Interpreter { frame.stack.push(Object::None); } debug_assert!(!frame.stack.is_empty()); - let out = match self.gen_fast_step(frame, snap_gen, depth, None) { + let out = match self.gen_fast_step(frame, snap_gen, depth, None, dead) { GenStep::Yielded(v) => GenNext::Yielded(v), GenStep::Bail if frame.pc == start => { // Nothing ran: the resume is undone. diff --git a/crates/weavepy-vm/src/hot_filter.rs b/crates/weavepy-vm/src/hot_filter.rs index 8f11da51..12af799a 100644 --- a/crates/weavepy-vm/src/hot_filter.rs +++ b/crates/weavepy-vm/src/hot_filter.rs @@ -1,10 +1,9 @@ //! Insert-only atomic bloom pre-filters — RFC 0065 (WS4). //! -//! Three registries (the weakref registry, the GC tracked-object -//! index, the prompt-reap suspect map) sit behind `GilCell`/mutex -//! borrows but are consulted on *usually-miss* probes from the -//! interpreter's drop paths: "is anything watching this object?", -//! "is this object tracked?", "is this id enrolled as a suspect?". +//! Registries (the weakref registry, the GC tracked-object index) sit +//! behind `GilCell`/mutex borrows but are consulted on *usually-miss* +//! probes from the interpreter's drop paths: "is anything watching this +//! object?", "is this object tracked?". //! RFC 0061 measured the borrow machinery of those misses at ~5% //! of drop-heavy fixtures. An [`AtomicBloom`] answers the miss with //! two relaxed loads, no lock, no TLS. diff --git a/crates/weavepy-vm/src/hot_gates.rs b/crates/weavepy-vm/src/hot_gates.rs index c3a4f070..aea4bde6 100644 --- a/crates/weavepy-vm/src/hot_gates.rs +++ b/crates/weavepy-vm/src/hot_gates.rs @@ -65,13 +65,11 @@ static HOT: AtomicU32 = AtomicU32::new(0); /// RFC 0065 (WS1): the dispatch loop's *generation* word. Bumped by /// every mutation that can change the loop prologue's decisions — /// hot-gate bits ([`set`]/[`clear`] below), observer registration -/// (`trace::bump_observer_gen`), the GC finalizable/suspect -/// population transitions (`gc_trace`), and frame materialization +/// (`trace::bump_observer_gen`), and frame materialization /// (`FrameShell::materialize`). The dispatch loop caches a snapshot /// of the prologue's inputs keyed by this generation: while the /// generation is unchanged *and* the snapshot says "quiet" (no -/// pending work, no finalizables, no suspects, no observers, no -/// materialized frame), the loop runs one relaxed load + compare per +/// pending work, no observers, no materialized frame), the loop runs one relaxed load + compare per /// instruction instead of the full ten-plus-probe prologue. /// /// Same producer/consumer discipline as [`HOT`]: a spurious bump is @@ -94,6 +92,13 @@ pub fn loop_gen() -> u64 { LOOP_GEN.load(Ordering::Relaxed) } +/// The loop generation's address, for native code that compares it at +/// its back edges. +#[cfg(feature = "jit")] +pub(crate) fn loop_gen_ptr() -> *const u64 { + LOOP_GEN.as_ptr() +} + /// Invalidate every dispatch loop's cached prologue snapshot /// (RFC 0065 WS1). Cheap (one relaxed RMW); call from any mutation /// site whose state the loop prologue consults. @@ -138,11 +143,6 @@ pub mod env_flags { } }; } - once_flag!( - /// `WEAVEPY_REAP_TRACE`: prompt-reap tracing. - reap_trace, - "WEAVEPY_REAP_TRACE" - ); once_flag!( /// `WEAVEPY_NO_QUIET`: pin the dispatch loop to its full prologue. no_quiet, diff --git a/crates/weavepy-vm/src/import.rs b/crates/weavepy-vm/src/import.rs index 98e05586..e53d928a 100644 --- a/crates/weavepy-vm/src/import.rs +++ b/crates/weavepy-vm/src/import.rs @@ -158,7 +158,7 @@ impl ModuleCache { // is True); a module's globals routinely close cycles (a function // defined in the module captures the module's dict). Track on install. if matches!(module, Object::Module(_)) { - crate::gc_trace::track(module.clone()); + crate::gc_trace::track(&module); } self.modules .borrow_mut() diff --git a/crates/weavepy-vm/src/inst_dict.rs b/crates/weavepy-vm/src/inst_dict.rs index a7f7e8a3..5485a91e 100644 --- a/crates/weavepy-vm/src/inst_dict.rs +++ b/crates/weavepy-vm/src/inst_dict.rs @@ -107,6 +107,14 @@ impl SharedKeys { }) } + /// How many leading names are not `name` (Python hash `hash`): its + /// position, or all of them when none is. An instance holding no more + /// values than this has no attribute `name`. + pub(crate) fn names_before(&self, name: &str, hash: i64) -> usize { + let n = self.len(); + self.position_hashed(n, name, hash).unwrap_or(n) + } + /// Publish the `str` name `name` as the next name; `None` when the /// table is full. The caller holds the GIL outside free-threaded mode. fn push(&self, name: &SharedStr) -> Option { @@ -225,6 +233,15 @@ impl Drop for SplitValues { } impl SplitValues { + /// Where native code finds a split block's parts: the block pointer + /// (null when empty) within the values, and the shared names' pointer, + /// the `u32` length and the first value within a block. + pub(crate) const BLOCK_OFFSET: usize = std::mem::offset_of!(Self, block); + pub(crate) const KEYS_OFFSET: usize = std::mem::offset_of!(SplitHeader, keys); + pub(crate) const LEN_OFFSET: usize = std::mem::offset_of!(SplitHeader, len); + pub(crate) const CAP_OFFSET: usize = std::mem::offset_of!(SplitHeader, cap); + pub(crate) const VALUES_OFFSET: usize = std::mem::size_of::(); + #[inline] fn layout(cap: usize) -> std::alloc::Layout { std::alloc::Layout::new::() @@ -439,6 +456,34 @@ impl SplitValues { self.push(keys, want, value); } + /// Make room for `cap` values before the first one arrives, adopting + /// the class's names (`keys`): a constructor that sets its fields in the + /// class's order then appends each in place, the first included (see + /// [`Self::append_over`] and compiled code's new-key stores). Nothing + /// when a block exists (values, or a recycled one) or `cap` is zero. + pub(crate) fn reserve_for(&mut self, keys: impl FnOnce() -> Rc, cap: usize) { + if self.block.is_some() || cap == 0 { + return; + } + let layout = Self::layout(cap); + // SAFETY: a nonzero-size layout (the header alone has a size). + #[allow(clippy::cast_ptr_alignment)] + let h = unsafe { std::alloc::alloc(layout) }.cast::(); + let Some(h) = std::ptr::NonNull::new(h) else { + std::alloc::handle_alloc_error(layout); + }; + // SAFETY: `h` is a fresh block of `cap` slots; the header owns one + // strong count of the names. + unsafe { + h.as_ptr().write(SplitHeader { + keys: Rc::into_raw(keys()), + len: 0, + cap: cap as u32, + }); + } + self.block = Some(h); + } + /// Whether [`Self::append_over`] of position `i` of `keys` succeeds /// once the positions from the current length up to `i` have been /// appended first (the next of a run of in-order appends). @@ -675,6 +720,12 @@ impl std::fmt::Debug for InstDict { } impl InstDict { + /// Where native code finds the published dictionary's pointer (null + /// while the values are split) and the split values' cell. + pub(crate) const LAZY_OFFSET: usize = + std::mem::offset_of!(Self, lazy) + LazyArc::>::POINTER_OFFSET; + pub(crate) const SPLIT_OFFSET: usize = std::mem::offset_of!(Self, split); + pub fn new() -> Self { Self { lazy: LazyArc::new(), @@ -1128,10 +1179,17 @@ impl crate::types::PyInstance { /// layout, no materializing). pub fn attr_position_str(&self, name: &str) -> Option { let i = match self.dict.published() { - Some(d) => d - .try_borrow() - .ok()? - .get_index_of(&crate::object::StrKey(name))?, + Some(d) => { + // Never runs Python: a stored key only a user `__eq__` + // could equate leaves the position unknown. + let probe = + crate::object::LeafNameProbe::new(name, crate::object::py_str_hash(name)); + let i = d.try_borrow().ok()?.get_index_of(&probe); + if probe.saw_exotic() { + return None; + } + i? + } None => self .dict .split_cell() diff --git a/crates/weavepy-vm/src/lazy_arc.rs b/crates/weavepy-vm/src/lazy_arc.rs index 728983c2..655a6251 100644 --- a/crates/weavepy-vm/src/lazy_arc.rs +++ b/crates/weavepy-vm/src/lazy_arc.rs @@ -10,14 +10,17 @@ use std::mem::ManuallyDrop; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; -pub struct LazyArc { +pub struct LazyArc { pointer: AtomicPtr, // AtomicPtr alone would allow Send/Sync regardless of T. Match Arc's // ownership, drop checking, and thread-safety requirements instead. owner: PhantomData>, } -impl LazyArc { +impl LazyArc { + /// Where the payload pointer (null until set) sits, for native code. + pub(crate) const POINTER_OFFSET: usize = std::mem::offset_of!(Self, pointer); + pub const fn new() -> Self { Self { pointer: AtomicPtr::new(ptr::null_mut()), @@ -101,7 +104,7 @@ impl LazyArc { } } -impl LazyArc { +impl LazyArc { #[inline] pub fn share(&self) -> Arc { let pointer = self.get_or_init_ptr(|| Arc::new(T::default())); @@ -113,13 +116,13 @@ impl LazyArc { } } -impl Default for LazyArc { +impl Default for LazyArc { fn default() -> Self { Self::new() } } -impl From> for LazyArc { +impl From> for LazyArc { fn from(value: Arc) -> Self { Self { pointer: AtomicPtr::new(Arc::into_raw(value).cast_mut()), @@ -135,7 +138,7 @@ impl From> for LazyArc { // (`PyInstance::dict_cell` for the instance dictionary), so every // creator is visible at the call site. -impl Clone for LazyArc { +impl Clone for LazyArc { fn clone(&self) -> Self { // A shallow instance clone must share even a previously empty dict: // subsequent writes through either owner must reach the same object. @@ -143,13 +146,13 @@ impl Clone for LazyArc { } } -impl fmt::Debug for LazyArc { +impl fmt::Debug for LazyArc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("LazyArc").field(&self.get()).finish() } } -impl Drop for LazyArc { +impl Drop for LazyArc { fn drop(&mut self) { let pointer = *self.pointer.get_mut(); if !pointer.is_null() { diff --git a/crates/weavepy-vm/src/leaf_plan.rs b/crates/weavepy-vm/src/leaf_plan.rs index 84115000..3ae8ec7f 100644 --- a/crates/weavepy-vm/src/leaf_plan.rs +++ b/crates/weavepy-vm/src/leaf_plan.rs @@ -12,6 +12,9 @@ use weavepy_compiler::{BinOpKind, CodeObject, CompareKind, OpCode, UnaryKind}; +#[cfg(feature = "jit")] +mod jit; + use crate::object::Object; use crate::sync::Rc; use crate::{CodeConstObjects, Interpreter}; @@ -107,6 +110,9 @@ pub(crate) fn compare(a: V, b: V, kind: CompareKind) -> Option { /// Registers a plan may address: the locals, then the operand stack. const REGS: usize = 32; +/// Nested leaf calls evaluated within one evaluation at most this deep. +const NEST: u8 = 3; + /// One plan operation. Registers are `u8` indices below [`REGS`]; `pc` /// is the source instruction (its inline caches and site slots), and /// `name` its `co_names` index. @@ -126,6 +132,12 @@ enum Op { dst: u8, pc: u16, }, + /// `regs[dst]` = the closure cell `idx`'s value (a free variable: a + /// leaf has no cell variables of its own). + Deref { + dst: u8, + idx: u8, + }, Attr { dst: u8, src: u8, @@ -167,6 +179,12 @@ enum Op { b: u8, kind: u8, }, + /// `regs[dst] = regs[a][regs[b]]` (see [`subscr`]). + Subscr { + dst: u8, + a: u8, + b: u8, + }, /// Exchange two registers. Swap { a: u8, @@ -229,6 +247,13 @@ pub(crate) struct LeafPlan { /// reassigned), each to a different name: the buffered stores share /// one receiver and each is the latest to its attribute. unique_stores: bool, + /// Evaluations so far, toward compiling the plan (see [`jit`]). + #[cfg(feature = "jit")] + runs: std::sync::atomic::AtomicU32, + /// The native code for a pure and an effect evaluation, once compiled + /// (`None` inside: the plan stays interpreted). + #[cfg(feature = "jit")] + native: [std::sync::OnceLock>; 2], } // SAFETY: the constants' pointers name the code extension's immutable @@ -376,7 +401,14 @@ impl Builder<'_> { fn step(&mut self, pc: usize, ins: weavepy_compiler::Instruction) -> Option { let arg = ins.arg; match ins.op { - OpCode::Resume | OpCode::Nop | OpCode::NotTaken => {} + OpCode::Resume | OpCode::Nop | OpCode::NotTaken | OpCode::CopyFreeVars => {} + OpCode::LoadDeref => { + let dst = self.push_new()?; + self.ops.push(Op::Deref { + dst, + idx: u8::try_from(arg).ok()?, + }); + } OpCode::LoadFast | OpCode::LoadFastBorrow | OpCode::LoadFastCheck => { self.load_local(arg)?; } @@ -490,6 +522,12 @@ impl Builder<'_> { kind: arg as u8, }); } + OpCode::BinarySubscr => { + let b = self.pop()?; + let a = self.pop()?; + let dst = self.push_new()?; + self.ops.push(Op::Subscr { dst, a, b }); + } OpCode::CopyTop => { let n = (arg as usize).max(1); let depth = self.stack.len(); @@ -679,6 +717,10 @@ impl Builder<'_> { nargs: u8::try_from(crate::leaf_arity(self.code)).ok()?, nregs: u8::try_from(self.top.get().clamp(self.nl, REGS)).ok()?, unique_stores: self.stored.is_some() && self.code.arg_count > 0, + #[cfg(feature = "jit")] + runs: std::sync::atomic::AtomicU32::new(0), + #[cfg(feature = "jit")] + native: [std::sync::OnceLock::new(), std::sync::OnceLock::new()], }) } } @@ -741,6 +783,14 @@ impl Owned { } impl Owned { + #[inline(always)] + fn new() -> Self { + Owned { + buf: [const { std::mem::MaybeUninit::uninit() }; OWNED], + n: 0, + } + } + /// Whether `p` names one of the held values. #[inline(always)] fn holds(&self, p: *const Object) -> bool { @@ -792,6 +842,15 @@ struct Pending { } impl Pending { + #[inline(always)] + fn new() -> Self { + Pending { + buf: [const { std::mem::MaybeUninit::uninit() }; PENDING], + n: 0, + moved: 0, + } + } + fn get(&self, k: usize) -> &Store { debug_assert!(k < self.n); // SAFETY: the first `n` entries are initialized. @@ -853,6 +912,22 @@ pub(crate) enum LeafRet { Owned(Object), } +/// The value of `f`'s closure cell `idx`, borrowed: nothing an evaluation +/// runs can rebind a cell (a leaf has no `STORE_DEREF`), and the function +/// holds its closure. `None` for an empty cell (the read raises). +#[inline] +pub(super) fn free_var(f: &crate::object::PyFunction, idx: u8) -> Option { + let Object::Cell(cell) = f.closure.get(usize::from(idx))? else { + return None; + }; + let p: *const Object = cell.as_ptr(); + // SAFETY: see above. + if matches!(unsafe { &*p }, Object::Unbound) { + return None; + } + Some(norm(p)) +} + impl LeafRet { /// The result as an owned object (a borrowed value is cloned). #[inline(always)] @@ -867,7 +942,7 @@ impl LeafRet { /// An owned object for a leaf value (the return value, a buffered /// store's value); `None` for the markers that are never values. #[inline(always)] -fn to_object(v: V) -> Option { +pub(super) fn to_object(v: V) -> Option { Some(match v { // SAFETY: as `norm` (an owned value is cloned before its holder // drops). @@ -903,6 +978,42 @@ impl Interpreter { .into_object() } + /// A frameless call's evaluation of the general leaf `code` (see + /// `Interpreter::pure_leaf_call`) straight in its plan's native + /// code, at recursion depth `depth`, once the callee's warm-up is past + /// (the frameless call bookkeeping is then only the hit count): + /// `None` when the plan has no native code, or the callee one of the + /// tiny shapes `leaf_eval` answers itself; otherwise the result, or + /// `None` inside having done nothing observable. + #[cfg(feature = "jit")] + #[inline] + pub(crate) fn leaf_site_native( + &self, + code: &CodeObject, + f: &crate::object::PyFunction, + args: &[*const Object], + depth: usize, + ) -> Option> { + let ext = crate::code_vm_ext(code)?; + if ext.pure_leaf.load(std::sync::atomic::Ordering::Relaxed) >= 3 + || code.jit_hint.lean_entries() <= crate::tier2::LEAN_WARM_COMPILE_THRESHOLD_CAP + { + return None; + } + let plan = ext.leaf_plan.get()?.as_deref()?; + if args.len() != usize::from(plan.nargs) || args.len() != crate::leaf_arity(code) { + return None; + } + let r = jit::run_compiled::(self, code, ext, plan, f, args, depth)? + .and_then(LeafRet::into_object); + if r.is_some() { + code.jit_hint.note_leaf_hit(); + } else { + code.jit_hint.note_leaf_miss(); + } + Some(r) + } + /// [`Self::leaf_run`] with its result borrowed when the caller may /// (see [`LeafRet`]). #[inline(never)] @@ -915,9 +1026,6 @@ impl Interpreter { args: &[*const Object], nest: u8, ) -> Option { - use weavepy_compiler::InlineCache as IC; - /// Nested leaf calls evaluated in place at most this deep. - const NEST: u8 = 3; /// A pure-leaf callee runs as a frame of this evaluation: its /// registers are the next `REGS` window, and its caller's state /// waits here until its return. @@ -936,6 +1044,14 @@ impl Interpreter { if args.len() != nargs { return None; } + // A warm plan runs as native code (a getter's or constructor's + // evaluation stays here). + #[cfg(feature = "jit")] + if !GETTER && !FRESH { + if let Some(r) = jit::run::(self, code, ext, plan, f, args, nest) { + return r; + } + } let (mut code, mut ext, mut plan, mut f) = (code, ext, plan, f); let mut regs = [const { std::mem::MaybeUninit::::uninit() }; REGS]; for (k, &a) in args.iter().enumerate() { @@ -960,15 +1076,8 @@ impl Interpreter { } let mut consts: &[V] = &plan.consts; let mut stamps: &[crate::StampSlot] = ext.stamp_slots.get().map_or(&[], |s| &s[..]); - let mut owned = Owned { - buf: [const { std::mem::MaybeUninit::uninit() }; OWNED], - n: 0, - }; - let mut pend = Pending { - buf: [const { std::mem::MaybeUninit::uninit() }; PENDING], - n: 0, - moved: 0, - }; + let mut owned = Owned::new(); + let mut pend = Pending::new(); let mut ops: &[Op] = &plan.ops; let mut ip = 0usize; loop { @@ -981,134 +1090,25 @@ impl Interpreter { Op::Move { dst, src } => set!(dst, get!(src)), // SAFETY: `k` names a plan constant (the translation). Op::Const { dst, k } => set!(dst, unsafe { *consts.get_unchecked(usize::from(k)) }), - Op::Global { dst, pc } => { - // The callee's own namespaces, stamp-validated as the - // core loop's `LOAD_GLOBAL` arm. - let pc = usize::from(pc); - let slot = stamps.get(pc)?; - let (gdict, bdict) = (f.globals.as_ptr(), f.builtins.as_ptr()); - let gid = crate::specialize::rc_id(&f.globals); - // SAFETY (raw dict reads): nothing runs code here. - let g_stamp = unsafe { (*gdict).mutation_stamp() }; - let hit = match code.caches.get(pc as u32) { - IC::LoadGlobalModule { - globals_id, - key_idx, - } if globals_id == gid && slot.get() == [gid, g_stamp, 0] => unsafe { - (*gdict).get_index(key_idx as usize) - }, - IC::LoadGlobalBuiltin { - builtins_id, - key_idx, - } if builtins_id == crate::specialize::rc_id(&f.builtins) - && !self.globals_missing_any.get() - && slot.get() - == [gid, g_stamp, unsafe { (*bdict).mutation_stamp() }] => - unsafe { (*bdict).get_index(key_idx as usize) }, - _ => return None, - }; - set!(dst, norm(hit?.1)); - } + Op::Global { dst, pc } => set!(dst, self.plan_global(code, stamps, f, pc)?), + Op::Deref { dst, idx } => set!(dst, free_var(f, idx)?), Op::Attr { dst, src, pc, name } => { - let V::R(p) = get!(src) else { - return None; - }; - let (pc, name) = (u32::from(pc), u32::from(name)); - // SAFETY: as `norm`. - let recv = unsafe { &*p }; - if EFFECT && pend.n > 0 { - // A buffered store to this attribute, the latest. - let hit = (0..pend.n) - .rev() - .map(|k| pend.get(k)) - // SAFETY: stable receivers (see `Store`). - .find(|(r, _, n, _)| *n == name && unsafe { (**r).is_same(recv) }); - if let Some((_, _, _, v)) = hit { - set!(dst, norm(v)); - continue; - } - } - let v = match recv { - Object::Instance(inst) => { - if GETTER && inst.cls_raw().native_kind.get() != 0 { - return None; - } - // SAFETY: the receiver remains rooted by an - // argument or owned scratch; no Python runs. - let hit = unsafe { - Self::leaf_cached_instance_field(ext, code, inst, pc, name) - } - .map(std::ptr::from_ref); - match hit { - Some(v) => norm(v), - // A stale or absent site cache resolves - // through the site's own entries. - None => owned.own(Self::leaf_attr_resolve_site( - code, inst, recv, pc, name, - )?)?, - } - } - Object::Type(cls) => { - if GETTER && !Self::plain_metaclass(cls) { - return None; - } - match stamps - .get(pc as usize) - .and_then(|s| crate::class_attr_hit(s, cls)) - { - Some(v) => owned.own(v)?, - None => { - owned.own(Self::leaf_load_type_attr(code, cls, pc, name)?)? - } - } - } - Object::Module(module) => { - if GETTER - && (crate::object::module_class(module).is_some() - || code.names.get(name as usize)?.starts_with("__")) - { - return None; - } - owned.own(Self::leaf_load_attr_recv(code, recv, pc, name)?)? - } - _ => return None, - }; + let v = self.plan_attr::( + code, + ext, + stamps, + &mut owned, + &pend, + get!(src), + pc, + name, + )?; set!(dst, v); } Op::Method { dst, src, pc, name } => { - // A method off the site's slot: the function under - // the receiver, or a class's function with an empty - // self slot (as the core loop's arm). - let V::R(p) = get!(src) else { - return None; - }; - let ms = crate::code_method_slot(code, u32::from(pc))?; - // SAFETY: as `norm`. - match unsafe { &*p } { - Object::Instance(inst) => { - let cls = inst.cls_raw(); - if !Self::default_getattribute(cls) { - return None; - } - let fp = ms.peek_fn(cls.attr_version.get())?; - // The instance's attributes must not shadow - // the method. - if crate::inst_may_shadow(inst, code, u32::from(name)) { - return None; - } - set!(dst, V::Fn(fp)); - set!(dst + 1, V::R(p)); - } - Object::Type(cls) => { - set!(dst, V::Fn(ms.peek_unbound(cls.attr_version.get())?)); - set!(dst + 1, V::Null); - } - recv => { - let b = self.leaf_builtin_method_ptr(ms, recv, code, name)?; - set!(dst, V::Bi(b)); - set!(dst + 1, V::R(p)); - } - } + let (func, recv) = self.plan_method(code, get!(src), pc, name)?; + set!(dst, func); + set!(dst + 1, recv); } Op::Compare { dst, a, b, kind } => { // SAFETY: the translation checked `kind` names a @@ -1117,77 +1117,12 @@ impl Interpreter { set!(dst, V::B(compare(get!(a), get!(b), kind)?)); } Op::Is { dst, a, b, invert } => { - let same = match (get!(a), get!(b)) { - (V::N, V::N) => true, - (V::B(x), V::B(y)) => x == y, - (V::I(x), V::I(y)) => Object::Int(x).is_same(&Object::Int(y)), - (V::F(x), V::F(y)) => Object::Float(x).is_same(&Object::Float(y)), - // SAFETY: as `norm`. - (V::R(p), V::R(q)) => unsafe { (*p).is_same(&*q) }, - _ => false, - }; - set!(dst, V::B(same != invert)); + set!(dst, V::B(is_same(get!(a), get!(b)) != invert)) } Op::Truth { dst, src } => set!(dst, V::B(truth(get!(src))?)), - Op::Unary { dst, src, kind } => { - let v = get!(src); - let r = match kind { - k if k == UnaryKind::Not as u8 => V::B(!truth(v)?), - k if k == UnaryKind::Neg as u8 => match v { - V::I(i) => V::I(i.checked_neg()?), - V::F(x) => V::F(-x), - _ => return None, - }, - k if k == UnaryKind::Pos as u8 => match v { - V::I(_) | V::F(_) => v, - _ => return None, - }, - k if k == UnaryKind::Invert as u8 => match v { - V::I(i) => V::I(!i), - _ => return None, - }, - _ => return None, - }; - set!(dst, r); - } - Op::Binary { dst, a, b, kind } => { - let r = match (get!(a), get!(b)) { - (V::I(a), V::I(b)) => V::I(match kind { - k if k == BinOpKind::Add as u8 => a.checked_add(b)?, - k if k == BinOpKind::Sub as u8 => a.checked_sub(b)?, - k if k == BinOpKind::Mult as u8 => a.checked_mul(b)?, - k if k == BinOpKind::BitAnd as u8 => a & b, - k if k == BinOpKind::BitOr as u8 => a | b, - k if k == BinOpKind::BitXor as u8 => a ^ b, - k if k == BinOpKind::RShift as u8 && (0..64).contains(&b) => a >> b, - k if k == BinOpKind::LShift as u8 - && (0..63).contains(&b) - && ((a << b) >> b) == a => - { - a << b - } - k if k == BinOpKind::FloorDiv as u8 && b > 0 && a >= 0 => a / b, - k if k == BinOpKind::Mod as u8 && b > 0 && a >= 0 => a % b, - _ => return None, - }), - (x, y) => { - let (x, y) = match (x, y) { - (V::F(x), V::F(y)) => (x, y), - (V::I(x), V::F(y)) => (x as f64, y), - (V::F(x), V::I(y)) => (x, y as f64), - _ => return None, - }; - // SAFETY: as the core loop's `BINARY_OP` arm - // (the compiler emits only valid kinds). - let kind: BinOpKind = unsafe { std::mem::transmute(kind) }; - match Self::leaf_float_op(x, y, kind)? { - Object::Float(x) => V::F(x), - _ => return None, - } - } - }; - set!(dst, r); - } + Op::Unary { dst, src, kind } => set!(dst, unary(get!(src), kind)?), + Op::Binary { dst, a, b, kind } => set!(dst, binary(get!(a), get!(b), kind)?), + Op::Subscr { dst, a, b } => set!(dst, subscr(&mut owned, get!(a), get!(b))?), Op::Swap { a, b } => { let (x, y) = (get!(a), get!(b)); set!(a, y); @@ -1204,28 +1139,7 @@ impl Interpreter { } } Op::Jump { target } => ip = usize::from(target), - Op::New { dst, dict } => { - // Tracked like the core loop's; an allocation that - // would trigger a collection is left to it. - if crate::stdlib::tracemalloc_real::is_tracking() - || crate::stdlib::testinternalcapi_mod::reftrace_print_active() - || crate::gc_trace::auto_collect_due() - { - return None; - } - let obj = if dict { - Object::Dict(Rc::new(crate::sync::RefCell::new( - crate::object::DictData::with_capacity_and_hasher( - 0, - crate::fasthash::FxBuildHasher, - ), - ))) - } else { - Object::new_list(Vec::new()) - }; - crate::gc_trace::track(obj.clone()); - set!(dst, owned.own(obj)?); - } + Op::New { dst, dict } => set!(dst, new_container(&mut owned, dict)?), Op::Decline => return None, Op::Call { at, argc, pc } => { // A pure-leaf callee, evaluated in place: only while @@ -1401,73 +1315,7 @@ impl Interpreter { set!(c.at, v); continue; } - // A value this evaluation's scratch doesn't hold outlives - // it (a pure body stores nothing), so a nested caller - // borrows it rather than taking a reference to release. - if !EFFECT && !matches!(v, V::R(p) if owned.holds(p)) { - return Some(LeafRet::Borrowed(v)); - } - let r = to_object(v)?; - if EFFECT && pend.n > 0 { - // The latest store to each attribute is the one - // that lands; every one of them must go through - // before any does (a decline touched nothing, and - // the ordinary call runs the body instead). A - // lone store is its own check: it declines whole. - let unique = plan.unique_stores; - if pend.n > 1 { - // With one receiver, appends of new attributes - // land in order: `cursor` is its split length - // once the earlier ones have. - let mut cursor = None; - // A store the shortcut can't vouch for may - // move the layout: the rest check in full. - let mut split_ok = unique; - for k in 0..pend.n { - if !unique && !pend.latest(k) { - continue; - } - let (rp, spc, name, _) = pend.get(k); - // SAFETY: stable receivers (see `Store`). - let Object::Instance(inst) = (unsafe { &**rp }) else { - return None; - }; - let (spc, name) = (*spc as usize, *name); - let ready = if split_ok { - Self::core_store_attr_ready_split(ext, inst, spc, &mut cursor) - } else { - None - }; - split_ok &= ready.is_some(); - if !ready.unwrap_or_else(|| { - Self::core_store_attr_ready(code, inst, spc, name) - }) { - return None; - } - } - } - let n = pend.n; - for k in 0..n { - if !unique && !pend.latest(k) { - continue; - } - let (rp, spc, name, value) = pend.get(k); - // SAFETY: stable receivers (see `Store`). - let Object::Instance(inst) = (unsafe { &**rp }) else { - return None; - }; - // On `true` the value moved into the dict. - if Self::core_store_attr(code, inst, *spc as usize, *name, value) { - pend.moved |= 1 << k; - } else if n == 1 { - // The lone store declined, untouched. - return None; - } else { - debug_assert!(false, "a ready store declined"); - } - } - } - return Some(LeafRet::Owned(r)); + return self.plan_finish::(code, ext, plan, &owned, &mut pend, v); } Op::StoreAttr { recv, @@ -1501,24 +1349,433 @@ impl Interpreter { } continue; } - if !matches!(recv, Object::Instance(_)) || pend.n == PENDING { - return None; + plan_store(args, &mut owned, &mut pend, rp, get!(val), pc, name)?; + } + } + } + } +} + +/// `a is b` on two leaf values. +#[inline(always)] +pub(crate) fn is_same(a: V, b: V) -> bool { + match (a, b) { + (V::N, V::N) => true, + (V::B(x), V::B(y)) => x == y, + (V::I(x), V::I(y)) => Object::Int(x).is_same(&Object::Int(y)), + (V::F(x), V::F(y)) => Object::Float(x).is_same(&Object::Float(y)), + // SAFETY: as `norm`. + (V::R(p), V::R(q)) => unsafe { (*p).is_same(&*q) }, + _ => false, + } +} + +/// A unary operator on a scalar; `None` for any other operand. +#[inline(always)] +pub(crate) fn unary(v: V, kind: u8) -> Option { + Some(match kind { + k if k == UnaryKind::Not as u8 => V::B(!truth(v)?), + k if k == UnaryKind::Neg as u8 => match v { + V::I(i) => V::I(i.checked_neg()?), + V::F(x) => V::F(-x), + _ => return None, + }, + k if k == UnaryKind::Pos as u8 => match v { + V::I(_) | V::F(_) => v, + _ => return None, + }, + k if k == UnaryKind::Invert as u8 => match v { + V::I(i) => V::I(!i), + _ => return None, + }, + _ => return None, + }) +} + +/// A binary operator on two scalars; `None` for any other operands or a +/// result the interpreter must produce (an overflow, a negative operand +/// of a floor division). +#[inline(always)] +pub(crate) fn binary(a: V, b: V, kind: u8) -> Option { + Some(match (a, b) { + (V::I(a), V::I(b)) => V::I(match kind { + k if k == BinOpKind::Add as u8 => a.checked_add(b)?, + k if k == BinOpKind::Sub as u8 => a.checked_sub(b)?, + k if k == BinOpKind::Mult as u8 => a.checked_mul(b)?, + k if k == BinOpKind::BitAnd as u8 => a & b, + k if k == BinOpKind::BitOr as u8 => a | b, + k if k == BinOpKind::BitXor as u8 => a ^ b, + k if k == BinOpKind::RShift as u8 && (0..64).contains(&b) => a >> b, + k if k == BinOpKind::LShift as u8 && (0..63).contains(&b) && ((a << b) >> b) == a => { + a << b + } + k if k == BinOpKind::FloorDiv as u8 && b > 0 && a >= 0 => a / b, + k if k == BinOpKind::Mod as u8 && b > 0 && a >= 0 => a % b, + _ => return None, + }), + (x, y) => { + let (x, y) = match (x, y) { + (V::F(x), V::F(y)) => (x, y), + (V::I(x), V::F(y)) => (x as f64, y), + (V::F(x), V::I(y)) => (x, y as f64), + _ => return None, + }; + // SAFETY: as the core loop's `BINARY_OP` arm (the compiler + // emits only valid kinds). + let kind: BinOpKind = unsafe { std::mem::transmute(kind) }; + match Interpreter::leaf_float_op(x, y, kind)? { + Object::Float(x) => V::F(x), + _ => return None, + } + } + }) +} + +/// `a[b]` that can't run code or raise: a list's or tuple's item at an +/// int index (borrowed: a leaf mutates no container), a dict's value for +/// a `str` or `int` key (the core loop's probe, which compares no other +/// key type), or an ASCII string's character (held by `owned`). Anything +/// else, a miss included, declines. +#[inline] +fn subscr(owned: &mut Owned, a: V, b: V) -> Option { + let V::R(c) = a else { + return None; + }; + let index = |i: i64, n: usize| { + let n = n as i64; + let i = if i < 0 { i + n } else { i }; + (0..n).contains(&i).then_some(i as usize) + }; + // SAFETY: as `norm`; the borrows end before anything could change the + // containers (no Python runs during an evaluation). + match (unsafe { &*c }, b) { + (Object::List(l), V::I(i)) => { + let xs = unsafe { l.peek() }?; + Some(norm(&raw const xs[index(i, xs.len())?])) + } + (Object::Tuple(t), V::I(i)) => Some(norm(&raw const t[index(i, t.len())?])), + (Object::Str(s), V::I(i)) => { + if crate::object::str_char_len(s) != s.len() { + return None; + } + owned.own(Object::from_char(s.as_bytes()[index(i, s.len())?] as char)) + } + (Object::Dict(d), V::I(_) | V::R(_)) => { + let int; + let key = match b { + V::I(i) => { + int = Object::Int(i); + &int + } + // SAFETY: as above. + V::R(k) if matches!(unsafe { &*k }, Object::Str(_)) => unsafe { &*k }, + _ => return None, + }; + let probe = crate::object::LeafProbe::new(key)?; + let v = unsafe { d.peek() }?.get(&probe)?; + Some(norm(v)) + } + _ => None, + } +} + +/// `BUILD_LIST 0` / `BUILD_MAP 0`: a new empty list or dict, tracked like +/// the core loop's and held by the owned scratch. An allocation that would +/// trigger a collection is left to the core loop. +#[inline(always)] +fn new_container(owned: &mut Owned, dict: bool) -> Option { + if crate::stdlib::tracemalloc_real::is_tracking() + || crate::stdlib::testinternalcapi_mod::reftrace_print_active() + || crate::gc_trace::auto_collect_due() + { + return None; + } + let obj = if dict { + Object::Dict(Rc::new(crate::sync::RefCell::new( + crate::object::DictData::with_capacity_and_hasher(0, crate::fasthash::FxBuildHasher), + ))) + } else { + Object::new_list(Vec::new()) + }; + crate::gc_trace::track(&obj); + owned.own(obj) +} + +/// An effect leaf's `recv.name = val` (at store `pc`), buffered until the +/// return commits it. An argument receiver is used in place; any other (a +/// field's value) is held by the owned scratch. +#[inline(always)] +fn plan_store( + args: &[*const Object], + owned: &mut Owned, + pend: &mut Pending, + rp: *const Object, + val: V, + pc: u16, + name: u16, +) -> Option<()> { + // SAFETY: as `norm`. + let recv = unsafe { &*rp }; + if !matches!(recv, Object::Instance(_)) || pend.n == PENDING { + return None; + } + let value = to_object(val)?; + let rp = if args.iter().any(|&a| std::ptr::eq(a, rp)) { + rp + } else { + let V::R(held) = owned.own(crate::clone_hot(recv))? else { + return None; + }; + held + }; + pend.buf[pend.n].write((rp, u32::from(pc), u32::from(name), value)); + pend.n += 1; + Some(()) +} + +impl Interpreter { + /// `LOAD_GLOBAL` at `pc` of `code` in `f`'s namespaces, stamp-validated + /// as the core loop's arm. + #[inline(always)] + fn plan_global( + &self, + code: &CodeObject, + stamps: &[crate::StampSlot], + f: &crate::object::PyFunction, + pc: u16, + ) -> Option { + self.plan_global_at(code, stamps.get(usize::from(pc))?, f, pc) + } + + /// [`Self::plan_global`] with the site's stamp slot in hand. + #[inline(always)] + fn plan_global_at( + &self, + code: &CodeObject, + slot: &crate::StampSlot, + f: &crate::object::PyFunction, + pc: u16, + ) -> Option { + use weavepy_compiler::InlineCache as IC; + let pc = usize::from(pc); + let (gdict, bdict) = (f.globals.as_ptr(), f.builtins.as_ptr()); + let gid = crate::specialize::rc_id(&f.globals); + // SAFETY (raw dict reads): nothing runs code here. + let g_stamp = unsafe { (*gdict).mutation_stamp() }; + let hit = match code.caches.get(pc as u32) { + IC::LoadGlobalModule { + globals_id, + key_idx, + } if globals_id == gid && slot.get() == [gid, g_stamp, 0] => unsafe { + (*gdict).get_index(key_idx as usize) + }, + IC::LoadGlobalBuiltin { + builtins_id, + key_idx, + } if builtins_id == crate::specialize::rc_id(&f.builtins) + && !self.globals_missing_any.get() + && slot.get() == [gid, g_stamp, unsafe { (*bdict).mutation_stamp() }] => + unsafe { (*bdict).get_index(key_idx as usize) }, + _ => return None, + }; + Some(norm(hit?.1)) + } + + /// `src.name` (the `LOAD_ATTR` at `pc`): the latest buffered store to + /// it, a cached instance field, or what the site resolves; a value the + /// read hands back owned is held by the owned scratch. + #[allow(clippy::too_many_arguments)] + #[inline(always)] + fn plan_attr( + &self, + code: &CodeObject, + ext: &CodeConstObjects, + stamps: &[crate::StampSlot], + owned: &mut Owned, + pend: &Pending, + src: V, + pc: u16, + name: u16, + ) -> Option { + let V::R(p) = src else { + return None; + }; + let (pc, name) = (u32::from(pc), u32::from(name)); + // SAFETY: as `norm`. + let recv = unsafe { &*p }; + if EFFECT && pend.n > 0 { + // A buffered store to this attribute, the latest. + let hit = (0..pend.n) + .rev() + .map(|k| pend.get(k)) + // SAFETY: stable receivers (see `Store`). + .find(|(r, _, n, _)| *n == name && unsafe { (**r).is_same(recv) }); + if let Some((_, _, _, v)) = hit { + return Some(norm(v)); + } + } + match recv { + Object::Instance(inst) => { + if GETTER && inst.cls_raw().native_kind.get() != 0 { + return None; + } + // SAFETY: the receiver remains rooted by an argument or + // owned scratch; no Python runs. + match unsafe { Self::leaf_cached_instance_field(ext, code, inst, pc, name) } { + Some(v) => Some(norm(v)), + // A stale or absent site cache resolves through the + // site's own entries. + None => owned.own(Self::leaf_attr_resolve_site(code, inst, recv, pc, name)?), + } + } + Object::Type(cls) => { + if GETTER && !Self::plain_metaclass(cls) { + return None; + } + match stamps + .get(pc as usize) + .and_then(|s| crate::class_attr_hit(s, cls)) + { + Some(v) => owned.own(v), + None => { + let v = Self::leaf_load_type_attr(code, cls, pc, name)?; + // A scalar is a plain class value (see + // `leaf_load_type_attr`): remembered for the next + // read, which native code makes in line. + crate::class_attr_fill(code, cls, pc as usize, &v); + owned.own(v) } - let value = to_object(get!(val))?; - // An argument receiver is used in place; any other - // (a field's value) is held by the owned scratch. - let rp = if args.iter().any(|&a| std::ptr::eq(a, rp)) { - rp + } + } + Object::Module(module) => { + if GETTER + && (crate::object::module_class(module).is_some() + || code.names.get(name as usize)?.starts_with("__")) + { + return None; + } + owned.own(Self::leaf_load_attr_recv(code, recv, pc, name)?) + } + _ => None, + } + } + + /// The method-form load of `src.name` off the site's slot: the function + /// and the receiver, or a class's function and the empty self slot (as + /// the core loop's arm). + #[inline(always)] + fn plan_method(&self, code: &CodeObject, src: V, pc: u16, name: u16) -> Option<(V, V)> { + let ms = crate::code_method_slot(code, u32::from(pc))?; + self.plan_method_at(code, ms, src, name) + } + + /// [`Self::plan_method`] with the site's method slot in hand. + #[inline(always)] + fn plan_method_at( + &self, + code: &CodeObject, + ms: &crate::MethodSlot, + src: V, + name: u16, + ) -> Option<(V, V)> { + let V::R(p) = src else { + return None; + }; + // SAFETY: as `norm`. + match unsafe { &*p } { + Object::Instance(inst) => { + let cls = inst.cls_raw(); + if !Self::default_getattribute(cls) { + return None; + } + let fp = ms.peek_fn(cls.attr_version.get())?; + // The instance's attributes must not shadow the method. + if crate::inst_may_shadow(inst, code, u32::from(name)) { + return None; + } + Some((V::Fn(fp), V::R(p))) + } + Object::Type(cls) => Some((V::Fn(ms.peek_unbound(cls.attr_version.get())?), V::Null)), + recv => { + let b = self.leaf_builtin_method_ptr(ms, recv, code, name)?; + Some((V::Bi(b), V::R(p))) + } + } + } + + /// The evaluation's return of `v`: borrowed when the evaluation's + /// scratch doesn't hold it (a pure body stores nothing, so it outlives + /// the evaluation), otherwise owned, with an effect leaf's buffered + /// stores committed first. The latest store to each attribute is the + /// one that lands; every one must go through before any does (a decline + /// touched nothing, and the ordinary call runs the body instead). A + /// lone store is its own check: it declines whole. + #[inline(always)] + fn plan_finish( + &self, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + owned: &Owned, + pend: &mut Pending, + v: V, + ) -> Option { + if !EFFECT && !matches!(v, V::R(p) if owned.holds(p)) { + return Some(LeafRet::Borrowed(v)); + } + let r = to_object(v)?; + if EFFECT && pend.n > 0 { + let unique = plan.unique_stores; + if pend.n > 1 { + // With one receiver, appends of new attributes land in + // order: `cursor` is its split length once the earlier ones + // have. A store the shortcut can't vouch for may move the + // layout: the rest check in full. + let mut cursor = None; + let mut split_ok = unique; + for k in 0..pend.n { + if !unique && !pend.latest(k) { + continue; + } + let (rp, spc, name, _) = pend.get(k); + // SAFETY: stable receivers (see `Store`). + let Object::Instance(inst) = (unsafe { &**rp }) else { + return None; + }; + let (spc, name) = (*spc as usize, *name); + let ready = if split_ok { + Self::core_store_attr_ready_split(ext, inst, spc, &mut cursor) } else { - let V::R(held) = owned.own(crate::clone_hot(recv))? else { - return None; - }; - held + None }; - pend.buf[pend.n].write((rp, u32::from(pc), u32::from(name), value)); - pend.n += 1; + split_ok &= ready.is_some(); + if !ready.unwrap_or_else(|| Self::core_store_attr_ready(code, inst, spc, name)) + { + return None; + } + } + } + let n = pend.n; + for k in 0..n { + if !unique && !pend.latest(k) { + continue; + } + let (rp, spc, name, value) = pend.get(k); + // SAFETY: stable receivers (see `Store`). + let Object::Instance(inst) = (unsafe { &**rp }) else { + return None; + }; + // On `true` the value moved into the dict. + if Self::core_store_attr(code, inst, *spc as usize, *name, value) { + pend.moved |= 1 << k; + } else if n == 1 { + // The lone store declined, untouched. + return None; + } else { + debug_assert!(false, "a ready store declined"); } } } + Some(LeafRet::Owned(r)) } } diff --git a/crates/weavepy-vm/src/leaf_plan/jit.rs b/crates/weavepy-vm/src/leaf_plan/jit.rs new file mode 100644 index 00000000..904d1b54 --- /dev/null +++ b/crates/weavepy-vm/src/leaf_plan/jit.rs @@ -0,0 +1,2202 @@ +//! Native code for leaf plans. +//! +//! A plan that keeps running is compiled, once per effect mode, into a +//! function over the same registers: each register is a pair of SSA +//! variables holding the [`V`] tag and payload words; moves, constants, +//! jumps, and scalar compares, truth tests and arithmetic run in line; so +//! do the common hits of a site's caches (an instance field, a method, a +//! class constant, a module global), which their helpers fill; and every +//! other operation calls a helper that runs the interpreter's own +//! implementation of it. Whatever the native code or a helper can't +//! settle declines the evaluation exactly where the interpreter's runner +//! would, so a native plan keeps the runner's contract: a result, or +//! nothing observable. +//! +//! A method call whose site last resolved a pure leaf is compiled with the +//! callee's plan in line, behind a guard on the function and its code; any +//! other call goes through [`Interpreter::leaf_eval_nested`] (and so runs +//! natively too once the callee's own plan is compiled). + +use super::{ + binary, compare, is_same, new_container, norm, plan_store, subscr, truth, unary, LeafPlan, + LeafRet, Op, Owned, Pending, NEST, V, +}; +use crate::object::{Object, PyFunction}; +use crate::sync::Rc; +use crate::{CodeConstObjects, Interpreter}; +use cranelift_codegen::ir::{ + condcodes::{FloatCC, IntCC}, + types, AbiParam, Block, InstBuilder, MemFlags, SigRef, Signature, StackSlotData, StackSlotKind, + Value, +}; +use cranelift_codegen::settings::{self, Configurable}; +use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext, Variable}; +use cranelift_jit::{JITBuilder, JITModule}; +use cranelift_module::{Linkage, Module}; +use weavepy_compiler::{BinOpKind, CodeObject, CompareKind}; + +/// Evaluations of a plan (per effect mode) before it is compiled. A +/// compile costs millions of instructions and saves tens to hundreds per +/// evaluation, so a plan compiles once interpreting it has cost about +/// what compiling would (the rent-or-buy rule): a short run doesn't pay +/// for code it never amortizes, and a long one loses little. +const WARM_RUNS: u32 = 20_000; + +/// How deep callee plans are compiled in line. +const INLINE_DEPTH: u8 = 2; + +/// The `repr(u64)` discriminants of [`V`]'s variants, in declaration order. +const T_FN: u64 = 0; +const T_BI: u64 = 1; +const T_NULL: u64 = 2; +const T_R: u64 = 3; +const T_I: u64 = 4; +const T_F: u64 = 5; +const T_B: u64 = 6; +const T_N: u64 = 7; + +/// A compiled plan: `(ctx, argument words) -> status`, where `0` returned +/// (the result's words in `ctx.out`) and anything else declined. +type NativeFn = unsafe extern "C" fn(*mut Ctx<'static>, *const u64) -> u32; + +/// A plan's native code and what it was compiled against. +pub(super) struct Native { + func: NativeFn, + /// The deepest in-line callee (the evaluation needs that much room + /// below the recursion limit). + depth: u8, + /// The in-line callees' frame descriptors, which the code addresses, + /// and what keeps their identity guards sound: a function's + /// allocation stays reserved (so no other function takes its + /// address), and its code stays alive. + // Boxed: the code embeds each descriptor's address. + #[allow(clippy::vec_box)] + _frames: Vec>, + _keep: Vec<(crate::sync::Weak, Rc)>, + /// The attribute and global sites' caches, which the code addresses. + #[allow(clippy::vec_box)] + _fields: Vec>, + #[allow(clippy::vec_box)] + _globals: Vec>, + #[allow(clippy::vec_box)] + _methods: Vec>, +} + +/// An attribute site's split-field positions by receiver class version: +/// a site several classes reach (methods a base class shares) keeps one +/// for each. A version names one class in one state, and a class's shared +/// names never move, so a hit needs only the instance to hold the field. +#[derive(Default)] +pub(super) struct FieldCache { + entries: [std::cell::Cell<(u64, u32)>; 4], + next: std::cell::Cell, +} + +/// A module global site's value as register words, with the globals +/// stamp it was read under (process-unique, so it names the dict and its +/// key layout: the entry stays put) and the global value epoch (which an +/// in-place rebinding advances). Native code reads it in line. +#[derive(Default)] +#[repr(C)] +pub(super) struct GlobalCache { + stamp: std::cell::Cell, + epoch: std::cell::Cell, + tag: std::cell::Cell, + pay: std::cell::Cell, +} + +const GC_STAMP: i32 = std::mem::offset_of!(GlobalCache, stamp) as i32; +const GC_EPOCH: i32 = std::mem::offset_of!(GlobalCache, epoch) as i32; +const GC_TAG: i32 = std::mem::offset_of!(GlobalCache, tag) as i32; +const GC_PAY: i32 = std::mem::offset_of!(GlobalCache, pay) as i32; +/// Where native code finds the evaluation's own function. +const TOP_F_OFFSET: i32 = TOP_OFFSET + std::mem::offset_of!(Frame, f) as i32; + +/// A class-attribute stamp's scalar tags (see `class_attr_fill`) and the +/// register tags of the values they hold. +const CLASS_ATTR_SCALARS: [(u64, u64); 4] = [ + (crate::CLASS_ATTR_INT, T_I), + (crate::CLASS_ATTR_FLOAT, T_F), + (crate::CLASS_ATTR_BOOL, T_B), + (crate::CLASS_ATTR_NONE, T_N), +]; + +/// A method site's latest resolutions, which native code reads in line: +/// for an instance receiver, its class's version, the function, and how +/// many of the class's shared names (from the first) come before the +/// method's name, so an instance holding no more fields than that can't +/// shadow it; for a class receiver, its version and the function called +/// with an empty self slot. Versions are process-unique (one class in one +/// state), so an equal one means the same resolution. +#[derive(Default)] +#[repr(C)] +pub(super) struct MethodCache { + inst_ver: std::cell::Cell, + inst_func: std::cell::Cell, + inst_limit: std::cell::Cell, + type_ver: std::cell::Cell, + type_func: std::cell::Cell, +} + +const MC_INST_VER: i32 = std::mem::offset_of!(MethodCache, inst_ver) as i32; +const MC_INST_FUNC: i32 = std::mem::offset_of!(MethodCache, inst_func) as i32; +const MC_INST_LIMIT: i32 = std::mem::offset_of!(MethodCache, inst_limit) as i32; +const MC_TYPE_VER: i32 = std::mem::offset_of!(MethodCache, type_ver) as i32; +const MC_TYPE_FUNC: i32 = std::mem::offset_of!(MethodCache, type_func) as i32; + +// SAFETY: the descriptors name immutable code objects and reserved +// function allocations; native code is only entered with the GIL held. +unsafe impl Send for Native {} +// SAFETY: as above. +unsafe impl Sync for Native {} + +/// `v` as its tag and payload words (the unused payload bytes zeroed). +#[inline(always)] +fn words(v: V) -> [u64; 2] { + match v { + V::Fn(p) => [T_FN, p as u64], + V::Bi(p) => [T_BI, p as u64], + V::Null => [T_NULL, 0], + V::R(p) => [T_R, p as u64], + V::I(i) => [T_I, i as u64], + V::F(x) => [T_F, x.to_bits()], + V::B(b) => [T_B, u64::from(b)], + V::N => [T_N, 0], + } +} + +/// The value with tag `t` and payload `p`. +#[inline(always)] +fn value(t: u64, p: u64) -> V { + match t { + T_FN => V::Fn(p as *const PyFunction), + T_BI => V::Bi(p as *const crate::object::BuiltinFn), + T_NULL => V::Null, + T_R => V::R(p as *const Object), + T_I => V::I(p as i64), + T_F => V::F(f64::from_bits(p)), + T_B => V::B((p & 0xff) != 0), + _ => V::N, + } +} + +/// Whether the discriminants above are `V`'s (checked once, before any +/// plan compiles). +fn tags_match() -> bool { + let tag = |v: V| { + // SAFETY: `V` is `repr(u64)`: its first word is the discriminant, + // always initialized. + unsafe { *std::ptr::from_ref(&v).cast::() } + }; + tag(V::Fn(std::ptr::null())) == T_FN + && tag(V::Bi(std::ptr::null())) == T_BI + && tag(V::Null) == T_NULL + && tag(V::R(std::ptr::null())) == T_R + && tag(V::I(0)) == T_I + && tag(V::F(0.0)) == T_F + && tag(V::B(false)) == T_B + && tag(V::N) == T_N + && std::mem::size_of::() == 16 +} + +/// What one frame of an evaluation runs: the code (its caches), its +/// constants and namespaces' function. +#[repr(C)] +pub(super) struct Frame { + code: *const CodeObject, + ext: *const CodeConstObjects, + f: *const PyFunction, +} + +impl Frame { + /// # Safety + /// + /// The frame's code and function are alive (the evaluation's own, or + /// an in-line callee whose guard passed). + #[inline(always)] + unsafe fn parts<'a>(&self) -> (&'a CodeObject, &'a CodeConstObjects, &'a PyFunction) { + // SAFETY: forwarded contract. + unsafe { (&*self.code, &*self.ext, &*self.f) } + } +} + +/// One native evaluation's state, shared with the helpers. +#[repr(C)] +pub(super) struct Ctx<'a> { + /// A helper's result words (a method load's two values; a return's + /// first). + out: [u64; 4], + /// The evaluation's own frame. + top: Frame, + interp: &'a Interpreter, + args: &'a [*const Object], + owned: Owned, + pend: Pending, + nest: u8, +} + +impl Ctx<'_> { + #[inline(always)] + fn put(&mut self, v: V) { + let [t, p] = words(v); + self.out[0] = t; + self.out[1] = p; + } +} + +/// `ctx.top`'s offset (native code passes the evaluation's own frame by +/// address). +const TOP_OFFSET: i32 = 32; +/// Where native code finds the count of buffered stores, and a field +/// cache's first entry's version and position. +const PEND_N_OFFSET: i32 = + (std::mem::offset_of!(Ctx<'static>, pend) + std::mem::offset_of!(Pending, n)) as i32; +const FIELD_VER_OFFSET: i32 = + (std::mem::offset_of!(FieldCache, entries) + std::mem::offset_of!((u64, u32), 0)) as i32; +const FIELD_IDX_OFFSET: i32 = + (std::mem::offset_of!(FieldCache, entries) + std::mem::offset_of!((u64, u32), 1)) as i32; +const _: () = assert!(std::mem::offset_of!(Ctx<'static>, top) == TOP_OFFSET as usize); + +/// Run `plan`'s native code for one evaluation, compiling it once it is +/// warm: `None` when it has none (the interpreter's runner evaluates), +/// otherwise the evaluation's outcome. +#[inline] +#[allow(clippy::too_many_arguments)] +pub(super) fn run( + interp: &Interpreter, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + f: &PyFunction, + args: &[*const Object], + nest: u8, +) -> Option> { + let slot = &plan.native[usize::from(EFFECT)]; + let native = match slot.get() { + Some(Some(native)) => native, + Some(None) => return None, + None => { + if plan.runs.fetch_add(1, std::sync::atomic::Ordering::Relaxed) < WARM_RUNS + || crate::tier2::jit_off_for_process() + || crate::gil::free_threading_enabled() + { + return None; + } + slot.get_or_init(|| { + // SAFETY: a function value is always an `Rc` allocation + // (`Object::Function`), alive for this evaluation: the + // count taken here is released with the handle. + let f = unsafe { + Rc::increment_strong_count(std::ptr::from_ref(f)); + Rc::from_raw(std::ptr::from_ref(f)) + }; + compile(code, ext, plan, &f, EFFECT) + }) + .as_ref()? + } + }; + enter::( + interp, + native, + (code, ext, plan, f), + args, + nest, + crate::recursion::current_depth(), + ) +} + +/// [`run`] for a plan whose native code is already compiled, with the +/// caller's recursion `depth` in hand: `None` when there is no native +/// code (yet), otherwise the evaluation's outcome. +#[inline] +pub(super) fn run_compiled( + interp: &Interpreter, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + f: &PyFunction, + args: &[*const Object], + depth: usize, +) -> Option> { + let native = plan.native[usize::from(EFFECT)].get()?.as_ref()?; + enter::(interp, native, (code, ext, plan, f), args, 0, depth) +} + +/// One evaluation of `native` (see [`run`]), at recursion depth `depth`. +#[inline(always)] +fn enter( + interp: &Interpreter, + native: &Native, + (code, ext, plan, f): (&CodeObject, &CodeConstObjects, &LeafPlan, &PyFunction), + args: &[*const Object], + nest: u8, + depth: usize, +) -> Option> { + // In-line callees run where the interpreter would nest frames: near + // the recursion limit, it decides. + if depth + usize::from(nest) + usize::from(native.depth) + 1 + >= crate::recursion::recursion_limit() + { + return None; + } + let mut aw = [0u64; 16]; + for (k, &a) in args.iter().enumerate() { + let [t, p] = words(norm(a)); + aw[2 * k] = t; + aw[2 * k + 1] = p; + } + let mut ctx = Ctx { + out: [0; 4], + top: Frame { code, ext, f }, + interp, + args, + owned: Owned::new(), + pend: Pending::new(), + nest, + }; + // SAFETY: the code was compiled from `plan` with this signature; the + // context and the argument words outlive the call, and the lifetime + // is only erased for its duration. + let status = unsafe { + (native.func)( + std::ptr::from_mut(&mut ctx).cast::>(), + aw.as_ptr(), + ) + }; + if status != 0 { + return Some(None); + } + let v = value(ctx.out[0], ctx.out[1]); + // In place: registers point into the scratch, which must not move. + Some(interp.plan_finish::(code, ext, plan, &ctx.owned, &mut ctx.pend, v)) +} + +// ---- helpers (the interpreter's operations, for native code) ---------- + +/// The context behind a native plan's first parameter. +/// +/// # Safety +/// +/// `ctx` is the live context [`run`] passed to the native code. +#[inline(always)] +unsafe fn cx<'a>(ctx: *mut Ctx<'static>) -> &'a mut Ctx<'a> { + // SAFETY: forwarded contract. + unsafe { &mut *ctx.cast::>() } +} + +/// A frame's stamp slots (its global and class attribute caches). +#[inline(always)] +fn stamps(ext: &CodeConstObjects) -> &[crate::StampSlot] { + ext.stamp_slots.get().map_or(&[], |s| &s[..]) +} + +unsafe extern "C" fn h_global( + ctx: *mut Ctx<'static>, + fr: *const Frame, + slot: *const crate::StampSlot, + cache: *const GlobalCache, + pc: u32, +) -> u32 { + // SAFETY: called by native code with its context, a live frame, the + // site's stamp slot (its code keeps the table), and its cache. + let (c, (code, _, f), slot, cache) = unsafe { (cx(ctx), (*fr).parts(), &*slot, &*cache) }; + let Some(v) = c.interp.plan_global_at(code, slot, f, pc as u16) else { + return 1; + }; + // A module global's value is remembered for the in-line read (a + // builtin's also depends on the globals not gaining the name, which + // the helper checks). + if matches!( + code.caches.get(pc), + weavepy_compiler::InlineCache::LoadGlobalModule { .. } + ) { + // SAFETY (raw dict read): nothing runs code here. + let gs = unsafe { (*f.globals.as_ptr()).mutation_stamp() }; + let [t, p] = words(v); + cache.stamp.set(gs); + cache.epoch.set(crate::object::global_value_epoch()); + cache.tag.set(t); + cache.pay.set(p); + } + c.put(v); + 0 +} + +unsafe extern "C" fn h_deref(ctx: *mut Ctx<'static>, fr: *const Frame, idx: u32) -> u32 { + // SAFETY: called by native code with its context and a live frame. + let (c, (_, _, f)) = unsafe { (cx(ctx), (*fr).parts()) }; + match super::free_var(f, idx as u8) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +unsafe extern "C" fn h_attr( + ctx: *mut Ctx<'static>, + fr: *const Frame, + t: u64, + p: u64, + pc_name: u32, +) -> u32 { + // SAFETY: called by native code with its context and a live frame. + let (c, (code, ext, _)) = unsafe { (cx(ctx), (*fr).parts()) }; + let (pc, name) = (pc_name as u16, (pc_name >> 16) as u16); + match c.interp.plan_attr::( + code, + ext, + stamps(ext), + &mut c.owned, + &c.pend, + value(t, p), + pc, + name, + ) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +/// [`h_attr`] through the site's [`FieldCache`]: a split field of a plain +/// instance whose class the site has seen answers without the general +/// read; a general read that found the instance's own field remembers it. +unsafe extern "C" fn h_field( + ctx: *mut Ctx<'static>, + fr: *const Frame, + cache: *const FieldCache, + t: u64, + p: u64, + pc_name: u32, +) -> u32 { + // SAFETY: called by native code with its context and the site's + // cache. + let (c, cache) = unsafe { (cx(ctx), &*cache) }; + // SAFETY: as `norm`. + let inst = match (t == T_R).then(|| unsafe { &*(p as *const Object) }) { + Some(Object::Instance(inst)) => Some(inst), + _ => None, + }; + if let Some(inst) = inst.filter(|_| !EFFECT || c.pend.n == 0) { + let ver = inst.cls_raw().attr_version.get(); + for e in &cache.entries { + let (v, idx) = e.get(); + if v == ver && v != 0 { + // SAFETY: the receiver is rooted and nothing runs code + // while the view is read. + if let Some(v) = unsafe { inst.split_field(idx as usize) } { + c.put(norm(v)); + return 0; + } + break; + } + } + } + // SAFETY: forwarded. + let status = unsafe { h_attr::(ctx, fr, t, p, pc_name) }; + if let (0, Some(inst)) = (status, inst) { + // SAFETY: a live frame. + let (code, _, _) = unsafe { (*fr).parts() }; + let ver = inst.cls_raw().attr_version.get(); + // The instance's own field (not a class value or descriptor), + // from its split layout. + if let Some((_, Some(idx))) = + Interpreter::leaf_resolve_instance_attr_ix(code, inst, pc_name >> 16) + { + if ver != 0 && inst.dict.published().is_none() { + let k = usize::from(cache.next.get()) % cache.entries.len(); + cache.entries[k].set((ver, idx)); + cache.next.set(cache.next.get().wrapping_add(1)); + } + } + } + status +} + +unsafe extern "C" fn h_method( + ctx: *mut Ctx<'static>, + fr: *const Frame, + ms: *const crate::MethodSlot, + cache: *const MethodCache, + t: u64, + p: u64, + name: u32, +) -> u32 { + // SAFETY: called by native code with its context, a live frame, the + // site's method slot (its code keeps the table), and its cache. + let (c, (code, ext, _), ms, cache) = unsafe { (cx(ctx), (*fr).parts(), &*ms, &*cache) }; + let Some((func, recv)) = c.interp.plan_method_at(code, ms, value(t, p), name as u16) else { + return 1; + }; + let [a, b] = words(func); + let [d, e] = words(recv); + c.out = [a, b, d, e]; + // Remember the resolution for the in-line check. + // SAFETY: the method load succeeded on a heap receiver (as `norm`). + match (func, unsafe { &*(p as *const Object) }) { + (V::Fn(fp), Object::Instance(inst)) if inst.dict.published().is_none() => { + let cls = inst.cls_raw(); + let i = name as usize; + if let (Some(Object::Str(n)), Some(&hash)) = + (ext.name_objs.get(i), ext.name_hashes.get(i)) + { + let limit = cls.shared_keys.get().map_or(0, |k| k.names_before(n, hash)); + cache.inst_ver.set(cls.attr_version.get()); + cache.inst_func.set(fp as u64); + cache.inst_limit.set(limit as u64); + } + } + (V::Fn(fp), Object::Type(cls)) if matches!(recv, V::Null) => { + cache.type_ver.set(cls.attr_version.get()); + cache.type_func.set(fp as u64); + } + _ => {} + } + 0 +} + +/// `0`/`1` for the comparison's result, `2` to decline. +extern "C" fn h_compare(ta: u64, pa: u64, tb: u64, pb: u64, kind: u32) -> u32 { + // SAFETY: the translation checked `kind` names a comparison. + let kind: CompareKind = unsafe { std::mem::transmute(kind as u8) }; + match compare(value(ta, pa), value(tb, pb), kind) { + Some(b) => u32::from(b), + None => 2, + } +} + +/// `0`/`1` for the operand's truth, `2` to decline. +extern "C" fn h_truth(t: u64, p: u64) -> u32 { + match truth(value(t, p)) { + Some(b) => u32::from(b), + None => 2, + } +} + +extern "C" fn h_is(ta: u64, pa: u64, tb: u64, pb: u64) -> u32 { + u32::from(is_same(value(ta, pa), value(tb, pb))) +} + +unsafe extern "C" fn h_unary(ctx: *mut Ctx<'static>, t: u64, p: u64, kind: u32) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + match unary(value(t, p), kind as u8) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +unsafe extern "C" fn h_binary( + ctx: *mut Ctx<'static>, + ta: u64, + pa: u64, + tb: u64, + pb: u64, + kind: u32, +) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + match binary(value(ta, pa), value(tb, pb), kind as u8) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +unsafe extern "C" fn h_new(ctx: *mut Ctx<'static>, dict: u32) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + match new_container(&mut c.owned, dict != 0) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +/// `a[b]` (see [`subscr`]). +unsafe extern "C" fn h_subscr(ctx: *mut Ctx<'static>, ta: u64, pa: u64, tb: u64, pb: u64) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + match subscr(&mut c.owned, value(ta, pa), value(tb, pb)) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + } +} + +/// The evaluation's own `recv.name = val` (only the top frame stores). +/// A store nothing after it can decline (`last`, decided when the plan +/// compiled), with no store buffered before it, lands at once: the +/// evaluation can no longer abandon it. Any other store is buffered for +/// the return to commit. +unsafe extern "C" fn h_store( + ctx: *mut Ctx<'static>, + tr: u64, + pr: u64, + tv: u64, + pv: u64, + pc_name: u32, + last: u32, +) -> u32 { + // SAFETY: called by native code with its context. + let c = unsafe { cx(ctx) }; + let V::R(rp) = value(tr, pr) else { + return 1; + }; + let (pc, name) = (pc_name as u16, (pc_name >> 16) as u16); + if last != 0 && c.pend.n == 0 { + // SAFETY: as `norm`; the receiver is rooted by an argument or a + // field of one, and nothing has run since it was read. A handle + // of its own keeps it while the store runs (a field's slot may + // move when the store grows the instance holding it). + let inst = match unsafe { &*rp } { + Object::Instance(inst) => inst.clone(), + _ => return 1, + }; + let Some(v) = super::to_object(value(tv, pv)) else { + return 1; + }; + // SAFETY: the evaluation's own frame (its code is alive). + let (code, _, _) = unsafe { c.top.parts() }; + // On `true` the value moved into the instance; a declined store + // touched nothing, and the evaluation declines whole. + let v = std::mem::ManuallyDrop::new(v); + if Interpreter::core_store_attr(code, &inst, usize::from(pc), u32::from(name), &v) { + return 0; + } + drop(std::mem::ManuallyDrop::into_inner(v)); + return 1; + } + match plan_store( + c.args, + &mut c.owned, + &mut c.pend, + rp, + value(tv, pv), + pc, + name, + ) { + Some(()) => 0, + None => 1, + } +} + +/// A leaf call at `pc` of frame `fr` (at in-line `depth`) on the registers +/// whose words start at `w`: the callee, its self slot, then `argc` +/// arguments. A pure-leaf Python callee is evaluated one level down +/// (natively, once compiled); a read-only builtin runs on borrowed copies +/// of its arguments. +unsafe extern "C" fn h_call( + ctx: *mut Ctx<'static>, + fr: *const Frame, + w: *const u64, + argc_depth: u32, + pc: u32, +) -> u32 { + // SAFETY: called by native code with its context and a live frame. + let (c, (code, _, _)) = unsafe { (cx(ctx), (*fr).parts()) }; + let (argc, depth) = (argc_depth & 0xff, (argc_depth >> 8) as u8); + let nest = c.nest + depth; + // Only while no store is buffered (the callee would not see one). + if (EFFECT && c.pend.n > 0) || nest >= NEST { + return 1; + } + // SAFETY: native code spilled `argc + 2` registers' words at `w`. + let reg = |k: usize| unsafe { value(*w.add(2 * k), *w.add(2 * k + 1)) }; + let first = if matches!(reg(1), V::Null) { 2 } else { 1 }; + let n = argc as usize + 2 - first; + if n > 8 { + return 1; + } + let (fp, native) = match reg(0) { + V::Fn(fp) => (fp, None), + V::Bi(b) => (std::ptr::null(), Some((b, None))), + // SAFETY: as `norm`. + V::R(p) => match unsafe { &*p } { + Object::Function(func) => (Rc::as_ptr(func), None), + Object::Builtin(b) => (std::ptr::null(), Some((Rc::as_ptr(b), Some(b)))), + _ => return 1, + }, + _ => return 1, + }; + // Scalar arguments are staged as objects (no drop glue); heap ones are + // borrowed in place. + let mut staged = [const { std::mem::MaybeUninit::::uninit() }; 8]; + let mut ptrs = [std::ptr::null::(); 8]; + for k in 0..n { + let o = match reg(first + k) { + V::R(p) => { + ptrs[k] = p; + continue; + } + V::I(i) => Object::Int(i), + V::F(x) => Object::Float(x), + V::B(b) => Object::Bool(b), + V::N => Object::None, + V::Fn(_) | V::Bi(_) | V::Null => return 1, + }; + ptrs[k] = staged[k].write(o); + } + if let Some((b, rc)) = native { + // A read-only builtin, on borrowed copies of the arguments: never + // dropped, so no reference moves. + let mut copies = [const { std::mem::MaybeUninit::::uninit() }; 8]; + for k in 0..n { + // SAFETY: each pointer names a live object; the copy is never + // dropped. + copies[k].write(unsafe { std::ptr::read(ptrs[k]) }); + } + // SAFETY: the first `n` copies were written. + let args = unsafe { std::slice::from_raw_parts(copies.as_ptr().cast::(), n) }; + let Some(r) = c.interp.leaf_pure_builtin(code, pc as usize, b, rc, args) else { + return 1; + }; + return match c.owned.own(r) { + Some(v) => { + c.put(v); + 0 + } + None => 1, + }; + } + // SAFETY: the class or the namespace holds the callee, and nothing + // here runs code that could release it. + let callee = unsafe { &*fp }; + // SAFETY: GIL-serialized raw read of the code cell. + let ccode: &Rc = unsafe { &*callee.code.as_ptr() }; + if !crate::code_is_pure_leaf(ccode) + || !Interpreter::leaf_code_ok(ccode) + || n != crate::leaf_arity(ccode) + || ccode.has_varkeywords + || crate::recursion::current_depth() + usize::from(nest) + 1 + >= crate::recursion::recursion_limit() + { + return 1; + } + let v = match c + .interp + .leaf_eval_nested(ccode, callee, &ptrs[..n], nest + 1) + { + Some(LeafRet::Borrowed(v)) => v, + Some(LeafRet::Owned(r)) => match c.owned.own(r) { + Some(v) => v, + None => return 1, + }, + None => return 1, + }; + c.put(v); + 0 +} + +// ---- the compiler ------------------------------------------------------- + +/// The process's plan compiler. Code it emits lives as long as the +/// process: plans are shared between threads, and their code is never +/// freed. +struct Engine { + module: JITModule, + ctx: cranelift_codegen::Context, + fbctx: FunctionBuilderContext, + ptr: types::Type, + next: u32, +} + +// SAFETY: the engine is only used under its mutex. +unsafe impl Send for Engine {} + +static ENGINE: std::sync::Mutex>> = std::sync::Mutex::new(None); + +impl Engine { + fn new() -> Option { + if !tags_match() { + return None; + } + let mut flags = settings::builder(); + flags.set("use_colocated_libcalls", "false").ok()?; + flags.set("is_pic", "false").ok()?; + // No mid-end optimization: a plan's lowering emits the code it + // wants, and the pass cost more to run than it saved. + // `WEAVEPY_JIT_OPT=1` turns it on. + let opt = if std::env::var_os("WEAVEPY_JIT_OPT").is_some_and(|v| v == "1") { + "speed" + } else { + "none" + }; + flags.set("opt_level", opt).ok()?; + if std::env::var_os("WEAVEPY_JIT_QUICK").is_some() { + flags.set("opt_level", "none").ok()?; + flags.set("regalloc_algorithm", "single_pass").ok()?; + } + if !cfg!(debug_assertions) && !verify() { + flags.set("enable_verifier", "false").ok()?; + } + let isa = cranelift_native::builder() + .ok()? + .finish(settings::Flags::new(flags)) + .ok()?; + let module = JITModule::new(JITBuilder::with_isa( + isa, + cranelift_module::default_libcall_names(), + )); + let ptr = module.target_config().pointer_type(); + let ctx = module.make_context(); + Some(Engine { + module, + ctx, + fbctx: FunctionBuilderContext::new(), + ptr, + next: 0, + }) + } +} + +/// Whether to verify the IR of every compiled plan and report failures +/// (`WEAVEPY_PLAN_VERIFY`, a debugging aid). +fn verify() -> bool { + static VERIFY: std::sync::OnceLock = std::sync::OnceLock::new(); + *VERIFY.get_or_init(|| std::env::var_os("WEAVEPY_PLAN_VERIFY").is_some()) +} + +/// Compile `plan` (the translation of `code`) for `effect` mode; `None` +/// leaves it to the runner. A compiler panic turns plan compilation off +/// for the process rather than taking the interpreter down. +fn compile( + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + f: &Rc, + effect: bool, +) -> Option { + crate::tier2::ensure_obj_layout(); + let mut guard = ENGINE + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let engine = guard.get_or_insert_with(Engine::new).as_mut()?; + let built = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + compile_with(engine, code, ext, plan, f, effect) + })); + match built { + Ok(native) => native, + Err(_) => { + *guard = Some(None); + None + } + } +} + +fn compile_with( + engine: &mut Engine, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + f: &Rc, + effect: bool, +) -> Option { + engine.module.clear_context(&mut engine.ctx); + let ptr = engine.ptr; + let sig = &mut engine.ctx.func.signature; + sig.params.push(AbiParam::new(ptr)); + sig.params.push(AbiParam::new(ptr)); + sig.returns.push(AbiParam::new(types::I32)); + let name = format!("wpplan_{}", engine.next); + engine.next += 1; + let id = engine + .module + .declare_function(&name, Linkage::Local, &engine.ctx.func.signature) + .ok()?; + let lowered = { + let b = FunctionBuilder::new(&mut engine.ctx.func, &mut engine.fbctx); + let mut lower = Lower { + b, + effect, + ptr, + ctx: Value::from_u32(0), + decline: Block::from_u32(0), + sigs: Sigs::default(), + depth: 0, + frames: Vec::new(), + keep: Vec::new(), + fields: Vec::new(), + globals: Vec::new(), + methods: Vec::new(), + }; + let done = lower.lower(code, ext, plan, f); + let Lower { + b, + depth, + frames, + keep, + fields, + globals, + methods, + .. + } = lower; + done.map(|()| { + b.finalize(); + (depth, frames, keep, fields, globals, methods) + }) + }; + let Some((depth, frames, keep, fields, globals, methods)) = lowered else { + // An unfinished function leaves the builder's state behind. + engine.fbctx = FunctionBuilderContext::new(); + engine.module.clear_context(&mut engine.ctx); + return None; + }; + let defined = engine.module.define_function(id, &mut engine.ctx); + if let (Err(e), true) = (&defined, verify()) { + eprintln!( + "weavepy: plan for {:?} failed to compile: {e:?}", + code.qualname + ); + } + engine.module.clear_context(&mut engine.ctx); + defined.ok()?; + engine.module.finalize_definitions().ok()?; + let func = engine.module.get_finalized_function(id); + // SAFETY: `func` is a finalized function with exactly the signature + // declared above, and the module (never dropped) keeps it alive. + let func = unsafe { std::mem::transmute::<*const u8, NativeFn>(func) }; + Some(Native { + func, + depth, + _frames: frames, + _keep: keep, + _fields: fields, + _globals: globals, + _methods: methods, + }) +} + +/// The helper signatures one function imports, by shape. +#[derive(Default)] +struct Sigs { + map: Vec<(Vec, SigRef)>, +} + +/// One plan's registers and blocks while it is lowered (the evaluation's +/// own, or an in-line callee's). +struct FrameLower<'p> { + plan: &'p LeafPlan, + code: &'p CodeObject, + /// The frame descriptor the helpers get. + fr: Value, + /// In-line depth (0 for the evaluation's own frame). + depth: u8, + tags: Vec, + pays: Vec, + blocks: Vec, + /// Where a callee's return goes: the caller's result register (in the + /// caller's variables) and the block after the call. + ret: Option<(Variable, Variable, Block)>, + /// Whether an attribute store may have run before an op (only the + /// evaluation's own frame stores). + may_store: Vec, + /// The function the frame runs: an in-line callee's (its guard proved + /// it), or the one the plan was compiled for (the evaluation's own + /// may be another with the same code). + func: *const PyFunction, +} + +struct Lower<'b> { + b: FunctionBuilder<'b>, + effect: bool, + ptr: types::Type, + ctx: Value, + decline: Block, + sigs: Sigs, + /// The deepest in-line callee. + depth: u8, + // Boxed: the code embeds each descriptor's address. + #[allow(clippy::vec_box)] + frames: Vec>, + keep: Vec<(crate::sync::Weak, Rc)>, + #[allow(clippy::vec_box)] + fields: Vec>, + #[allow(clippy::vec_box)] + globals: Vec>, + #[allow(clippy::vec_box)] + methods: Vec>, +} + +impl Lower<'_> { + fn sig(&mut self, params: &[types::Type]) -> SigRef { + if let Some((_, s)) = self.sigs.map.iter().find(|(p, _)| p == params) { + return *s; + } + let mut sig = Signature::new(self.b.func.signature.call_conv); + for &t in params { + sig.params.push(AbiParam::new(t)); + } + sig.returns.push(AbiParam::new(types::I32)); + let s = self.b.import_signature(sig); + self.sigs.map.push((params.to_vec(), s)); + s + } + + /// Call the helper at `addr`: its `u32` result. + fn call(&mut self, addr: usize, args: &[Value]) -> Value { + let params: Vec = args + .iter() + .map(|&a| self.b.func.dfg.value_type(a)) + .collect(); + let sig = self.sig(¶ms); + let callee = self.b.ins().iconst(self.ptr, addr as i64); + let inst = self.b.ins().call_indirect(sig, callee, args); + self.b.inst_results(inst)[0] + } + + /// Continue in a fresh block when `status` is `0`; decline otherwise. + fn check(&mut self, status: Value) { + let ok = self.b.create_block(); + self.b.ins().brif(status, self.decline, &[], ok, &[]); + self.b.switch_to_block(ok); + } + + fn get(&mut self, fl: &FrameLower<'_>, r: u8) -> (Value, Value) { + let r = usize::from(r); + (self.b.use_var(fl.tags[r]), self.b.use_var(fl.pays[r])) + } + + fn set(&mut self, fl: &FrameLower<'_>, r: u8, t: Value, p: Value) { + let r = usize::from(r); + self.b.def_var(fl.tags[r], t); + self.b.def_var(fl.pays[r], p); + } + + /// Branch to `miss` when `cond`; continue in a fresh block otherwise. + fn miss_if(&mut self, cond: Value, miss: Block) { + let ok = self.b.create_block(); + self.b.ins().brif(cond, miss, &[], ok, &[]); + self.b.switch_to_block(ok); + } + + /// [`h_field`]'s first cache entry in line: the receiver register + /// `(t, p)` is a plain instance whose class has the entry's version + /// and whose values are split over its class's names, holding the + /// entry's position; the field's value as register words. Anything + /// else (and, with buffered stores, everything) branches to `miss`. + fn inline_field( + &mut self, + l: &weavepy_jit::ObjLayout, + t: Value, + p: Value, + cache: i64, + effect: bool, + miss: Block, + ) -> (Value, Value) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + // (The checks are grouped by what each group's loads need proven: + // fewer blocks compile faster.) + let not_ref = self.b.ins().icmp_imm(IntCC::NotEqual, t, T_R as i64); + self.miss_if(not_ref, miss); + let otag = self.b.ins().uload8(types::I32, f, p, 0); + let mut bad = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + if effect { + let n = self.b.ins().load(types::I64, f, self.ctx, PEND_N_OFFSET); + let pending = self.b.ins().icmp_imm(IntCC::NotEqual, n, 0); + bad = self.b.ins().bor(bad, pending); + } + self.miss_if(bad, miss); + // An instance: its class, its split values, and the site's entry. + let inst = self.b.ins().load(ptr, f, p, 8); + let cls = self.b.ins().load(ptr, f, inst, l.inst_class); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let at = self.b.ins().iconst(ptr, cache); + let want = self.b.ins().load(types::I64, f, at, FIELD_VER_OFFSET); + let idx = self.b.ins().uload32(f, at, FIELD_IDX_OFFSET); + let lazy = self.b.ins().load(ptr, f, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, f, flag, 0); + let borrow = self.b.ins().sload32(f, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, f, inst, l.inst_split_block); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, miss); + // A block over the class's names that holds the position. + let keys = self.b.ins().load(ptr, f, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, f, cls, l.type_shared_keys); + let len = self.b.ins().uload32(f, block, l.split_len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let bad = self.b.ins().bor(foreign, absent); + self.miss_if(bad, miss); + let off = self.b.ins().ishl_imm(idx, 4); + let v = self.b.ins().iadd(block, off); + let v = self.b.ins().iadd_imm(v, i64::from(l.split_values)); + // `norm`: the scalars by value, anything else by reference. + let vt = self.b.ins().uload8(types::I64, f, v, 0); + let word = self.b.ins().load(types::I64, f, v, 8); + let byte = self.b.ins().uload8(types::I64, f, v, 1); + let zero = self.b.ins().iconst(types::I64, 0); + let mut rt = self.b.ins().iconst(types::I64, T_R as i64); + let mut rp = v; + for (tag, vtag, pay) in [ + (l.tag_int, T_I, word), + (l.tag_float, T_F, word), + (l.tag_bool, T_B, byte), + (l.tag_none, T_N, zero), + ] { + let hit = self.b.ins().icmp_imm(IntCC::Equal, vt, i64::from(tag)); + let tv = self.b.ins().iconst(types::I64, vtag as i64); + rt = self.b.ins().select(hit, tv, rt); + rp = self.b.ins().select(hit, pay, rp); + } + (rt, rp) + } + + /// [`h_method`]'s cache in line, for the receiver register `(t, p)`: an + /// instance whose class has the cached version and that holds none of + /// the class's names from the method's on (in its split values over + /// the class's names, with no dictionary of its own), or a class with + /// the cached version. Jumps to `done` with the function's and the + /// self slot's words, or to `slow`. + fn inline_method( + &mut self, + l: &weavepy_jit::ObjLayout, + t: Value, + p: Value, + cache: i64, + done: Block, + slow: Block, + ) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + let not_ref = self.b.ins().icmp_imm(IntCC::NotEqual, t, T_R as i64); + self.miss_if(not_ref, slow); + let otag = self.b.ins().uload8(types::I32, f, p, 0); + let at = self.b.ins().iconst(ptr, cache); + let is_inst = self + .b + .ins() + .icmp_imm(IntCC::Equal, otag, i64::from(l.tag_instance)); + let inst_b = self.b.create_block(); + let other = self.b.create_block(); + self.b.ins().brif(is_inst, inst_b, &[], other, &[]); + // A class: its version. + self.b.switch_to_block(other); + let is_type = self + .b + .ins() + .icmp_imm(IntCC::Equal, otag, i64::from(l.tag_type)); + let type_b = self.b.create_block(); + self.b.ins().brif(is_type, type_b, &[], slow, &[]); + self.b.switch_to_block(type_b); + let cls = self.b.ins().load(ptr, f, p, 8); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let want = self.b.ins().load(types::I64, f, at, MC_TYPE_VER); + let func = self.b.ins().load(types::I64, f, at, MC_TYPE_FUNC); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let bad = self.b.ins().bor(unset, stale); + self.miss_if(bad, slow); + let tf = self.b.ins().iconst(types::I64, T_FN as i64); + let tn = self.b.ins().iconst(types::I64, T_NULL as i64); + let zero = self.b.ins().iconst(types::I64, 0); + self.b + .ins() + .jump(done, &[tf.into(), func.into(), tn.into(), zero.into()]); + // An instance: its class's version, and its fields. + self.b.switch_to_block(inst_b); + let inst = self.b.ins().load(ptr, f, p, 8); + let cls = self.b.ins().load(ptr, f, inst, l.inst_class); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let want = self.b.ins().load(types::I64, f, at, MC_INST_VER); + let func = self.b.ins().load(types::I64, f, at, MC_INST_FUNC); + let limit = self.b.ins().load(types::I64, f, at, MC_INST_LIMIT); + let lazy = self.b.ins().load(ptr, f, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, f, flag, 0); + let borrow = self.b.ins().sload32(f, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, f, inst, l.inst_split_block); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let busy = self.b.ins().icmp_imm(IntCC::SignedLessThan, borrow, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, busy); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, slow); + let tf = self.b.ins().iconst(types::I64, T_FN as i64); + let tr = self.b.ins().iconst(types::I64, T_R as i64); + let hit_args = [tf.into(), func.into(), tr.into(), p.into()]; + // No fields at all, or none from the method's name on. + let fields = self.b.create_block(); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + self.b.ins().brif(empty, done, &hit_args, fields, &[]); + self.b.switch_to_block(fields); + let keys = self.b.ins().load(ptr, f, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, f, cls, l.type_shared_keys); + // (`uload32` zero-extends to `i64` itself.) + let len = self.b.ins().uload32(f, block, l.split_len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let shadows = self.b.ins().icmp(IntCC::UnsignedGreaterThan, len, limit); + let bad = self.b.ins().bor(foreign, shadows); + self.miss_if(bad, slow); + self.b.ins().jump(done, &hit_args); + } + + /// [`h_global`]'s cache in line: the frame runs its compile-time + /// function, whose globals still have the cached stamp, under the + /// cached value epoch. Jumps to `done` with the value's words, or to + /// `slow`. + fn inline_global(&mut self, fl: &FrameLower<'_>, cache: i64, done: Block, slow: Block) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + if fl.depth == 0 { + // The evaluation's own function may be another with this code. + let top = self.b.ins().load(ptr, f, self.ctx, TOP_F_OFFSET); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, top, fl.func as i64); + self.miss_if(other, slow); + } + // SAFETY: the frame's function is alive while its code runs (see + // `FrameLower::func`), and so its globals. + let stamp_at = + unsafe { (*fl.func).globals.as_ptr() } as usize + crate::object::DictData::STAMP_OFFSET; + let stamp_at = self.b.ins().iconst(ptr, stamp_at as i64); + let epoch_at = self + .b + .ins() + .iconst(ptr, crate::object::global_value_epoch_ptr() as i64); + let at = self.b.ins().iconst(ptr, cache); + let stamp = self.b.ins().load(types::I64, f, stamp_at, 0); + let epoch = self.b.ins().load(types::I64, f, epoch_at, 0); + let want = self.b.ins().load(types::I64, f, at, GC_STAMP); + let want_epoch = self.b.ins().load(types::I64, f, at, GC_EPOCH); + let t = self.b.ins().load(types::I64, f, at, GC_TAG); + let p = self.b.ins().load(types::I64, f, at, GC_PAY); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, stamp, want); + let moved = self.b.ins().icmp(IntCC::NotEqual, epoch, want_epoch); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, moved); + self.miss_if(bad, slow); + self.b.ins().jump(done, &[t.into(), p.into()]); + } + + /// The interpreter's indexed store (`core_store_attr` through the + /// site's [`crate::FieldSlot`] at `slot`) in line, for a scalar value + /// over a scalar field: the receiver register is a plain instance whose + /// class has the slot's version and whose values, split over its + /// class's names and borrowed by nobody, hold the slot's position. Its + /// old value owns nothing, and neither does the new one (no write + /// barrier, no release). Jumps to `done` stored, or to `slow` untouched. + fn inline_store( + &mut self, + l: &weavepy_jit::ObjLayout, + (tr, pr): (Value, Value), + (tv, pv): (Value, Value), + slot: i64, + done: Block, + slow: Block, + ) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + // The value: a scalar register. + let is_i = self.b.ins().icmp_imm(IntCC::Equal, tv, T_I as i64); + let is_f = self.b.ins().icmp_imm(IntCC::Equal, tv, T_F as i64); + let is_b = self.b.ins().icmp_imm(IntCC::Equal, tv, T_B as i64); + let is_n = self.b.ins().icmp_imm(IntCC::Equal, tv, T_N as i64); + let any = self.b.ins().bor(is_i, is_f); + let any = self.b.ins().bor(any, is_b); + let any = self.b.ins().bor(any, is_n); + let not_scalar = self.b.ins().icmp_imm(IntCC::Equal, any, 0); + let not_ref = self.b.ins().icmp_imm(IntCC::NotEqual, tr, T_R as i64); + let bad = self.b.ins().bor(not_scalar, not_ref); + self.miss_if(bad, slow); + let otag = self.b.ins().uload8(types::I32, f, pr, 0); + let not_inst = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_instance)); + self.miss_if(not_inst, slow); + let inst = self.b.ins().load(ptr, f, pr, 8); + let cls = self.b.ins().load(ptr, f, inst, l.inst_class); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let at = self.b.ins().iconst(ptr, slot); + let want = self + .b + .ins() + .load(types::I64, f, at, crate::FIELD_SLOT_VER as i32); + let idx = self.b.ins().uload32(f, at, crate::FIELD_SLOT_IDX as i32); + let lazy = self.b.ins().load(ptr, f, inst, l.inst_dict_lazy); + let flag = self.b.ins().iconst(ptr, l.cells_unguarded as i64); + let shared = self.b.ins().uload8(types::I64, f, flag, 0); + let watched = self.b.ins().iconst(ptr, l.dict_watchers as i64); + let watched = self.b.ins().uload8(types::I64, f, watched, 0); + let borrow = self.b.ins().sload32(f, inst, l.inst_split_borrow); + let block = self.b.ins().load(ptr, f, inst, l.inst_split_block); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let busy = self.b.ins().icmp_imm(IntCC::NotEqual, borrow, 0); + let empty = self.b.ins().icmp_imm(IntCC::Equal, block, 0); + let other = self.b.ins().bor(lazy, shared); + let other = self.b.ins().bor(other, watched); + let other = self.b.ins().icmp_imm(IntCC::NotEqual, other, 0); + let bad = self.b.ins().bor(unset, stale); + let bad = self.b.ins().bor(bad, busy); + let bad = self.b.ins().bor(bad, empty); + let bad = self.b.ins().bor(bad, other); + self.miss_if(bad, slow); + let keys = self.b.ins().load(ptr, f, block, l.split_keys); + let ckeys = self.b.ins().load(ptr, f, cls, l.type_shared_keys); + let len = self.b.ins().uload32(f, block, l.split_len); + let foreign = self.b.ins().icmp(IntCC::NotEqual, keys, ckeys); + let absent = self + .b + .ins() + .icmp(IntCC::UnsignedGreaterThanOrEqual, idx, len); + let bad = self.b.ins().bor(foreign, absent); + self.miss_if(bad, slow); + let off = self.b.ins().ishl_imm(idx, 4); + let v = self.b.ins().iadd(block, off); + let v = self.b.ins().iadd_imm(v, i64::from(l.split_values)); + // The old value: a scalar, which owns nothing. + let old = self.b.ins().uload8(types::I32, f, v, 0); + let mut owns = self.b.ins().iconst(types::I8, 1); + for tag in [l.tag_int, l.tag_float, l.tag_bool, l.tag_none] { + let hit = self.b.ins().icmp_imm(IntCC::Equal, old, i64::from(tag)); + let clear = self.b.ins().bxor_imm(hit, 1); + owns = self.b.ins().band(owns, clear); + } + self.miss_if(owns, slow); + // The new value's tag byte and payload (a bool's byte at offset 1). + let t_int = self.b.ins().iconst(types::I32, i64::from(l.tag_int)); + let t_float = self.b.ins().iconst(types::I32, i64::from(l.tag_float)); + let t_bool = self.b.ins().iconst(types::I32, i64::from(l.tag_bool)); + let t_none = self.b.ins().iconst(types::I32, i64::from(l.tag_none)); + let tag = self.b.ins().select(is_i, t_int, t_none); + let tag = self.b.ins().select(is_f, t_float, tag); + let tag = self.b.ins().select(is_b, t_bool, tag); + self.b.ins().istore8(f, tag, v, 0); + let byte = self.b.ins().band_imm(pv, 1); + self.b.ins().istore8(f, byte, v, 1); + self.b.ins().store(f, pv, v, 8); + self.b.ins().jump(done, &[]); + } + + /// A class attribute the site's stamp remembers, in line: the receiver + /// register `(t, p)` is a class at the stamp's version, which resolved + /// the name to a scalar (see `class_attr_fill`). Jumps to `done` with + /// its words, or to `slow`. + fn inline_class_attr( + &mut self, + l: &weavepy_jit::ObjLayout, + t: Value, + p: Value, + slot: i64, + done: Block, + slow: Block, + ) { + let f = MemFlags::trusted(); + let ptr = self.ptr; + let not_ref = self.b.ins().icmp_imm(IntCC::NotEqual, t, T_R as i64); + self.miss_if(not_ref, slow); + let otag = self.b.ins().uload8(types::I32, f, p, 0); + let not_type = self + .b + .ins() + .icmp_imm(IntCC::NotEqual, otag, i64::from(l.tag_type)); + self.miss_if(not_type, slow); + let cls = self.b.ins().load(ptr, f, p, 8); + let ver = self.b.ins().load(types::I64, f, cls, l.type_attr_version); + let at = self.b.ins().iconst(ptr, slot); + let want = self.b.ins().load(types::I64, f, at, 0); + let tag = self.b.ins().load(types::I64, f, at, 8); + let bits = self.b.ins().load(types::I64, f, at, 16); + let unset = self.b.ins().icmp_imm(IntCC::Equal, want, 0); + let stale = self.b.ins().icmp(IntCC::NotEqual, ver, want); + let bad = self.b.ins().bor(unset, stale); + self.miss_if(bad, slow); + for (stamp, vtag) in CLASS_ATTR_SCALARS { + let next = self.b.create_block(); + let hit = self.b.ins().icmp_imm(IntCC::Equal, tag, stamp as i64); + let tv = self.b.ins().iconst(types::I64, vtag as i64); + let pay = if vtag == T_N { + self.b.ins().iconst(types::I64, 0) + } else { + bits + }; + self.b + .ins() + .brif(hit, done, &[tv.into(), pay.into()], next, &[]); + self.b.switch_to_block(next); + } + self.b.ins().jump(slow, &[]); + } + + /// Load the result words a helper left at `ctx.out[k..k + 2]` into `r`. + fn set_out(&mut self, fl: &FrameLower<'_>, r: u8, k: i32) { + let flags = MemFlags::trusted(); + let t = self.b.ins().load(types::I64, flags, self.ctx, 8 * k); + let p = self.b.ins().load(types::I64, flags, self.ctx, 8 * k + 8); + self.set(fl, r, t, p); + } + + /// A boolean (`i8`) into register `r` as `V::B`. + fn set_bool(&mut self, fl: &FrameLower<'_>, r: u8, bit: Value) { + let t = self.b.ins().iconst(types::I64, T_B as i64); + let p = self.b.ins().uextend(types::I64, bit); + self.set(fl, r, t, p); + } + + /// A payload word as the float it holds. + fn as_f64(&mut self, p: Value) -> Value { + self.b.ins().bitcast(types::F64, MemFlags::new(), p) + } + + fn u32c(&mut self, x: u32) -> Value { + self.b.ins().iconst(types::I32, i64::from(x)) + } + + /// Decline unless the `0`/`1` helper result `r` is not `2`; the `i8` + /// bit. + fn bit_or_decline(&mut self, r: Value) -> Value { + let declines = self.b.ins().icmp_imm(IntCC::Equal, r, 2); + let ok = self.b.create_block(); + self.b.ins().brif(declines, self.decline, &[], ok, &[]); + self.b.switch_to_block(ok); + self.b.ins().ireduce(types::I8, r) + } + + /// A register's truth as an `i8`, declining where the interpreter's + /// `truth` would: in line for the scalar tags, through the helper for + /// a heap value. + fn truth(&mut self, fl: &FrameLower<'_>, src: u8) -> Value { + let (t, p) = self.get(fl, src); + let merge = self.b.create_block(); + self.b.append_block_param(merge, types::I8); + let not_b = self.b.create_block(); + let not_i = self.b.create_block(); + let not_n = self.b.create_block(); + let is_b = self.b.ins().icmp_imm(IntCC::Equal, t, T_B as i64); + let low = self.b.ins().band_imm(p, 0xff); + let b_truth = self.b.ins().icmp_imm(IntCC::NotEqual, low, 0); + self.b + .ins() + .brif(is_b, merge, &[b_truth.into()], not_b, &[]); + self.b.switch_to_block(not_b); + let is_i = self.b.ins().icmp_imm(IntCC::Equal, t, T_I as i64); + let i_truth = self.b.ins().icmp_imm(IntCC::NotEqual, p, 0); + self.b + .ins() + .brif(is_i, merge, &[i_truth.into()], not_i, &[]); + self.b.switch_to_block(not_i); + let is_n = self.b.ins().icmp_imm(IntCC::Equal, t, T_N as i64); + let falsy = self.b.ins().iconst(types::I8, 0); + self.b.ins().brif(is_n, merge, &[falsy.into()], not_n, &[]); + self.b.switch_to_block(not_n); + let r = self.call(h_truth as *const () as usize, &[t, p]); + let bit = self.bit_or_decline(r); + self.b.ins().jump(merge, &[bit.into()]); + self.b.switch_to_block(merge); + self.b.block_params(merge)[0] + } + + fn lower( + &mut self, + code: &CodeObject, + ext: &CodeConstObjects, + plan: &LeafPlan, + f: &Rc, + ) -> Option<()> { + let entry = self.b.create_block(); + self.b.append_block_params_for_function_params(entry); + self.b.switch_to_block(entry); + self.ctx = self.b.block_params(entry)[0]; + let argw = self.b.block_params(entry)[1]; + self.decline = self.b.create_block(); + let fr = self.b.ins().iadd_imm(self.ctx, i64::from(TOP_OFFSET)); + let fl = self.frame(plan, code, ext, fr, 0, None, Rc::as_ptr(f)); + self.keep.push((Rc::downgrade(f), f.code())); + let flags = MemFlags::trusted(); + for k in 0..usize::from(plan.nargs) { + let t = self.b.ins().load(types::I64, flags, argw, (16 * k) as i32); + let p = self + .b + .ins() + .load(types::I64, flags, argw, (16 * k + 8) as i32); + self.set(&fl, k as u8, t, p); + } + self.b.ins().jump(fl.blocks[0], &[]); + self.body(&fl)?; + self.b.switch_to_block(self.decline); + let one = self.b.ins().iconst(types::I32, 1); + self.b.ins().return_(&[one]); + self.b.seal_all_blocks(); + Some(()) + } + + /// Declare a frame's registers (zeroed in the current block) and its + /// op blocks. + fn frame<'p>( + &mut self, + plan: &'p LeafPlan, + code: &'p CodeObject, + ext: &'p CodeConstObjects, + fr: Value, + depth: u8, + ret: Option<(Variable, Variable, Block)>, + func: *const PyFunction, + ) -> FrameLower<'p> { + let nregs = usize::from(plan.nregs); + let mut tags = Vec::with_capacity(nregs); + let mut pays = Vec::with_capacity(nregs); + let zero = self.b.ins().iconst(types::I64, 0); + for _ in 0..nregs { + let t = self.b.declare_var(types::I64); + let p = self.b.declare_var(types::I64); + self.b.def_var(t, zero); + self.b.def_var(p, zero); + tags.push(t); + pays.push(p); + } + let blocks = (0..plan.ops.len()).map(|_| self.b.create_block()).collect(); + // Conservatively: a store anywhere earlier in the op order. + let mut seen = false; + let may_store = plan + .ops + .iter() + .map(|op| { + let before = seen; + seen |= matches!(op, Op::StoreAttr { .. }); + before + }) + .collect(); + let _ = ext; + FrameLower { + plan, + code, + fr, + depth, + tags, + pays, + blocks, + ret, + may_store, + func, + } + } + + /// Lower every op of a frame into its blocks. + fn body(&mut self, fl: &FrameLower<'_>) -> Option<()> { + for (i, op) in fl.plan.ops.iter().enumerate() { + self.b.switch_to_block(fl.blocks[i]); + if self.op(fl, i, op)? { + // Falls through (the translation ends every path in a + // return or a jump, so a next op exists). + let next = *fl.blocks.get(i + 1)?; + self.b.ins().jump(next, &[]); + } + } + Some(()) + } + + /// The pure-leaf callee to compile in line for the `CALL` at op `i` + /// whose callee register `at` a method load filled: what that load's + /// site last resolved, with its plan and arity (self included). + fn inline_target( + &self, + fl: &FrameLower<'_>, + i: usize, + at: u8, + argc: u8, + ) -> Option<( + Rc, + Rc, + &'static CodeConstObjects, + &'static LeafPlan, + )> { + if fl.depth >= INLINE_DEPTH || (self.effect && fl.depth == 0 && fl.may_store[i]) { + return None; + } + let (pc, _) = fl.plan.ops[..i].iter().rev().find_map(|op| match *op { + Op::Method { dst, pc, name, .. } if dst == at => Some((pc, name)), + _ => None, + })?; + let f = crate::code_method_slot(fl.code, u32::from(pc))?.last_fn()?; + let ccode = f.code(); + if !crate::code_is_pure_leaf(&ccode) + || !Interpreter::leaf_code_ok(&ccode) + || crate::leaf_arity(&ccode) != usize::from(argc) + 1 + || ccode.has_varkeywords + { + return None; + } + let cext = crate::code_vm_ext(&ccode)?; + let cplan = cext + .leaf_plan + .get_or_init(|| super::build(&ccode, cext).map(Box::new)) + .as_deref()?; + // A pure callee stores nothing. + if cplan + .ops + .iter() + .any(|op| matches!(op, Op::StoreAttr { .. })) + { + return None; + } + // SAFETY: `ccode` (kept alive with the compiled code) owns its + // extension and plan, which never move. + let (cext, cplan) = unsafe { (&*std::ptr::from_ref(cext), &*std::ptr::from_ref(cplan)) }; + Some((f, ccode, cext, cplan)) + } + + /// Lower one op; whether it falls through to the next. + #[allow(clippy::too_many_lines)] + fn op(&mut self, fl: &FrameLower<'_>, i: usize, op: &Op) -> Option { + let effect = self.effect && fl.depth == 0; + match *op { + Op::Move { dst, src } => { + let (t, p) = self.get(fl, src); + self.set(fl, dst, t, p); + } + Op::Const { dst, k } => { + let [t, p] = words(*fl.plan.consts.get(usize::from(k))?); + let t = self.b.ins().iconst(types::I64, t as i64); + let p = self.b.ins().iconst(types::I64, p as i64); + self.set(fl, dst, t, p); + } + Op::Global { dst, pc } => { + let Some(slot) = crate::code_stamp_slot(fl.code, u32::from(pc)) else { + self.b.ins().jump(self.decline, &[]); + return Some(false); + }; + let slot = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(slot) as i64); + let cache = Box::::default(); + let cache_at = std::ptr::from_ref(&*cache) as i64; + self.globals.push(cache); + let done = self.b.create_block(); + self.b.append_block_param(done, types::I64); + self.b.append_block_param(done, types::I64); + let slow = self.b.create_block(); + self.inline_global(fl, cache_at, done, slow); + self.b.switch_to_block(slow); + let cache_v = self.b.ins().iconst(self.ptr, cache_at); + let pc = self.u32c(u32::from(pc)); + let st = self.call( + h_global as *const () as usize, + &[self.ctx, fl.fr, slot, cache_v, pc], + ); + self.check(st); + let flags = MemFlags::trusted(); + let t = self.b.ins().load(types::I64, flags, self.ctx, 0); + let p = self.b.ins().load(types::I64, flags, self.ctx, 8); + self.b.ins().jump(done, &[t.into(), p.into()]); + self.b.switch_to_block(done); + let (t, p) = (self.b.block_params(done)[0], self.b.block_params(done)[1]); + self.set(fl, dst, t, p); + } + Op::Deref { dst, idx } => { + let idx = self.u32c(u32::from(idx)); + let st = self.call(h_deref as *const () as usize, &[self.ctx, fl.fr, idx]); + self.check(st); + let flags = MemFlags::trusted(); + let t = self.b.ins().load(types::I64, flags, self.ctx, 0); + let p = self.b.ins().load(types::I64, flags, self.ctx, 8); + self.set(fl, dst, t, p); + } + Op::Attr { dst, src, pc, name } => { + let (t, p) = self.get(fl, src); + let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); + let cache = Box::::default(); + let cache_at = std::ptr::from_ref(&*cache) as i64; + let cache_v = self.b.ins().iconst(self.ptr, cache_at); + self.fields.push(cache); + // The site's most recent field, read in line. + let done = self.b.create_block(); + self.b.append_block_param(done, types::I64); + self.b.append_block_param(done, types::I64); + if let Some(l) = crate::tier2::published_obj_layout() { + let slow = self.b.create_block(); + let (vt, vp) = self.inline_field(l, t, p, cache_at, effect, slow); + self.b.ins().jump(done, &[vt.into(), vp.into()]); + self.b.switch_to_block(slow); + if let Some(slot) = crate::code_stamp_slot(fl.code, u32::from(pc)) { + let slow = self.b.create_block(); + let slot = std::ptr::from_ref(slot) as i64; + self.inline_class_attr(l, t, p, slot, done, slow); + self.b.switch_to_block(slow); + } + } + let addr = if effect { + h_field:: as *const () as usize + } else { + h_field:: as *const () as usize + }; + let st = self.call(addr, &[self.ctx, fl.fr, cache_v, t, p, pn]); + self.check(st); + let flags = MemFlags::trusted(); + let rt = self.b.ins().load(types::I64, flags, self.ctx, 0); + let rp = self.b.ins().load(types::I64, flags, self.ctx, 8); + self.b.ins().jump(done, &[rt.into(), rp.into()]); + self.b.switch_to_block(done); + let (rt, rp) = (self.b.block_params(done)[0], self.b.block_params(done)[1]); + self.set(fl, dst, rt, rp); + } + Op::Method { dst, src, pc, name } => { + let Some(ms) = crate::code_method_slot(fl.code, u32::from(pc)) else { + self.b.ins().jump(self.decline, &[]); + return Some(false); + }; + let (t, p) = self.get(fl, src); + let cache = Box::::default(); + let cache_at = std::ptr::from_ref(&*cache) as i64; + self.methods.push(cache); + let done = self.b.create_block(); + for _ in 0..4 { + self.b.append_block_param(done, types::I64); + } + let slow = self.b.create_block(); + if let Some(l) = crate::tier2::published_obj_layout() { + self.inline_method(l, t, p, cache_at, done, slow); + } else { + self.b.ins().jump(slow, &[]); + } + self.b.switch_to_block(slow); + let ms = self.b.ins().iconst(self.ptr, std::ptr::from_ref(ms) as i64); + let cache_v = self.b.ins().iconst(self.ptr, cache_at); + let nm = self.u32c(u32::from(name)); + let st = self.call( + h_method as *const () as usize, + &[self.ctx, fl.fr, ms, cache_v, t, p, nm], + ); + self.check(st); + let flags = MemFlags::trusted(); + let w: Vec<_> = (0..4) + .map(|k| self.b.ins().load(types::I64, flags, self.ctx, 8 * k).into()) + .collect(); + self.b.ins().jump(done, &w); + self.b.switch_to_block(done); + let ps = self.b.block_params(done).to_vec(); + self.set(fl, dst, ps[0], ps[1]); + self.set(fl, dst + 1, ps[2], ps[3]); + } + Op::Compare { dst, a, b, kind } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + let cc = match kind { + k if k == CompareKind::Lt as u8 => IntCC::SignedLessThan, + k if k == CompareKind::LtE as u8 => IntCC::SignedLessThanOrEqual, + k if k == CompareKind::Eq as u8 => IntCC::Equal, + k if k == CompareKind::NotEq as u8 => IntCC::NotEqual, + k if k == CompareKind::Gt as u8 => IntCC::SignedGreaterThan, + k if k == CompareKind::GtE as u8 => IntCC::SignedGreaterThanOrEqual, + _ => return None, + }; + let fcc = match kind { + k if k == CompareKind::Lt as u8 => FloatCC::LessThan, + k if k == CompareKind::LtE as u8 => FloatCC::LessThanOrEqual, + k if k == CompareKind::Eq as u8 => FloatCC::Equal, + k if k == CompareKind::NotEq as u8 => FloatCC::NotEqual, + k if k == CompareKind::Gt as u8 => FloatCC::GreaterThan, + _ => FloatCC::GreaterThanOrEqual, + }; + // Two ints, or two floats, compare in line. + let ia = self.b.ins().icmp_imm(IntCC::Equal, ta, T_I as i64); + let ib = self.b.ins().icmp_imm(IntCC::Equal, tb, T_I as i64); + let both = self.b.ins().band(ia, ib); + let fast = self.b.create_block(); + let not_int = self.b.create_block(); + let float = self.b.create_block(); + let slow = self.b.create_block(); + let merge = self.b.create_block(); + self.b.append_block_param(merge, types::I8); + self.b.ins().brif(both, fast, &[], not_int, &[]); + self.b.switch_to_block(fast); + let bit = self.b.ins().icmp(cc, pa, pb); + self.b.ins().jump(merge, &[bit.into()]); + self.b.switch_to_block(not_int); + let fa = self.b.ins().icmp_imm(IntCC::Equal, ta, T_F as i64); + let fb = self.b.ins().icmp_imm(IntCC::Equal, tb, T_F as i64); + let both = self.b.ins().band(fa, fb); + self.b.ins().brif(both, float, &[], slow, &[]); + self.b.switch_to_block(float); + let (xa, xb) = (self.as_f64(pa), self.as_f64(pb)); + // A NaN declines, as the interpreter's `compare` does. + let nan = self.b.ins().fcmp(FloatCC::Unordered, xa, xb); + let ordered = self.b.create_block(); + self.b.ins().brif(nan, self.decline, &[], ordered, &[]); + self.b.switch_to_block(ordered); + let bit = self.b.ins().fcmp(fcc, xa, xb); + self.b.ins().jump(merge, &[bit.into()]); + self.b.switch_to_block(slow); + let k = self.u32c(u32::from(kind)); + let r = self.call(h_compare as *const () as usize, &[ta, pa, tb, pb, k]); + let bit = self.bit_or_decline(r); + self.b.ins().jump(merge, &[bit.into()]); + self.b.switch_to_block(merge); + let bit = self.b.block_params(merge)[0]; + self.set_bool(fl, dst, bit); + } + Op::Is { dst, a, b, invert } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + let r = self.call(h_is as *const () as usize, &[ta, pa, tb, pb]); + let r = if invert { + self.b.ins().bxor_imm(r, 1) + } else { + r + }; + let bit = self.b.ins().ireduce(types::I8, r); + self.set_bool(fl, dst, bit); + } + Op::Truth { dst, src } => { + let bit = self.truth(fl, src); + self.set_bool(fl, dst, bit); + } + Op::Unary { dst, src, kind } => { + let (t, p) = self.get(fl, src); + let k = self.u32c(u32::from(kind)); + let st = self.call(h_unary as *const () as usize, &[self.ctx, t, p, k]); + self.check(st); + self.set_out(fl, dst, 0); + } + Op::Binary { dst, a, b, kind } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + let int_op = [BinOpKind::Add, BinOpKind::Sub, BinOpKind::Mult] + .into_iter() + .find(|k| *k as u8 == kind); + let float_div = kind == BinOpKind::Div as u8; + let merge = self.b.create_block(); + self.b.append_block_param(merge, types::I64); + self.b.append_block_param(merge, types::I64); + let slow = self.b.create_block(); + if let Some(k) = int_op { + // Two ints add, subtract and multiply in line; an + // overflow declines (the interpreter's `binary` does). + let ia = self.b.ins().icmp_imm(IntCC::Equal, ta, T_I as i64); + let ib = self.b.ins().icmp_imm(IntCC::Equal, tb, T_I as i64); + let both = self.b.ins().band(ia, ib); + let fast = self.b.create_block(); + let not_int = self.b.create_block(); + self.b.ins().brif(both, fast, &[], not_int, &[]); + self.b.switch_to_block(fast); + let (r, ovf) = match k { + BinOpKind::Add => { + let r = self.b.ins().iadd(pa, pb); + let axr = self.b.ins().bxor(pa, r); + let bxr = self.b.ins().bxor(pb, r); + let and = self.b.ins().band(axr, bxr); + (r, self.b.ins().icmp_imm(IntCC::SignedLessThan, and, 0)) + } + BinOpKind::Sub => { + let r = self.b.ins().isub(pa, pb); + let axb = self.b.ins().bxor(pa, pb); + let axr = self.b.ins().bxor(pa, r); + let and = self.b.ins().band(axb, axr); + (r, self.b.ins().icmp_imm(IntCC::SignedLessThan, and, 0)) + } + _ => { + let lo = self.b.ins().imul(pa, pb); + let hi = self.b.ins().smulhi(pa, pb); + let sign = self.b.ins().sshr_imm(lo, 63); + (lo, self.b.ins().icmp(IntCC::NotEqual, hi, sign)) + } + }; + let ok = self.b.create_block(); + self.b.ins().brif(ovf, self.decline, &[], ok, &[]); + self.b.switch_to_block(ok); + let ti = self.b.ins().iconst(types::I64, T_I as i64); + self.b.ins().jump(merge, &[ti.into(), r.into()]); + // Two floats in line too; a NaN result takes the helper + // (which gives it its identity). + self.b.switch_to_block(not_int); + let fa = self.b.ins().icmp_imm(IntCC::Equal, ta, T_F as i64); + let fb = self.b.ins().icmp_imm(IntCC::Equal, tb, T_F as i64); + let both = self.b.ins().band(fa, fb); + let float = self.b.create_block(); + self.b.ins().brif(both, float, &[], slow, &[]); + self.b.switch_to_block(float); + let (xa, xb) = (self.as_f64(pa), self.as_f64(pb)); + let r = match k { + BinOpKind::Add => self.b.ins().fadd(xa, xb), + BinOpKind::Sub => self.b.ins().fsub(xa, xb), + _ => self.b.ins().fmul(xa, xb), + }; + let nan = self.b.ins().fcmp(FloatCC::Unordered, r, r); + let fine = self.b.create_block(); + self.b.ins().brif(nan, slow, &[], fine, &[]); + self.b.switch_to_block(fine); + let bits = self.b.ins().bitcast(types::I64, MemFlags::new(), r); + let tf = self.b.ins().iconst(types::I64, T_F as i64); + self.b.ins().jump(merge, &[tf.into(), bits.into()]); + } else if float_div { + // Two floats divide in line by a nonzero divisor; a NaN + // result takes the helper. + let fa = self.b.ins().icmp_imm(IntCC::Equal, ta, T_F as i64); + let fb = self.b.ins().icmp_imm(IntCC::Equal, tb, T_F as i64); + let both = self.b.ins().band(fa, fb); + let float = self.b.create_block(); + self.b.ins().brif(both, float, &[], slow, &[]); + self.b.switch_to_block(float); + let (xa, xb) = (self.as_f64(pa), self.as_f64(pb)); + let zero = self.b.ins().f64const(0.0); + let by_zero = self.b.ins().fcmp(FloatCC::Equal, xb, zero); + let nonzero = self.b.create_block(); + self.b.ins().brif(by_zero, slow, &[], nonzero, &[]); + self.b.switch_to_block(nonzero); + let r = self.b.ins().fdiv(xa, xb); + let nan = self.b.ins().fcmp(FloatCC::Unordered, r, r); + let fine = self.b.create_block(); + self.b.ins().brif(nan, slow, &[], fine, &[]); + self.b.switch_to_block(fine); + let bits = self.b.ins().bitcast(types::I64, MemFlags::new(), r); + let tf = self.b.ins().iconst(types::I64, T_F as i64); + self.b.ins().jump(merge, &[tf.into(), bits.into()]); + } else { + self.b.ins().jump(slow, &[]); + } + self.b.switch_to_block(slow); + let kv = self.u32c(u32::from(kind)); + let st = self.call( + h_binary as *const () as usize, + &[self.ctx, ta, pa, tb, pb, kv], + ); + self.check(st); + let flags = MemFlags::trusted(); + let t = self.b.ins().load(types::I64, flags, self.ctx, 0); + let p = self.b.ins().load(types::I64, flags, self.ctx, 8); + self.b.ins().jump(merge, &[t.into(), p.into()]); + self.b.switch_to_block(merge); + let (t, p) = (self.b.block_params(merge)[0], self.b.block_params(merge)[1]); + self.set(fl, dst, t, p); + } + Op::Swap { a, b } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + self.set(fl, a, tb, pb); + self.set(fl, b, ta, pa); + } + Op::BranchIf { src, when, target } => { + let bit = self.truth(fl, src); + let to = *fl.blocks.get(usize::from(target))?; + let next = self.b.create_block(); + if when { + self.b.ins().brif(bit, to, &[], next, &[]); + } else { + self.b.ins().brif(bit, next, &[], to, &[]); + } + self.b.switch_to_block(next); + } + Op::BranchNone { src, when, target } => { + let (t, _) = self.get(fl, src); + let is_none = self.b.ins().icmp_imm(IntCC::Equal, t, T_N as i64); + let to = *fl.blocks.get(usize::from(target))?; + let next = self.b.create_block(); + if when { + self.b.ins().brif(is_none, to, &[], next, &[]); + } else { + self.b.ins().brif(is_none, next, &[], to, &[]); + } + self.b.switch_to_block(next); + } + Op::Jump { target } => { + let to = *fl.blocks.get(usize::from(target))?; + self.b.ins().jump(to, &[]); + return Some(false); + } + Op::Call { at, argc, pc } => self.call_op(fl, i, at, argc, pc)?, + Op::Return { src } => { + let (t, p) = self.get(fl, src); + if let Some((rt, rp, cont)) = fl.ret { + // An in-line callee's return: the caller's result + // register, and on past its call. + self.b.def_var(rt, t); + self.b.def_var(rp, p); + self.b.ins().jump(cont, &[]); + } else { + let flags = MemFlags::trusted(); + self.b.ins().store(flags, t, self.ctx, 0); + self.b.ins().store(flags, p, self.ctx, 8); + let zero = self.b.ins().iconst(types::I32, 0); + self.b.ins().return_(&[zero]); + } + return Some(false); + } + Op::New { dst, dict } => { + let d = self.u32c(u32::from(dict)); + let st = self.call(h_new as *const () as usize, &[self.ctx, d]); + self.check(st); + self.set_out(fl, dst, 0); + } + Op::Subscr { dst, a, b } => { + let (ta, pa) = self.get(fl, a); + let (tb, pb) = self.get(fl, b); + let st = self.call(h_subscr as *const () as usize, &[self.ctx, ta, pa, tb, pb]); + self.check(st); + self.set_out(fl, dst, 0); + } + Op::Decline => { + self.b.ins().jump(self.decline, &[]); + return Some(false); + } + Op::StoreAttr { + recv, + val, + pc, + name, + } => { + if !effect { + self.b.ins().jump(self.decline, &[]); + return Some(false); + } + let (tr, pr) = self.get(fl, recv); + let (tv, pv) = self.get(fl, val); + let last = Self::infallible_after(fl.plan, i); + // The plan's only store, with nothing after it that can + // decline: a scalar over a scalar field lands in line. + let done = self.b.create_block(); + if last && !fl.may_store[i] { + if let (Some(l), Some(slot)) = ( + crate::tier2::published_obj_layout(), + crate::code_field_slot(fl.code, usize::from(pc)), + ) { + let slow = self.b.create_block(); + self.inline_store(l, (tr, pr), (tv, pv), slot as i64, done, slow); + self.b.switch_to_block(slow); + } + } + let pn = self.u32c(u32::from(pc) | u32::from(name) << 16); + let last = self.u32c(u32::from(last)); + let st = self.call( + h_store as *const () as usize, + &[self.ctx, tr, pr, tv, pv, pn, last], + ); + self.check(st); + self.b.ins().jump(done, &[]); + self.b.switch_to_block(done); + } + } + Some(true) + } + + /// Whether nothing after op `i` can decline the evaluation: every + /// later op is a move, a constant, a return or a forward jump. + fn infallible_after(plan: &LeafPlan, i: usize) -> bool { + plan.ops + .iter() + .enumerate() + .skip(i + 1) + .all(|(k, op)| match *op { + Op::Move { .. } | Op::Const { .. } | Op::Return { .. } => true, + Op::Jump { target } => usize::from(target) > k, + _ => false, + }) + } + + /// A `CALL` (op `i`): the callee its site last resolved in line behind + /// a guard, falling back to the helper. + fn call_op(&mut self, fl: &FrameLower<'_>, i: usize, at: u8, argc: u8, pc: u16) -> Option<()> { + let cont = self.b.create_block(); + if let Some((f, ccode, cext, cplan)) = self.inline_target(fl, i, at, argc) { + // The guard: the register holds this function, called with an + // instance receiver, and the function still runs this code. + let (tf, pf) = self.get(fl, at); + let (ts, _) = self.get(fl, at + 1); + let is_fn = self.b.ins().icmp_imm(IntCC::Equal, tf, T_FN as i64); + let same = self + .b + .ins() + .icmp_imm(IntCC::Equal, pf, Rc::as_ptr(&f) as i64); + let has_self = self.b.ins().icmp_imm(IntCC::Equal, ts, T_R as i64); + let ok = self.b.ins().band(is_fn, same); + let ok = self.b.ins().band(ok, has_self); + let code_check = self.b.create_block(); + let inline = self.b.create_block(); + let generic = self.b.create_block(); + self.b.ins().brif(ok, code_check, &[], generic, &[]); + self.b.switch_to_block(code_check); + let cell = f.code.as_ptr(); + // SAFETY: the cell holds an `Rc` (one pointer word); the guard + // above proved the function alive wherever this load runs. + let bits = unsafe { *cell.cast::() }; + let addr = self.b.ins().iconst(self.ptr, cell as i64); + let now = self.b.ins().load(types::I64, MemFlags::trusted(), addr, 0); + let same_code = self.b.ins().icmp_imm(IntCC::Equal, now, bits as i64); + self.b.ins().brif(same_code, inline, &[], generic, &[]); + self.b.switch_to_block(inline); + let frame = Box::new(Frame { + code: Rc::as_ptr(&ccode), + ext: cext, + f: Rc::as_ptr(&f), + }); + let fr = self + .b + .ins() + .iconst(self.ptr, std::ptr::from_ref(&*frame) as i64); + let depth = fl.depth + 1; + self.depth = self.depth.max(depth); + let ret = Some((fl.tags[usize::from(at)], fl.pays[usize::from(at)], cont)); + let cfl = self.frame(cplan, &ccode, cext, fr, depth, ret, Rc::as_ptr(&f)); + // The receiver and the arguments become the callee's first + // registers. + for k in 0..=usize::from(argc) { + let (t, p) = self.get(fl, at + 1 + k as u8); + self.set(&cfl, k as u8, t, p); + } + self.b.ins().jump(cfl.blocks[0], &[]); + self.frames.push(frame); + self.keep.push((Rc::downgrade(&f), ccode.clone())); + self.body(&cfl)?; + self.b.switch_to_block(generic); + } + let n = usize::from(argc) + 2; + let slot = self.b.create_sized_stack_slot(StackSlotData::new( + StackSlotKind::ExplicitSlot, + (16 * n) as u32, + 3, + )); + for k in 0..n { + let (t, p) = self.get(fl, at + k as u8); + self.b.ins().stack_store(t, slot, (16 * k) as i32); + self.b.ins().stack_store(p, slot, (16 * k + 8) as i32); + } + let w = self.b.ins().stack_addr(self.ptr, slot, 0); + let ad = self.u32c(u32::from(argc) | u32::from(fl.depth) << 8); + let pc = self.u32c(u32::from(pc)); + let addr = if self.effect && fl.depth == 0 { + h_call:: as *const () as usize + } else { + h_call:: as *const () as usize + }; + let st = self.call(addr, &[self.ctx, fl.fr, w, ad, pc]); + self.check(st); + self.set_out(fl, at, 0); + self.b.ins().jump(cont, &[]); + self.b.switch_to_block(cont); + Some(()) + } +} diff --git a/crates/weavepy-vm/src/lib.rs b/crates/weavepy-vm/src/lib.rs index dcbfecb3..ed8cba52 100644 --- a/crates/weavepy-vm/src/lib.rs +++ b/crates/weavepy-vm/src/lib.rs @@ -51,6 +51,8 @@ pub mod error; pub mod ext_loader; pub mod fasthash; pub mod foreign; +#[cfg(feature = "jit")] +mod frame_jit; pub mod frozen_code_cache; pub mod frozen_table; pub mod gc_trace; @@ -90,6 +92,7 @@ pub mod types; pub mod unicode_case; pub mod unicode_numeric; pub mod vm_singletons; +pub mod weak_object; pub mod weakref_registry; use crate::builtin_types::{builtin_types, instance_is_subclass, make_exception_with_class}; @@ -483,7 +486,12 @@ fn py_frame_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) { /// test_ssl.test_handshake_timeout_handler_leak). fn py_traceback_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) { let Object::Traceback(tb) = obj else { return }; - visit(&Object::Frame(tb.frame.clone())); + // A lazy entry's activation is still running (its locals are roots + // the collector reaches through the spine); materializing here would + // only add a frame object for the traversal to find. + if let Some(frame) = tb.frame_if_ready() { + visit(&Object::Frame(frame)); + } if let Ok(next) = tb.next.try_borrow() { if let Some(n) = next.as_ref() { visit(&Object::Traceback(n.clone())); @@ -491,63 +499,6 @@ fn py_traceback_traverse(obj: &Object, visit: &mut dyn FnMut(&Object)) { } } -/// Snapshot the objects in a generator's suspended frame that will -/// need finalization when the frame is dropped (instances with a -/// `__del__`). Captured *before* `close()` tears the frame down so -/// the caller can emulate CPython's prompt refcount-driven -/// finalization afterwards. -fn frame_reapables(g: &Rc) -> Vec { - let Ok(state) = g.state.try_borrow() else { - return Vec::new(); - }; - let boxed = match &*state { - GeneratorState::Created(b) | GeneratorState::Suspended(b) => b, - _ => return Vec::new(), - }; - let frame: &Frame = boxed; - // CPython clears a generator's frame the instant it finishes/closes - // (`gen_send`/`gen_close` → `_PyFrame_ClearExceptCode`), so every - // local and value-stack entry is decref'd promptly. WeavePy mirrors - // that by running each prompt-reapable entry through the cascade - // after the frame is torn down — not just `__del__`-bearing - // instances but anything that anchors a finalizable acyclic - // subgraph *or* whose only surviving reference (once the frame is - // gone) is a weakref slot's strong clone in our registry. The - // latter is what lets `weakref.ref(obj)` zero out by refcount the - // way CPython's does: without it a suspended generator that - // referenced `obj` would keep `obj` pinned through the registry's - // clone until the next cyclic collection (RFC 0040: - // `test_concurrent_futures.test_init` leaked semaphores because the - // `_assert_logged`/`sleeping_retry` generators pinned the test - // instance → its `multiprocessing.Queue` → SemLocks). - let mut out: Vec = Vec::new(); - let mut seen: std::collections::HashSet = - std::collections::HashSet::new(); - let locals = frame.locals.borrow(); - for o in locals.iter().chain(frame.stack.iter()) { - if (Interpreter::local_needs_prompt_reap(o) - || crate::weakref_registry::count_for(crate::weakref_registry::id_of(o)) > 0) - && seen.insert(crate::weakref_registry::id_of(o)) - { - out.push(o.clone()); - } - } - // RFC 0073 WS4 — a parked native activation's pin table holds the - // live values (the locals above are the stale park-time snapshot). - #[cfg(feature = "jit")] - if let Some(act) = &frame.parked_native { - for o in act.pinned_objects() { - if (Interpreter::local_needs_prompt_reap(&o) - || crate::weakref_registry::count_for(crate::weakref_registry::id_of(&o)) > 0) - && seen.insert(crate::weakref_registry::id_of(&o)) - { - out.push(o); - } - } - } - out -} - /// Warm the JIT's code generator on a background thread when the JIT /// is on (see `tier2::prewarm_codegen`): a program's first compile then /// doesn't pay the generator's cold start. @@ -981,7 +932,10 @@ pub struct Interpreter { /// The core loop's last native iterator `__next__`, keyed by the /// (process-unique) attribute version of the class that resolved it /// (see [`Interpreter::core_leaf_next`]). - core_next: ThreadCell>)>>, + core_next: ThreadCell>, u8)>>, + /// `_seqtools`' lazy `map` and `filter` classes, once imported (see + /// [`Interpreter::seqtools_step`]). + seqtools: std::cell::OnceCell>, /// How the last class seen in a boolean context answers it (see /// [`Interpreter::leaf_instance_truth`]), by attribute version. core_truth: ThreadCell>, @@ -1214,6 +1168,7 @@ impl Default for Interpreter { leaf_fns: std::cell::OnceCell::new(), leaf_opaque: ThreadCell::new((0, leaf_builtins::LeafMap::default())), core_next: ThreadCell::new(None), + seqtools: std::cell::OnceCell::new(), core_truth: ThreadCell::new(None), dbg_sample: crate::hot_gates::env_flags::dbg_sample(), }; @@ -1341,6 +1296,7 @@ impl Interpreter { leaf_fns: std::cell::OnceCell::new(), leaf_opaque: ThreadCell::new((0, leaf_builtins::LeafMap::default())), core_next: ThreadCell::new(None), + seqtools: std::cell::OnceCell::new(), core_truth: ThreadCell::new(None), dbg_sample: crate::hot_gates::env_flags::dbg_sample(), } @@ -2451,16 +2407,13 @@ impl Interpreter { result } - /// Emulate CPython's prompt refcount-driven death for an object - /// whose binding was just dropped (a `del` statement, or a frame - /// torn down by `close()`). If `dropped` holds the last - /// program-visible reference — everything else is the GC - /// registry's handle and weakref slots' strong clones — run its - /// finalizer now and clear its weakrefs, exactly as CPython's - /// `tp_dealloc` would. Conservative: any extra reference anywhere - /// (containers, caches, other bindings) skips the reap and leaves - /// the object to the cycle collector. - fn prompt_reap_dropped(&mut self, dropped: Object) { + /// Release `dropped`: CPython's `Py_DECREF` at the end of a binding. + /// Objects die by reference count (the collector and the weakref + /// registry hold no strong references), so this is a plain drop, plus + /// the deallocation side effects a `Drop` impl can't produce itself, + /// plus the freelists, which take a dead plain instance's or tuple's + /// allocation for the next one. + fn release(&self, dropped: Object) { // C-API watcher probes (test_capi.test_watchers): a watched dict // losing its last binding fires PyDict_EVENT_DEALLOCATED; a dying // function fires PyFunction_EVENT_DESTROY. Both gates are relaxed @@ -2474,1256 +2427,48 @@ impl Interpreter { if crate::stdlib::testinternalcapi_mod::reftrace_print_active() { println!("DESTROY {}", dropped.type_name_owned()); } - // A discarded never-driven `asend`/`athrow`/`aclose` awaitable - // warns at finalization (gh-113753; CPython's - // `async_gen_asend_finalize` family). - if let Object::AsyncGenAwait(a) = &dropped { - if !a.started.get() && !a.consumed.get() && Rc::strong_count(a) <= 1 { - a.consumed.set(true); - let method = match a.kind { - crate::object::AgenAwaitKind::Send => "asend", - crate::object::AgenAwaitKind::Throw => "athrow", - crate::object::AgenAwaitKind::Close => "aclose", - }; - let qualname = match &a.agen { - Object::AsyncGenerator(g) => g.qualname.borrow().to_str(), - other => other.type_name_owned(), - }; - let _ = self.emit_runtime_warning(format!( - "coroutine method '{method}' of '{qualname}' was never awaited" - )); - } - return; - } - // A `BoundMethod` is itself untracked and carries no finalizer, so - // the fast-path early-return below would merely `Rc`-drop it — - // reclaiming the method object but leaving a *tracked* receiver - // pinned by its own GC handle (the early-return only accounts for - // *untracked* children a value anchors). Release the method's hold - // and run the receiver through the cascade, exactly as - // `reap_call_receiver` does for call temporaries. CPython decrefs a - // `method`'s `__self__` the instant the method object dies; this is - // what reaps a stored bound `__exit__` — the compiler's `.with_exit` - // synthetic local, dropped by `DeleteFast` on every `with` exit path - // — when the context manager is a plain instance that anchors - // resources. `unittest._AssertRaisesContext` holds the test case, - // which in `test_concurrent_futures.test_init` pinned the log - // `Queue` and its SemLocks until the next cyclic collection - // (RFC 0040: resource-tracker exited nonzero on leaked semaphores). - if matches!(&dropped, Object::BoundMethod(_)) { - // A bound method that escaped into an instance attribute - // (`self.cb = self.handler`) was GC-tracked at store time (see - // `generic_setattr_instance`); merely dropping our clone would - // leave it pinned by its own collector handle — and, through - // `__self__`, its receiver too. Route it through the cascade, - // which untracks and frees it. pickle's `_Unpickler` is the - // motivating case: it stores `self.append = self.stack.append` - // and rebinds it on every `load`, and the displaced method - // otherwise pinned the previous stack list and every - // freshly-unpickled object on it until the next cyclic - // collection (test_concurrent_futures - // `test_ressources_gced_in_workers`). - // - // Receivers can themselves be bound methods, arbitrarily deep - // (`f = f.__call__` wrapper towers — test_descr - // test_wrapper_segfault builds a million of them), so the walk - // is a loop, not recursion, and stops as soon as a link is - // still referenced elsewhere (its receiver isn't dead). - let mut cur = dropped; - loop { - let (tracked, alive, recv) = { - let Object::BoundMethod(bm) = &cur else { break }; - let id = crate::weakref_registry::id_of(&cur); - // `cur` is our only handle when nothing else holds the - // Rc (modulo registry strong clones); a method still - // aliased — e.g. as an outer wrapper's `__self__` — - // keeps its receiver alive, so the cascade must not - // descend into it. - let alive = - Rc::strong_count(bm) > 1 + crate::weakref_registry::strong_clone_count(id); - (gc_trace::is_tracked(id), alive, bm.receiver.clone()) - }; - if tracked { - self.reap_dead_subgraph(cur); - return; - } - if alive { - return; - } - drop(cur); - if !(Self::local_needs_prompt_reap(&recv) && Self::looks_reapable_temporary(&recv)) - { - return; - } - cur = recv; - } - self.prompt_reap_dropped(cur); - return; - } - // A tuple of scalars anchors nothing, is never GC-tracked at - // birth (the filter has no false negatives) and can't be weakly - // referenced: the freelist (or the plain drop) is its teardown. - if let Object::Tuple(t) = &dropped { - if t.iter().all(Object::is_gc_atomic) - && !gc_trace::maybe_tracked(crate::weakref_registry::id_of(&dropped)) + match dropped { + // A discarded never-driven `asend`/`athrow`/`aclose` awaitable + // warns at finalization (gh-113753; CPython's + // `async_gen_asend_finalize` family), at the next safe point. + Object::AsyncGenAwait(a) => { + if !a.started.get() && !a.consumed.get() && Rc::strong_count(&a) <= 1 { + a.consumed.set(true); + crate::vm_singletons::push_pending_finalizer(Object::AsyncGenAwait(a)); + } + } + // A dead plain tuple's allocation is recycled (CPython's tuple + // freelist; see `maybe_donate_tuple`). + dropped @ Object::Tuple(_) => self.maybe_donate_tuple(dropped), + // A natively served `datetime` value goes back to its pool + // (see `stdlib::datetime_native::recycle`). + Object::Instance(i) + if Rc::strong_count(&i) == 1 && crate::stdlib::datetime_native::pooled_kind(&i) => { - self.maybe_donate_tuple(dropped); - return; - } - } - // A deferred-tracking instance holds only atomic values, has no - // weakref (creating one tracks it) and no collector handle: the - // plain `Rc` drop is its whole teardown. - if let Object::Instance(i) = &dropped { - if i.dies_by_plain_drop() { - if let Object::Instance(i) = dropped { - PyInstance::try_recycle(i); + if let Err(i) = crate::stdlib::datetime_native::recycle(i) { + drop(i); } - return; } - } - let id0 = crate::weakref_registry::id_of(&dropped); - // Nothing parked on this object needs prompt handling: no - // finalizer to run, no weakref to clear, and no cycle-GC strong - // handle pinning it. Ordinary `Rc` drop reclaims it (and any - // *untracked* children it anchors, recursively, by refcount). - // - // An instance that escaped into a C extension (`c_body != 0`) is - // excluded from this fast path even when untracked/unwatched: its - // `tp_dealloc` releases references held in *C struct fields* - // (invisible to the plain `Rc` drop's recursion), and a *tracked* - // child freed that way is left pinned by its own GC handle unless - // the cascade below walks to it and reaps it in turn. pandas' - // chained `df[cols].iloc[rows]` is the motivating case: the - // discarded `_iLocIndexer` temporary holds the intermediate - // DataFrame only through Cython's `self->obj` field, and the - // intermediate's weakref (`subset._is_copy`) must clear the - // instant the indexer dies, as it does on CPython. - if !Self::object_is_finalizable(&dropped) - && crate::weakref_registry::count_for(id0) == 0 - && !gc_trace::is_tracked(id0) - && !Self::instance_escaped_to_c(&dropped) - { - // One more container-shaped exception to the fast path: an - // *untracked* container can be the sole holder of a *tracked* - // child, and the plain `Rc` drop below would leave that child - // refcount-dead but pinned by its own collector handle until - // the next cyclic collection. Tuples are the canonical case — - // they are never GC-tracked at `BuildTuple` time — and - // `multiprocessing`'s wire format is exactly this shape: - // `conn.recv()` unpickles `('#ERROR', exc)`, the manager proxy - // re-raises `exc`, and the handled exception must die at - // handler exit even with `gc.disable()` in force - // (test_multiprocessing_* `TestManagerExceptions.test_queue_get`, - // gh-106558). Route such containers through the cascade, which - // recurses through untracked containers to reap the tracked - // descendants. - // - // Untracked *instances* are the same shape (RFC 0076 WS3): the - // canonical case is a dying handled exception. `except E as e:` - // compiles to POP_EXCEPT (while `e` still binds the exception — - // the reap there sees it alive) followed by `e = None; del e`, - // so the actual death site is the STORE displacing `e` — this - // path. The exception is untracked and non-finalizable, but - // through `__traceback__` → frame → locals it is routinely the - // *sole* holder of tracked instances from the unwound frames. - // Pillow's `Image.open` is the motivating case: each failed - // plugin candidate (`ImImageFile(fp)` raising SyntaxError in - // `__init__`) lives only in the constructor frame the traceback - // pins; skipping the cascade here left the candidates handle- - // pinned holding `fp`, so the "unclosed file" ResourceWarning - // fired at the next amortized acyclic sweep instead of at - // handler exit (mpo/psd/spider/tiff `test_unclosed_file`). - // The deadness pre-check keeps the common alive-rebind - // (`self.x = ...` displacing a shared value) at two cheap - // lookups; `anchors_tracked_child` is node-capped. - if !(matches!( - dropped, - Object::Tuple(_) - | Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::FrozenSet(_) - | Object::Instance(_) - | Object::Frame(_) - ) - // A container of nothing but scalars anchors no tracked - // child by inspection, so it can skip both the deadness - // test and the traversal — the shape most temporaries - // have, and (since tracking such a container is deferred) - // the one that reaches here most. - && !Self::is_scalar_leaf_container(&dropped) - && Self::is_refcount_dead(&dropped, 1) - && Self::anchors_tracked_child(&dropped, 6)) - { - // A dead plain tuple's allocation is recycled (CPython's - // tuple freelist; see `maybe_donate_tuple`). - if matches!(dropped, Object::Tuple(_)) && Self::is_refcount_dead(&dropped, 1) { - self.maybe_donate_tuple(dropped); - } - return; + // Likewise a dead instance's (see `PyInstance::try_recycle`). + Object::Instance(i) if Rc::strong_count(&i) == 1 && i.dies_by_plain_drop() => { + PyInstance::try_recycle(i); } + dropped => drop(dropped), } - // Is `dropped` actually dead — i.e. is this its last - // program-visible reference? The caller still holds `dropped` - // (one reference), the GC may hold a handle, and weakrefs may - // hold strong clones; anything beyond that is a live binding and - // we must leave the object to the cycle collector. - if !Self::is_refcount_dead(&dropped, 1) { - // Still referenced — but possibly only by Rust-side transients - // that die between safe points (the same shape the cascade's - // child check handles via `note_suspect`, but at the *root*: a - // `del obj` racing another thread's in-flight teardown clones). - // A weakly-referenced tracked root that misses its reap here is - // user-visible — the weakref never clears - // (test_threading.test_no_refcycle_through_target under load) — - // so enroll borderline cases for the eval loop's re-probe. - // Finalizable objects stay excluded, as everywhere on the - // suspect path: their `__del__` must not fire from a probe. - return; - } - self.reap_dead_subgraph(dropped); - } - - /// The cascade's commonest root, settled without it: a refcount-dead - /// plain instance (no `__del__`, no weakref, no C body or native - /// value, no `__slots__` values) whose every attribute is a scalar - /// leaf, a value that outlives it, or a scalar-leaf container only it - /// holds. Nothing in that subgraph finalizes, clears a weakref or - /// anchors a tracked object the drop would strand, so untracking the - /// instance and those containers (the cascade's next links, which it - /// would retire the same way) and dropping it is the whole teardown. - /// A tracked attribute value near its dead line (the cascade enrolls - /// those as suspects) or anything else the cascade walks into hands - /// `dropped` back in `Err`. - fn reap_plain_instance(dropped: Object) -> Result<(), Object> { - let Some((leaves, n)) = Self::plain_instance_leaves(&dropped) else { - return Err(dropped); - }; - for cid in &leaves[..n] { - gc_trace::with_state(|s| s.untrack_id(*cid)); - } - let id = crate::weakref_registry::id_of(&dropped); - if gc_trace::is_tracked(id) { - gc_trace::with_state(|s| s.untrack_id(id)); - } - drop(dropped); - Ok(()) } - /// A dead function (a rebound lambda or closure — CPython frees it by - /// refcount on the spot) whose death strands nothing: no weakref, and - /// every edge it releases — namespaces, defaults, closure cells, its - /// `__dict__` and getset slots — either outlives it or holds only - /// scalars. Untracking and dropping it is then the whole teardown. - fn plain_function_dies_alone(obj: &Object) -> bool { - let Object::Function(f) = obj else { - return false; - }; - if crate::vm_singletons::cext_call_active() - || crate::vm_singletons::has_pending_cext_drops() - || crate::weakref_registry::may_have_weakrefs(crate::weakref_registry::id_of(obj)) - { - return false; - } - // A value this function held: fine when it holds nothing a cascade - // would visit, or when another reference keeps it alive past the - // suspect band (a collector handle is one of the references). - fn released_ok(v: &Object) -> bool { - if gc_trace::is_atomic(v) { - return true; - } - let id = crate::weakref_registry::id_of(v); - if crate::weakref_registry::may_have_weakrefs(id) { - return false; - } - let sc = gc_trace::strong_count_for(v); - if gc_trace::is_tracked(id) { - sc > 4 - } else { - sc >= 2 - } - } - // The namespaces are module-owned in every ordinary case. - let namespaces = [&f.globals, &f.builtins]; - if namespaces.iter().any(|d| Rc::strong_count(d) < 3) { - return false; - } - if !f.defaults.iter().all(released_ok) || !f.kw_defaults.iter().all(|(_, v)| released_ok(v)) - { - return false; - } - // A closure cell only this function holds dies with it. - if !f.closure.iter().all(|c| match c { - Object::Cell(cell) => { - Rc::strong_count(cell) >= 2 - || cell.try_borrow().is_ok_and(|v| gc_trace::is_atomic(&v)) - } - other => released_ok(other), - }) { - return false; - } - let Ok(attrs) = f.attrs.try_borrow() else { - return false; - }; - if let Some(attrs) = attrs.as_ref() { - if Rc::strong_count(attrs) == 1 && !attrs.try_borrow().is_ok_and(|d| d.is_empty()) { - return false; - } + /// Release a finished call's arguments now, before the caller handles + /// the result (CPython's `CALL` decrefs its operands first): an + /// argument only the call held dies here, finalizers and all. + fn reap_call_args(&mut self, args: &mut [Object]) { + for a in args { + drop(std::mem::replace(a, Object::Unbound)); } - let Ok(slots) = f.slots.try_borrow() else { - return false; - }; - slots - .iter() - .all(|(k, v)| gc_trace::is_atomic(&k.0) && released_ok(v)) } - /// [`Self::reap_plain_instance`]'s test: the tracked scalar-leaf - /// containers `obj` alone holds (ids, count), or `None` when the - /// general cascade must run. - fn plain_instance_leaves( - obj: &Object, - ) -> Option<([crate::weakref_registry::ObjectId; 4], usize)> { - let Object::Instance(inst) = obj else { - return None; - }; - if inst.c_body.get() != 0 - || inst.native.get().is_some() - || crate::vm_singletons::cext_call_active() - || crate::vm_singletons::has_pending_cext_drops() - || crate::weakref_registry::may_have_weakrefs(crate::weakref_registry::id_of(obj)) - { - return None; - } - match inst.class.try_borrow() { - Ok(cls) if cls.native_kind.get() == 0 && !cls.instances_need_finalize() => {} - _ => return None, - } - if !inst - .slots - .try_borrow() - .is_ok_and(|s| s.iter().next().is_none()) - { - return None; - } - let mut leaves = [0; 4]; - let mut n = 0; - // A dict shared through `vars(obj)` outlives the instance, and - // with it every value. - let split; - let d; - let attrs: &mut dyn Iterator = match inst.dict.published() { - Some(dict) if inst.dict.strong_count() == 1 => { - d = dict.try_borrow().ok()?; - &mut d.iter() - } - Some(_) => return Some((leaves, n)), - None => { - split = inst.dict.split_cell().try_borrow().ok()?; - &mut split.iter() - } - }; - for (k, v) in attrs { - if !gc_trace::is_atomic(&k.0) { - return None; - } - if gc_trace::is_atomic(v) { - continue; - } - let cid = crate::weakref_registry::id_of(v); - if crate::weakref_registry::may_have_weakrefs(cid) { - return None; - } - let sc = gc_trace::strong_count_for(v); - if gc_trace::is_tracked(cid) { - // Past the cascade's suspect band: stays alive. - if sc > 4 { - continue; - } - // Its handle and this dict: it dies here, and a container - // of scalars is retired by untracking. - if sc == 2 && n < leaves.len() && Self::is_scalar_leaf_container(v) { - leaves[n] = cid; - n += 1; - continue; - } - return None; - } - // Untracked: another holder keeps it alive, or it dies by - // plain drop holding nothing but scalars. - if sc >= 2 - || Self::is_scalar_leaf_container(v) - || matches!(v, Object::Tuple(t) if t.iter().all(gc_trace::is_atomic)) - { - continue; - } - return None; - } - Some((leaves, n)) - } - - /// The dead-acyclic-subgraph cascade — the shared tail of - /// [`Self::prompt_reap_dropped`], split out so a call site that has - /// already established `dropped` is refcount-dead and anchors - /// finalizable/tracked children, yet is itself an untracked, - /// non-finalizable *leaf* the fast-path early-return in - /// `prompt_reap_dropped` would skip, can drive the cascade directly. - /// The motivating case is a handled exception cleared at `POP_EXCEPT`: - /// an `AttributeError` carries no `__del__` and is not GC-tracked, but - /// it anchors `AttributeError.obj` (and values reachable only through - /// its saved `__traceback__` frames) whose `__del__` must run at - /// handler exit, matching CPython's refcount timing. Every node is - /// independently refcount-guarded, so anything that stays externally - /// reachable is left to the cycle collector. - fn reap_dead_subgraph(&mut self, dropped: Object) { - if !Self::is_refcount_dead(&dropped, 1) { - return; - } - let Err(dropped) = Self::reap_plain_instance(dropped) else { - return; - }; - if Self::plain_function_dies_alone(&dropped) { - let id = crate::weakref_registry::id_of(&dropped); - if gc_trace::is_tracked(id) { - gc_trace::with_state(|s| s.untrack_id(id)); - } - drop(dropped); - return; - } - // RFC 0045 (wave 5): if the dead subgraph contains any instance - // that has escaped into a C extension (owns a faithful inline - // body), reclaiming it is only safe when no extension frame is - // live on this thread. The reaper's deadness test is - // `Rc`-refcount only and cannot see a *borrowed* C reference — a - // raw body pointer an extension holds across a re-entrant call - // (pandas caches `Index._engine` and reads it back through - // `PyDict_GetItem` as a borrowed ref; an `Index`/`DataFrame` holds - // its `ndarray`/`BlockManager` the same way). Freeing a body such - // a borrow still points at hands the slot to the next `ndarray` - // and the extension's pointer becomes a type-confused - // use-after-free (`'ndarray' has no attribute 'is_unique'` in - // `Index.unique` during `merge`). - // - // A borrowed pointer, however, can only be *in flight* while the - // extension's own stack frame is: C-owned references are always - // visible to the deadness test (an inline body's C refcount pins - // the instance through the capi `STRONG` map; an identity box - // pins through its strong payload), so a refcount-dead object has - // no C *owner*, and with no extension frame below us no borrow - // can outlive its owner either (the one C idiom that stashes a - // raw pointer past refcount zero — Cython's `@cython.freelist` — - // is neutralised at the dealloc bridge). `cext_call_active()` - // distinguishes the two: `weavepy-capi` marks every bridged C - // invocation for its duration, including bytecode the extension - // re-enters. At depth 0 — plain bytecode, the overwhelmingly - // common site of a `del`/rebind/frame-exit reap — the full - // cascade below is exactly CPython's refcount-driven - // `tp_dealloc` chain, and pandas *depends* on that timing: a - // chained `df[cols][rows]` stamps the subset with - // `_is_copy = weakref.ref(intermediate)` and the indexer - // temporary holds the intermediate in a Cython C field, so the - // intermediate only dies (and its weakref only reads `None`, - // which is what `_check_setitem_copy` accepts) if the indexer's - // `tp_dealloc` runs promptly and releases that C field. - // - // Inside a live C call the memory must wait for the tracing - // cycle collector — but the object is still *semantically* dead, - // and CPython runs `tp_finalize` and clears weakrefs the instant - // the refcount hits zero regardless. So run the root's finalizer - // and clear its weakrefs now, leaving the GC handle (and - // therefore the allocation) in place. Inert for pure-Python - // programs (no instance ever owns a body). - if crate::vm_singletons::cext_call_active() && Self::subgraph_contains_escaped(&dropped) { - let id = crate::weakref_registry::id_of(&dropped); - if Self::object_is_finalizable(&dropped) { - let claimed = if gc_trace::is_tracked(id) { - gc_trace::mark_finalized(id) - } else { - true - }; - if claimed { - crate::vm_singletons::push_pending_finalizer(dropped.clone()); - self.run_pending_finalizers(); - if !Self::is_refcount_dead(&dropped, 1) { - return; // resurrected by its own __del__ - } - } - } - crate::weakref_registry::queue_callbacks(crate::weakref_registry::notify_clear(id)); - self.run_pending_finalizers(); - // The allocation itself is parked (GC handle / this queue clone - // keep it alive while C may still borrow into its subgraph), but - // CPython would have freed the whole chain inside the extension - // call. Requeue it for the between-bytecodes safe point — the - // first moment with no extension frame live — where the drain - // re-runs this cascade and the full teardown proceeds. Without - // this, a temporary that died *inside* a C call (pandas' - // `mgr.iget` dropping its transient wrappers) pins everything it - // anchors (the manager, its `blocks` tuple, the `Block`s and - // their weakrefs) until the next full collection — and pandas' - // `_is_view_after_cow_rules` reads a stale live Block weakref - // within the very statement that dropped the intermediate. - // - // The *unfiltered* queue entry: the park must accept every - // object kind the reaper handles. A closure function (the - // compiler's ``/`` temporary) is GC-tracked - // and holds `cell(self)`, but fell through the C-drop kind - // filter — the discarded park left it handle-pinned, leaking - // one `self` reference per C-initiated call of any method - // whose parameter is promoted to a cell (pandas' - // `BlockManager.shape` via Cython's `PyObject_GetAttr`). - crate::vm_singletons::queue_parked_drop(&dropped); - return; - } - // RFC 0039 (WS4): cascade through the dead *acyclic* subgraph, - // emulating CPython's refcount-driven `tp_dealloc` chain. Freeing - // `dropped` drops the last reference to its tracked children, - // which become dead in turn; each is finalized (in `tp_finalize` - // order — `__del__` before the object's contents are released) - // and untracked here instead of lingering in the GC's tracked set - // pinned by its own handle. Without this, a deep acyclic chain of - // finalizable/tracked objects (`test_gc.test_trashcan_threads`, - // `test_weakref`'s len-race) is only drained one layer per - // O(heap) collection — quadratic, and easily never finished for a - // quiescing interpreter. A reference *cycle* never enters the - // cascade: its members reference one another, so the refcount - // test below keeps `strong` above the dead threshold and leaves - // the cycle to the tracing collector. - let mut work = vec![dropped]; - // Each node's scratch, emptied per node and allocated once per - // cascade (a dead graph of many small containers paid four - // allocations and a hash set per node). - let mut children: Vec> = Vec::new(); - let mut weakref_candidates: Vec = Vec::new(); - let mut pool_candidates: Vec = Vec::new(); - let mut scan_through: Vec = Vec::new(); - let mut scanned: std::collections::HashSet< - crate::weakref_registry::ObjectId, - std::hash::BuildHasherDefault, - > = std::collections::HashSet::default(); - loop { - let Some(obj) = work.pop() else { - // Freeing the objects above may have parked C-side dealloc - // products in the pending-drop queue: `free_mirror` queues a - // container a C `tp_dealloc` just released (a Cython - // `BlockManager`'s `blocks` tuple), and identity boxes minted - // transiently by this very cascade (`run_external_traverse`) - // queue their payloads. CPython runs those dealloc chains - // inside the same instruction, so pull them into this cascade - // rather than leaving them for the next between-bytecodes - // safe point — pandas' `_check_setitem_copy` / - // `_is_view_after_cow_rules` read the dropped intermediate's - // `Block` weakref *within the statement* that dropped it - // (`zed["eyes"]["right"].fillna(..., inplace=True)`), one - // bytecode too early for the deferred drain. Deadness-guarded - // exactly like the safe-point drain: anything still alive - // just sheds the queue's clone. Skipped while an extension - // frame is live — the safe-point path owns that case (the - // escaped-subgraph park above). - if crate::hot_gates::env_flags::reap_trace() { - eprintln!( - "[DRAIN] cext_active={} pending={}", - crate::vm_singletons::cext_call_active(), - crate::vm_singletons::has_pending_cext_drops() - ); - } - if crate::vm_singletons::cext_call_active() - || !crate::vm_singletons::has_pending_cext_drops() - { - break; - } - for d in crate::vm_singletons::drain_pending_cext_drops() { - if crate::hot_gates::env_flags::reap_trace() { - eprintln!( - "[DRAIN-ITEM] {} sc={} dead={}", - d.type_name_owned(), - gc_trace::strong_count_for(&d), - Self::is_refcount_dead(&d, 1) - ); - } - if Self::is_refcount_dead(&d, 1) { - work.push(d); - } - } - continue; - }; - let id = crate::weakref_registry::id_of(&obj); - // RFC 0077 (WS2): a dead exact `list`/`dict`/`set` holding only - // scalar leaves is CPython's `list_dealloc` of a list of ints: - // untrack and free, nothing else. Every step of the cascade - // body below is a no-op for it — no `__del__` (only instances - // and unfinished generators finalize), no weakrefs (the - // registry filter says so), no child that could be tracked or - // anchor one (scalars never are), and no C crossing to guard - // with the cascade marker — yet the body still paid a - // cascade-set insert, a weakref clear, two finalizer drains - // and a per-element `id_of`/`handle_for` traversal. Since - // every list is GC-tracked, that overhead was the dominant - // cost of *dying* for the commonest containers in the language - // (`list(range(n))` / `[...]` temporaries measured at ~35% of - // the fannkuch fixture once `list(it)` results were tracked). - // `try_borrow` failing (a re-entrant mutation in flight) just - // takes the general path. - if Self::is_scalar_leaf_container(&obj) && crate::weakref_registry::count_for(id) == 0 { - if gc_trace::is_tracked(id) { - gc_trace::with_state(|s| s.untrack_id(id)); - } - drop(obj); - continue; - } - // Mark `obj` as mid-cascade for the rest of this iteration: the - // teardown below crosses it back into C (the GC bridge's - // `tp_traverse` mints and releases a transient self box), and - // that release must not re-queue the object into the pending - // C-drop queue — the queued clone would make the deadness - // re-checks see it as alive, abort the teardown, and restart - // the cascade from the drain: a livelock (RFC 0047, wave 5). - let _cascade_guard = crate::vm_singletons::enter_cascade(id); - let finalizable = Self::object_is_finalizable(&obj); - // Run `__del__` once, while `obj` and its children are still - // intact (CPython runs `tp_finalize` before `tp_clear`). A - // tracked object claims its finalizer via the GC handle's - // one-shot flag; an untracked finalizable object (a generator - // the GC never saw) runs directly. - if finalizable { - let claimed = if gc_trace::is_tracked(id) { - gc_trace::mark_finalized(id) - } else { - true - }; - if claimed { - crate::vm_singletons::push_pending_finalizer(obj.clone()); - self.run_pending_finalizers(); - // The finalizer ran arbitrary Python and may have - // resurrected `obj` (stashed it somewhere reachable). - // If so it is alive again: leave it tracked and stop - // cascading through it. - if !Self::is_refcount_dead(&obj, 1) { - continue; - } - } - } - // Clear weakrefs and queue their callbacks (CPython clears - // weakrefs after `tp_finalize`). - crate::weakref_registry::queue_callbacks(crate::weakref_registry::notify_clear(id)); - self.run_pending_finalizers(); - if !Self::is_refcount_dead(&obj, 1) { - continue; - } - // Snapshot the tracked children before `obj` is freed: those - // are the only ones a strong GC handle would otherwise pin - // (untracked children are reclaimed by ordinary `Rc` drop). - // - // Crucially, recurse *through* untracked containers to reach the - // tracked descendants hidden behind them. A container is left - // GC-untracked either because it was built all-scalar - // (`BuildTuple`/`BuildSet`) or — the subtler case — because it was - // created *empty* (so `container_can_cycle` saw nothing to track) - // and only later populated with objects: a dict literal `{}` grown - // by `d[k] = obj` is the canonical example, and - // `pickle._Unpickler.memo` (`self.memo = {}`, then memoized objects - // stuffed in) is exactly that. When such a container is the sole - // holder of a tracked, finalizable object, freeing the container by - // refcount drops that object to "GC handle only" — refcount-dead, - // but pinned by its own handle and so invisible to the next link of - // this cascade unless we walk into the container here. RFC 0040: - // `test_concurrent_futures.test_ressources_gced_in_workers` submits - // an object whose `__del__` signals a `Manager` event; the worker - // unpickles it (memoized in `_Unpickler.memo`), runs the job, and - // `del`s the `_CallItem`, expecting the argument's `__del__` to have - // fired — which only happens if dropping the call item (and the - // unpickler temporary holding the memo) cascades through the - // untracked `memo` dict to the tracked argument. The refcount guard - // below still filters anything that stays externally reachable. - debug_assert!(children.is_empty()); - // Untracked descendants with live weakrefs: CPython clears an - // object's weakrefs at refcount zero whether or not the GC ever - // tracked it, but this cascade's "next link" set is (otherwise) - // tracked-only, and a plain `Rc` drop cannot clear weakref - // slots — the registry's strong slot clones would keep the - // referent alive and the weakref reading it forever. The - // canonical victims are C-minted Cython instances (RFC 0047): - // `PyType_GenericAlloc` bodies are not GC-tracked on the VM - // side, and pandas' `Block`s — weakly referenced by every - // `BlockValuesRefs` — die exactly this way when a setter drops - // the old `blocks` tuple (`test_consolidate` asserts - // `has_reference()` flips to False). Collect them during the - // scan and run the dead ones through the cascade so their - // weakrefs clear at death, like any tracked link. - debug_assert!(weakref_candidates.is_empty()); - // Plain-tuple children about to die with `obj` are recycled - // through the tuple pool (see `maybe_donate_tuple`): the plain - // `Rc` drop below frees them invisibly, and this cascade is - // where the common `f(*args)`-shaped temporaries actually die - // (the argument tuple is anchored by a dead list/frame local). - debug_assert!(pool_candidates.is_empty()); - scan_through.push(obj.clone()); - scanned.clear(); - while let Some(parent) = scan_through.pop() { - gc_trace::traverse_object(&parent, &mut |c| { - // Scalars are never tracked, weakly referenced, or - // anchors of anything; a deferred-tracking instance - // holds only scalars and has no collector handle. - // Neither can lead the cascade anywhere, which matters - // for the commonest large death: a list of plain - // objects falling out of scope. - if gc_trace::is_atomic(c) { - return; - } - if let Object::Instance(i) = c { - if i.is_gc_deferred() && i.c_body.get() == 0 { - let cid = crate::weakref_registry::id_of(c); - if crate::weakref_registry::count_for(cid) > 0 && scanned.insert(cid) { - weakref_candidates.push(c.clone()); - } - return; - } - } - let cid = crate::weakref_registry::id_of(c); - if let Object::Tuple(t) = c { - if !t.is_empty() - && t.len() <= 16 - && !gc_trace::is_tracked(cid) - && crate::weakref_registry::count_for(cid) == 0 - { - pool_candidates.push(c.clone()); - } - } - if let Some(h) = gc_trace::find_handle(cid) { - children.push(h); - } else if matches!( - c, - Object::Tuple(_) - | Object::FrozenSet(_) - | Object::Dict(_) - | Object::List(_) - | Object::Set(_) - | Object::MappingProxy(_) - | Object::SimpleNamespace(_) - // An *untracked instance* (and `Cell`/`BoundMethod` - // hop) can sit between `obj` and a tracked - // finalizable just like an untracked container does. - // The canonical case: a caught exception stashed on - // an instance attribute — `unittest`'s - // `_AssertRaisesContext.exception` — where the - // `AttributeError` is itself untracked yet anchors - // `AttributeError.obj` (a finalizable io temporary). - // When the context manager is the sole holder, - // dropping it frees the exception by `Rc`, which - // drops the temporary to "GC handle only" — - // refcount-dead but pinned by its own handle, and - // invisible to the next cascade link unless we walk - // into the untracked instance here - // (`test_io.test_error_through_destructor`). The - // refcount guard below still filters anything that - // stays externally reachable. - | Object::Instance(_) - | Object::Cell(_) - | Object::BoundMethod(_) - // A dying exception owns its traceback, and the - // traceback's frames own the unwound activations' - // locals — where a cancelled asyncio call chain - // keeps its transports and SSL contexts. CPython - // frees all of it by refcount the moment the - // exception dies; walk through so the tracked - // descendants become the cascade's next links - // (test_ssl.test_handshake_timeout_handler_leak). - // Live frames (still on the stack / owned by a - // suspended generator) survive the per-child - // refcount guard below unharmed. - | Object::Traceback(_) - | Object::Frame(_) - ) && scanned.insert(cid) - { - if crate::weakref_registry::count_for(cid) > 0 { - weakref_candidates.push(c.clone()); - } - scan_through.push(c.clone()); - } - }); - } - // Drop the GC's strong handle, then our own reference: `obj` - // is reclaimed by `Rc` here and its children lose a reference. - if gc_trace::is_tracked(id) { - gc_trace::with_state(|s| s.untrack_id(id)); - } - drop(obj); - // A tuple child whose last holder was `obj` is now ours alone - // (strong count 1 = the scan's clone); park its allocation. - // Anything still referenced elsewhere just sheds the clone. - for cand in pool_candidates.drain(..) { - if matches!(&cand, Object::Tuple(t) if ThinArc::strong_count(t) == 1) { - self.maybe_donate_tuple(cand); - } - } - // Any tracked child that just lost its last program reference - // is the next link in the chain. - for h in children.drain(..) { - if !h.untracked.load(std::sync::atomic::Ordering::Acquire) { - let weak = crate::weakref_registry::strong_clone_count(h.id); - // `h.object` is the GC's own strong reference; the - // child is dead iff nothing beyond that handle and its - // weakref clones still points at it. - let sc = gc_trace::strong_count_for(&h.object); - if sc <= 1 + weak { - work.push(h.object.clone()); - } else if sc <= 4 + weak && !Self::object_is_finalizable(&h.object) { - // Still referenced — but only just. The extra refs are - // often Rust-side transients (an in-flight `PyException` - // clone in the task machinery, a native call's argument) - // that die between safe points, where no opcode-level - // reap can see the drop. Enroll for the eval loop's - // suspect re-probe so the object still dies at CPython's - // refcount timing (asyncio cancellation chains: - // test_ssl.test_handshake_timeout_handler_leak). - gc_trace::note_suspect(h.clone()); - } - } - } - // An untracked, weakly-referenced descendant whose last - // program reference just died: push it through the cascade so - // its weakrefs clear now (the refcount guard at the top of the - // loop re-checks liveness — our `weakref_candidates` clone is - // accounted for by the cascade's `extra` slack of 1). - for cand in weakref_candidates.drain(..) { - if Self::is_refcount_dead(&cand, 1) { - work.push(cand); - } - } - } - } - - /// CPython finalizes a frame's locals by refcount the instant the - /// frame is torn down: a local holding the *last* reference to a - /// finalizable object runs its `__del__` / weakref callback promptly - /// on `return`, not at the next cycle collection. WeavePy otherwise - /// only reaps on an explicit rebind (`STORE_FAST` over a live slot, - /// `del`), so a function that lets a finalizable local fall out of - /// scope by *returning* (`tempfile.NamedTemporaryFile` with - /// `delete=True`, `test_tempfile.test_del_by_finalizer`) leaked the - /// reclamation to the tracing GC. Move each interesting local out and - /// run it through the prompt-reap cascade; the refcount guard inside - /// leaves anything still reachable (including the return value, which - /// the caller still holds) untouched. - /// - /// Gated to the object kinds that can carry a finalizer (or anchor a - /// finalizable acyclic subgraph) so the common all-scalars frame pays - /// nothing on the hot return path. - fn reap_frame_locals_on_exit(&mut self, frame: &mut Frame, retval: Option<&Object>) { - // Class/module bodies keep their names in a namespace dict, not in - // fast locals, so this is a no-op for them; only real function - // activations (incl. comprehensions/lambdas) carry fast locals. - // Operands still on the stack at `RETURN_VALUE` (an iterator the - // compiler unwinds past, a `with` exit pair) are dropped with the - // frame too; grade them like the slots below. - for v in &frame.stack { - gc_trace::note_dropped(v); - } - let nlocals = frame.locals.borrow().len(); - if nlocals == 0 && frame.cells.is_empty() { - return; - } - let _ = retval; - // Fast path: no local can be at its dead line. A candidate the - // cascade below could reap has nothing but this frame's slots, its - // collector handle, and weakref clones holding it, so its strong - // count is at most `nlocals + 1` when no weakref watches it. - // Every candidate above that (the usual `self`, an argument the - // caller still holds) is plainly escaped, which the passes below - // would conclude too, one allocation and two scans later. Grading - // each slot is all that is left to do. - // (An escaped `PyFrame` sharing the locals storage keeps the - // slot-clearing passes below.) - if frame.code.cellvars.is_empty() && Rc::strong_count(&frame.locals) == 1 { - // SAFETY: RFC 0065 (WS3) — see `load_fast_value`; nothing in - // this block runs Python code or touches the locals storage - // through another path. - let locals: &Vec = unsafe { &*frame.locals.as_ptr() }; - let escaped = |v: &Object| { - let sc = gc_trace::strong_count_for(v); - if sc <= 1 { - return false; - } - let id = crate::weakref_registry::id_of(v); - // Held past every slot and a collector handle: no count. - if sc > nlocals + 1 { - return !crate::weakref_registry::may_have_weakrefs(id); - } - // How many of this frame's slots hold `v`: exact for - // small frames (the usual `self` is held once here and - // once by the caller), the slot count otherwise. - let held_here = if nlocals <= 8 { - locals - .iter() - .filter(|w| { - std::mem::discriminant(*w) == std::mem::discriminant(v) - && crate::weakref_registry::id_of(w) == id - }) - .count() - } else { - nlocals - }; - // A tracked object's collector handle is one more holder - // (the miss-filter answer: a false positive only sends - // the value down the careful path below). - let handle = usize::from(gc_trace::maybe_tracked(id)); - sc > held_here + handle && !crate::weakref_registry::may_have_weakrefs(id) - }; - if locals.iter().all(|v| { - !(Self::local_needs_prompt_reap(v) || matches!(v, Object::Function(_))) - || escaped(v) - || matches!(v, Object::Instance(i) if i.dies_by_plain_drop()) - || Self::atomic_container_dies_plainly(v) - || Self::tracked_container_dies_inertly(v) - }) { - for v in locals { - // A dying tracked container sheds its collector handle - // now (the cascade would find nothing else to do). - if Self::tracked_container_dies_inertly(v) { - gc_trace::untrack_id(crate::weakref_registry::id_of(v)); - } else { - gc_trace::note_dropped(v); - } - } - return; - } - } - // Pass 1: move every candidate local out of its slot. A *closure - // function* local is a candidate too: it is GC-tracked - // (`MakeFunction`), so merely dropping the frame's `Rc` leaves it - // pinned by its own GC handle — and still holding its captured - // cells — until the next cyclic collection. Routing a - // uniquely-held (non-escaped) function through the cascade - // untracks and frees it now, releasing its share of this frame's - // cells so the cellvar sweep below can reclaim a captured-but- - // unescaped finalizable (`test_tempfile.test_writelines_rollover`). - // - // Two slots naming the *same* object (`x = C(); for y in [x]: …`, - // or test_interpreters' `second = create(); for interp in - // list_all(): …`) each see the other's reference and would both - // fail the escaped-value pre-filter below, so the object would - // fall to the plain `Rc` drop — whose `Drop` net queues `__del__` - // *after* the weakrefs have been cleared (CPython's - // `subtype_dealloc` finalizes first). Dedupe by identity here so - // the frame contributes exactly one reference per object. - // - // The locals borrow is released before any cascade runs: the - // cascade can execute arbitrary `__del__` code, which may - // re-enter this very frame's shared locals storage (e.g. via a - // traceback's `f_locals`). - let mut candidates: Vec = Vec::new(); - { - let mut locals = frame.locals.borrow_mut(); - for i in 0..nlocals { - let needs = Self::local_needs_prompt_reap(&locals[i]) - || matches!(locals[i], Object::Function(_)); - if !needs { - // RFC 0077 (WS2): `RETURN_VALUE` is exempt from the - // eval loop's coarse stack-shrink mark; the frame's - // slots are the references it actually releases, so - // grade each one here (no-op for scalar locals). - gc_trace::note_dropped(&locals[i]); - continue; - } - let v = std::mem::replace(&mut locals[i], Object::Unbound); - if candidates.iter().any(|c| c.is_same(&v)) { - // Duplicate slot: releasing this clone is the frame - // letting go of its second reference. - drop(v); - } else { - candidates.push(v); - } - } - } - // Pass 2: cheap pre-filter, then cascade. A candidate holding the - // *last* program reference has `strong_count == our clone (1) + GC - // handle (<=1)`, plus any weakref slots' strong clones (which the - // cascade discounts, so an object reachable *only* through - // `weakref.ref(obj)` is still dead). Anything larger means the - // value escaped (it was returned, stored on an instance, captured - // by a closure, …), so leave it alone without paying for the full - // refcount/weakref test or a cascade. Keeps the hot return path of - // scalar-only frames free. - for v in candidates { - if Self::looks_reapable_temporary(&v) { - self.prompt_reap_dropped(v); - } else { - gc_trace::note_dropped(&v); - } - } - // A value captured by a nested closure does not live in a fast local - // — it lives in one of this frame's *cell variables*, and that cell is - // shared with the closure object(s) that closed over it. CPython - // decrefs the frame's cells together with those closures when the - // frame is torn down, so a captured-but-unescaped finalizable dies - // promptly. The locals sweep above has already reaped the dying - // closures (functions, and generators/coroutines via - // `local_needs_prompt_reap`), dropping their share of the cells; now - // reap any *own* cellvar (never a freevar, which belongs to an - // enclosing frame) that is left uniquely held by this frame and whose - // contents are a dead finalizable. - let ncellvars = frame.code.cellvars.len(); - if ncellvars == 0 || ncellvars > frame.cells.len() { - return; - } - for idx in 0..ncellvars { - let cell = frame.cells[idx].clone(); - // Only a cell uniquely held by this frame can have its contents - // die with the frame; one still shared with an escaped closure - // (returned/stored) keeps `strong_count > 1` (our local `cell` - // clone is the +1 we account for). - if Rc::strong_count(&cell) > 2 { - continue; - } - let reapable = { - let guard = cell.borrow(); - Self::local_needs_prompt_reap(&guard) && Self::looks_reapable_temporary(&guard) - }; - if reapable { - let v = std::mem::replace(&mut *cell.borrow_mut(), Object::Unbound); - self.prompt_reap_dropped(v); - } - } - } - - /// Reap a generator/coroutine frame the instant it *finishes* (a - /// normal return, an exhausting `next()`, or an exception escaping - /// the body). CPython clears the frame on completion, so a local or - /// value-stack entry holding the last reference to a finalizable - /// object dies promptly — and, in WeavePy's `Rc`-rooted model, an - /// object whose only other reference is a weakref slot's strong - /// clone in the registry becomes refcount-dead, which clears its - /// weakref (so `weakref.ref(obj)()` zeroes out without waiting for a - /// cyclic collection). The reapables are moved *out* of the frame - /// first so the frame no longer counts toward their refcount when - /// the cascade's deadness test runs; the frame is dropped (now - /// holding `Unbound` placeholders) by the caller. Mirrors - /// [`Self::reap_frame_locals_on_exit`] for the suspended-frame world. - fn reap_dead_frame(&mut self, frame: &mut Frame) { - let mut reap: Vec = Vec::new(); - // Same-object slots collapse to one entry (see - // `reap_frame_locals_on_exit`): the deadness test must see the - // frame as a single reference. - let take = |slot: &mut Object, reap: &mut Vec| { - if Self::local_needs_prompt_reap(slot) - || crate::weakref_registry::count_for(crate::weakref_registry::id_of(slot)) > 0 - { - let v = std::mem::replace(slot, Object::Unbound); - if !reap.iter().any(|c| c.is_same(&v)) { - reap.push(v); - } - } else { - gc_trace::note_dropped(slot); - } - }; - { - let mut locals = frame.locals.borrow_mut(); - for slot in locals.iter_mut() { - take(slot, &mut reap); - } - } - for slot in &mut frame.stack { - take(slot, &mut reap); - } - for v in reap { - self.prompt_reap_dropped(v); - } - } - - /// After a bound-method call returns, give the receiver a prompt-reap - /// chance. While the call executed, the `BoundMethod` object held a - /// strong reference to its receiver, so the callee's frame-exit reaping - /// of the `self` local saw the receiver as still-referenced and left it - /// to the cycle collector. Now that the method object is being - /// discarded, a receiver that was only a *temporary* — `Maker().make()`, - /// or `_Unpickler(buf).load()` inside `pickle.loads` — becomes - /// refcount-dead and must run its (or its referents') finalizers - /// immediately, mirroring CPython decref'ing a `method`'s `__self__` the - /// instant the call frame is torn down. RFC 0040: - /// `test_concurrent_futures.test_ressources_gced_in_workers` submits an - /// object whose `__del__` signals a `Manager` event; the worker unpickles - /// its `_CallItem`, runs it, and `del`s it expecting the argument's - /// `__del__` to have fired — but the unpickler's `memo` (pinned by the - /// unpickler temporary, in turn pinned only by the GC handle once the - /// `loads` bound-method call returns) kept the argument alive until the - /// next cyclic collection. - /// - /// Cheap on the hot path: only `BoundMethod` callables pay anything, and - /// a receiver still bound elsewhere (the overwhelmingly common `self`) - /// fails the `strong_count` pre-check without touching the GC's maps. - fn reap_call_receiver(&mut self, callable: Object) { - match &callable { - Object::BoundMethod(bm) => { - // A native method can release references the interpreter - // never sees (`lst.clear()`, `d.pop(k)`, `sys.setprofile(f)` - // dropping the previous hook — test_sys_setprofile's - // `test_reentrancy` expects the displaced hook's `__del__` - // to have run by the next statement). `CALL` no longer - // takes the coarse stack-shrink mark (RFC 0077 WS2), so - // native callees keep it explicitly; the mark is a - // thread-local flag and the scan it schedules is the - // hot-set-only pass. - if matches!(bm.function, Object::Builtin(_)) { - gc_trace::mark_maybe_dead(); - } - let recv = bm.receiver.clone(); - // End the borrow and release the BoundMethod's hold on the - // receiver so `strong_count` reflects only real bindings + the - // GC handle. - drop(callable); - // `recv` (1) + the GC's tracking handle (<=1) + any weakref - // clones is the fingerprint of a dead temporary; anything - // larger means a live binding still holds it. The - // authoritative refcount/weakref test lives in - // `prompt_reap_dropped`; this is just a cheap pre-filter. - if Self::local_needs_prompt_reap(&recv) && Self::looks_reapable_temporary(&recv) { - self.prompt_reap_dropped(recv); - } else { - gc_trace::note_dropped(&recv); - } - } - // A *function* built as a call temporary — the canonical case is a - // comprehension's anonymous ``/``/``/ - // ``, emitted by `MakeFunction` and consumed by the very - // next `Call` — is GC-tracked when it captures cells (its closure). - // A plain `Rc` drop then leaves it pinned by its own GC handle, - // still holding those cells and, through them, the enclosing locals - // it closed over (e.g. `self` in `[self.submit(x) for x in it]`, - // promoted to a cell). That defers the finalization of anything - // uniquely reachable through the capture until the next cyclic - // collection. RFC 0040: `ThreadPoolExecutor.map`'s `result_iterator` - // closes over the list comprehension's `self`, so each `map` call - // leaked one reference to the executor — `del executor` then never - // dropped its refcount to zero, so the weakref callback that wakes - // idle workers never fired and `test_concurrent_futures` - // `test_del_shutdown` (and the other prompt-finalization shutdown - // cases) hung. Route a uniquely-held (non-escaped) function through - // the same cascade the frame-exit locals sweep uses for - // closure-function *locals*, so its cells — and any finalizable they - // uniquely hold — are released now, matching CPython's refcount - // timing. A function still bound elsewhere (a global `foo` in - // `foo()`, an escaped/returned closure) keeps `strong_count` above - // the dead threshold and fails the cheap pre-filter untouched, so - // the hot path of ordinary named-function calls pays only a single - // `strong_count` read. `prompt_reap_dropped` early-returns for an - // untracked function (no cells to leak), so it is reclaimed by the - // ordinary `Rc` drop of `callable` here. - Object::Function(_) => { - if Self::looks_reapable_temporary(&callable) { - self.prompt_reap_dropped(callable); - } else { - gc_trace::note_dropped(&callable); - } - } - // A callable *instance* temporary — `C()()` dispatching - // `__call__` — is GC-tracked and would otherwise stay pinned - // by its own collector handle until the next cyclic - // collection, exactly like the bound-method receiver above. - // CPython decrefs the callee operand as the CALL completes. - Object::Instance(_) => { - if Self::local_needs_prompt_reap(&callable) - && Self::looks_reapable_temporary(&callable) - { - self.prompt_reap_dropped(callable); - } else { - gc_trace::note_dropped(&callable); - } - } - // Native function: same reasoning as the builtin-method arm - // above (`sys.setprofile(f)` is a module-level `Builtin`). - Object::Builtin(_) => gc_trace::mark_maybe_dead(), - // RFC 0077 (WS2): `CALL` is exempt from the eval loop's coarse - // stack-shrink mark (see `note_dropped`); every operand it - // discards is graded here or in `reap_call_args` instead. - _ => gc_trace::note_dropped(&callable), - } - } - - /// After a call returns, give each *argument temporary* a prompt-reap - /// chance. While the call executed, the caller's argument vector held a - /// strong reference to every argument, so the callee's frame-exit reaping - /// of the corresponding parameter local saw the value as still-referenced - /// and left it to the cycle collector. Now that the caller is dropping its - /// argument vector, an argument that was only a *temporary* — `f(Make())`, - /// `runner.run(test_class('m'))` — becomes refcount-dead and must run its - /// (or its referents') finalizers immediately, mirroring CPython clearing - /// a call's stack slots the instant the call returns. RFC 0040: - /// `test_concurrent_futures.test_init` runs a child test inline as - /// `runner.run(test_class(...))`; the test case is never bound to a name, - /// so without reaping it here its `ProcessPoolExecutor` (and the log - /// `Queue`'s SemLocks) lingered to the next cyclic collection and the - /// resource tracker reported leaked semaphores. Moves each dead-looking - /// argument *out* of the vector before reaping so the vector slot no - /// longer inflates its refcount. - fn reap_call_args(&mut self, args: &mut [Object]) { - for a in args { - // `Function` mirrors `reap_call_receiver`'s comprehension-function - // arm, for a function consumed as an *argument*: the canonical - // case is `__build_class__(, name, *bases)` — the - // body function is GC-tracked when it captures cells (any `class` - // in a closure-bearing scope) and CPython frees it the instant the - // call returns, while a plain drop left it pinned by its own - // collector handle. test_descr `test_slots` observes exactly that: - // `len(gc.get_objects())` taken before/after a compare loop - // differed by the stale `G` body function dying in between. - // `prompt_reap_dropped` early-returns for untracked functions. - if (Self::local_needs_prompt_reap(a) || matches!(a, Object::Function(_))) - && Self::looks_reapable_temporary(a) - { - let v = std::mem::replace(a, Object::Unbound); - self.prompt_reap_dropped(v); - } else { - gc_trace::note_dropped(a); - } - } - } - - /// Prompt-reap the value an assignment just displaced. CPython decrefs - /// the previous attribute/subscript value the instant a store - /// overwrites it (`self.x = y` decrefs the old `self.x`); a value whose - /// last program-visible binding was that slot dies immediately, running - /// its `__del__`/weakref callbacks rather than waiting for the next - /// cyclic collection. The crucial case is a slot holding the only - /// strong reference to an object that *itself* anchors a weakref false - /// cycle: `ExecutorMixin.tearDown` does `self.executor = None` after - /// `shutdown()`, and the `ProcessPoolExecutor`↔`_ExecutorManagerThread` - /// pair is only kept reachable by the thread's `weakref.ref(executor)` - /// (a registry strong-clone in WeavePy's `Rc` model). Reaping the - /// displaced executor clears that weakref, breaks the cycle, and - /// cascades through the manager thread into the call/result queues so - /// their SemLocks unlink before the test stops the resource tracker - /// (RFC 0040: `test_concurrent_futures.test_init` - /// `FailingInitializerResourcesTest`). The cheap `local_needs_prompt_reap` - /// gate keeps scalar stores (`self.n = i` in a loop) off this path. - fn maybe_prompt_reap_replaced(&mut self, old: Object) { - static NO_STORE_REAP: std::sync::OnceLock = std::sync::OnceLock::new(); - let reap = !*NO_STORE_REAP.get_or_init(|| std::env::var_os("WP_NO_STORE_REAP").is_some()); - if reap && Self::local_needs_prompt_reap(&old) && Self::looks_reapable_temporary(&old) { - self.prompt_reap_dropped(old); - } else { - // RFC 0077 (WS2): the specialized `STORE_ATTR` hits are exempt - // from the coarse stack-shrink mark; the displaced value is - // the reference they release. - gc_trace::note_dropped(&old); - } - } - - /// Cheap "looks like a dead temporary" pre-filter shared by the - /// prompt-reap call sites. An object holding the single program reference - /// about to be dropped has `strong_count == that reference (1) + the GC's - /// tracking handle (<=1)`. A live weakref additionally keeps one strong - /// *clone* per slot in the registry (see [`crate::weakref_registry`]); - /// the cascade discounts those (an object reachable only through - /// `weakref.ref(obj)` is dead in CPython too), so they must not veto the - /// reap here. Anything beyond `2 + weak_clones` is a live binding. The - /// registry lookup is skipped on the common `strong_count <= 2` fast path. /// Park a dead tuple's allocation in [`Self::tuple_pool`] (CPython's - /// tuple freelist analogue). Only called from the prompt reaper's - /// fast path, which has already established the tuple carries no - /// finalizer, no weakref slot, no GC handle, and anchors no tracked - /// child; the `strong_count == 1` check here (we own the only - /// handle) plus `Rc::get_mut` (which additionally refuses if weak + /// tuple freelist analogue). The `strong_count == 1` check (we own the + /// only handle) plus `Rc::get_mut` (which additionally refuses if weak /// references exist) make the park equivalent to a free. Elements /// are cleared in place so their refcounts fall now, exactly as a /// plain drop would. @@ -3799,493 +2544,6 @@ impl Interpreter { Object::new_tuple_array(items) } - /// Is an exhausted iterator's backing list refcount-dead — held only by - /// the iterator itself (plus the cycle collector's tracking handle, - /// which `BuildList` installs on every list, and any weakref slots' - /// strong clones)? True exactly when the loop iterated a temporary — - /// `for x in f():` — so the elements die with the iterator, and CPython - /// would free them by refcount right now. - fn iter_backing_list_dead(items: &Rc>>) -> bool { - let probe = Object::List(items.clone()); - let id = crate::weakref_registry::id_of(&probe); - // 2 = the iterator's own handle + our `probe` clone. - let spare = 2 - + usize::from(gc_trace::is_tracked(id)) - + crate::weakref_registry::strong_clone_count(id); - Rc::strong_count(items) <= spare - } - - /// Prompt-reap the elements of an exhausted `FOR_ITER` iterator's - /// backing list, harvested at exhaustion because the iterator held the - /// last reference (the loop iterated a call-expression temporary). - /// CPython frees these by refcount the moment the iterator dies; our - /// detach-and-`Rc`-drop path skipped `__del__`s. Each element is - /// independently refcount-guarded by the cascade, so one still bound - /// (e.g. the final loop variable) is left alone. - fn reap_iter_backing(&mut self, items: Vec) { - for v in items { - if !(Self::local_needs_prompt_reap(&v) && Self::looks_reapable_temporary(&v)) { - continue; - } - if Self::exc_has_finalizable(&v, 4) { - // A finalizable behind an untracked container (a tuple of - // instances) would be skipped by `prompt_reap_dropped`'s - // fast path; drive the cascade directly. - self.reap_dead_subgraph(v); - } else { - self.prompt_reap_dropped(v); - } - } - } - - /// Whether the caller's reference to `o` may be its last program - /// reference: nothing holds it beyond the caller, its collector - /// handle (when tracked), and weakref-registry clones. - #[inline] - fn looks_reapable_temporary(o: &Object) -> bool { - let sc = gc_trace::strong_count_for(o); - if sc <= 1 { - return true; - } - let id = crate::weakref_registry::id_of(o); - let handle = usize::from(gc_trace::maybe_tracked(id)); - if sc <= 1 + handle { - return true; - } - crate::weakref_registry::may_have_weakrefs(id) - && sc <= 1 + handle + crate::weakref_registry::strong_clone_count(id) - } - - /// Whether a frame local is worth running through the prompt-reap - /// cascade on frame exit. Mirrors the `PopTop` discard gate: instances - /// (possible `__del__` / weakref finalizer), and live - /// generators/coroutines/async-generators (their `close()` delivers - /// `GeneratorExit`). Kept deliberately narrow so the hot return path of - /// scalar-only frames pays a single cheap `matches!` per local. - /// A small untracked list, tuple or dict of atomic values held only - /// here: its release frees nothing a reap would visit (no finalizer, - /// weakref or collector handle can hang off it or its values). - fn atomic_container_dies_plainly(o: &Object) -> bool { - const MAX: usize = 16; - fn atomic(o: &Object) -> bool { - matches!( - o, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None | Object::Str(_) - ) - } - let (sc, id) = match o { - Object::List(l) => (Rc::strong_count(l), Rc::as_ptr(l) as usize as u64), - Object::Dict(d) => (Rc::strong_count(d), Rc::as_ptr(d) as usize as u64), - Object::Tuple(t) => ( - ThinArc::strong_count(t), - ThinArc::as_ptr(t).cast::<()>() as usize as u64, - ), - _ => return false, - }; - if sc != 1 || gc_trace::maybe_tracked(id) || crate::weakref_registry::may_have_weakrefs(id) - { - return false; - } - match o { - Object::List(l) => l - .try_borrow() - .is_ok_and(|v| v.len() <= MAX && v.iter().all(atomic)), - Object::Dict(d) => d - .try_borrow() - .is_ok_and(|d| d.len() <= MAX && d.iter().all(|(k, v)| atomic(&k.0) && atomic(v))), - Object::Tuple(t) => t.len() <= MAX && t.iter().all(atomic), - _ => false, - } - } - - /// A collector-tracked list or dict whose only holders are the caller's - /// reference and its collector handle, watched by no weakref, holding - /// only values its death can't finalize (atomic ones, or instances - /// others still hold): untracking it and dropping the reference is its - /// whole teardown, as in CPython's `list_dealloc`. - fn tracked_container_dies_inertly(o: &Object) -> bool { - const MAX: usize = 16; - fn inert(v: &Object) -> bool { - matches!( - v, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None | Object::Str(_) - ) || gc_trace::drop_survives_plainly(v) - } - let (sc, id) = match o { - Object::List(l) => (Rc::strong_count(l), Rc::as_ptr(l) as usize as u64), - Object::Dict(d) => (Rc::strong_count(d), Rc::as_ptr(d) as usize as u64), - _ => return false, - }; - if sc != 2 || crate::weakref_registry::may_have_weakrefs(id) || !gc_trace::is_tracked(id) { - return false; - } - match o { - Object::List(l) => l - .try_borrow() - .is_ok_and(|v| v.len() <= MAX && v.iter().all(inert)), - Object::Dict(d) => d - .try_borrow() - .is_ok_and(|d| d.len() <= MAX && d.iter().all(|(k, v)| inert(&k.0) && inert(v))), - _ => false, - } - } - - fn local_needs_prompt_reap(o: &Object) -> bool { - matches!( - o, - Object::Instance(_) - | Object::Generator(_) - | Object::Coroutine(_) - | Object::AsyncGenerator(_) - | Object::AsyncGenAwait(_) - // A container local can anchor a finalizable acyclic subgraph - // that dies with the frame: pickle's `_dict_op` builds - // `items = [key, value]` via `_pop_to_mark`, stuffs it into a - // dict, and lets `items` fall out of scope at return — leaving - // the only non-GC reference to a freshly-unpickled, finalizable - // `value` on a dead list local. CPython frees it by refcount on - // return; reap it here so its `__del__` runs promptly instead of - // lingering to the next cyclic collection (RFC 0040: - // `test_concurrent_futures.test_ressources_gced_in_workers`). - // The `strong_count` pre-filter at the call site keeps escaped - // containers off the cascade. - | Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::Tuple(_) - | Object::FrozenSet(_) - // A bound method anchors its receiver (`__self__`); a dead - // one must release it promptly — and a *stored* one - // (GC-tracked at attribute-store time) must shed its own - // collector handle. See the `BoundMethod` arm of - // `prompt_reap_dropped`. - | Object::BoundMethod(_) - // A function is GC-tracked: a rebound lambda or closure must - // shed its collector handle (and whatever it alone holds) - // now, as CPython frees it by refcount. - | Object::Function(_) - // An escaped frame owns locals after its activation exits. - // Releasing the last frame alias must retire those locals, - // including tracked containers and weakly held instances. - | Object::Frame(_) - ) - } - - /// Does `obj` carry a finalizer that prompt reclamation must run - /// before freeing it? Instances with `__del__`, and unfinished - /// generators/coroutines/async-generators (their `close()` delivers - /// `GeneratorExit`). - fn object_is_finalizable(obj: &Object) -> bool { - match obj { - // Cached per class (`TypeObject::has_del`), invalidated on - // `__del__` assignment / MRO changes — the uncached - // `lookup("__del__")` walk clones the MRO on every reaped - // instance and dominated the prompt-reap path. - Object::Instance(i) => i.cls().instances_need_finalize(), - // An unstarted generator has nothing to finalize: closing it - // runs no Python (see `gc_trace::has_finalizer`). A coroutine - // still does — the never-awaited RuntimeWarning. - Object::Generator(g) => !g.is_finished() && !g.is_unstarted(), - Object::Coroutine(g) | Object::AsyncGenerator(g) => !g.is_finished(), - _ => false, - } - } - - /// An exact builtin `list`/`dict`/`set` whose every element (and key) - /// is a scalar leaf ([`gc_trace::is_atomic`]): a container that can - /// neither finalize nor anchor anything the prompt reaper would have - /// to cascade into. Subclass instances are `Object::Instance` and never - /// match. A container that is currently borrowed reports `false` so - /// the caller takes the general path. - fn is_scalar_leaf_container(obj: &Object) -> bool { - // Capped like `anchors_tracked_child`'s node budget: the answer is - // an optimisation, never a correctness input, and an unbounded - // scan would make dropping a large container O(len) on a path that - // runs for every displaced binding. - const SCAN_CAP: usize = 64; - match obj { - Object::List(l) => l - .try_borrow() - .map(|v| v.len() <= SCAN_CAP && v.iter().all(gc_trace::is_atomic)) - .unwrap_or(false), - Object::Set(s) => s - .try_borrow() - .map(|m| m.len() <= SCAN_CAP && m.iter().all(|k| gc_trace::is_atomic(&k.0))) - .unwrap_or(false), - Object::Dict(d) => d - .try_borrow() - .map(|m| { - m.len() <= SCAN_CAP - && m.iter() - .all(|(k, v)| gc_trace::is_atomic(&k.0) && gc_trace::is_atomic(v)) - }) - .unwrap_or(false), - _ => false, - } - } - - /// The common handled exception, recognized without the reap walks: a - /// class with no finalizer, no instance dict, and slots holding only - /// atomic values, a tuple of them (`args`), or a traceback whose frames - /// are all still executing. Neither `exc_has_finalizable` nor - /// `anchors_tracked_child` can find anything in it. - fn exc_plainly_inert(obj: &Object) -> bool { - let Object::Instance(inst) = obj else { - return false; - }; - if inst.cls().instances_need_finalize() - || inst - .dict - .get() - .is_some_and(|d| d.try_borrow().map_or(true, |d| !d.is_empty())) - { - return false; - } - let Ok(slots) = inst.slots.try_borrow() else { - return false; - }; - // (Bound, not returned directly: the iterator borrows `slots`.) - #[allow(clippy::let_and_return)] - let inert = slots.iter().all(|(_, v)| match v { - Object::Tuple(t) => t.iter().all(Object::is_gc_atomic), - Object::Traceback(tb) => { - let mut cur = Some(tb.clone()); - while let Some(node) = cur { - if node.frame.on_stack.get() == 0 { - return false; - } - cur = node.next.borrow().clone(); - } - true - } - other => other.is_gc_atomic(), - }); - inert - } - - /// Cheap, allocation-free pre-check for the `POP_EXCEPT` reap: does the - /// acyclic subgraph rooted at `obj` contain a finalizable object - /// (`__del__` / an unfinished generator) reachable through value - /// containers and user instances, up to `depth` links? The - /// overwhelmingly common handled exception — `ValueError(i)`, - /// `KeyError(k)`, scalar/`str` args, no `__del__` anywhere — returns - /// `false` here in pure recursion (no heap traffic, no weakref-registry - /// work), so the hot handler-exit path stays at baseline cost and only - /// an exception that actually anchors a destructor pays for the full - /// [`Self::reap_dead_subgraph`] cascade. Tracebacks/frames are - /// deliberately *not* traversed: walking the captured call stack on - /// every caught exception would dominate the path, and a finalizable - /// pinned only by a traceback frame is left to the cycle collector - /// (CPython's own traceback-clearing is likewise lazy here). - fn exc_has_finalizable(obj: &Object, depth: u8) -> bool { - if Self::object_is_finalizable(obj) { - return true; - } - if depth == 0 { - return false; - } - if matches!( - obj, - Object::Tuple(_) - | Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::FrozenSet(_) - | Object::Instance(_) - | Object::Cell(_) - | Object::MappingProxy(_) - | Object::SimpleNamespace(_) - ) { - let mut found = false; - crate::gc_trace::traverse_object(obj, &mut |c| { - if !found && Self::exc_has_finalizable(c, depth - 1) { - found = true; - } - }); - return found; - } - false - } - - /// Does the container subgraph rooted at `root` contain a *GC-tracked* - /// child within `depth` links? Used by [`Self::prompt_reap_dropped`] to - /// decide whether an untracked, unwatched, non-finalizable container is - /// still worth the cascade: a plain `Rc` drop frees the container but - /// leaves a tracked child it solely anchored pinned by its own - /// collector handle (weakrefs unclear, `__del__` deferred to the next - /// cyclic collection). Bounded — both in depth and in total nodes - /// visited — so a huge scalar container (a million-int list dying at - /// frame exit) costs a few dozen `is_tracked` lookups at most; hitting - /// the cap conservatively reports `false` (skip the prompt reap; the - /// tracing collector reclaims later). - fn anchors_tracked_child(root: &Object, depth: u8) -> bool { - const NODE_CAP: usize = 64; - fn walk(obj: &Object, depth: u8, budget: &mut usize) -> bool { - if depth == 0 || *budget == 0 { - return false; - } - let mut found = false; - crate::gc_trace::traverse_object(obj, &mut |c| { - // Scalars are never tracked and hold nothing; a frame still - // on the stack keeps its locals whatever dies here (a - // traceback leads to the very frame that is handling it). - if found - || *budget == 0 - || c.is_gc_atomic() - || matches!(c, Object::Frame(f) if f.on_stack.get() > 0) - || matches!(c, Object::Instance(i) if i.is_gc_deferred() && i.c_body.get() == 0) - { - return; - } - *budget -= 1; - if crate::gc_trace::is_tracked(crate::weakref_registry::id_of(c)) { - found = true; - return; - } - if matches!( - c, - Object::Tuple(_) - | Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::FrozenSet(_) - | Object::Instance(_) - | Object::Cell(_) - // Follow `__traceback__` → frame → locals: a dying - // exception is routinely the last holder of tracked - // objects that live only in the frames it unwound. - | Object::Traceback(_) - | Object::Frame(_) - ) && walk(c, depth - 1, budget) - { - found = true; - } - }); - found - } - let mut budget = NODE_CAP; - walk(root, depth, &mut budget) - } - - /// Is `obj` dead in refcount terms — does the caller hold its last - /// program-visible reference? `local_refs` is the number of strong - /// references the caller itself holds (the binding being dropped, or - /// a work-list entry). The GC's own handle and weakref strong clones - /// are discounted: neither is a program-visible binding. - fn is_refcount_dead(obj: &Object, local_refs: usize) -> bool { - let id = crate::weakref_registry::id_of(obj); - let registry_holds = usize::from(gc_trace::is_tracked(id)); - let weak_clones = crate::weakref_registry::strong_clone_count(id); - gc_trace::strong_count_for(obj) <= local_refs + registry_holds + weak_clones - } - - /// Has `obj` escaped into a C extension — does it own a faithful - /// inline instance body (RFC 0045, `c_body != 0`)? Such an instance's - /// pointer may be held *borrowed* (uncounted) by a C extension across - /// a re-entrant call into the VM, so the prompt reaper — whose - /// deadness test sees only VM `Rc` references — must not reclaim it - /// (that would free the body and let the allocator reuse it under the - /// extension's live pointer). The tracing cycle collector reclaims it - /// later, once C's transient borrows are gone. A single `Cell` read; - /// `false` for every pure-Python instance and non-instance object. - #[inline] - fn instance_escaped_to_c(obj: &Object) -> bool { - matches!(obj, Object::Instance(inst) if inst.c_body.get() != 0) - } - - /// Does the acyclic subgraph rooted at `root` contain any instance - /// that has escaped into a C extension (see [`instance_escaped_to_c`])? - /// Used by the prompt reaper to leave such subgraphs to the tracing - /// cycle collector (a C extension may hold a borrowed pointer into one - /// of the bodies). A bounded breadth-first walk over the same edges - /// `gc_trace::traverse_object` exposes, deduplicated by object id and - /// capped so a pathological structure can never make frame exit - /// quadratic; hitting the cap conservatively reports `true` (skip the - /// reap — always safe). Returns immediately for the overwhelmingly - /// common all-scalar / pure-Python subgraph. - fn subgraph_contains_escaped(root: &Object) -> bool { - // Fast path: a root that can hold no references (scalar, string, - // …) trivially has no escaped instance in its subgraph. This - // covers the overwhelming majority of reaped temporaries at zero - // allocation cost. - if Self::instance_escaped_to_c(root) { - return true; - } - if !Self::object_can_hold_refs(root) { - return false; - } - // Bounded DFS reusing a thread-local scratch stack + visited set so - // the hot reap path allocates nothing. Cap chosen well above - // pandas' shallow `_cache`/`__dict__` fan-out; on overflow we - // proceed with the reap (return `false`) — the only structures big - // enough to hit it are large escaped-free graphs, and a pandas - // escaped body is always shallow. - const VISIT_CAP: usize = 2048; - thread_local! { - static SCRATCH: std::cell::RefCell<( - Vec, - std::collections::HashSet, - )> = std::cell::RefCell::new((Vec::new(), std::collections::HashSet::new())); - } - SCRATCH.with(|cell| { - let Ok(mut guard) = cell.try_borrow_mut() else { - // Re-entrant (a finalizer walked here): fall back to a - // conservative "skip the reap" rather than risk aliasing - // the scratch buffers. - return true; - }; - let (stack, seen) = &mut *guard; - stack.clear(); - seen.clear(); - gc_trace::traverse_object(root, &mut |child| stack.push(child.clone())); - let mut visited = 0usize; - let mut found = false; - while let Some(obj) = stack.pop() { - if !seen.insert(crate::weakref_registry::id_of(&obj)) { - continue; - } - if Self::instance_escaped_to_c(&obj) { - found = true; - break; - } - visited += 1; - if visited > VISIT_CAP { - break; - } - if Self::object_can_hold_refs(&obj) { - gc_trace::traverse_object(&obj, &mut |child| stack.push(child.clone())); - } - } - stack.clear(); - seen.clear(); - found - }) - } - - /// Cheap discriminator: can `obj` hold references to other objects - /// (and therefore possibly reach an escaped instance)? `false` for - /// scalars/strings/bytes and other leaves, letting - /// [`subgraph_contains_escaped`] skip the walk entirely. - #[inline] - fn object_can_hold_refs(obj: &Object) -> bool { - matches!( - obj, - Object::List(_) - | Object::Tuple(_) - | Object::Dict(_) - | Object::Set(_) - | Object::FrozenSet(_) - | Object::MappingProxy(_) - | Object::SimpleNamespace(_) - | Object::Instance(_) - | Object::Cell(_) - | Object::BoundMethod(_) - | Object::Function(_) - | Object::Type(_) - | Object::Frame(_) - ) - } - /// Emulate CPython deleting a terminating thread's thread-state /// dict: drop `ident`'s slot from every `_threading_local.local`. /// Run from the native worker teardown (`spawn_python_worker`) the @@ -4338,63 +2596,17 @@ impl Interpreter { self.run_pending_finalizers(); } - /// Invoke any `__del__` finalizers queued by the cycle GC. - /// Each finalizer runs at most once. Exceptions from a - /// finalizer are routed through `sys.unraisablehook` (the - /// default hook prints `Exception ignored in: …` to stderr, - /// exactly like CPython) so they don't propagate. - /// Prompt (deterministic) finalization driver — the interpreter half of - /// RFC 0040's GC arc. CPython runs an object's `__del__` the instant its - /// refcount reaches zero; weavepy's tracing collector pins every tracked - /// object on a strong handle, so a refcount-dead finalizable object would - /// otherwise linger until the next cyclic collection. This closes that gap - /// by, at each reference-drop safe point, running `__del__` for objects - /// whose last *program* reference just dropped (`strong_count <= 1`). - /// - /// Drives a small fixpoint loop: queue the dead finalizables' `__del__`, - /// run them, reclaim the (non-resurrected) corpses — which cascades the - /// refcount drop into their referents and can expose the next layer of - /// now-dead finalizables — and repeat until quiescent. Bounded by - /// `MAX_COLLECT_PASSES` against a pathological `__del__` that keeps - /// resurrecting and re-killing objects. - /// - /// Gated by [`gc_trace::has_any_finalizable`] at every call site, so when - /// no `__del__`-bearing object is live (the common case) it costs one - /// relaxed atomic load. Re-entrancy is suppressed via a thread-local: a - /// `__del__` body re-enters the eval loop, whose own safe points would - /// otherwise recurse here. + /// Run the work queued for this safe point: finalizers a dying + /// object's `Drop` resurrected, and weakref callbacks. pub fn drain_prompt_finalizers(&mut self) { - thread_local! { - static IN_PROMPT_FINALIZE: std::cell::Cell = const { std::cell::Cell::new(false) }; - } - if IN_PROMPT_FINALIZE.with(std::cell::Cell::get) { - return; - } - IN_PROMPT_FINALIZE.with(|c| c.set(true)); - for _ in 0..gc_trace::MAX_COLLECT_PASSES { - let progressed = gc_trace::reap_dead_finalizable(); - if progressed == 0 { - break; - } - // Run the `__del__`s and weakref callbacks just queued. They may - // drop the last reference to — or resurrect — other finalizables, - // which the next pass re-evaluates. - self.run_pending_finalizers(); - // Cascade through *non*-finalizable tracked objects too: freeing - // a `__del__`-bearing object often exposes a chain of plain - // containers (transport → protocol → context dicts) that CPython's - // refcounting would free in the same instant, and whose survival - // would keep further finalizables/weakref targets alive past this - // drain (asyncio's SSL leak tests watch such a chain). Only runs - // on the rare "a finalizable just died" path, so the O(tracked) - // scan stays off the hot loop; amortized further when the - // tracked population itself is huge (see - // `reap_dead_acyclic_amortized`). - gc_trace::reap_dead_acyclic_amortized(); - } - IN_PROMPT_FINALIZE.with(|c| c.set(false)); + self.run_pending_finalizers(); } + /// Invoke any `__del__` finalizers and weakref callbacks queued by a + /// dying object's `Drop` or by the cycle collector. Each finalizer runs + /// at most once. Exceptions from a finalizer are routed through + /// `sys.unraisablehook` (the default hook prints `Exception ignored + /// in: …` to stderr, exactly like CPython) so they don't propagate. pub fn run_pending_finalizers(&mut self) -> usize { // RFC 0077 (WS2): the prompt reaper drains after every node it // tears down, and for the overwhelming majority nothing was @@ -4753,19 +2965,20 @@ impl Interpreter { Some(Object::Module(m)) => m.dict.borrow().iter().map(|(_k, v)| v.clone()).collect(), _ => Vec::new(), }; - let mut sys_deferred: Vec> = Vec::new(); + let mut sys_deferred: Vec<(crate::sync::Rc, Object)> = + Vec::new(); for _ in 0..8 { let candidates = crate::gc_trace::finalization_candidates(); if candidates.is_empty() { break; } - for handle in candidates { - if sys_values.iter().any(|v| v.is_same(&handle.object)) { + for (handle, obj) in candidates { + if sys_values.iter().any(|v| v.is_same(&obj)) { if !sys_deferred .iter() - .any(|h| crate::sync::Rc::ptr_eq(h, &handle)) + .any(|(h, _)| crate::sync::Rc::ptr_eq(h, &handle)) { - sys_deferred.push(handle); + sys_deferred.push((handle, obj)); } continue; } @@ -4777,21 +2990,21 @@ impl Interpreter { { continue; } - self.invoke_finalizer(&handle.object); + self.invoke_finalizer(&obj); } // A finalizer may have queued cyclic finalizers of its own. self.run_pending_finalizers(); } if !sys_deferred.is_empty() { crate::vm_singletons::set_unraisable_silenced(true); - for handle in sys_deferred { + for (handle, obj) in sys_deferred { if handle .finalized .swap(true, std::sync::atomic::Ordering::AcqRel) { continue; } - self.invoke_finalizer(&handle.object); + self.invoke_finalizer(&obj); } crate::vm_singletons::set_unraisable_silenced(false); } @@ -4826,9 +3039,6 @@ impl Interpreter { if std::env::var_os("WEAVEPY_VM_STATS").is_some() { let snap = crate::specialize::snapshot(); eprint!("{}", crate::specialize::format_stats_markdown(&snap)); - if let Some(md) = gc_trace::floor_stats::markdown() { - eprint!("{md}"); - } if let Some(md) = jit_stats_markdown() { eprint!("{md}"); } @@ -4864,6 +3074,23 @@ impl Interpreter { /// prompt, collector, and shutdown requests may refer to the same object. /// Only the winning request may publish finalizer completion. fn invoke_finalizer(&mut self, obj: &Object) -> bool { + // A discarded never-driven `asend`/`athrow`/`aclose` awaitable (see + // `release`). + if let Object::AsyncGenAwait(a) = obj { + let method = match a.kind { + crate::object::AgenAwaitKind::Send => "asend", + crate::object::AgenAwaitKind::Throw => "athrow", + crate::object::AgenAwaitKind::Close => "aclose", + }; + let qualname = match &a.agen { + Object::AsyncGenerator(g) => g.qualname.borrow().to_str(), + other => other.type_name_owned(), + }; + let _ = self.emit_runtime_warning(format!( + "coroutine method '{method}' of '{qualname}' was never awaited" + )); + return true; + } let finalize_ran = match obj { Object::Instance(inst) => &inst.finalize_ran, Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { @@ -5092,7 +3319,7 @@ impl Interpreter { let mut cur = Some(tb); while let Some(node) = cur { entries.push(crate::error::TracebackEntry { - code: node.frame.code.clone(), + code: node.code(), lineno: node.lineno, }); cur = node.next.borrow().clone(); @@ -5731,7 +3958,6 @@ impl Interpreter { fn run_frame(&mut self, frame: &mut Frame) -> Result { match self.run_until_yield_or_return(frame, None)? { FrameOutcome::Returned(v) => { - self.reap_frame_locals_on_exit(frame, Some(&v)); self.recycle_frame_allocs(frame); Ok(v) } @@ -6176,12 +4402,6 @@ impl Interpreter { let quiet_off = self.quiet_off; let mut loop_snap_gen: u64 = 0; // stale → first iteration derives let mut loop_quiet = false; - let mut loop_fin_quiet = false; - // Within fin-quiet mode: is a finalizable object live (the coarse - // stack-shrink drop heuristic applies), or only active suspects - // (which the full prologue probes at marked safe points without - // it)? - let mut loop_coarse = false; // The loops are off only because a frame object is observed (see // `frame_object_observed`): re-derive once nothing outside the // activation holds it any more (an `except` clause dropped the @@ -6197,21 +4417,6 @@ impl Interpreter { self.gil_countdown = self.gil_countdown.wrapping_sub(1); if self.gil_countdown == 0 { self.gil_countdown = crate::gil::GIL_CHECK_INTERVAL; - // RFC 0065 (WS1): the quiet paths skip the per- - // instruction suspect probes, so aged-out (dormant) - // suspects are instead re-probed on this ~128- - // instruction cadence. `has_suspects` keeps its - // dormant-stride admission, so a dormant-only - // population pays two relaxed loads here and a real - // probe only every DORMANT_STRIDE-th checkpoint. - if (loop_quiet || loop_fin_quiet) - && !gc_trace::active_suspects_present() - && gc_trace::has_suspects() - { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } crate::gil::yield_checkpoint(); } // RFC 0065 (WS1): re-derive the prologue snapshot only when @@ -6236,27 +4441,7 @@ impl Interpreter { let frame_observed = Self::frame_object_observed(&shell, py_frame_slot.as_ref(), frame); frame_blocks_quiet = unobserved && frame_observed; - let base_quiet = unobserved && !frame_observed; - // Graded: fully quiet when no finalizable object exists - // anywhere; "fin-quiet" — everything cold except the - // CPython-timing-bearing prompt-finalization probe — - // when the only live input is the finalizable - // population (in practice ≥ 1 in any real program: the - // ABC machinery keeps a weakref-callback-enrolled type - // alive for the process lifetime). - let fin_live = gc_trace::has_any_finalizable(); - // An *active* suspect (one still inside its probe budget) - // is the same kind of input: it is re-probed at every - // marked drop safe point, which is exactly the fin-quiet - // path's `take_maybe_dead` gate. Pinning the loop to the - // full prologue for it instead made every instruction of - // a drop-free hot loop pay that prologue for as long as - // the suspect's budget lasted, since nothing there probes - // it. - let suspects_live = gc_trace::active_suspects_present(); - loop_quiet = base_quiet && !fin_live && !suspects_live; - loop_fin_quiet = base_quiet && (fin_live || suspects_live); - loop_coarse = fin_live; + loop_quiet = unobserved && !frame_observed; } // The leaf burst: while the loop is quiet, run the straight // run of leaf instructions at `frame.pc` in one tight loop @@ -6268,43 +4453,24 @@ impl Interpreter { && !crate::gil::free_threading_enabled(); let cur_pc: usize; let mut instruction_ran = false; - let watch_drops: bool; - let stack_before: usize; - let op_before: Option; let stepped = if let Some((stepped, at)) = pending.take() { // An outcome produced before this loop took over (see the // method docs). Its producer ran the per-instruction - // protocol, drop heuristic included. - watch_drops = false; - stack_before = 0; - op_before = None; + // protocol. cur_pc = at; instruction_ran = true; stepped - } else if loop_quiet || loop_fin_quiet { - // The quiet paths: nothing is pending, nobody is - // watching. Semantically identical to the full prologue - // with the corresponding gates cold — each gate's - // producer bumps the loop generation before its gate - // can read hot, so the full prologue resumes on the - // very next instruction. - // The one live subsystem in fin-quiet mode: prompt - // finalization. Same probe order and cadence as the full - // prologue's finalizable arm (`take_maybe_dead` gate, then - // the finalizer drain, then the stride-gated suspect - // re-probe), and the same drop-watch bookkeeping in the - // shared tail below. - let drained = loop_fin_quiet && self.drain_if_maybe_dead(); - // Nothing pending: hand the frame to the quiet loop proper. - // (After a drain the finalizers may have changed any loop - // input, so that iteration single-steps.) - let quiet_exit = if burst_ok && !drained { + } else if loop_quiet { + // The quiet path: nothing is pending, nobody is watching. + // Semantically identical to the full prologue with the + // corresponding gates cold — each gate's producer bumps the + // loop generation before its gate can read hot, so the full + // prologue resumes on the very next instruction. + let quiet_exit = if burst_ok { Some(self.quiet_run( frame, QuietShell::Ready(&shell), loop_snap_gen, - loop_fin_quiet, - loop_coarse, &mut prev_pc, )) } else { @@ -6328,6 +4494,10 @@ impl Interpreter { if let Some(py) = shell.materialized.borrow().as_ref() { py.lasti.set(at); } + } else if let Some(exit) = &quiet_exit { + // Likewise the shell a lazy traceback entry will + // materialize from (see `pop_frame_shell`). + Self::sync_lazy_tb_lasti(Some(&shell), &frame.code, exit); } match quiet_exit { Some(QuietExit::Yield) => continue, @@ -6335,25 +4505,11 @@ impl Interpreter { stepped, cur_pc: at, }) => { - // `quiet_run` already applied the drop heuristic. - watch_drops = false; - stack_before = 0; - op_before = None; cur_pc = at; instruction_ran = true; stepped } None => { - if loop_fin_quiet && loop_coarse { - watch_drops = true; - stack_before = frame.stack.len(); - op_before = - frame.code.instructions.get(frame.pc as usize).map(|i| i.op); - } else { - watch_drops = false; - stack_before = 0; - op_before = None; - } cur_pc = frame.pc as usize; shell .lasti @@ -6370,6 +4526,25 @@ impl Interpreter { self.step_hot(frame) } } + } else if frame_blocks_quiet { + // Quiet but for a frame object something outside holds (an + // exception's traceback, while its handler runs): nothing + // the full prologue serves is pending, so each instruction + // only brings the frame's `lasti` current before it runs. + cur_pc = frame.pc as usize; + shell + .lasti + .store(frame.pc, std::sync::atomic::Ordering::Relaxed); + if shell + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) + { + self.ensure_top_py_frame(&mut py_frame_slot) + .lasti + .set(frame.pc); + } + instruction_ran = true; + self.step_hot(frame) } else { // RFC 0059 (WS2): the unified eval-breaker word. One relaxed // load answers "is *any* deferred work pending?" for the six @@ -6379,42 +4554,13 @@ impl Interpreter { // Each bit-guarded arm below re-checks its subsystem's // precise gate, so a stale-set bit is harmless. let hot = crate::hot_gates::load(); - // RFC 0040 (GC arc): prompt finalization. Between bytecodes — a - // safe point with no outstanding container borrows — run `__del__` - // and fire callback-weakrefs for any object whose last reference - // dropped during the previous instruction, matching CPython's - // refcount-driven `tp_dealloc` timing. Two gates keep this free in - // the common case: a finalizable object must be live *and* the - // previous instruction must have dropped a reference (`take_maybe_ - // _dead`, made variant-precise by RFC 0059 WS1b), so a hot loop - // that neither allocates `__del__` objects nor drops heap-graph - // references pays only the relaxed loads. - let any_finalizable = gc_trace::has_any_finalizable(); - if (any_finalizable || gc_trace::has_suspects()) && gc_trace::take_maybe_dead() { - if any_finalizable { - self.drain_prompt_finalizers(); - } - // Re-probe cascade-skipped suspects (see `note_suspect`): a - // Rust-side transient that pinned one during its cascade has - // typically died by now, and CPython would have freed the - // object (and everything it anchors) at that instant. - if gc_trace::has_suspects() { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } - } else if hot & crate::hot_gates::PENDING_FINALIZERS != 0 - && crate::vm_singletons::has_pending_finalizers() - { - // An *untracked* finalizable instance freed by a plain `Rc` - // drop (its container temporary died between bytecodes — a - // `for k, v in d.items()` loop rebinding its tuple, dropping - // the overwritten value) parks its `__del__` via - // `PyInstance::drop`'s resurrection net. It never appears in - // the finalizable index, so the gate above stays cold; run - // the parked finalizers here to keep CPython's refcount - // timing (`test_dict.test_oob_indexing_dictiter_iternextitem` - // clears the dict from such a `__del__` mid-iteration). + // Prompt finalization. Between bytecodes — a safe point with no + // outstanding container borrows — run the `__del__`s and + // weakref callbacks queued by objects that died during the + // previous instruction (their `Drop` raised the gate), + // matching CPython's refcount-driven `tp_dealloc` timing. + if hot & crate::hot_gates::PENDING_FINALIZERS != 0 { + gc_trace::take_maybe_dead(); self.run_pending_finalizers(); } // A tracking site crossed the young threshold without polling @@ -6425,58 +4571,19 @@ impl Interpreter { self.run_pending_finalizers(); } } - // RFC 0047 (wave 5): reap objects whose last C-side reference - // was dropped *inside* an extension call since the previous - // instruction (a Cython setter decref'ing the old value of a - // `cdef` field). CPython would have run their dealloc chains in - // the call itself; the capi boundary parks them instead, and - // this between-bytecodes safe point — the first moment with no - // extension frame live — runs the refcount-guarded reap so - // weakrefs to anything that died clear before the next - // instruction can observe them. Gated on one thread-local read. - // - // Crucially, the drain waits for `cext_call_active()` to clear: - // bytecode a C extension re-enters (a Cython module-exec, a - // `PyObject_Call` back into Python) hits this safe point too, - // where every parked escaped subgraph would just be re-parked - // by `reap_dead_subgraph`'s C-frame guard after an O(subgraph) - // escape scan — re-scanning the whole queue on *every bytecode* - // of the re-entered code (pandas' import runs minutes long - // under that quadratic churn). Draining only at depth 0 runs - // each parked object through the cascade exactly once. - // Probe order matters: `has_pending_cext_drops` is one - // relaxed load when empty (the overwhelming case), while - // `cext_call_active` is a thread-local read — so ask - // "is there anything to do" before "may we do it". - // Native side exits share this queue: their obsolete pins - // can be reaped now that the continuation's frame is on - // the stack and its recursion depth is accounted for. + // Release the references C extension code dropped since the + // previous instruction (RFC 0047, wave 5). The capi boundary + // parks them rather than releasing them inside the call, + // where C may still borrow a body they back; this safe point + // with no extension frame live is the first moment they can + // go (`has_pending_cext_drops` is the cheap probe, so it + // comes first). if hot & crate::hot_gates::PENDING_CEXT != 0 && crate::vm_singletons::has_pending_cext_drops() && !crate::vm_singletons::cext_call_active() { - static REAP_TRACE: std::sync::OnceLock = std::sync::OnceLock::new(); - let reap_trace = *REAP_TRACE - .get_or_init(|| std::env::var_os("WEAVEPY_REAP_TRACE").is_some()); for dropped in crate::vm_singletons::drain_pending_cext_drops() { - if reap_trace { - let dead = Self::is_refcount_dead(&dropped, 1); - let sc = gc_trace::strong_count_for(&dropped); - eprintln!( - "[CEXT-REAP] {} id={:#x} sc={} dead={}", - dropped.type_name_owned(), - crate::weakref_registry::id_of(&dropped), - sc, - dead - ); - } - // Straight to the cascade: `prompt_reap_dropped`'s - // fast-path early return would plain-drop an untracked, - // finalizer-free container — but a C-dropped container - // (the old `blocks` tuple) is queued here precisely - // because its *tracked children* (Cython `Block`s) need - // the cascade to clear their weakrefs. - self.reap_dead_subgraph(dropped); + self.release(dropped); } } // RFC 0040 (WS7): surface any "unclosed file" `ResourceWarning` @@ -6659,21 +4766,6 @@ impl Interpreter { // always — stays free. Feeding any handler-raised error // through the same arm as `step`'s `Err` keeps it catchable // by a surrounding `try/except`, exactly as CPython. - // RFC 0040 (GC arc): note whether this instruction can drop the - // last reference to a finalizable object, so the next safe point's - // prompt-finalization sweep runs only when warranted. Gated on a - // live finalizable so non-`__del__` workloads pay nothing. A - // dropped reference shows up either as the operand stack shrinking - // (`POP_TOP`, every `STORE_*`/`*_SUBSCR`, `POP_EXCEPT`, `END_FOR`, - // a net-consuming `CALL`, …) or as one of the stack-neutral - // `DELETE_*`-name opcodes. - watch_drops = gc_trace::has_any_finalizable(); - stack_before = if watch_drops { frame.stack.len() } else { 0 }; - op_before = if watch_drops { - frame.code.instructions.get(frame.pc as usize).map(|i| i.op) - } else { - None - }; // `PyThreadState_SetAsyncExc`: an exception scheduled for this // thread from another thread (or itself) is raised here, at the // next instruction boundary — CPython's eval-breaker async-exc @@ -6779,39 +4871,6 @@ impl Interpreter { if instruction_ran { prev_pc = Some(cur_pc); } - // RFC 0059 WS1b: opcodes whose handlers account for every value - // they discard via `gc_trace::note_dropped` (audited: `POP_TOP`, - // `STORE_FAST`, and the `BINARY_OP`/`COMPARE_OP` arms — the - // specialized scalar fast paths of the latter two consume only - // pure-leaf variants by construction) are exempt from the coarse - // stack-shrink heuristic, so scalar-churning hot loops stop - // scheduling finalizer sweeps. Every other opcode keeps the - // conservative mark. - // RFC 0077 (WS2) widened the audited set to the opcodes the - // deltablue census showed driving ~95% of the coarse marks: - // `CALL` (operands graded by `reap_call_receiver`/`reap_call_ - // args`, moved arguments by the callee's frame-exit sweep, the - // raise path re-marks), `RETURN_VALUE` (the frame-exit sweep - // grades every slot), `STORE_ATTR` (receiver + displaced value - // on the instance hits; slot/generic paths re-mark), the - // `POP_JUMP_IF_*` family and `IS_OP` (operands noted in the - // handlers). - // One table byte per opcode (RFC 0077 WS3): bit 0 marks the - // audited (exempt) set above, bit 1 the stack-neutral - // `DELETE_*` opcodes that count as a drop regardless. - let drop_class = match op_before { - Some(op) => COARSE_DROP_CLASS[op as u8 as usize], - None => 0, - }; - if watch_drops - && drop_class & COARSE_EXEMPT == 0 - && (frame.stack.len() < stack_before || drop_class & COARSE_DELETE != 0) - { - if crate::specialize::stats_enabled() { - gc_trace::floor_stats::coarse_mark(op_before.map_or("?", OpCode::name)); - } - gc_trace::mark_maybe_dead(); - } match stepped { Ok(StepOutcome::Continue) => { // PEP 669 BRANCH/JUMP: fired after a control-flow @@ -6985,35 +5044,7 @@ impl Interpreter { Ok(StepOutcome::StartGenerator) => break Ok(FrameOutcome::StartGenerator), Err(err) => { if let RuntimeError::PyException(mut exc) = err { - // CPython's `_PyErr_SetObject` chains *every* - // fresh exception — including ones raised from C - // (here: Rust opcodes and builtins) — to the - // currently handled exception. `RAISE_VARARGS` - // already did this at the raise site - // (`context_settled`); a fresh Rust-raised error - // has empty traceback and unset context/cause. - if !exc.context_settled - && exc.context.is_none() - && exc.cause.is_none() - && exc.traceback.is_empty() - && !exc.suppress_tb_once - && !instance_has_nonnull_attr(&exc.instance, "__context__") - { - self.attach_implicit_context(&mut exc); - if exc.context.is_some() { - Self::sync_exc_attrs(&exc); - } - } else if exc.traceback.is_empty() - && (exc.cause.is_some() || exc.context.is_some()) - { - // A fresh Rust-raised error that already carries an - // explicit `cause`/`context` (e.g. `_io` chaining a - // misbehaving raw `readinto`'s `TypeError` into an - // `OSError`, `test_bad_readinto_type`) must still - // mirror those onto the instance dict so Python's - // `except` sees `e.__cause__`. - Self::sync_exc_attrs(&exc); - } + self.chain_fresh_exception(&mut exc); // CPython's RERAISE fires the separate RERAISE // monitoring event, which `sys.settrace` does // *not* subscribe to — only fresh raises @@ -7108,15 +5139,11 @@ impl Interpreter { break 'activation Err(e); } } - // The frame's locals and operands drop - // wholesale below; see `mark_bulk_drop`. - gc_trace::mark_bulk_drop(); break Err(e); } } - } else { - break Err(err); } + break Err(err); } } }; @@ -7388,6 +5415,7 @@ impl Interpreter { lasti: std::sync::atomic::AtomicU32::new(frame.pc), has_materialized: std::sync::atomic::AtomicBool::new(materialized.is_some()), materialized: RefCell::new(materialized), + tb_refs: std::sync::atomic::AtomicU32::new(0), }); self.frame_stack.borrow_mut().push(shell.clone()); shell @@ -7559,6 +5587,7 @@ impl Interpreter { on_stack: Cell::new(0), extra_locals: RefCell::new(None), cleared: Cell::new(false), + lazy_back: RefCell::new(None), }) } @@ -7731,27 +5760,15 @@ impl Interpreter { } // Tear down the never-started generator: its frame // locals (bound arguments) die now, like CPython's - // refcount-driven dealloc. Drop the snapshot's own - // strong clones (mirror, materialised dict) *first* - // so the reap sees the true remaining refcount. - let caps = frame_reapables(&g); + // refcount-driven dealloc. *g.state.borrow_mut() = GeneratorState::Finished; Self::wipe_frame_locals(py); - for cap in caps { - self.prompt_reap_dropped(cap); - } return Ok(Object::None); } _ => {} } } Self::wipe_frame_locals(py); - // The wiped locals were pinned by a traceback until now - // (unittest's `assertRaises` clears them on exit); any of them - // may have been the last holder of a finalizable the hot/cold - // gate already demoted. Re-grade the whole index at the next - // safe point (see `gc_trace::mark_bulk_drop`). - gc_trace::mark_bulk_drop(); Ok(Object::None) } @@ -7926,7 +5943,80 @@ impl Interpreter { } } + /// A quiet loop leaves its shell's `lasti` behind the instruction it + /// stopped on. A lazy traceback entry materializes from the shell + /// once the activation leaves the spine (see `pop_frame_shell`), so + /// bring it current for the frame object's final `f_lineno`. + /// A `RERAISE` leaves the restored raise site there itself (PEP 626). + #[inline] + fn sync_lazy_tb_lasti( + shell: Option<&crate::object::FrameShell>, + code: &CodeObject, + exit: &QuietExit, + ) { + if let (Some(shell), QuietExit::Outcome { cur_pc, stepped }) = (shell, exit) { + if shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) != 0 + && !(stepped.is_err() + && code + .instructions + .get(*cur_pc) + .is_some_and(|i| i.op == OpCode::Reraise)) + { + shell + .lasti + .store(*cur_pc as u32, std::sync::atomic::Ordering::Relaxed); + } + } + } + + /// Materialize the departing top activation for the lazy references + /// that outlive it (see `pop_frame_shell`). Its `f_back` is its + /// caller's frame object when one exists, else a lazy link to the + /// caller's shell (CPython's `take_ownership` links `f_back` at exit + /// too, but its frame objects cost nothing to keep current; here a + /// materialized caller would leave its quiet loop). + #[cold] + #[inline(never)] + fn materialize_departing_top(&self) { + let (top, caller) = { + let stack = self.frame_stack.borrow(); + let n = stack.len(); + ( + stack[n - 1].clone(), + n.checked_sub(2).map(|i| stack[i].clone()), + ) + }; + let py = top.materialize_departing(None); + let Some(caller) = caller else { return }; + let existing = caller.materialized.borrow().clone(); + match existing { + Some(back) => { + // A new holder of the caller's frame object: its loop + // re-derives whether the object is observed. + caller.refresh_materialized(&back); + py.set_back(Some(back)); + crate::hot_gates::bump_loop_gen(); + } + None => { + *py.lazy_back.borrow_mut() = Some(crate::object::LazyShellRef::new(caller)); + } + } + } + fn pop_frame_shell(&self) -> Option> { + // A lazy reference to this activation (a traceback entry, see + // `PyTraceback::new_lazy`, or a callee frame's `f_back`) still + // lives: build the frame object CPython would have by now, while + // the shell is still whole. + let lazy_top = self.frame_stack.borrow().last().is_some_and(|top| { + top.tb_refs.load(std::sync::atomic::Ordering::Relaxed) != 0 + && !top + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) + }); + if lazy_top { + self.materialize_departing_top(); + } let popped = self.frame_stack.borrow_mut().pop(); if let Some(shell) = &popped { // Borrowed slots (see `FrameSlot`) read through the frame, @@ -7953,7 +6043,7 @@ impl Interpreter { // generator-family frames are re-entered and observed // while suspended. if shell.is_gen { - *py.back.borrow_mut() = None; + py.set_back(None); } } } @@ -8483,13 +6573,14 @@ impl Interpreter { _ => None, }; let tb_arg = if synthesize_head { - Object::Traceback(Rc::new(PyTraceback { - lineno: py_frame.last_line.get().unwrap_or(1), - lasti: py_frame.lasti.get(), - frame: py_frame.clone(), - raw_lasti: None, - next: RefCell::new(inherited_tb.clone()), - })) + let tb = PyTraceback::new( + py_frame.clone(), + py_frame.last_line.get().unwrap_or(1), + py_frame.lasti.get(), + None, + ); + *tb.next.borrow_mut() = inherited_tb.clone(); + Object::Traceback(Rc::new(tb)) } else { inherited_tb.clone().map_or(Object::None, Object::Traceback) }; @@ -8881,8 +6972,6 @@ impl Interpreter { // consume the thread-local flag before the operands // actually drop — and before the `?`, so the error path's // unwind drop is covered too. - gc_trace::note_dropped(&a); - gc_trace::note_dropped(&b); let r = r_res?; // tracemalloc: a binary op that built a fresh container / // string / bytes (`b'x' * n`, `s1 + s2`, `list * 3`) is an @@ -8918,15 +7007,7 @@ impl Interpreter { /// keeps bound containers off the cascade. #[inline] fn retire_operand(&mut self, o: Object) { - match &o { - Object::Instance(_) => self.prompt_reap_dropped(o), - Object::List(_) | Object::Dict(_) | Object::Set(_) => { - if Self::looks_reapable_temporary(&o) { - self.prompt_reap_dropped(o); - } - } - _ => {} - } + self.release(o); } /// The `COMPARE_OP` handler body, split out of `step`. @@ -8953,8 +7034,6 @@ impl Interpreter { let r_res = self.rich_compare_obj(&a, &b, kind, &frame.globals); // RFC 0059 WS1b: same operand accounting (and ordering) // as BINARY_OP above. - gc_trace::note_dropped(&a); - gc_trace::note_dropped(&b); let r = r_res?; let r = if to_bool && !matches!(r, Object::Bool(_)) { let truthy = match &r { @@ -9061,37 +7140,9 @@ impl Interpreter { #[inline] fn pop_top_step(&mut self, frame: &mut Frame) -> Result<(), RuntimeError> { let v = frame.pop()?; - // RFC 0059 WS1b: POP_TOP is exempt from the eval loop's - // coarse stack-shrink mark; note the discarded value here - // (no-op for pure leaves — e.g. a discarded expression - // statement's int/str result). - gc_trace::note_dropped(&v); - // Discarding the last reference to a temporary mirrors - // CPython's refcount-driven finalization (`f()` as a - // statement finalizes the result immediately). This covers - // not just directly-finalizable values (instances with - // `__del__`, live generators/coroutines) but *containers* - // that anchor a dead finalizable acyclic subgraph: a - // discarded `pool.map(...)` returns a `list` of unpickled, - // GC-tracked result copies, and if it isn't reaped here the - // copies linger as dead-but-tracked garbage until *some* - // thread's next collection finalizes them — which, across the - // statement's trailing `del`/`sleep`, is frequently a - // background pool thread, so the copies' `__del__` then races - // the main thread's (CPython runs every one on the thread that - // dropped the ref). The `local_needs_prompt_reap` kinds match - // `reap_frame_locals_on_exit`; `prompt_reap_dropped`'s refcount - // guard leaves anything still referenced elsewhere alone - // (RFC 0040: `test_multiprocessing_*` `test_release_task_refs`). - // No `sync_py_locals` is needed: a discarded statement result - // is an operand-stack temporary that was never a frame local, - // so the live-locals mirror cannot hold a stale clone of it, - // and `frame.locals` always reflects live bindings — the - // refcount guard therefore cannot false-positive on a value - // still bound to a local. - if Self::local_needs_prompt_reap(&v) { - self.prompt_reap_dropped(v); - } + // A discarded temporary dies here, finalizers and all (`f()` as a + // statement finalizes its result at once, as in CPython). + self.release(v); Ok(()) } @@ -9143,9 +7194,7 @@ impl Interpreter { } } }; - if let Some(stale) = stale { - gc_trace::note_dropped(&stale); - } + drop(stale); return Ok(()); } } @@ -9175,7 +7224,6 @@ impl Interpreter { // loop's coarse stack-shrink mark, so the displaced value // must be noted here (no-op for pure-leaf rebinds — loop // counters, accumulators, first bindings over `Unbound`). - gc_trace::note_dropped(&old); // CPython decrefs the value previously bound to the local; // when that was the last reference to a finalizable object // its `__del__` runs at the rebind, not at frame exit @@ -9192,10 +7240,7 @@ impl Interpreter { // held elsewhere" rebind (`for t in tasks:` displacing // the previous element) on one strong-count load before // the cascade's registry probes (RFC 0077 WS2). - if Self::local_needs_prompt_reap(&old) && Self::looks_reapable_temporary(&old) { - self.sync_py_locals(frame); - self.prompt_reap_dropped(old); - } + self.release(old); } Ok(()) } @@ -9216,15 +7261,13 @@ impl Interpreter { let truthy = match &v { Object::Instance(_) | Object::MappingProxyObj(_) => { let g = frame.globals.clone(); - self.obj_truthy(&v, &g) - .inspect_err(|_| gc_trace::mark_maybe_dead())? + self.obj_truthy(&v, &g)? } _ => v.is_truthy(), }; // RFC 0077 (WS2): the POP_JUMP family audits its one // discarded operand (exempt from the coarse stack-shrink // mark); the usual `bool` is a leaf and schedules nothing. - gc_trace::note_dropped(&v); if truthy == when { frame.pc += arg; } @@ -9295,8 +7338,6 @@ impl Interpreter { frame: &mut Frame, mut shell: QuietShell<'_>, snap_gen: u64, - fin: bool, - coarse: bool, prev_pc: &mut Option, ) -> QuietExit { // SAFETY (for the reads in `quiet_frame`): the flag is this @@ -9315,7 +7356,6 @@ impl Interpreter { root: frame, root_shell: (&raw mut shell).cast(), root_last: &raw mut last, - fin, maybe_dead, }; let mut entry = QuietEntry::Fresh; @@ -9330,7 +7370,6 @@ impl Interpreter { &mut *roots.root, &mut *roots.root_shell.cast::>(), snap_gen, - coarse, prev_pc, &mut *roots.root_last, entry, @@ -9354,7 +7393,6 @@ impl Interpreter { top_frame, &mut top_shell, snap_gen, - coarse, &mut top_prev, &mut top_last, entry, @@ -9407,7 +7445,7 @@ impl Interpreter { } else { let result = self.inline_finish(&mut done, exit); match inl.last_mut() { - None => self.inline_deliver(frame, shell, done, result), + None => self.inline_deliver(frame, done, result), Some(caller) => { let caller = &mut **caller; // SAFETY: as above. @@ -9416,8 +7454,7 @@ impl Interpreter { } else { &mut caller.frame }; - let mut caller_shell = QuietShell::Lazy(&mut caller.act); - self.inline_deliver(caller_frame, &mut caller_shell, done, result) + self.inline_deliver(caller_frame, done, result) } } } @@ -9426,6 +7463,20 @@ impl Interpreter { } } + /// After the quiet loop caught an exception: whether the activation + /// yields as the outer loop's per-instruction prologue would (the GIL + /// countdown, a hot gate). Code that only calls and catches, such as + /// recursion retried in its `RecursionError` handler, would otherwise + /// never let another thread or a signal in. + #[inline] + fn quiet_caught_yields(&mut self, snap_gen: u64) -> bool { + if self.gil_countdown <= 2 { + return true; + } + self.gil_countdown -= 1; + crate::hot_gates::loop_gen() != snap_gen + } + /// One activation's stretch of the quiet loop (see /// [`Self::quiet_run`]): runs `frame` from `entry` until it makes an /// inline call or exits. @@ -9436,13 +7487,12 @@ impl Interpreter { frame: &mut Frame, shell: &mut QuietShell<'_>, snap_gen: u64, - coarse: bool, prev_pc: &mut Option, last: &mut usize, entry: QuietEntry, roots: &SwitchRoots, ) -> FrameEv { - let (fin, maybe_dead) = (roots.fin, roots.maybe_dead); + let maybe_dead = roots.maybe_dead; // A burst's stop handed over from the activation it moved away // from (see `FrameEv::Switched`). let mut pending: Option = None; @@ -9462,26 +7512,33 @@ impl Interpreter { break 'run QuietExit::Yield; } // SAFETY: see `quiet_run`. - if fin && unsafe { (*maybe_dead).get() } { + if unsafe { (*maybe_dead).get() } { self.flush_lean(frame, shell); if self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen { break 'run QuietExit::Yield; } } } - QuietEntry::Raised { err, cur_pc } => { - // The raise path drops the staged operands wholesale - // (see the `CALL` arm of `step`). - gc_trace::mark_maybe_dead(); - self.flush_lean(frame, shell); - if *last != usize::MAX { - *prev_pc = Some(*last); + QuietEntry::Raised { err, cur_pc } => match self.quiet_catch(frame, shell, err) { + Ok(()) => { + *last = cur_pc; + if self.quiet_caught_yields(snap_gen) { + break 'run QuietExit::Yield; + } } - return FrameEv::Exit(QuietExit::Outcome { - stepped: Err(err), - cur_pc, - }); - } + Err(err) => { + // The raise path drops the staged operands wholesale + // (see the `CALL` arm of `step`). + self.flush_lean(frame, shell); + if *last != usize::MAX { + *prev_pc = Some(*last); + } + return FrameEv::Exit(QuietExit::Outcome { + stepped: Err(err), + cur_pc, + }); + } + }, } loop { let stop = match pending.take() { @@ -9498,10 +7555,19 @@ impl Interpreter { match stop { LeafStop::Breaker => break 'run QuietExit::Yield, LeafStop::Raised(err) => { + let cur_pc = frame.pc as usize - 1; + let err = match self.quiet_catch(frame, shell, err) { + Ok(()) => { + *last = cur_pc; + if self.quiet_caught_yields(snap_gen) { + break 'run QuietExit::Yield; + } + continue; + } + Err(err) => err, + }; // The `CALL` arm of `step` marks on the raise path. - gc_trace::mark_maybe_dead(); self.flush_lean(frame, shell); - let cur_pc = frame.pc as usize - 1; if *last != usize::MAX { *prev_pc = Some(*last); } @@ -9511,12 +7577,10 @@ impl Interpreter { }); } LeafStop::Marked => { - if fin { - // Finalizers are arbitrary code: the spine must be - // complete before they run. - self.flush_lean(frame, shell); - self.drain_if_maybe_dead(); - } + // Finalizers are arbitrary code: the spine must be + // complete before they run. + self.flush_lean(frame, shell); + self.drain_if_maybe_dead(); // The drop may also have queued work of its own (an // unclosed file's ResourceWarning), which CPython // delivers at the instruction that dropped it. @@ -9582,9 +7646,18 @@ impl Interpreter { }; if let Some(called) = called { if let Err(err) = called { + let err = match self.quiet_catch(frame, shell, err) { + Ok(()) => { + *last = cur_pc; + if self.quiet_caught_yields(snap_gen) { + break 'run QuietExit::Yield; + } + continue; + } + Err(err) => err, + }; // The raise path drops the staged operands // wholesale (see the `CALL` arm of `step`). - gc_trace::mark_maybe_dead(); self.flush_lean(frame, shell); if *last != usize::MAX { *prev_pc = Some(*last); @@ -9603,7 +7676,7 @@ impl Interpreter { break 'run QuietExit::Yield; } // SAFETY: see `quiet_run`. - if fin && unsafe { (*maybe_dead).get() } { + if unsafe { (*maybe_dead).get() } { self.flush_lean(frame, shell); if self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen @@ -9630,8 +7703,14 @@ impl Interpreter { return FrameEv::Call(act); } } - // `inst.prop` for a plain `property`: its getter runs as a - // lean activation, like a lean call; `for x in gen` resumes + // `inst.prop` for a plain `property`: its getter runs as an + // inline activation, like an inline call. + if op_before == Some(OpCode::LoadAttr) && self.inline_calls_ok() { + if let Some(act) = self.try_inline_property(frame, shell, cur_pc) { + return FrameEv::Call(act); + } + } + // Otherwise as a lean activation; `for x in gen` resumes // the generator as one. if matches!(op_before, Some(OpCode::LoadAttr | OpCode::ForIter)) { let loaded = if op_before == Some(OpCode::LoadAttr) { @@ -9641,7 +7720,16 @@ impl Interpreter { }; if let Some(loaded) = loaded { if let Err(err) = loaded { - gc_trace::mark_maybe_dead(); + let err = match self.quiet_catch(frame, shell, err) { + Ok(()) => { + *last = cur_pc; + if self.quiet_caught_yields(snap_gen) { + break 'run QuietExit::Yield; + } + continue; + } + Err(err) => err, + }; self.flush_lean(frame, shell); if *last != usize::MAX { *prev_pc = Some(*last); @@ -9660,7 +7748,7 @@ impl Interpreter { break 'run QuietExit::Yield; } // SAFETY: see `quiet_run`. - if fin && unsafe { (*maybe_dead).get() } { + if unsafe { (*maybe_dead).get() } { self.flush_lean(frame, shell); if self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen @@ -9689,22 +7777,16 @@ impl Interpreter { } } let live_shell = self.flush_lean(frame, shell); - let stack_before = frame.stack.len(); live_shell .lasti .store(frame.pc, std::sync::atomic::Ordering::Relaxed); - let stepped = self.step_hot(frame); - if coarse { - let drop_class = match op_before { - Some(op) => COARSE_DROP_CLASS[op as u8 as usize], - None => 0, - }; - if drop_class & COARSE_EXEMPT == 0 - && (frame.stack.len() < stack_before || drop_class & COARSE_DELETE != 0) - { - gc_trace::mark_maybe_dead(); - } - } + let stepped = match self.step_hot(frame) { + Err(err) => match self.quiet_catch(frame, shell, err) { + Ok(()) => Ok(StepOutcome::Continue), + Err(err) => Err(err), + }, + stepped => stepped, + }; if !matches!(stepped, Ok(StepOutcome::Continue)) { return FrameEv::Exit(QuietExit::Outcome { stepped, cur_pc }); } @@ -9718,8 +7800,7 @@ impl Interpreter { break 'run QuietExit::Yield; } // SAFETY: see `quiet_run`. - if fin - && unsafe { (*maybe_dead).get() } + if unsafe { (*maybe_dead).get() } && self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen { @@ -9770,6 +7851,12 @@ impl Interpreter { let Object::Function(f) = &callable else { unreachable!("inline callees are plain functions") }; + if !code.cellvars.is_empty() { + // SAFETY: a bound slot's locals storage is its own. + let cells = fresh_cells(f, &code, unsafe { &mut *act.frame.locals.as_ptr() }); + return self + .inline_bind_with(frame, shell, pc, act, code, callable, guard, cells, true); + } let cells: *const Rc>>> = f.lean_cells_ref(&code).expect("checked by the shape"); self.inline_bind_cells(frame, shell, pc, act, code, callable, guard, cells) @@ -9784,31 +7871,53 @@ impl Interpreter { frame: &mut Frame, shell: &mut QuietShell<'_>, pc: usize, - mut act: Box, + act: Box, code: Rc, callable: Object, guard: crate::recursion::Guard, cells: *const Rc>>>, + ) -> Box { + // SAFETY: the caller's contract — `callable` (moved into the + // slot below) or the process keeps it alive. + let cells = borrowed_rc(unsafe { &*cells }); + self.inline_bind_with(frame, shell, pc, act, code, callable, guard, cells, false) + } + + /// [`Self::inline_bind_cells`] with the frame's cells handle built: + /// borrowed (`owned` false: the callee's lean cells, see + /// [`borrowed_rc`]) or the activation's own fresh cells, which the + /// slot releases when it parks. + #[inline] + #[allow(clippy::too_many_arguments)] + fn inline_bind_with( + &mut self, + frame: &mut Frame, + shell: &mut QuietShell<'_>, + pc: usize, + mut act: Box, + code: Rc, + callable: Object, + guard: crate::recursion::Guard, + cells: Rc>>>, + owned: bool, ) -> Box { let Object::Function(f) = &callable else { unreachable!("inline callees are plain functions") }; - // SAFETY: the caller's contract — `callable` (moved into the - // slot below) or the process keeps it alive. - let cells = unsafe { &*cells }; // SAFETY: the slot is parked: its code, cells, globals and // builtins fields hold stale non-owning copies, overwritten here // without a drop. The new handles are `f`'s (borrowed: the slot - // holds `callable`, so `f` outlives the activation) and the code - // (owned). + // holds `callable`, so `f` outlives the activation), the cells + // as given, and the code (owned). unsafe { let fr: &mut Frame = &mut act.frame; std::ptr::write(&raw mut fr.code, code); - std::ptr::write(&raw mut fr.cells, borrowed_rc(cells)); + std::ptr::write(&raw mut fr.cells, cells); std::ptr::write(&raw mut fr.globals, borrowed_rc(&f.globals)); std::ptr::write(&raw mut fr.builtins, borrowed_rc(&f.builtins)); std::ptr::write(&raw mut act.callable, callable); } + act.owns_cells = owned; act.frame.pc = 0; act.parked = false; act.call_pc = pc; @@ -9895,8 +8004,14 @@ impl Interpreter { } // SAFETY: the code handle is owned while active; release it and // leave the stale copy (never read again until `inline_bind` - // overwrites it; `InlineAct::drop` restores an owned one). + // overwrites it; `InlineAct::drop` restores an owned one). Owned + // cells are released the same way. unsafe { drop(std::ptr::read(&raw const fr.code)) }; + if act.owns_cells { + act.owns_cells = false; + // SAFETY: as for the code handle. + unsafe { drop(std::ptr::read(&raw const act.frame.cells)) }; + } act.parked = true; act.clean = false; act.act.shell = None; @@ -9904,7 +8019,29 @@ impl Interpreter { act.caller_pending = None; act.init_inst = None; if self.inline_pool.len() < INLINE_POOL_CAP { - self.inline_pool.push(act); + push_fast(&mut self.inline_pool, act); + } + } + + /// [`Self::inline_park`] for a clean slot whose operands and locals + /// the caller already released (the in-loop return's common case): + /// only the code handle and the slot's own fields remain. + #[inline(always)] + fn inline_park_clean(&mut self, mut act: Box) { + debug_assert!(act.clean && act.frame.stack.is_empty()); + // SAFETY: as `inline_park`: the code handle is owned while active. + unsafe { drop(std::ptr::read(&raw const act.frame.code)) }; + if act.owns_cells { + act.owns_cells = false; + // SAFETY: as `inline_park`. + unsafe { drop(std::ptr::read(&raw const act.frame.cells)) }; + } + act.parked = true; + act.clean = false; + act.guard = None; + act.caller_pending = None; + if self.inline_pool.len() < INLINE_POOL_CAP { + push_fast(&mut self.inline_pool, act); } } @@ -10078,7 +8215,6 @@ impl Interpreter { Ok(FrameOutcome::Returned(v)) => { *gen.state.borrow_mut() = GeneratorState::Finished; let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(&gen); Ok(GenStep::Exhausted(v)) @@ -10093,7 +8229,6 @@ impl Interpreter { *gen.state.borrow_mut() = GeneratorState::Finished; let escaped = self.pep479_escape(&gen, err); let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(&gen); Err(escaped) @@ -10139,7 +8274,6 @@ impl Interpreter { // The exhausted generator leaves the stack (a full handler // takes the coarse mark for it). drop(it); - gc_trace::mark_maybe_dead(); QuietEntry::Returned { cur_pc: call_pc } } Err(err) => QuietEntry::Raised { @@ -10159,6 +8293,7 @@ impl Interpreter { ) -> Result { let exc_depth = done.exc_depth; let frame: &mut Frame = &mut done.frame; + Self::sync_lazy_tb_lasti(done.act.shell.as_deref(), &frame.code, &exit); let outcome = match (exit, done.act.shell.take()) { // The whole activation ran without ever needing its shell. ( @@ -10169,31 +8304,17 @@ impl Interpreter { None, ) => { // Only leaf instructions and inline calls ran here, so the - // frame's other fields are untouched. Scalar-only locals - // (and no leftover operands or cells) owe the exit reap - // nothing: they die here with no drop glue. + // frame's other fields are untouched: with no leftover + // operands or cells, the locals are released here. done.clean = true; if frame.stack.is_empty() - && frame.code.cellvars.is_empty() + && (frame.code.cellvars.is_empty() || done.owns_cells) && Rc::strong_count(&frame.locals) == 1 { // SAFETY: sole owner; nothing else reaches the vector. - let locals = unsafe { &mut *frame.locals.as_ptr() }; - if locals.iter().all(|o| { - matches!( - o, - Object::Int(_) - | Object::Float(_) - | Object::Bool(_) - | Object::None - | Object::Unbound - ) - }) { - // SAFETY: scalars own nothing (no drop skipped). - unsafe { locals.set_len(0) }; - drop(done.guard.take()); - return Ok(v); - } + unsafe { (*frame.locals.as_ptr()).clear() }; + drop(done.guard.take()); + return Ok(v); } Ok(FrameOutcome::Returned(v)) } @@ -10210,9 +8331,26 @@ impl Interpreter { && self.exc_info_len() <= exc_depth => { self.pop_frame_shell(); + // A lazy traceback entry still holding the shell had its + // frame object built by the pop. + Self::settle_exited_frame_object(&shell, frame, true); self.recycle_frame_shell(shell); Ok(FrameOutcome::Returned(v)) } + // An exception leaving an unobserved activation with no handler + // for it: the general loop's unwind, without entering it. + ( + QuietExit::Outcome { + stepped: Err(RuntimeError::PyException(exc)), + .. + }, + Some(shell), + ) if !crate::trace::any_observers_active() + && find_handler(&frame.code.exception_table, frame.pc.wrapping_sub(1)) + .is_none() => + { + Err(self.quiet_unwind(frame, shell, exc, exc_depth)) + } // Everything else finishes in the general loop, which takes // over exactly where the quiet loop stopped. (exit, shell) => { @@ -10235,10 +8373,7 @@ impl Interpreter { } }; let result = match outcome { - Ok(FrameOutcome::Returned(v)) => { - self.reap_frame_locals_on_exit(frame, Some(&v)); - Ok(v) - } + Ok(FrameOutcome::Returned(v)) => Ok(v), Ok(FrameOutcome::Yielded(_)) => Err(RuntimeError::Internal( "lean frame yielded to a non-generator caller".to_owned(), )), @@ -10257,7 +8392,6 @@ impl Interpreter { fn inline_deliver( &mut self, frame: &mut Frame, - shell: &mut QuietShell<'_>, done: Box, result: Result, ) -> QuietEntry { @@ -10284,7 +8418,7 @@ impl Interpreter { match result { Ok(v) => { frame.stack.push(v); - self.drop_lean_callable(frame, shell, callable); + self.release(callable); QuietEntry::Returned { cur_pc: call_pc } } Err(err) => { @@ -10297,49 +8431,6 @@ impl Interpreter { } } - /// Release a lean call's callable after a normal return - /// (`reap_call_receiver`'s function arm): a function that was only a - /// call temporary goes through the cascade (which can run arbitrary - /// code); a bound one is just graded. The usual callee — a function - /// its namespace still holds (our clone, that binding, its collector - /// handle: three or more) with no weakref — needs neither. - #[inline(always)] - fn drop_lean_callable( - &mut self, - frame: &mut Frame, - shell: &mut QuietShell<'_>, - callable: Object, - ) { - let alive_elsewhere = match &callable { - Object::Function(f) => { - Rc::strong_count(f) > 2 - && !crate::weakref_registry::may_have_weakrefs(Rc::as_ptr(f) as usize as u64) - } - _ => false, - }; - if alive_elsewhere { - drop_hot(callable); - } else { - self.drop_lean_callable_slow(frame, shell, callable); - } - } - - /// [`Self::drop_lean_callable`] for a callable that may die here. - #[inline(never)] - fn drop_lean_callable_slow( - &mut self, - frame: &mut Frame, - shell: &mut QuietShell<'_>, - callable: Object, - ) { - if Self::looks_reapable_temporary(&callable) { - self.flush_lean(frame, shell); - self.prompt_reap_dropped(callable); - } else { - gc_trace::note_dropped(&callable); - } - } - /// Push the frame shells of every pending lean activation, outermost /// first, then this activation's own, and return it (see [`LeanAct`]). /// A no-op for an activation that already has its shell. @@ -10473,6 +8564,7 @@ impl Interpreter { lasti: std::sync::atomic::AtomicU32::new(lasti), has_materialized: std::sync::atomic::AtomicBool::new(false), materialized: RefCell::new(None), + tb_refs: std::sync::atomic::AtomicU32::new(0), }); self.frame_stack.borrow_mut().push(shell.clone()); shell @@ -10562,7 +8654,7 @@ impl Interpreter { Some(match result { Ok(v) => { frame.stack.push(v); - self.drop_lean_callable(frame, shell, callable); + self.release(callable); Ok(()) } Err(e) => Err(e), @@ -10575,6 +8667,17 @@ impl Interpreter { /// code, and how many trailing parameters its compiled defaults fill. #[inline] fn lean_call_shape(f: &PyFunction, eff_argc: usize) -> Option<(Rc, usize)> { + Self::lean_call_shape_cells(f, eff_argc, false) + } + + /// [`Self::lean_call_shape`] for an inline activation, which with + /// `cells` may also own cell variables (see [`fresh_cells`]). + #[inline] + fn lean_call_shape_cells( + f: &PyFunction, + eff_argc: usize, + cells: bool, + ) -> Option<(Rc, usize)> { let code = f.code(); let total = code.arg_count as usize; if code.is_generator @@ -10593,7 +8696,13 @@ impl Interpreter { { return None; } - f.lean_cells_ref(&code)?; + if cells && !code.cellvars.is_empty() { + if !closure_cells_ok(f, &code) { + return None; + } + } else { + f.lean_cells_ref(&code)?; + } let missing = if eff_argc < total { if f.defaults.len() < total - eff_argc { return None; @@ -10624,6 +8733,14 @@ impl Interpreter { code.has_varargs || code.has_varkeywords || code.kwonly_count != 0 } + /// Whether `code` binds keyword-only parameters or `**kwargs` (the + /// extended parameters a positional call's cached shape can't bind; + /// `*args` it can). + #[inline(always)] + fn has_keyword_params(code: &CodeObject) -> bool { + code.has_varkeywords || code.kwonly_count != 0 + } + /// How many trailing positional defaults a call of `f` (running /// `code`) with `given` positional arguments, self included, takes /// from `f.defaults`. `None` when too many arguments come, too few @@ -10800,13 +8917,13 @@ impl Interpreter { _ => None, }; let (code, missing) = match &bound { - Some((f, _)) => Self::lean_call_shape(f, argc + 1)?, + Some((f, _)) => Self::lean_call_shape_cells(f, argc + 1, true)?, None => { let Object::Function(f) = &frame.stack[callee_slot] else { return None; }; let has_self = !matches!(frame.stack[self_slot], Object::Unbound); - Self::lean_call_shape(f, argc + usize::from(has_self))? + Self::lean_call_shape_cells(f, argc + usize::from(has_self), true)? } }; // Past the recursion limit the nested lean path raises. @@ -10817,11 +8934,6 @@ impl Interpreter { if let Some((f, receiver)) = bound { let bm = std::mem::replace(&mut frame.stack[callee_slot], Object::Function(f)); frame.stack[self_slot] = receiver; - // The bound method was a call temporary (or is still held - // elsewhere): grade its release like any dropped operand. - if gc_trace::note_dropped_marks(&bm) { - gc_trace::mark_maybe_dead(); - } drop(bm); } let has_self = !matches!(frame.stack[self_slot], Object::Unbound); @@ -11043,7 +9155,7 @@ impl Interpreter { Some(match result { Ok(v) => { frame.stack.push(v); - self.drop_lean_callable(frame, shell, callable); + self.release(callable); Ok(()) } Err(e) => Err(e), @@ -11262,7 +9374,7 @@ impl Interpreter { /// the callee's own references rather than at the next collection. fn release_call_operands(&mut self, stack: &mut Vec, callee_slot: usize) { let callee = std::mem::replace(&mut stack[callee_slot], Object::Unbound); - self.reap_call_receiver(callee); + self.release(callee); self.reap_call_args(&mut stack[callee_slot + 1..]); stack.truncate(callee_slot); } @@ -11271,13 +9383,69 @@ impl Interpreter { /// the call didn't use (its locals must be empty again). fn inline_unslot(&mut self, act: Box) { if self.inline_pool.len() < INLINE_POOL_CAP { - self.inline_pool.push(act); + push_fast(&mut self.inline_pool, act); + } + } + + /// The getter of the `property` that the `LOAD_ATTR` at `pc` of + /// `frame` loads off the plain instance at TOS, when it's a plain + /// one-argument Python function a lean activation can run. + fn lean_property_getter( + frame: &Frame, + pc: usize, + ) -> Option<( + Rc, + Rc, + Rc, + )> { + use weavepy_compiler::InlineCache as IC; + let name_idx = frame.code.instructions.get(pc)?.arg; + // A first execution still specializes the site. + if matches!(frame.code.caches.get(pc as u32), IC::Empty) { + return None; + } + let Some(Object::Instance(inst)) = frame.stack.last() else { + return None; + }; + // A getter's `AttributeError` falls back to `__getattr__`, which + // the inline return can't do. + if inst + .cls_raw() + .dunder(crate::types::Dunder::GetAttr) + .present() + { + return None; + } + let LeafAttr::Property(prop) = + Self::leaf_resolve_instance_attr(&frame.code, inst, name_idx)? + else { + return None; + }; + // SAFETY: GIL-serialized raw read of the getter cell (the burst is + // off in free-threaded mode); the clone is taken before anything + // else runs. + let Object::Function(f) = (unsafe { &*prop.fget.as_ptr() }) else { + return None; + }; + let code = f.code(); + if code.arg_count != 1 + || code.is_generator + || code.is_coroutine + || code.is_async_generator + || code.has_varargs + || code.has_varkeywords + || code.kwonly_count != 0 + || !Self::lean_code_ok(&code) + { + return None; } + f.lean_cells_ref(&code)?; + let f = f.clone(); + Some((prop, f, code)) } /// `LOAD_ATTR` of a `property` on a plain instance, from the quiet - /// loop: the getter — a plain one-argument Python function the lean - /// path can run — is called with the receiver, and its result + /// loop: the getter is called with the receiver, and its result /// replaces the receiver at TOS. `None` leaves the load to the full /// handler with nothing touched. fn try_lean_property( @@ -11287,54 +9455,66 @@ impl Interpreter { snap_gen: u64, pc: usize, ) -> Option> { - use weavepy_compiler::InlineCache as IC; - let name_idx = frame.code.instructions.get(pc)?.arg; - // A first execution still specializes the site. - if matches!(frame.code.caches.get(pc as u32), IC::Empty) { - return None; - } - let getter = { - let Some(Object::Instance(inst)) = frame.stack.last() else { - return None; - }; - let LeafAttr::Property(prop) = - Self::leaf_resolve_instance_attr(&frame.code, inst, name_idx)? - else { - return None; - }; - // SAFETY: GIL-serialized raw read of the getter cell (the - // burst is off in free-threaded mode); the clone is taken - // before anything else runs. - let Object::Function(f) = (unsafe { &*prop.fget.as_ptr() }) else { - return None; - }; - let f = f.clone(); - let code = f.code(); - if code.arg_count != 1 - || code.is_generator - || code.is_coroutine - || code.is_async_generator - || code.has_varargs - || code.has_varkeywords - || code.kwonly_count != 0 - || !Self::lean_code_ok(&code) - { - return None; - } - f.lean_cells_ref(&code)?; - (f, code) - }; - let (getter, code) = (getter.0, getter.1); + let (_, getter, code) = Self::lean_property_getter(frame, pc)?; let receiver = frame.stack.last()?.clone(); let result = self.run_lean_fn(frame, shell, snap_gen, pc, &getter, code, receiver); Some(result.map(|v| { if let Some(top) = frame.stack.last_mut() { - let old = std::mem::replace(top, v); - gc_trace::note_dropped(&old); + *top = v; } })) } + /// The getter the `LOAD_ATTR` site at `pc` of `frame` cached for the + /// class of the instance at TOS (see [`MethodSlot::getter`]). + #[inline] + fn cached_property_getter( + frame: &Frame, + pc: usize, + ) -> Option<(Rc, Rc)> { + let Some(Object::Instance(inst)) = frame.stack.last() else { + return None; + }; + let slot = code_vm_ext(&frame.code)?.method_slots.get()?.get(pc)?; + slot.getter(inst.cls_raw().attr_version.get()) + } + + /// [`Self::try_lean_property`] as an inline activation (CPython's + /// `LOAD_ATTR_PROPERTY`): the receiver leaves the stack as the + /// getter's argument, and the getter's return pushes its result. + fn try_inline_property( + &mut self, + frame: &mut Frame, + shell: &mut QuietShell<'_>, + pc: usize, + ) -> Option> { + let (getter, code) = match Self::cached_property_getter(frame, pc) { + Some(hit) => hit, + None => { + let (prop, getter, code) = Self::lean_property_getter(frame, pc)?; + if let (Some(Object::Instance(inst)), Some(slot)) = + (frame.stack.last(), code_method_slot(&frame.code, pc as u32)) + { + slot.set_getter(inst.cls_raw().attr_version.get(), &prop, &getter, &code); + } + (getter, code) + } + }; + // Past the recursion limit the nested lean path raises. + let crate::recursion::Enter::Ok(guard) = crate::recursion::enter() else { + return None; + }; + // Committed. + let receiver = frame.stack.pop()?; + let act = self.inline_slot(); + // SAFETY: a parked slot's locals storage is its own, and empty. + let locals = unsafe { &mut *act.frame.locals.as_ptr() }; + locals.push(receiver); + fill_unbound(locals, code.varnames.len()); + frame.pc = pc as u32 + 1; + Some(self.inline_bind(frame, shell, pc, act, code, Object::Function(getter), guard)) + } + /// `FOR_ITER` over a generator, from the quiet loop: the generator /// resumes as a lean activation (see `generator_send_lean`); its /// value is pushed, or on exhaustion the loop exits exactly as the @@ -11372,7 +9552,6 @@ impl Interpreter { // The exhausted generator leaves the stack (a full handler // takes the coarse mark for it). drop(it); - gc_trace::mark_maybe_dead(); Ok(()) } Err(e) => Err(e), @@ -11496,7 +9675,7 @@ impl Interpreter { /// `lasti` from the advanced `pc`); one that has its shell keeps the /// mirror current itself, as the general path's pre-step store does. /// Returns the pending entry for [`Self::lean_pending_exit`]. - #[inline] + #[inline(always)] fn lean_pending_enter( &mut self, frame: &mut Frame, @@ -11631,7 +9810,7 @@ impl Interpreter { /// warms that compile after a few entries): a native entry is then /// worth its framing — a recursive body calls its native self /// directly from there. - #[inline] + #[inline(always)] fn lean_code_ok(code: &CodeObject) -> bool { if code.wire.as_ref().is_some_and(|w| w.exec_error.is_some()) { return false; @@ -11754,21 +9933,13 @@ impl Interpreter { crate::tier2::warm_compile(self, frame); } let exc_depth_on_entry = self.exc_info_len(); - let fin_live = gc_trace::has_any_finalizable(); - let fin = fin_live || gc_trace::active_suspects_present(); let mut act = LeanAct { frame: std::ptr::from_mut(frame), shell: None, }; let mut prev_pc: Option = None; - let exit = self.quiet_run( - frame, - QuietShell::Lazy(&mut act), - snap_gen, - fin, - fin_live, - &mut prev_pc, - ); + let exit = self.quiet_run(frame, QuietShell::Lazy(&mut act), snap_gen, &mut prev_pc); + Self::sync_lazy_tb_lasti(act.shell.as_deref(), &frame.code, &exit); let outcome = match (exit, act.shell) { // The whole activation ran without ever needing its shell. ( @@ -11791,6 +9962,9 @@ impl Interpreter { && self.exc_info_len() <= exc_depth_on_entry => { self.pop_frame_shell(); + // A lazy traceback entry still holding the shell had its + // frame object built by the pop. + Self::settle_exited_frame_object(&shell, frame, true); self.recycle_frame_shell(shell); FrameOutcome::Returned(v) } @@ -11816,7 +9990,6 @@ impl Interpreter { match outcome { FrameOutcome::Returned(v) => { // The owner retires the frame (`retire_lean_frame`). - self.reap_frame_locals_on_exit(frame, Some(&v)); Ok(v) } FrameOutcome::Yielded(_) => Err(RuntimeError::Internal( @@ -11836,14 +10009,7 @@ impl Interpreter { if !gc_trace::take_maybe_dead() { return false; } - if gc_trace::has_any_finalizable() { - self.drain_prompt_finalizers(); - } - if gc_trace::has_suspects() { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } + self.run_pending_finalizers(); true } @@ -11876,13 +10042,30 @@ impl Interpreter { /// the full leaf arms and hands back at the next core instruction. #[inline(never)] fn leaf_burst(&mut self, sw: &mut CoreSwitch, snap_gen: u64) -> LeafStop { + // Which copy of the core loop runs: the native one drives the + // running code's native form (see `frame_jit`). + #[cfg(feature = "jit")] + let mut native = false; loop { - match self.leaf_core(sw, snap_gen) { + #[cfg(feature = "jit")] + let exit = if native { + self.leaf_core::(sw, snap_gen) + } else { + self.leaf_core::(sw, snap_gen) + }; + #[cfg(not(feature = "jit"))] + let exit = self.leaf_core::(sw, snap_gen); + match exit { + #[cfg(feature = "jit")] + CoreExit::Switch => { + native = !native; + continue; + } CoreExit::Stop(stop) => return stop, CoreExit::Helper => { // SAFETY: see `CoreSwitch` (the running activation's // handles; nothing else borrows them here). - let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) }; + let frame = unsafe { &mut *sw.cur }; if burst_stats::enabled() { if let Some(ins) = frame.code.instructions.get(frame.pc as usize) { burst_stats::note_helper(ins.op); @@ -11890,12 +10073,53 @@ impl Interpreter { burst_stats::note_site(&frame.code.qualname, frame.pc, ins.op, recv); } } + // A cached property site skips the helper's resolution: a + // pure leaf getter is evaluated in place, any other runs + // as an inline activation. + let pc = frame.pc as usize; + if frame + .code + .instructions + .get(pc) + .is_some_and(|i| i.op == OpCode::LoadAttr) + { + if let Some((getter, code)) = Self::cached_property_getter(frame, pc) { + // SAFETY: as above. + let last_pc = unsafe { &mut *sw.last }; + if self.core_leaf_property(frame, last_pc, pc, &getter, &code) { + // SAFETY: the running thread's own flag. + if unsafe { (*sw.maybe_dead).get() } { + return LeafStop::Marked; + } + continue; + } + if self.core_property(sw) { + continue; + } + } + } + // SAFETY: as above (`core_property` declined untouched). + let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) }; match self.leaf_core_helper(frame, last_pc) { + // A released receiver or argument may have queued + // a finalizer (see `after_release!` in the core). + // SAFETY: the running thread's own flag. + CoreAttr::Done if unsafe { (*sw.maybe_dead).get() } => { + return LeafStop::Marked; + } CoreAttr::Done => continue, - CoreAttr::Marked => return LeafStop::Marked, CoreAttr::Raised(e) => return LeafStop::Raised(e), CoreAttr::Full => return LeafStop::Step, - CoreAttr::Decline => {} + CoreAttr::Decline => { + let load_attr = frame + .code + .instructions + .get(frame.pc as usize) + .is_some_and(|i| i.op == OpCode::LoadAttr); + if load_attr && self.core_property(sw) { + continue; + } + } } } CoreExit::Slow => {} @@ -11904,6 +10128,8 @@ impl Interpreter { // SAFETY: as above. let (frame, last_pc) = unsafe { (&mut *sw.cur, &mut *sw.last) }; match self.leaf_burst_slow(frame, snap_gen, last_pc) { + // SAFETY: as above. + LeafStop::Core if unsafe { (*sw.maybe_dead).get() } => return LeafStop::Marked, LeafStop::Core => {} stop => return stop, } @@ -11963,10 +10189,11 @@ impl Interpreter { } let done = match ins.op { OpCode::Call => self.leaf_core_call(frame, ins, pc), + OpCode::LoadSpecial => Self::leaf_load_special(frame, ins.arg, pc), _ => self.leaf_core_attr(frame, ins, pc), }; match done { - CoreAttr::Done | CoreAttr::Marked => { + CoreAttr::Done => { *last_pc = pc; frame.pc = pc as u32 + 1; } @@ -11978,6 +10205,158 @@ impl Interpreter { done } + /// `LOAD_SPECIAL` at `pc` of an instance whose class resolves the + /// method to a plain function: the function and the instance as its + /// self (what the full handler's bound method unpacks to at the call), + /// remembered in the site's method slot for the core loop. Declines + /// any other owner or class attribute, touching nothing. + fn leaf_load_special(frame: &mut Frame, special: u32, pc: usize) -> CoreAttr { + use weavepy_compiler::bytecode::{SPECIAL_ENTER, SPECIAL_EXIT}; + let name = match special { + SPECIAL_ENTER => "__enter__", + SPECIAL_EXIT => "__exit__", + _ => return CoreAttr::Decline, + }; + let Some(Object::Instance(inst)) = frame.stack.last() else { + return CoreAttr::Decline; + }; + let cls = inst.cls_raw(); + let ver = cls.attr_version.get(); + let Some(Object::Function(f)) = cls.lookup(name) else { + return CoreAttr::Decline; + }; + if let Some(ms) = code_method_slot(&frame.code, pc as u32) { + ms.set(ver, &f); + } + let owner = frame.stack.pop().expect("checked above"); + frame.stack.push(Object::Function(f)); + frame.stack.push(owner); + CoreAttr::Done + } + + /// The core loop's `LOAD_FAST x` (at `pc`, of the heap local `other`) + /// followed by `x.m()`: a native method the site + /// cached for the receiver's class, a leaf method the site verified, + /// a pure leaf method, or a container's site-cached leaf method, + /// called on the borrowed operands. Returns the call's result and the + /// `CALL`'s pc; `None` touches nothing (the instructions run one by + /// one). + /// + /// # Safety + /// + /// `lbase` is the running activation's `nlocals` locals; nothing else + /// borrows them. + #[inline(always)] + #[allow(clippy::too_many_arguments)] + unsafe fn core_local_method( + &self, + code: &CodeObject, + ext: Option<&CodeConstObjects>, + other: &Object, + pc: usize, + lbase: *mut Object, + nlocals: usize, + consts: &[Object], + depth_cell: *const std::cell::Cell, + ) -> Option<(Result, usize)> { + let next = *code.instructions.get(pc + 1)?; + let mslots: &[MethodSlot] = ext + .and_then(|e| e.method_slots.get()) + .map_or(&[][..], |s| &s[..]); + match other { + Object::Instance(_) | Object::Module(_) => { + // A site that verified its callee for this class version + // (its call checks the argument shape itself). + let mut missed = false; + if let (Object::Instance(i), Some(ext)) = (other, ext) { + if let Some(site) = leaf_site_hit(ext, pc + 1, i.cls_raw().attr_version.get()) { + match self.core_leaf_site_call( + code, + i, + other, + site, + pc + 1, + next.arg, + lbase, + nlocals, + consts, + depth_cell, + ) { + SiteCall::Done(v, call_pc) => return Some((Ok(v), call_pc)), + SiteCall::Declined => {} + SiteCall::Missed => missed = true, + } + } + } + // `x.m(...)` of a pure leaf method with simple arguments: its + // result, evaluated on the borrowed operands. + if !simple_args_prefix(&code.instructions, pc + 2) { + return None; + } + // A native method the site cached for this class version: + // straight to its fused call (the Python leaf probes below + // would miss). + if let (Object::Instance(i), Some(ms)) = (other, mslots.get(pc + 1)) { + if ms + .peek_inst_builtin(i.cls_raw().attr_version.get()) + .is_some() + { + if let Some(hit) = self.core_native_method( + code, + other, + pc + 1, + next.arg, + mslots, + lbase, + nlocals, + consts, + ) { + return Some(hit); + } + } + } + let pure_site = !missed + && match (other, mslots.get(pc + 1)) { + (Object::Instance(i), Some(ms)) => ms + .peek_fn(i.cls_raw().attr_version.get()) + // SAFETY: the slot's function is alive (its class + // version matched). + .is_some_and(|fp| fn_is_leaf(unsafe { &*fp })), + _ => false, + }; + if pure_site { + let (v, call_pc) = self.core_pure_method( + code, + other, + pc + 1, + next.arg, + mslots, + lbase, + nlocals, + consts, + depth_cell, + )?; + return Some((Ok(v), call_pc)); + } + self.core_native_method( + code, + other, + pc + 1, + next.arg, + mslots, + lbase, + nlocals, + consts, + ) + } + // A container's site-cached leaf method. + Object::List(_) | Object::Dict(_) | Object::Set(_) | Object::Str(_) => { + self.core_builtin_method(code, other, pc + 1, mslots, lbase, nlocals, consts) + } + _ => None, + } + } + /// [`Self::leaf_fused_len`] for the `LOAD_GLOBAL` at `pc` of `frame`, /// when the four-instruction `len(x)` shape is there. #[inline] @@ -12020,7 +10399,7 @@ impl Interpreter { /// the frame first and reload everything after (see /// [`Self::core_call`], [`Self::core_return`]). #[inline(always)] - fn leaf_core(&mut self, sw: &mut CoreSwitch, snap_gen: u64) -> CoreExit { + fn leaf_core(&mut self, sw: &mut CoreSwitch, snap_gen: u64) -> CoreExit { #[inline(always)] fn scalar(v: &Object) -> bool { matches!( @@ -12089,7 +10468,84 @@ impl Interpreter { let mut len = stack.len(); let mut pc = frame.pc as usize; let mut last = *last_pc; + let pending_work = sw.maybe_dead; + // A dying object's `Drop` queued a finalizer or callback: the + // quiet loop runs it before the next instruction, as CPython + // does inside the releasing one's `Py_DECREF`. Checked after the + // arms that end a binding; any other release is noticed at the + // next back edge, call or return (queueing bumps the loop + // generation). + macro_rules! after_release { + () => { + // SAFETY: the running thread's own flag (see `quiet_run`). + #[allow(unused_unsafe)] + let queued = unsafe { (*pending_work).get() }; + if queued { + break Some(CoreExit::Stop(LeafStop::Marked)); + } + }; + } + // The code's native form (see `frame_jit`): the native copy of + // this loop runs it from every pc the arms below leave it at, + // and the arms only run the instructions it returns at. + #[cfg(feature = "jit")] + let native = ext.and_then(|e| e.frame_jit.get(nlocals)); + #[cfg(feature = "jit")] + if NATIVE != native.is_some() { + // (The activation is synced at every reload.) + return CoreExit::Switch; + } + #[cfg(feature = "jit")] + if pc == 0 && !NATIVE { + if let Some(ext) = ext { + ext.frame_jit + .warm(code, ext, nlocals, frame_jit::Heat::Call); + } + } + // The native code's view of the activation, built at its first + // entry (only the native copy of this loop enters it). + #[cfg(feature = "jit")] + let mut nst: Option = None; + // The pc the native code last returned at (the arms run it). + #[cfg(feature = "jit")] + let mut handed = usize::MAX; let stop = loop { + #[cfg(feature = "jit")] + if NATIVE && pc != handed { + if let Some(native) = native.filter(|n| n.enters_at(pc)) { + let nst = nst.get_or_insert_with(|| frame_jit::State { + locals: lbase, + stack: base, + len, + cap, + pc, + last, + countdown: std::ptr::addr_of_mut!(self.gil_countdown), + snap_gen, + maybe_dead: pending_work, + interp: std::ptr::from_ref(self), + out: 0, + depth_cell: sw.depth_cell, + err: None, + }); + nst.len = len; + nst.pc = pc; + nst.last = last; + // SAFETY: the running activation's state, as this + // loop holds it (its locals count checked above). + let status = unsafe { native.run(nst) }; + len = nst.len; + pc = nst.pc; + last = nst.last; + if status == frame_jit::MARKED { + break Some(CoreExit::Stop(LeafStop::Marked)); + } + if let Some(e) = nst.err.take() { + break Some(CoreExit::Stop(LeafStop::Raised(e))); + } + handed = pc; + } + } if pc >= ninstrs { break Some(CoreExit::Stop(LeafStop::Step)); } @@ -12246,148 +10702,42 @@ impl Interpreter { // off the local (no receiver clone pushed and // released); a miss loads the receiver and // leaves `LOAD_ATTR` to its own arm. - if matches!(other, Object::Instance(_) | Object::Module(_)) - && pc + 1 < ninstrs + // `x.m(...)` of a leaf or native method on the + // borrowed local (see `core_local_method`). + if pc + 1 < ninstrs + && len < cap + && (*instrs.add(pc + 1)).op == OpCode::LoadMethodAttr { - let next = *instrs.add(pc + 1); - // `x.m(...)` of a pure leaf method with - // simple arguments: its result, - // evaluated on the borrowed operands. - if next.op == OpCode::LoadMethodAttr - && len < cap - && simple_args_prefix(&code.instructions, pc + 2) - { - // A native method the site cached - // for this class version: straight - // to its fused call (the Python - // leaf probes below would miss). - if let (Object::Instance(i), Some(ms)) = - (other, mslots!(cold_mslots, ext).get(pc + 1)) - { - if ms - .peek_inst_builtin( - i.cls_raw().attr_version.get(), - ) - .is_some() - { - if let Some((r, call_pc)) = self - .core_native_method( - code, - other, - pc + 1, - next.arg, - mslots!(cold_mslots, ext), - lbase, - nlocals, - consts, - ) - { - last = call_pc; - pc = call_pc + 1; - match r { - Ok(v) => { - base.add(len).write(v); - len += 1; - continue; - } - Err(e) => { - break Some(CoreExit::Stop( - LeafStop::Raised(e), - )); - } - } - } - } - } - // A site that verified its callee - // for this class version. - let mut missed = false; - if let (Object::Instance(i), Some(ext)) = (other, ext) { - if let Some(site) = leaf_site_hit( - ext, - pc + 1, - i.cls_raw().attr_version.get(), - ) { - match self.core_leaf_site_call( - code, - i, - other, - site, - pc + 1, - next.arg, - lbase, - nlocals, - consts, - sw.depth_cell, - ) { - SiteCall::Done(v, call_pc) => { - base.add(len).write(v); - len += 1; - last = call_pc; - pc = call_pc + 1; - continue; - } - SiteCall::Declined => {} - SiteCall::Missed => missed = true, - } - } - } - let pure_site = !missed - && match ( - other, - mslots!(cold_mslots, ext).get(pc + 1), - ) { - (Object::Instance(i), Some(ms)) => ms - .peek_fn(i.cls_raw().attr_version.get()) - .is_some_and(|fp| fn_is_leaf(&*fp)), - _ => false, - }; - if pure_site { - if let Some((v, call_pc)) = self.core_pure_method( - code, - other, - pc + 1, - next.arg, - mslots!(cold_mslots, ext), - lbase, - nlocals, - consts, - sw.depth_cell, - ) { + if let Some((r, call_pc)) = self.core_local_method( + code, + ext, + other, + pc, + lbase, + nlocals, + consts, + sw.depth_cell, + ) { + last = call_pc; + pc = call_pc + 1; + match r { + Ok(v) => { base.add(len).write(v); len += 1; - last = call_pc; - pc = call_pc + 1; continue; } - } else if let Some((r, call_pc)) = self - .core_native_method( - code, - other, - pc + 1, - next.arg, - mslots!(cold_mslots, ext), - lbase, - nlocals, - consts, - ) - { - last = call_pc; - pc = call_pc + 1; - match r { - Ok(v) => { - base.add(len).write(v); - len += 1; - continue; - } - Err(e) => { - break Some(CoreExit::Stop( - LeafStop::Raised(e), - )); - } + Err(e) => { + break Some(CoreExit::Stop(LeafStop::Raised( + e, + ))); } } } + } + if matches!(other, Object::Instance(_) | Object::Module(_)) + && pc + 1 < ninstrs + { + let next = *instrs.add(pc + 1); if next.op == OpCode::LoadAttr { if code .instructions @@ -12450,43 +10800,6 @@ impl Interpreter { continue; } } - // `x.m(...)` of a container's site-cached - // leaf method on the borrowed local. - if matches!( - other, - Object::List(_) - | Object::Dict(_) - | Object::Set(_) - | Object::Str(_) - ) && pc + 1 < ninstrs - && len < cap - && (*instrs.add(pc + 1)).op == OpCode::LoadMethodAttr - { - if let Some((r, call_pc)) = self.core_builtin_method( - code, - other, - pc + 1, - mslots!(cold_mslots, ext), - lbase, - nlocals, - consts, - ) { - last = call_pc; - pc = call_pc + 1; - match r { - Ok(v) => { - base.add(len).write(v); - len += 1; - continue; - } - Err(e) => { - break Some(CoreExit::Stop(LeafStop::Raised( - e, - ))); - } - } - } - } clone_hot(other) } }; @@ -12553,76 +10866,20 @@ impl Interpreter { // Overwriting a scalar needs no drop. len -= 1; slot.write(base.add(len).read()); - } else if Self::core_shared_instance(&*slot) { - // A shared instance: the grading below would - // answer "escaped, no mark"; its release is a - // bare decrement. - len -= 1; - drop(std::mem::replace(&mut *slot, base.add(len).read())); } else { - // A displaced heap value the full arm would - // reap stays there — except a deferred-tracking - // instance, whose teardown is its plain drop - // (recycled when this was its last reference); - // any other is graded like every burst drop. - if Self::local_needs_prompt_reap(&*slot) - && Self::looks_reapable_temporary(&*slot) - { - match &*slot { - Object::Instance(i) if i.dies_by_plain_drop() => { - len -= 1; - if let Object::Instance(i) = - std::mem::replace(&mut *slot, base.add(len).read()) - { - PyInstance::try_recycle(i); - } - last = pc; - pc += 1; - continue; - } - // A tuple of scalars: the freelist is - // its teardown (see `prompt_reap_dropped`). - Object::Tuple(t) - if t.iter().all(Object::is_gc_atomic) - && !gc_trace::maybe_tracked( - crate::weakref_registry::id_of(&*slot), - ) => - { - len -= 1; - let old = - std::mem::replace(&mut *slot, base.add(len).read()); - self.maybe_donate_tuple(old); - last = pc; - pc += 1; - continue; - } - // An untracked container of scalars: - // a plain drop. - Object::List(_) | Object::Dict(_) - if Self::core_plain_last_container(&*slot) => - { - len -= 1; - drop(std::mem::replace( - &mut *slot, - base.add(len).read(), - )); - last = pc; - pc += 1; - continue; - } - _ => break None, - } + // The displaced value dies here if this was + // its last reference (the freelists take a + // dead plain instance's or tuple's + // allocation). + if !Self::core_droppable(&*slot) { + break None; } len -= 1; - let old = std::mem::replace(&mut *slot, base.add(len).read()); - let marked = gc_trace::note_dropped_marks(&old); - drop(old); - if marked { - gc_trace::mark_maybe_dead(); - last = pc; - pc += 1; - break Some(CoreExit::Stop(LeafStop::Marked)); - } + self.release(std::mem::replace(&mut *slot, base.add(len).read())); + last = pc; + pc += 1; + after_release!(); + continue; } } last = pc; @@ -12647,7 +10904,7 @@ impl Interpreter { .collect(); len -= n; let obj = Object::new_list(items); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); // SAFETY: `len < cap` (the operands' slots were freed). unsafe { base.add(len).write(obj) }; len += 1; @@ -12671,6 +10928,10 @@ impl Interpreter { ) { // SAFETY: as above. unsafe { drop_hot(base.add(len).read()) }; + last = pc; + pc += 1; + after_release!(); + continue; } last = pc; pc += 1; @@ -12910,6 +11171,9 @@ impl Interpreter { } _ => break None, }; + if let Some(e) = ext { + native_site_note(e, ninstrs, pc); + } // SAFETY: both operands leave by plain // decrements (checked). unsafe { @@ -12932,6 +11196,33 @@ impl Interpreter { } } } + // Arithmetic on a big int: the full handler's own + // bignum path (exact ints run no Python), out of + // line. + (Object::Long(_), Object::Int(_) | Object::Long(_)) + | (Object::Int(_), Object::Long(_)) => { + let r = core_bignum_op(a, b, kind); + // SAFETY: ints leave by plain decrements. + unsafe { + drop_hot(base.add(len - 1).read()); + drop_hot(base.add(len - 2).read()); + } + len -= 2; + match r { + Ok(v) => { + // SAFETY: the operands' slots are free. + unsafe { base.add(len).write(v) }; + len += 1; + last = pc; + pc += 1; + continue; + } + Err(e) => { + pc += 1; + break Some(CoreExit::Stop(LeafStop::Raised(e))); + } + } + } // String concatenation, repetition and `%` // formatting over scalars, out of line. (Object::Str(_), _) | (Object::Int(_), Object::Str(_)) => { @@ -12974,10 +11265,22 @@ impl Interpreter { len = n; last = pc; pc += 1; + after_release!(); } // An instance whose class's `__getitem__` is a // native fast subscript the site cached. None if ins.op == OpCode::BinarySubscr && len >= 2 => { + // A plain dict without the key raises its + // KeyError here, for the quiet loop's handler + // dispatch (the `try: d[k] except KeyError:` + // idiom). + // SAFETY: the `len >= 2` operand slots are + // initialized; on a raise both are consumed. + if let Some(e) = unsafe { Self::core_dict_miss(base, len) } { + len -= 2; + pc += 1; + break Some(CoreExit::Stop(LeafStop::Raised(e))); + } let Some(fast) = self.core_native_subscript( code, pc, @@ -13007,6 +11310,7 @@ impl Interpreter { len += 1; last = pc; pc += 1; + after_release!(); } Err(e) => { pc += 1; @@ -13046,6 +11350,9 @@ impl Interpreter { } _ => break None, }; + if let Some(e) = ext { + native_site_note(e, ninstrs, pc); + } // SAFETY: both operands leave by plain // decrements (checked); the result takes the // lower one's slot. @@ -13143,6 +11450,22 @@ impl Interpreter { self.gil_countdown -= 1; last = pc; pc = (pc + 1).saturating_sub(ins.arg as usize); + // A hot loop compiles its code, and switches to the + // native copy of this loop on the spot. + #[cfg(feature = "jit")] + if !NATIVE { + if let Some(ext) = ext { + ext.frame_jit.warm( + code, + ext, + nlocals, + frame_jit::Heat::BackEdge(last), + ); + if ext.frame_jit.get(nlocals).is_some() { + break Some(CoreExit::Switch); + } + } + } } OpCode::ForIter => { // A range iterator's next value; anything else (another @@ -13158,12 +11481,10 @@ impl Interpreter { // the full arm, which asks again (a leaf iterator // stays exhausted) and ends the loop. obj @ Object::Instance(inst) => { - let Some(b) = self.core_leaf_next_ptr(inst) else { + let Some(step) = self.leaf_next_step(obj, inst) else { break None; }; - // SAFETY: the cache and the class keep the - // builtin alive; its body runs no Python. - match unsafe { ((*b).call)(std::slice::from_ref(obj)) } { + match step { Ok(v) => { // SAFETY: `len < cap`. unsafe { base.add(len).write(v) }; @@ -13189,9 +11510,10 @@ impl Interpreter { Object::Generator(g) => { // A simple body runs to its next yield in // place, without switching (see `gen_fast`). - let g = g.clone(); + // (The stack slot keeps `g` alive: the step + // runs no code that could reach this frame.) if let gen_fast::GenNext::Yielded(v) = - self.gen_fast_next(&g, snap_gen, 0) + self.gen_fast_next(g, snap_gen, 0, pending_work) { // SAFETY: `len < cap` (checked above). unsafe { base.add(len).write(v) }; @@ -13200,7 +11522,6 @@ impl Interpreter { pc += 1; continue; } - drop(g); // SAFETY: `len <= cap`, every slot initialized. unsafe { frame.stack.set_len(len) }; frame.pc = pc as u32; @@ -13259,11 +11580,7 @@ impl Interpreter { *index += 1; v } - None if unique - && owner.is_none() - && Rc::strong_count(items) >= 2 - && !Self::iter_backing_list_dead(items) => - { + None if unique && owner.is_none() => { retire = true; Object::None } @@ -13454,11 +11771,7 @@ impl Interpreter { // `POP_ITER` pair (the full handler's shape). len -= 1; // SAFETY: the iterator slot leaves the stack. - let it = unsafe { base.add(len).read() }; - let marked = - gc_trace::maybe_tracked(crate::weakref_registry::id_of(&it)) - && gc_trace::note_dropped_marks(&it); - drop(it); + drop(unsafe { base.add(len).read() }); last = pc; pc += 1 + ins.arg as usize; let op_at = |pc: usize| { @@ -13471,10 +11784,6 @@ impl Interpreter { pc += 1; } } - if marked { - gc_trace::mark_maybe_dead(); - break Some(CoreExit::Stop(LeafStop::Marked)); - } continue; } // SAFETY: `len < cap`. @@ -13698,8 +12007,15 @@ impl Interpreter { // SAFETY: `start + kwc + argc + 3 == len`. let ops = unsafe { std::slice::from_raw_parts(base.add(start), len - start) }; - let Some(r) = self.core_pure_kw_call(code, pc, ops, argc, sw.depth_cell) - else { + let r = match &ops[0] { + // A natively served class's constructor (see + // `stdlib::datetime_native`). + Object::Type(ty) if ty.native_kind.get() != 0 => { + Self::core_native_ctor_kw(ops, argc, true) + } + _ => self.core_pure_kw_call(code, pc, ops, argc, sw.depth_cell), + }; + let Some(r) = r else { // A plain Python callee switches in place, as // `CALL`'s does (see `core_call_kw`). if !matches!(&ops[0], Object::Function(_)) { @@ -13788,6 +12104,35 @@ impl Interpreter { } } } + // A natively served class's constructor or bound + // class method (see `stdlib::datetime_native`), in + // place. + if python != 1 { + let start = len - argc - 2; + // SAFETY: `len >= argc + 2`: the call's operands. + let ops = + unsafe { std::slice::from_raw_parts(base.add(start), argc + 2) }; + let r = match &ops[0] { + Object::Type(_) => Self::core_native_ctor_kw(ops, argc, false), + Object::BoundMethod(_) => Self::core_native_bound_call(ops), + _ => None, + }; + if let Some(r) = r { + // SAFETY: every operand was checked to leave by a + // plain decrement; the result takes the callee's + // slot. + unsafe { + for k in start..len { + drop_hot(base.add(k).read()); + } + base.add(start).write(r); + } + len = start + 1; + last = pc; + pc += 1; + continue; + } + } if python != 0 { // SAFETY: as above. unsafe { frame.stack.set_len(len) }; @@ -14229,6 +12574,38 @@ impl Interpreter { last = pc; pc += 1; } + // `with`'s `__enter__` / `__exit__` of an instance: the + // plain function its class resolved at this site, under + // the instance (special lookup reads only the type, so + // the class version alone keys it). The helper fills + // the site. + OpCode::LoadSpecial => { + if len == 0 || len == cap { + break Some(CoreExit::Helper); + } + // SAFETY: `len > 0`. + let top = unsafe { base.add(len - 1) }; + let Object::Instance(inst) = (unsafe { &*top }) else { + break Some(CoreExit::Helper); + }; + let ver = inst.cls_raw().attr_version.get(); + let Some(f) = mslots!(cold_mslots, ext) + .get(pc) + .and_then(|ms| ms.get_held(ver)) + else { + break Some(CoreExit::Helper); + }; + // SAFETY: `len < cap`; the instance moves up into + // the self slot. + unsafe { + let recv = top.read(); + top.write(Object::Function(f)); + base.add(len).write(recv); + } + len += 1; + last = pc; + pc += 1; + } // The method-form load's per-site function (the helper's // `leaf_load_method` / class-receiver hits): an instance // receiver moves up into the self slot under the function; @@ -14380,7 +12757,7 @@ impl Interpreter { // shared module leaves by a plain decrement). Object::Module(m) => { match Self::core_module_attr(code, m, pc, ins.arg) { - Some(v) if !gc_trace::note_dropped_marks(unsafe { &*top }) => { + Some(v) => { // SAFETY: the receiver is replaced in place. unsafe { drop_hot(std::mem::replace(&mut *top, v)) }; last = pc; @@ -14395,7 +12772,7 @@ impl Interpreter { // A natively served instance's public field (see // `stdlib::datetime_native`). if inst.cls_raw().native_kind.get() != 0 { - match Self::core_native_field(ext, inst, ins.arg) { + match Self::core_native_field(ext, inst, ins.arg, pc, ninstrs) { Some(v) if Self::core_droppable(unsafe { &*top }) => { // SAFETY: the receiver (droppable) is // replaced in place. @@ -14536,6 +12913,9 @@ impl Interpreter { // SAFETY: see `CoreSwitch`: the running activation is synced and // unborrowed here. Self::unpack_bound_callee(unsafe { &mut *sw.cur }, pc); + if self.core_call_plain(sw, pc) { + return true; + } let mut tmp = None; // SAFETY: see `CoreSwitch`: the running activation is the // innermost; its handles are live and unborrowed here. @@ -14553,7 +12933,7 @@ impl Interpreter { if let (Some(act), Some((func, has_self, eff_argc))) = (&act, shape) { let code = &act.frame.code; if let Some(slot) = code_call_slot(&frame.code, pc) - .filter(|_| !Self::has_extended_params(code)) + .filter(|_| !Self::has_keyword_params(code)) { slot.set(CallShape { func, @@ -14572,7 +12952,137 @@ impl Interpreter { }; let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; + sw.cur = callee; + sw.scratch = usize::MAX; + sw.last = &raw mut sw.scratch; + true + } + + /// [`Self::core_call`]'s common case, with every check the site's + /// [`CallSlot`] settles done once: a plain function (no cells, no + /// closure, positional parameters only) that the site saw before, + /// called with exactly its arity, switched to in a pooled slot. The + /// general path's work, without its probes: `false` touches nothing. + #[inline(always)] + fn core_call_plain(&mut self, sw: &mut CoreSwitch, pc: usize) -> bool { + // SAFETY: see `CoreSwitch`: the running activation is synced and + // unborrowed here. + let frame = unsafe { &mut *sw.cur }; + let Some(ins) = frame.code.instructions.get(pc) else { + return false; + }; + let argc = ins.arg as usize; + let n = frame.stack.len(); + let Some(callee_slot) = n.checked_sub(argc + 2) else { + return false; + }; + let Object::Function(f) = &frame.stack[callee_slot] else { + return false; + }; + let fp: *const crate::object::PyFunction = Rc::as_ptr(f); + let has_self = !matches!(frame.stack[callee_slot + 1], Object::Unbound); + // SAFETY: GIL-serialized raw read of the function's code cell (see + // `PyFunction::code`); only compared, then cloned below. + let code_rc: &Rc = unsafe { &*f.code.as_ptr() }; + let Some(slot) = code_call_slot(&frame.code, pc) else { + return false; + }; + if slot.hit(fp, Rc::as_ptr(code_rc)) != Some((0, has_self)) + || !code_rc.cellvars.is_empty() + || !code_rc.freevars.is_empty() + || !f.closure.is_empty() + || Self::has_extended_params(code_rc) + || !Self::lean_code_ok(code_rc) + || crate::gil::free_threading_enabled() + { + return false; + } + let Some(mut act) = self.inline_pool.pop() else { + return false; + }; + let crate::recursion::Enter::Ok(guard) = crate::recursion::enter_with(sw.depth_cell) else { + self.inline_pool.push(act); + return false; + }; + // Committed. The arguments move into the slot's (empty, sole-owned) + // locals; the callable and the empty self slot leave the stack. + let code = code_rc.clone(); + let nlocals = code.varnames.len(); + // SAFETY: a parked slot's locals storage is its own, and empty. + let locals = unsafe { &mut *act.frame.locals.as_ptr() }; + let first = if has_self { + callee_slot + 1 + } else { + callee_slot + 2 + }; + let nargs = n - first; + locals.reserve(nlocals.max(nargs)); + // SAFETY: the `nargs` operands above `first` move into the locals + // (reserved above) and the stack forgets them; the callable moves + // out of its slot, and an empty self slot owns nothing. + let callable = unsafe { + let (src, dst) = (frame.stack.as_ptr().add(first), locals.as_mut_ptr()); + for k in 0..nargs { + dst.add(k).write(src.add(k).read()); + } + locals.set_len(nargs); + let callable = frame.stack.as_ptr().add(callee_slot).read(); + frame.stack.set_len(callee_slot); + callable + }; + fill_unbound(locals, nlocals); + frame.pc = pc as u32 + 1; + // SAFETY: the slot is parked (see `inline_bind_cells`): its handles + // are stale copies, overwritten without a drop. `fp` stays alive + // while the slot holds `callable`. + unsafe { + let fr: &mut Frame = &mut act.frame; + std::ptr::write(&raw mut fr.code, code); + std::ptr::write( + &raw mut fr.cells, + borrowed_rc(crate::object::empty_cells_ref()), + ); + std::ptr::write(&raw mut fr.globals, borrowed_rc(&(*fp).globals)); + std::ptr::write(&raw mut fr.builtins, borrowed_rc(&(*fp).builtins)); + std::ptr::write(&raw mut act.callable, callable); + } + act.frame.pc = 0; + act.parked = false; + act.call_pc = pc; + // The caller waits on the pending list (see `lean_pending_enter`). + // SAFETY: see `CoreSwitch`: the caller's handles are live, and + // nothing else borrows them here. + act.caller_pending = unsafe { + let depth = (*sw.inl).len(); + if depth == 0 { + self.lean_pending_enter(frame, &mut *sw.root_shell, pc) + } else if depth == sw.entry_depth && !sw.entry_dead { + self.lean_pending_enter(frame, &mut *sw.entry_shell, pc) + } else { + let inl: &mut Vec> = &mut *sw.inl; + let caller: *mut InlineAct = &raw mut *inl[depth - 1]; + self.lean_pending_enter(frame, &mut QuietShell::Lazy(&mut (*caller).act), pc) + } + }; + act.exc_depth = self.exc_info_len(); + act.guard = Some(guard); + // As `inline_bind_cells`: a hot body is compiled once, so native + // callers can take the direct lanes. + #[cfg(feature = "jit")] + { + let n = act.frame.code.jit_hint.bump_lean_entries(); + if n <= crate::tier2::LEAN_WARM_COMPILE_THRESHOLD_CAP + && n == crate::tier2::lean_warm_at() + && !crate::tier2::jit_off_for_process() + && !act.frame.code.jit_hint.is_not_jitable() + { + crate::tier2::warm_compile(self, &mut act.frame); + } + } + let callee: *mut Frame = &raw mut *act.frame; + // SAFETY: as above. + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -14609,14 +13119,73 @@ impl Interpreter { }; let bm = std::mem::replace(&mut frame.stack[callee_slot], Object::Function(f)); frame.stack[self_slot] = receiver; - // The bound method was a call temporary (or is still held - // elsewhere): grade its release like any dropped operand. - if gc_trace::note_dropped_marks(&bm) { - gc_trace::mark_maybe_dead(); - } drop(bm); } + /// The `LOAD_ATTR` at `pc` of `frame` of a property whose cached + /// getter is a pure leaf: the getter evaluated on the receiver at TOS + /// (no activation), its result replacing the receiver. `false` + /// touches nothing. + #[inline(never)] + fn core_leaf_property( + &self, + frame: &mut Frame, + last_pc: &mut usize, + pc: usize, + getter: &PyFunction, + code: &Rc, + ) -> bool { + if !code_is_pure_leaf(code) + || !pure_leaf_warm(code) + || crate::recursion::current_depth() >= crate::recursion::recursion_limit() + { + return false; + } + let Some(recv) = frame.stack.last() else { + return false; + }; + let Some(v) = + self.pure_leaf_eval::(code, getter, &[std::ptr::from_ref(recv)]) + else { + return false; + }; + if let Some(top) = frame.stack.last_mut() { + drop_hot(std::mem::replace(top, v)); + } + *last_pc = pc; + frame.pc = pc as u32 + 1; + true + } + + /// The core loop's `LOAD_ATTR` of a `property` at `sw`'s (synced) + /// running activation's pc: the getter's activation + /// ([`Self::try_inline_property`]) is pushed and made the running one, + /// as [`Self::core_call`] does for a call. `false` touches nothing. + #[inline(never)] + fn core_property(&mut self, sw: &mut CoreSwitch) -> bool { + if !self.inline_calls_ok() { + return false; + } + let mut tmp = None; + // SAFETY: see `CoreSwitch` (as in `core_call`). + let act = unsafe { + let depth = (*sw.inl).len(); + let (frame, _, shell) = sw.activation(depth, &mut tmp); + let pc = (*frame).pc as usize; + self.try_inline_property(&mut *frame, &mut *shell.cast::>(), pc) + }; + let Some(mut act) = act else { + return false; + }; + let callee: *mut Frame = &raw mut *act.frame; + // SAFETY: as above. + unsafe { push_fast(&mut *sw.inl, act) }; + sw.cur = callee; + sw.scratch = usize::MAX; + sw.last = &raw mut sw.scratch; + true + } + /// [`Self::core_call`] for the `CALL_KW` at `pc`: the quiet loop's /// inline keyword call ([`Self::try_inline_call_kw`]), pushed and made /// the running activation. `false` touches nothing. @@ -14637,7 +13206,7 @@ impl Interpreter { }; let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -14666,7 +13235,7 @@ impl Interpreter { }; let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -14710,7 +13279,7 @@ impl Interpreter { }; let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -14759,7 +13328,7 @@ impl Interpreter { _ => return None, }; let eff_argc = items.len() + usize::from(receiver.is_some()); - let (code, missing) = Self::lean_call_shape(f, eff_argc)?; + let (code, missing) = Self::lean_call_shape_cells(f, eff_argc, true)?; // Past the recursion limit the nested lean path raises. let crate::recursion::Enter::Ok(guard) = crate::recursion::enter() else { return None; @@ -14781,7 +13350,7 @@ impl Interpreter { // The spread tuple and a bound method were call temporaries (or // are still held elsewhere): grade their releases like any // dropped operand. - self.reap_call_receiver(callee); + self.release(callee); self.reap_call_args(&mut [spread, mapping]); let self_slot = callee_slot + 1; let act = self.inline_slot(); @@ -15106,7 +13675,7 @@ impl Interpreter { act.init_inst = Some(inst); let callee: *mut Frame = &raw mut *act.frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = callee; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -15183,7 +13752,6 @@ impl Interpreter { // A declined `__init__` may have stored into the instance // before it stopped; nothing else ever saw it, so it goes // (the framed call starts over on a fresh one). - gc_trace::note_dropped(&inst); drop(inst); false } @@ -15303,6 +13871,11 @@ impl Interpreter { let d = d.try_borrow().ok()?; clone_hot(d.get(&probe)?) } + (Object::Dict(d), Object::Instance(_)) if Self::core_droppable(k) => { + let probe = crate::object::LeafProbe::new(k)?; + let d = d.try_borrow().ok()?; + clone_hot(d.get(&probe)?) + } // `seq[a:b:c]` with plain int (or omitted) bounds: the // full handler's own slicing, out of line (a bad step // declines, and the full handler raises). @@ -15316,10 +13889,10 @@ impl Interpreter { _ => return None, }; // SAFETY: both operand slots are initialized. The key is a - // scalar, a string or a slice of ints, and the container a - // shared value - // (`core_droppable`), so neither release runs code; the - // result takes the container's slot. + // scalar, a string, a slice of ints or a droppable instance, + // and the container a shared value (`core_droppable`), so + // neither release runs code; the result takes the + // container's slot. unsafe { drop_hot(base.add(len - 1).read()); drop_hot(base.add(len - 2).read()); @@ -15560,53 +14133,18 @@ impl Interpreter { DictStep::Item(item.0, item.1) } - /// Whether the core loop may release `v` with a plain drop: a scalar, - /// a string (freeing one runs no code), or a shared heap value whose - /// release is a bare decrement the collector need not hear about (the - /// leaf arms' `note_dropped_marks` grade answers no). + /// Whether the core loop may release `v` with a plain drop. Releasing + /// a value never runs Python (a dying object's `Drop` only queues work + /// for the next safe point), so only the fixtures that observe deaths + /// as they happen (the C-API dict/function watchers, the reftrace + /// printer) send the release to the full path. #[inline(always)] fn core_droppable(v: &Object) -> bool { - match v { - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None | Object::Str(_) => { - true - } - Object::Instance(i) => { - // Deferred tracking is an exact proof that the collector - // and weakref registry hold no owner. A stale Bloom-filter - // hit must not reject this nonfinal release. The last - // owner still takes ordinary teardown, and every store or - // weakref operation that requires tracking revokes the flag. - // Past two owners only a weakref-watched object can be at - // its dead line (see `gc_trace::note_dropped_marks`). - gc_trace::drop_survives_plainly(v) - || Rc::strong_count(i) > 1 - && (i.is_gc_deferred() || !gc_trace::note_dropped_marks(v)) - } - Object::List(l) if Rc::strong_count(l) > 1 => !gc_trace::note_dropped_marks(v), - Object::Dict(d) if Rc::strong_count(d) > 1 => !gc_trace::note_dropped_marks(v), - // The last owner of a container the collector never took, and - // no weakref watches, holding only scalars: freeing it runs no - // code (`prompt_reap_dropped` reaches the same plain drop the - // long way round). - Object::List(_) | Object::Dict(_) => Self::core_plain_last_container(v), - Object::Tuple(t) => ThinArc::strong_count(t) > 1 && !gc_trace::note_dropped_marks(v), - Object::Function(f) => Rc::strong_count(f) > 1 && !gc_trace::note_dropped_marks(v), - Object::Type(t) => Rc::strong_count(t) > 1 && !gc_trace::note_dropped_marks(v), - _ => false, - } - } - - /// [`Self::core_droppable`] for the last owner of an exact `list` or - /// `dict`: untracked (the miss filter proves it), unwatched, and - /// holding only scalars. - #[inline(never)] - fn core_plain_last_container(v: &Object) -> bool { - let id = crate::weakref_registry::id_of(v); - !gc_trace::maybe_tracked(id) - && !crate::weakref_registry::may_have_weakrefs(id) + // (A never-driven `asend` awaitable warns as it goes.) + !matches!(v, Object::AsyncGenAwait(_)) && !crate::capi_watchers::dicts_active() + && !crate::capi_watchers::funcs_active() && !crate::stdlib::testinternalcapi_mod::reftrace_print_active() - && Self::is_scalar_leaf_container(v) } /// Module scope for the core loop's `LOAD_NAME` / `STORE_NAME` arms: @@ -15624,105 +14162,36 @@ impl Interpreter { /// shapes, see `leaf_builtin_call`); `None` declines untouched. #[inline(never)] fn core_isinstance(ops: &[Object]) -> Option> { - let [obj, Object::Type(cls)] = ops else { + let [obj, spec] = ops else { return None; }; - if !cls.metaclass_is_type() { - return None; - } - let r = match obj { - Object::Instance(inst) => { - if !inst.cls_raw().is_subclass_of(cls) { - return None; - } - true - } - Object::File(_) => return None, - obj => builtins::class_of(obj).is_subclass_of(cls), + // One class, or a flat tuple of them (`isinstance(x, (str, int))` + // asks each in turn). + let classes: &[Object] = match spec { + Object::Type(_) => std::slice::from_ref(spec), + Object::Tuple(items) if items.len() <= 8 => items, + _ => return None, }; - Some(Ok(Object::Bool(r))) - } - - /// Whether a returning frame's `locals` owe the exit reap - /// (`reap_frame_locals_on_exit`) nothing: scalars and strings, and - /// instances or containers held beyond every slot of the frame that - /// could name them plus the collector's handle, with no weakref - /// watching — the reap's own "escaped" verdict, and a grade of no mark - /// for each release. - #[inline] - fn core_escaped_locals(locals: &[Object]) -> bool { - // Each heap local's identity and strong count: up to eight are - // compared exactly (how many slots here name each object); past - // that, every heap slot is assumed to name every object. - const EXACT: usize = 8; - let mut heap = [const { std::mem::MaybeUninit::<(u64, usize)>::uninit() }; EXACT]; - let mut n = 0usize; - for o in locals { - let (id, sc) = match o { - Object::Int(_) - | Object::Float(_) - | Object::Bool(_) - | Object::None - | Object::Unbound - | Object::Str(_) => continue, - Object::Instance(i) => (Rc::as_ptr(i) as usize as u64, Rc::strong_count(i)), - Object::List(l) => (Rc::as_ptr(l) as usize as u64, Rc::strong_count(l)), - Object::Dict(d) => (Rc::as_ptr(d) as usize as u64, Rc::strong_count(d)), - Object::Tuple(t) => ( - ThinArc::as_ptr(t).cast::<()>() as usize as u64, - ThinArc::strong_count(t), - ), - _ => return false, + let mut hit = false; + for cls in classes { + let Object::Type(cls) = cls else { + return None; }; - if n < EXACT { - heap[n].write((id, sc)); + if !cls.metaclass_is_type() { + return None; } - n += 1; - } - if n == 0 { - return true; + hit |= match obj { + // A miss on an instance is left to the full check, which + // also consults `__class__`. + Object::Instance(inst) => inst.cls_raw().is_subclass_of(cls), + Object::File(_) => return None, + obj => builtins::class_of(obj).is_subclass_of(cls), + }; } - if n > EXACT { - return locals.iter().all(|o| { - let (sc, id) = match o { - Object::Instance(i) => (Rc::strong_count(i), Rc::as_ptr(i) as usize as u64), - Object::List(l) => (Rc::strong_count(l), Rc::as_ptr(l) as usize as u64), - Object::Dict(d) => (Rc::strong_count(d), Rc::as_ptr(d) as usize as u64), - Object::Tuple(t) => ( - ThinArc::strong_count(t), - ThinArc::as_ptr(t).cast::<()>() as usize as u64, - ), - _ => return true, - }; - sc > n + usize::from(gc_trace::maybe_tracked(id)) - && !crate::weakref_registry::may_have_weakrefs(id) - }); + if !hit && matches!(obj, Object::Instance(_)) { + return None; } - // SAFETY: the first `n` entries were written. - let heap: &[(u64, usize)] = unsafe { std::slice::from_raw_parts(heap.as_ptr().cast(), n) }; - heap.iter().all(|&(id, sc)| { - // Held past every slot here and a collector handle needs no - // count of the slots that name it. - (sc > n + 1 || { - let held = heap.iter().filter(|&&(other, _)| other == id).count(); - sc > held + usize::from(gc_trace::maybe_tracked(id)) - }) && !crate::weakref_registry::may_have_weakrefs(id) - }) - } - - /// Whether `v` is an instance the frame-slot and drop graders would - /// both pass over (`looks_reapable_temporary` false and - /// `note_dropped_marks` false): held elsewhere beyond the collector's - /// handle and watched by no weakref — its release is a bare decrement. - #[inline(always)] - fn core_shared_instance(v: &Object) -> bool { - let Object::Instance(i) = v else { - return false; - }; - let sc = Rc::strong_count(i); - let id = Rc::as_ptr(i) as usize as u64; - (sc >= 3 || (sc == 2 && !gc_trace::maybe_tracked(id))) - && !crate::weakref_registry::may_have_weakrefs(id) + Some(Ok(Object::Bool(hit))) } /// The `(function, has_self, effective argc)` of a plain-function @@ -15773,7 +14242,7 @@ impl Interpreter { let (missing, slot_self) = slot.hit(Rc::as_ptr(f), Rc::as_ptr(code_rc))?; if slot_self != has_self || !Self::lean_code_ok(code_rc) - || Self::has_extended_params(code_rc) + || Self::has_keyword_params(code_rc) { return None; } @@ -15784,7 +14253,15 @@ impl Interpreter { { return None; } - let cells: *const Rc>>> = f.lean_cells_ref(code_rc)?; + let own_cells = !code_rc.cellvars.is_empty(); + let cells: *const Rc>>> = if own_cells { + if !closure_cells_ok(f, code_rc) { + return None; + } + std::ptr::null() + } else { + f.lean_cells_ref(code_rc)? + }; let crate::recursion::Enter::Ok(guard) = crate::recursion::enter_with(depth_cell) else { return None; }; @@ -15814,14 +14291,37 @@ impl Interpreter { } frame.stack.pop().expect("callee slot checked above") }; + // A `*args` callee's surplus positionals become its tuple, which + // follows the positional parameters (there are no keyword ones). + let star = code.has_varargs.then(|| { + let npos = code.arg_count as usize; + if locals.len() > npos { + Object::Tuple(crate::tuple_storage::TupleStorage::from_exact_iter( + locals.drain(npos..), + )) + } else { + // The interned empty tuple. + Object::new_tuple(Vec::new()) + } + }); if missing > 0 { let Object::Function(f) = &callable else { unreachable!("checked above") }; locals.extend(f.defaults[f.defaults.len() - missing..].iter().cloned()); } + locals.extend(star); fill_unbound(locals, nlocals); frame.pc = pc as u32 + 1; + if own_cells { + let Object::Function(f) = &callable else { + unreachable!("checked above") + }; + let cells = fresh_cells(f, &code, locals); + return Some( + self.inline_bind_with(frame, shell, pc, act, code, callable, guard, cells, true), + ); + } // The cells handle is `f`'s (now `callable`'s) or the shared empty // vector: moving the callable leaves it where it was. Some(self.inline_bind_cells(frame, shell, pc, act, code, callable, guard, cells)) @@ -15965,6 +14465,40 @@ impl Interpreter { } } + /// `dict[key]` on the two operand slots ending at `len` when a plain + /// dict misses a str or int key: the full handler's `KeyError(key)`, + /// with both operands released. `None` (nothing touched) otherwise. + /// + /// # Safety + /// + /// The `len >= 2` slots at `base` are initialized; on `Some` the top + /// two are moved out (the caller drops them from its count). + #[cold] + #[inline(never)] + unsafe fn core_dict_miss(base: *mut Object, len: usize) -> Option { + // SAFETY: the caller's contract. + let (c, k) = unsafe { (&*base.add(len - 2), &*base.add(len - 1)) }; + let (Object::Dict(d), Object::Str(_) | Object::Int(_)) = (c, k) else { + return None; + }; + if !Self::core_droppable(c) { + return None; + } + let probe = crate::object::LeafProbe::new(k)?; + // A stored key of another kind sharing the bucket might equal the + // probe through Python (`__eq__`, which may also raise): only a + // miss that met none is the full lookup's miss. + if d.try_borrow().ok()?.get(&probe).is_some() || !probe.miss_is_exact() { + return None; + } + // SAFETY: both slots are initialized and leave the stack here; the + // key is a str or int and the container a shared dict, so neither + // release runs code. + let (c, k) = unsafe { (base.add(len - 2).read(), base.add(len - 1).read()) }; + drop_hot(c); + Some(crate::error::key_error_object(k)) + } + /// The native fast `__getitem__` the `BINARY_SUBSCR` at `pc` cached for /// `ops[0]`'s class (see [`Self::leaf_instance_subscript`]), when both /// operands leave by plain decrements. @@ -16165,11 +14699,9 @@ impl Interpreter { for (k, o) in f.defaults[f.defaults.len() - missing..].iter().enumerate() { args[nargs + k] = o; } - if effect { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - } else { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - } + // SAFETY: this thread's own depth cell. + let depth = unsafe { (*depth_cell).get() }; + self.pure_leaf_call(code_rc, f, &args[..total], effect, depth) } /// The core loop's fused `LOAD_FAST x; LOAD_ATTR m (method); (code_rc, f, &args[..total]) - } else { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - }; + // SAFETY: as above. + let depth = unsafe { (*depth_cell).get() }; + let r = self.pure_leaf_call(code_rc, f, &args[..total], effect, depth); let Some(v) = r else { if let Some(ext) = code_vm_ext(code) { leaf_site_set(ext, code.instructions.len(), attr_pc, None); @@ -16436,11 +14966,10 @@ impl Interpreter { if values[index] == 0 { if let Object::Instance(instance) = operand { let id = Rc::as_ptr(instance) as usize as u64; - eprintln!("First predicate instance drop miss: owners={}, deferred={}, maybe_tracked={}, tracked={}, weakrefs={}, weak_clones={}", + eprintln!("First predicate instance drop miss: owners={}, deferred={}, maybe_tracked={}, tracked={}, weakrefs={}", Rc::strong_count(instance), instance.is_gc_deferred(), gc_trace::maybe_tracked(id), gc_trace::is_tracked(id), - crate::weakref_registry::may_have_weakrefs(id), - crate::weakref_registry::strong_clone_count(id)); + crate::weakref_registry::may_have_weakrefs(id)); } } values[index] += 1; @@ -16471,11 +15000,9 @@ impl Interpreter { for (k, o) in f.defaults[f.defaults.len() - missing..].iter().enumerate() { args[nargs + k] = o; } - if effect { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - } else { - self.pure_leaf_eval::(code_rc, f, &args[..total]) - } + // SAFETY: this thread's own depth cell. + let depth = unsafe { (*depth_cell).get() }; + self.pure_leaf_call(code_rc, f, &args[..total], effect, depth) } /// [`Self::core_pure_call`] for a `CALL_KW` at `pc`: `ops` is the @@ -16578,6 +15105,106 @@ impl Interpreter { self.pure_leaf_eval::(code_rc, f, &args[..arity]) } + /// A tier-2 keyword call (the `CALL_KW` at `pc` of `caller`) of the warm + /// pure leaf `f`, bound straight onto the evaluation's argument list: + /// `vals` holds the positional run (a bound receiver first) and then + /// the keyword values in call order, `names` the keywords. The site's + /// `CallPyKwNames` permutation places them, as [`Self::core_pure_kw_call`] + /// does for the interpreter. Its result, or `None` having done nothing + /// observable. + #[allow(clippy::too_many_arguments)] + pub(crate) fn kw_pure_leaf_call( + &self, + caller: &CodeObject, + pc: usize, + f: &Rc, + names: &[Object], + vals: &[Object], + eff_argc: usize, + depth_cell: *const std::cell::Cell, + ) -> Option { + use weavepy_compiler::InlineCache as IC; + let IC::CallPyKwNames { + func_id, + perm, + argc: ca, + kwc: ck, + } = caller.caches.get(pc as u32) + else { + return None; + }; + let kwc = names.len(); + if ca as usize != eff_argc + || ck as usize != kwc + || vals.len() != eff_argc + kwc + || !fn_is_pure_leaf(f) + || !vals.iter().all(Self::core_droppable) + { + return None; + } + // SAFETY: GIL-serialized raw read of the function's code cell; the + // caller keeps the function alive across the evaluation. + let code_rc: &Rc = unsafe { &*f.code.as_ptr() }; + if !code_is_pure_leaf(code_rc) || !pure_leaf_warm(code_rc) || !Self::leaf_code_ok(code_rc) { + return None; + } + let (_, covered) = + Self::kw_names_bind_cached(caller, pc, f, func_id, perm, names, eff_argc)?; + // The ordinary call's `RecursionError` check. + // SAFETY: this thread's own depth cell. + if unsafe { (*depth_cell).get() } >= crate::recursion::recursion_limit() { + return None; + } + // A pure leaf has no keyword-only parameters; its `**kwargs` + // dictionary, if any, follows the positional ones. + let total = code_rc.arg_count as usize; + let arity = leaf_arity(code_rc); + if arity > 8 { + return None; + } + let mut args: [*const Object; 8] = [std::ptr::null(); 8]; + for (k, o) in vals[..eff_argc].iter().enumerate() { + args[k] = o; + } + // The `**kwargs` dictionary is built where it lives: no move of a + // filled table into its cell. It reuses the last call's when that + // one didn't escape (see `recycle_kw_dict`). + let varkw = code_rc + .has_varkeywords + .then(|| Object::Dict(spare_kw_dict(kwc))); + for (j, o) in vals[eff_argc..].iter().enumerate() { + let slot = ((perm >> (4 * j)) & 0xF) as usize; + if slot == specialize::KW_TO_VARKW as usize { + // The bind check proved the callee collects it. + let Some(Object::Dict(d)) = &varkw else { + return None; + }; + d.borrow_mut() + .insert(DictKey(names.get(j)?.clone()), o.clone()); + continue; + } + if slot >= total { + return None; + } + args[slot] = o; + } + for (slot, arg) in args.iter_mut().enumerate().take(total).skip(eff_argc) { + if covered & (1 << slot) == 0 { + *arg = f + .defaults + .get(f.defaults.len().checked_sub(total - slot)?)?; + } + } + if let Some(d) = &varkw { + args[total] = d; + } + let result = self.pure_leaf_eval::(code_rc, f, &args[..arity]); + if let Some(Object::Dict(d)) = varkw { + recycle_kw_dict(d); + } + result + } + /// A keyword call's parameters, bound as `kw_names_fill_locals` leaves /// them in `locals` (a `**kwargs` dictionary included), evaluated /// frameless when `f` is a warm pure leaf: its result, or `None` @@ -16703,6 +15330,40 @@ impl Interpreter { r } + /// [`Self::pure_leaf_eval`] (with `effect`, of an effect leaf) at + /// recursion depth `depth`: a warm callee's compiled plan is entered + /// directly (see [`Self::leaf_site_native`]). + #[inline(always)] + fn pure_leaf_call( + &self, + code: &CodeObject, + f: &crate::object::PyFunction, + args: &[*const Object], + effect: bool, + depth: usize, + ) -> Option { + // (The tiny shapes `leaf_eval` answers itself skip the probe.) + #[cfg(feature = "jit")] + if code_vm_ext(code) + .is_some_and(|e| e.pure_leaf.load(std::sync::atomic::Ordering::Relaxed) < 3) + { + let fast = if effect { + self.leaf_site_native::(code, f, args, depth) + } else { + self.leaf_site_native::(code, f, args, depth) + }; + if let Some(r) = fast { + return r; + } + } + let _ = depth; + if effect { + self.pure_leaf_eval::(code, f, args) + } else { + self.pure_leaf_eval::(code, f, args) + } + } + /// [`Self::pure_leaf_eval`] for a constructor's leaf `__init__` on the /// fresh instance `args[0]` (see `leaf_run`'s `FRESH`): `None` may /// leave that instance half-initialized, for the caller to discard. @@ -16958,7 +15619,7 @@ impl Interpreter { }; let gen_frame = act.gen_frame; // SAFETY: as above. - unsafe { (*sw.inl).push(act) }; + unsafe { push_fast(&mut *sw.inl, act) }; sw.cur = gen_frame; sw.scratch = usize::MAX; sw.last = &raw mut sw.scratch; @@ -17019,7 +15680,7 @@ impl Interpreter { // SAFETY: the consumer is the innermost remaining activation. let (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; // SAFETY: as above. - unsafe { (*cframe).stack.push(v) }; + unsafe { push_fast(&mut (*cframe).stack, v) }; sw.cur = cframe; sw.last = if clast == &raw mut sw.scratch { sw.scratch = usize::MAX; @@ -17040,7 +15701,7 @@ impl Interpreter { return true; } // SAFETY: see `quiet_run`. - if sw.fin && unsafe { (*sw.maybe_dead).get() } { + if unsafe { (*sw.maybe_dead).get() } { // SAFETY: as above. unsafe { self.flush_lean(&mut *cframe, &mut *cshell.cast::>()); @@ -17067,11 +15728,15 @@ impl Interpreter { let Some(top) = inl.last_mut() else { return false; }; - // A generator resume, or an activation that needed its shell, - // finishes through `quiet_run`; so does a return into a caller - // whose protocol would yield straight away. + // A generator resume, or an activation whose shell the full + // epilogue would act on, finishes through `quiet_run`; so does a + // return into a caller whose protocol would yield straight away. if top.gen.is_some() - || top.act.shell.is_some() + || top + .act + .shell + .as_deref() + .is_some_and(|s| !self.shell_pops_quietly(s, top.exc_depth)) || self.gil_countdown <= 2 || crate::hot_gates::loop_gen() != snap_gen { @@ -17088,24 +15753,30 @@ impl Interpreter { if depth == sw.entry_depth { sw.entry_dead = true; } + // An activation that caught an exception got its shell then (see + // `quiet_catch`): retire it as `inline_finish` does. Such a frame + // is not clean (the handler path set its exception fields). + let had_shell = match done.act.shell.take() { + Some(shell) => { + self.retire_quiet_shell(shell, frame, cur_pc); + true + } + None => false, + }; // `inline_finish`'s shell-less return, its common case in line: no - // leftover operands or cells, and locals that die here with no drop - // glue (scalars) or whose release the exit reap would pass over - // (see `core_escaped_locals`). + // leftover operands or cells, and locals only this activation + // holds. They are released here; an object that dies with them + // queues its finalizer, which runs below before the caller goes on. let clean = frame.stack.is_empty() - && frame.code.cellvars.is_empty() - && Rc::strong_count(&frame.locals) == 1 - // SAFETY: sole owner; nothing else reaches the vector. - && Self::core_escaped_locals(unsafe { &*frame.locals.as_ptr() }); + && (frame.code.cellvars.is_empty() || done.owns_cells) + && Rc::strong_count(&frame.locals) == 1; let mut tmp = None; let (cframe, clast, cshell, entry); if clean && done.init_inst.is_none() { - done.clean = true; - // SAFETY: as above; the values drop by plain decrements (or - // own nothing). - for v in unsafe { (*frame.locals.as_ptr()).drain(..) } { - drop_hot(v); - } + done.clean = !had_shell; + // SAFETY: as above. + // SAFETY: as above. + unsafe { release_locals(&mut *frame.locals.as_ptr()) }; drop(done.guard.take()); // SAFETY: the caller is the innermost remaining activation. (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; @@ -17115,47 +15786,74 @@ impl Interpreter { // SAFETY: the callable is moved out exactly once; the parked // slot treats the field as stale (see `inline_deliver`). let callable = unsafe { std::ptr::read(&raw const done.callable) }; - self.inline_park(done); - // SAFETY: as above. - unsafe { - (*cframe).stack.push(v); - self.drop_lean_callable( - &mut *cframe, - &mut *cshell.cast::>(), - callable, - ); + // A frame that caught an exception carries the handler path's + // state, which only the full park resets. + if had_shell { + self.inline_park(done); + } else { + self.inline_park_clean(done); } - entry = QuietEntry::Returned { cur_pc: call_pc }; - } else { - let result = if clean { - done.clean = true; - // SAFETY: as above. - for v in unsafe { (*frame.locals.as_ptr()).drain(..) } { - drop_hot(v); - } - drop(done.guard.take()); - Ok(v) + // SAFETY: as above. + unsafe { push_fast(&mut (*cframe).stack, v) }; + match callable { + // A function something else still holds (its class, its + // module) can't die here: a plain decrement. + Object::Function(f) if Rc::strong_count(&f) > 1 => drop(f), + callable => self.release(callable), + } + sw.cur = cframe; + sw.last = if clast == &raw mut sw.scratch { + sw.scratch = usize::MAX; + &raw mut sw.scratch } else { - self.inline_finish( - &mut done, - QuietExit::Outcome { - stepped: Ok(StepOutcome::Return(v)), - cur_pc, - }, - ) + clast }; - // SAFETY: the caller is the innermost remaining activation. - (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; + // The caller's `Returned` entry protocol (`quiet_frame`), as + // below. // SAFETY: as above. - entry = unsafe { - self.inline_deliver( - &mut *cframe, - &mut *cshell.cast::>(), - done, - result, - ) - }; + unsafe { *sw.last = call_pc }; + if self.gil_countdown <= 2 { + sw.pending = Some(CoreExit::Stop(LeafStop::Breaker)); + return true; + } + self.gil_countdown -= 1; + if crate::hot_gates::loop_gen() != snap_gen { + sw.pending = Some(CoreExit::Stop(LeafStop::Breaker)); + return true; + } + // SAFETY: see `quiet_run`. + if unsafe { (*sw.maybe_dead).get() } { + // SAFETY: as above. + unsafe { + self.flush_lean(&mut *cframe, &mut *cshell.cast::>()); + } + if self.drain_if_maybe_dead() && crate::hot_gates::loop_gen() != snap_gen { + sw.pending = Some(CoreExit::Stop(LeafStop::Breaker)); + } + } + return true; } + let result = if clean { + done.clean = !had_shell; + // SAFETY: as above. + // SAFETY: as above. + unsafe { release_locals(&mut *frame.locals.as_ptr()) }; + drop(done.guard.take()); + Ok(v) + } else { + self.inline_finish( + &mut done, + QuietExit::Outcome { + stepped: Ok(StepOutcome::Return(v)), + cur_pc, + }, + ) + }; + done.clean &= !had_shell; + // SAFETY: the caller is the innermost remaining activation. + (cframe, clast, cshell) = unsafe { sw.activation(depth - 1, &mut tmp) }; + // SAFETY: as above. + entry = unsafe { self.inline_deliver(&mut *cframe, done, result) }; sw.cur = cframe; sw.last = if clast == &raw mut sw.scratch { sw.scratch = usize::MAX; @@ -17178,7 +15876,7 @@ impl Interpreter { return true; } // SAFETY: see `quiet_run`. - if sw.fin && unsafe { (*sw.maybe_dead).get() } { + if unsafe { (*sw.maybe_dead).get() } { // SAFETY: as above. unsafe { self.flush_lean(&mut *cframe, &mut *cshell.cast::>()); @@ -17239,11 +15937,6 @@ impl Interpreter { return CoreAttr::Decline; }; let v = std::mem::replace(top, it); - if gc_trace::note_dropped_marks(&v) { - gc_trace::mark_maybe_dead(); - drop(v); - return CoreAttr::Marked; - } drop(v); CoreAttr::Done } @@ -17270,8 +15963,12 @@ impl Interpreter { // scalar one is remembered for the core loop's arm. Some(Object::Type(cls)) => { let v = Self::leaf_load_type_attr(code, cls, pc as u32, ins.arg); + // (The stamp is keyed by the class's version alone, + // which a metaclass's changes don't move.) if let Some(v) = &v { - class_attr_fill(code, cls, pc, v); + if Self::plain_metaclass(cls) { + class_attr_fill(code, cls, pc, v); + } } v } @@ -17284,11 +15981,6 @@ impl Interpreter { return CoreAttr::Decline; }; let receiver = std::mem::replace(top, v); - if gc_trace::note_dropped_marks(&receiver) { - gc_trace::mark_maybe_dead(); - drop(receiver); - return CoreAttr::Marked; - } drop(receiver); CoreAttr::Done } @@ -17304,15 +15996,8 @@ impl Interpreter { let Some(old) = Self::leaf_store_attr(code, stack, pc as u32, ins.arg) else { return CoreAttr::Decline; }; - let receiver = stack.pop().expect("receiver checked above"); - let marked = gc_trace::note_dropped_marks(&receiver) - | old.as_ref().is_some_and(gc_trace::note_dropped_marks); - drop(receiver); + drop(stack.pop().expect("receiver checked above")); drop(old); - if marked { - gc_trace::mark_maybe_dead(); - return CoreAttr::Marked; - } CoreAttr::Done } OpCode::LoadMethodAttr => { @@ -17337,11 +16022,6 @@ impl Interpreter { }; let cls = std::mem::replace(top, v); stack.push(Object::Unbound); - if gc_trace::note_dropped_marks(&cls) { - gc_trace::mark_maybe_dead(); - drop(cls); - return CoreAttr::Marked; - } drop(cls); return CoreAttr::Done; } @@ -17403,16 +16083,8 @@ impl Interpreter { let Some(v) = self.leaf_type_call(ty, &stack[first..]) else { return CoreAttr::Decline; }; - let mut marked = false; - for arg in stack.drain(self_slot..) { - marked |= gc_trace::note_dropped_marks(&arg); - } - stack.pop(); // the callee + stack.truncate(callee_slot); stack.push(v); - if marked { - gc_trace::mark_maybe_dead(); - return CoreAttr::Marked; - } return CoreAttr::Done; } let r = if argc == 0 @@ -17450,24 +16122,12 @@ impl Interpreter { let stack = &mut frame.stack; match r { Ok(v) => { - // The operands leave the stack: grade each one (the callee - // is a builtin: a leaf). A registered native body marks for - // any reference it releases itself (its registration - // contract). - let mut marked = false; + // The operands leave the stack. for arg in stack.drain(self_slot..) { - marked |= gc_trace::note_dropped_marks(&arg); drop_operand(arg); } - if matches!(stack[callee_slot], Object::BoundMethod(_)) { - marked |= gc_trace::note_dropped_marks(&stack[callee_slot]); - } stack.pop(); // the callee stack.push(v); - if marked { - gc_trace::mark_maybe_dead(); - return CoreAttr::Marked; - } CoreAttr::Done } Err(e) => { @@ -17499,18 +16159,93 @@ impl Interpreter { crate::stdlib::datetime_native::leaf_compare(kind, a, b) } + /// `cls(*args, **kwargs)` for a natively served class (see + /// [`crate::stdlib::datetime_native::construct_names`]): `ops` is the + /// core loop's `CALL` operands (callee, self slot, `argc` positional), + /// or with `kw` its `CALL_KW` operands (the keyword values and the + /// names tuple follow). The new instance, or `None` (nothing touched) + /// for the full handler, which also raises every error. + #[inline(never)] + fn core_native_ctor_kw(ops: &[Object], argc: usize, kw: bool) -> Option { + let (Object::Type(cls), Object::Unbound) = (ops.first()?, ops.get(1)?) else { + return None; + }; + if cls.native_kind.get() == 0 { + return None; + } + let args = ops.get(2..2 + argc)?; + let (names, values): (&[Object], &[Object]) = if kw { + let Object::Tuple(names) = ops.last()? else { + return None; + }; + (names, ops.get(2 + argc..ops.len() - 1)?) + } else { + (&[], &[]) + }; + if !ops + .iter() + .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o)) + { + return None; + } + crate::stdlib::datetime_native::construct_names(cls, args, names, values)?.ok() + } + + /// A call of a bound natively served class method + /// (`datetime.fromisoformat(s)`; see + /// [`crate::stdlib::datetime_native::bound_fast`]): `ops` is the core + /// loop's `CALL` operands (the bound method, an empty self slot and + /// the arguments). The result, or `None` (nothing touched) for the + /// full handler, which also raises every error. + #[inline(never)] + fn core_native_bound_call(ops: &[Object]) -> Option { + let (Object::BoundMethod(bm), Object::Unbound) = (ops.first()?, ops.get(1)?) else { + return None; + }; + let args = &ops[2..]; + if args.len() > 7 || bm.redispatch_descriptor { + return None; + } + let fast = crate::stdlib::datetime_native::bound_fast(&bm.function, &bm.receiver)?; + if !ops + .iter() + .all(|o| matches!(o, Object::Unbound) || Self::core_droppable(o)) + { + return None; + } + // Bitwise views of the receiver and the arguments, never dropped: + // the fast half reads them (and clones what it keeps). + let mut views = [const { std::mem::MaybeUninit::::uninit() }; 8]; + // SAFETY: the method and the core loop's stack keep them alive. + views[0].write(unsafe { std::ptr::read(&raw const bm.receiver) }); + for (v, a) in views[1..].iter_mut().zip(args) { + // SAFETY: as above. + v.write(unsafe { std::ptr::read(a) }); + } + // SAFETY: the first `args.len() + 1` entries were written. + let views = + unsafe { std::slice::from_raw_parts(views.as_ptr().cast::(), args.len() + 1) }; + fast(views)?.ok() + } + /// A natively served instance's public field (`dt.hour`; see - /// [`crate::stdlib::datetime_native::leaf_field`]), or `None`. + /// [`crate::stdlib::datetime_native::leaf_field`]), or `None`. A read + /// marks the site at `pc` native (see [`native_site`]). #[inline(never)] fn core_native_field( ext: Option<&CodeConstObjects>, inst: &PyInstance, name_idx: u32, + pc: usize, + ninstrs: usize, ) -> Option { - let Object::Str(name) = ext?.name_objs.get(name_idx as usize)? else { + let ext = ext?; + let Object::Str(name) = ext.name_objs.get(name_idx as usize)? else { return None; }; - crate::stdlib::datetime_native::leaf_field(inst, name) + let v = crate::stdlib::datetime_native::leaf_field(inst, name)?; + native_site_note(ext, ninstrs, pc); + Some(v) } /// A module attribute through the `LOAD_ATTR` site's cached index: @@ -17702,10 +16437,11 @@ impl Interpreter { } /// The core loop's `STORE_ATTR` into an instance: an indexed store - /// into the receiver's `__dict__` (the warm shape), or the single - /// probe a `StoreAttrNewKey` site's insert-or-overwrite needs — the + /// into the receiver's `__dict__` (the warm shape), the single probe + /// a `StoreAttrNewKey` site's insert-or-overwrite needs — the /// constructor shape, which would otherwise leave the loop for the - /// helper on every `self.x = …`. Nothing here runs Python code. + /// helper on every `self.x = …` — or a `__slots__` member's store. + /// Nothing here runs Python code. /// /// On `true` the value *moved* into the dict: the caller drops its /// stack slot without dropping the value. @@ -17778,10 +16514,83 @@ impl Interpreter { IC::StoreAttrNewKey { ver } => { Self::core_store_new_attr(code, inst, cls, attr_pc, name_idx, ver, value) } + IC::StoreAttrSlot { key_idx, ver } if cls.attr_version.get() == ver => { + Self::core_store_slot(code, inst, key_idx, name_idx, value) + } _ => false, } } + /// A validated `__slots__` member's store (the site's + /// `StoreAttrSlot`, whose class version the caller checked), as + /// `leaf_store_attr` makes it: over the member's droppable value, or + /// as its first value (the constructor shape). `true` moved `value` + /// in; `false` touched nothing. + #[inline(never)] + fn core_store_slot( + code: &CodeObject, + inst: &PyInstance, + key_idx: u32, + name_idx: u32, + value: &Object, + ) -> bool { + let Some(name) = code.names.get(name_idx as usize) else { + return false; + }; + let Ok(mut slots) = inst.slots.try_borrow_mut() else { + return false; + }; + match slots.get_index_mut(key_idx as usize) { + Some((key, slot)) if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => + { + if !Self::core_droppable(slot) { + return false; + } + // SAFETY: the value moves out of the caller's slot (it + // forgets it on `true`); the displaced one is droppable. + let v = unsafe { std::ptr::read(value) }; + inst.note_slot_store(&v); + drop(std::mem::replace(slot, v)); + true + } + _ => { + // Held elsewhere (another layout): the full store decides. + if slots.get(name).is_some() { + return false; + } + // SAFETY: as above. + let v = unsafe { std::ptr::read(value) }; + inst.note_slot_store(&v); + match code_name_obj(code, name_idx) { + Some(Object::Str(shared)) => slots.insert_shared(shared, v), + _ => slots.insert(name, v), + }; + true + } + } + } + + /// [`Self::core_store_slot`]'s verdict, without storing. + fn core_store_slot_ready( + code: &CodeObject, + inst: &PyInstance, + key_idx: u32, + name_idx: u32, + ) -> bool { + let Some(name) = code.names.get(name_idx as usize) else { + return false; + }; + let Ok(slots) = inst.slots.try_borrow() else { + return false; + }; + match slots.get_index(key_idx as usize) { + Some((key, old)) if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => { + Self::core_droppable(old) + } + _ => slots.get(name).is_none(), + } + } + /// [`Self::core_store_attr_ready`] through the site's [`FieldSlot`], /// for the next of an effect leaf's stores to one receiver in order /// (`cursor` as [`PyInstance::split_store_ready`]): `None` when the @@ -17815,6 +16624,10 @@ impl Interpreter { let cls = inst.cls_raw(); let ver = match code.caches.get(attr_pc as u32) { IC::StoreAttrInstance { ver, .. } | IC::StoreAttrNewKey { ver } => ver, + IC::StoreAttrSlot { key_idx, ver } => { + return cls.attr_version.get() == ver + && Self::core_store_slot_ready(code, inst, key_idx, name_idx); + } _ => return false, }; if cls.native_kind.get() != 0 @@ -18159,6 +16972,7 @@ impl Interpreter { globals, builtins, builtins_obj, + exc_handlers, .. } = frame; let code: &CodeObject = code_rc; @@ -18197,6 +17011,81 @@ impl Interpreter { last = pc; pc += 1; } + // A `def` or `lambda` (CPython 3.14's bare `MAKE_FUNCTION`): + // the function, with the defaults and closure the + // `SET_FUNCTION_ATTRIBUTE`s right after it attach built in + // (each takes the value next below the function). No Python + // runs, unless a function watcher would see the creation. + OpCode::MakeFunction if ins.arg == 0 => { + let n = stack.len(); + if crate::capi_watchers::funcs_active() + || !matches!(stack.last(), Some(Object::Code(_))) + { + break; + } + let mut defaults: Option> = None; + let mut closure: Option> = None; + let mut k = 0; + while let (Some(next), Some(value)) = ( + instrs.get(pc + 1 + k), + n.checked_sub(2 + k).map(|i| &stack[i]), + ) { + let (Object::Tuple(items), OpCode::SetFunctionAttribute) = (value, next.op) + else { + break; + }; + let field = match next.arg { + 0x01 => &mut defaults, + 0x08 => &mut closure, + _ => break, + }; + if field.is_some() { + break; + } + *field = Some(items.iter().cloned().collect()); + k += 1; + } + let Some(Object::Code(code)) = stack.pop() else { + unreachable!("checked above") + }; + for _ in 0..k { + let v = stack.pop().expect("checked above"); + self.release(v); + } + let obj = new_function( + code, + globals, + builtins, + defaults.unwrap_or_default(), + Vec::new(), + closure.unwrap_or_default(), + None, + ); + stack.push(obj); + last = pc + k; + pc += 1 + k; + } + // A closure tuple's cell (`LOAD_FAST` of a cell variable in + // CPython 3.14's listing). + OpCode::LoadClosure | OpCode::LoadClosureBorrow => { + let Some(cell) = cells.get(ins.arg as usize) else { + break; + }; + stack.push(Object::Cell(cell.clone())); + last = pc; + pc += 1; + } + // A prologue `MAKE_CELL`: the activation's cells are built + // with it (a later one, for an inlined comprehension, takes + // the full handler's fresh cell). + OpCode::MakeCell + if instrs[..pc] + .iter() + .all(|i| matches!(i.op, OpCode::MakeCell | OpCode::CopyFreeVars)) => + { + last = pc; + pc += 1; + } OpCode::LoadConst => { let Some(v) = consts.get(ins.arg as usize) else { break; @@ -18214,79 +17103,20 @@ impl Interpreter { if matches!(new, Object::Cell(_)) { break; } - // A displaced value the full handler would run through - // the prompt-reap cascade stays there too — except a - // deferred-tracking instance, whose teardown is its - // plain drop (recycled when this was its last ref). - if Self::local_needs_prompt_reap(old) && Self::looks_reapable_temporary(old) { - // A container of nothing but scalars, untracked and - // unwatched, frees nothing that could finalize: the - // plain drop below is its whole teardown, so the - // burst need not end for a cascade that would find - // nothing. This is the shape a `x = [i, i]` loop - // displaces on every iteration. - if !gc_trace::dies_inert(old) { - match old { - Object::Instance(i) if i.dies_by_plain_drop() => { - let Some(v) = stack.pop() else { break }; - if let Object::Instance(i) = - std::mem::replace(&mut locals[slot], v) - { - PyInstance::try_recycle(i); - } - last = pc; - pc += 1; - continue; - } - _ => break, - } - } + if !Self::core_droppable(old) { + break; } let Some(v) = stack.pop() else { break }; let old = std::mem::replace(&mut locals[slot], v); last = pc; pc += 1; - // The usual displaced value is an unboxed scalar: no - // heap to release, nothing to grade (`Object` has no - // `Drop` of its own). - if matches!( - old, - Object::Int(_) - | Object::Float(_) - | Object::Bool(_) - | Object::None - | Object::Unbound - ) { - std::mem::forget(old); - } else if gc_trace::note_dropped_marks(&old) { - gc_trace::mark_maybe_dead(); - drop(old); - stop = LeafStop::Marked; - break; - } + self.release(old); } OpCode::PopTop => { - // Only pure leaves: every other discard runs the full - // handler's prompt-reap gate. match stack.last() { - Some( - Object::None - | Object::Bool(_) - | Object::Int(_) - | Object::Float(_) - | Object::Str(_), - ) => { - drop_operand(stack.pop().expect("checked above")); - } - Some(Object::Instance(_)) => { - if !Self::leaf_pop_instance(stack) { - break; - } - } - Some(Object::List(l)) if Rc::strong_count(l) == 2 => { - if !Self::leaf_pop_dead_list(stack) { - break; - } + Some(v) if Self::core_droppable(v) => { + let v = stack.pop().expect("checked above"); + self.release(v); } _ => break, } @@ -18540,12 +17370,6 @@ impl Interpreter { if is_none == (ins.op == OpCode::PopJumpIfNone) { pc += ins.arg as usize; } - if gc_trace::note_dropped_marks(&v) { - gc_trace::mark_maybe_dead(); - drop(v); - stop = LeafStop::Marked; - break; - } } OpCode::JumpForward => { last = pc; @@ -18612,7 +17436,6 @@ impl Interpreter { } // The iterator leaves the stack (a full handler // takes the coarse mark for it). - gc_trace::mark_maybe_dead(); drop(it); stop = LeafStop::Marked; break; @@ -18675,9 +17498,7 @@ impl Interpreter { // plain drop. range_done = it_unique && owner.is_none() - && items.try_borrow().is_ok_and(|xs| *index >= xs.len()) - && Rc::strong_count(items) >= 2 - && !Self::iter_backing_list_dead(items); + && items.try_borrow().is_ok_and(|xs| *index >= xs.len()); } v } @@ -18730,8 +17551,6 @@ impl Interpreter { // An untracked iterator holds nothing whose death it // could cause (its list survives, see above); any // other is graded like every burst drop. - let marked = gc_trace::maybe_tracked(crate::weakref_registry::id_of(&it)) - && gc_trace::note_dropped_marks(&it); drop(it); last = pc; pc += 1 + ins.arg as usize; @@ -18744,11 +17563,6 @@ impl Interpreter { pc += 1; } } - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } continue; }; stack.push(v); @@ -18810,7 +17624,7 @@ impl Interpreter { } let items = stack.split_off(stack.len() - n); let obj = Object::new_list(items); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); stack.push(obj); last = pc; pc += 1; @@ -18847,7 +17661,7 @@ impl Interpreter { } stack.truncate(split); let obj = Object::Dict(Rc::new(RefCell::new(d))); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); stack.push(obj); last = pc; pc += 1; @@ -18892,15 +17706,9 @@ impl Interpreter { t.insert(k.clone(), v.clone()); } drop((t, s)); - let other = stack.pop().expect("length checked above"); + drop(stack.pop().expect("length checked above")); last = pc; pc += 1; - if gc_trace::note_dropped_marks(&other) { - gc_trace::mark_maybe_dead(); - drop(other); - stop = LeafStop::Marked; - break; - } } OpCode::Swap => { let depth = ins.arg as usize; @@ -19057,15 +17865,9 @@ impl Interpreter { _ => break, }; stack.truncate(n - 1); - let container = std::mem::replace(&mut stack[n - 2], r); + stack[n - 2] = r; last = pc; pc += 1; - if gc_trace::note_dropped_marks(&container) { - gc_trace::mark_maybe_dead(); - drop(container); - stop = LeafStop::Marked; - break; - } } OpCode::StoreSubscr => { // Stack: value, container, key (TOS). `list[int] = v` @@ -19091,11 +17893,6 @@ impl Interpreter { break; } let slot = &mut xs[i as usize]; - if Self::local_needs_prompt_reap(slot) - && Self::looks_reapable_temporary(slot) - { - break; - } let v = std::mem::replace(value_slot, Object::Unbound); std::mem::replace(slot, v) } @@ -19111,11 +17908,6 @@ impl Interpreter { }; let (old, changed) = match d.get_mut(&probe) { Some(slot) => { - if Self::local_needs_prompt_reap(slot) - && Self::looks_reapable_temporary(slot) - { - break; - } let v = std::mem::replace(value_slot, Object::Unbound); let old = std::mem::replace(slot, v); let changed = !old.is_same(slot); @@ -19144,15 +17936,8 @@ impl Interpreter { drop(key); last = pc; pc += 1; - let marked = gc_trace::note_dropped_marks(&container) - | gc_trace::note_dropped_marks(&old); drop(container); drop(old); - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } OpCode::LoadAttr => { // The instance-dict, slot, and module cache hits: a @@ -19191,15 +17976,9 @@ impl Interpreter { }, }; let Some(top) = stack.last_mut() else { break }; - let receiver = std::mem::replace(top, v); + *top = v; last = pc; pc += 1; - if gc_trace::note_dropped_marks(&receiver) { - gc_trace::mark_maybe_dead(); - drop(receiver); - stop = LeafStop::Marked; - break; - } } OpCode::LoadSuperAttr => { // Zero-argument `super().name` where the name resolves @@ -19338,19 +18117,11 @@ impl Interpreter { let Some(v) = self.leaf_type_call(ty, &stack[first..]) else { break; }; - let mut marked = false; - for arg in stack.drain(self_slot..) { - marked |= gc_trace::note_dropped_marks(&arg); - } + stack.truncate(self_slot); stack.pop(); // the callee stack.push(v); last = pc; pc += 1; - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } continue; } let r = if argc == 0 @@ -19391,23 +18162,13 @@ impl Interpreter { // (the callee is a builtin: a leaf). A registered // native body marks for any reference it // releases itself (its registration contract). - let mut marked = false; for arg in stack.drain(self_slot..) { - marked |= gc_trace::note_dropped_marks(&arg); drop_operand(arg); } - if matches!(stack[callee_slot], Object::BoundMethod(_)) { - marked |= gc_trace::note_dropped_marks(&stack[callee_slot]); - } stack.pop(); // the callee stack.push(v); last = pc; pc += 1; - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } Err(e) => { // As the full handler: the staged operands drop @@ -19438,15 +18199,8 @@ impl Interpreter { let receiver = stack.pop().expect("receiver checked above"); last = pc; pc += 1; - let marked = gc_trace::note_dropped_marks(&receiver) - | old.as_ref().is_some_and(gc_trace::note_dropped_marks); drop(receiver); drop(old); - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } OpCode::UnpackSequence => { let n = ins.arg as usize; @@ -19462,16 +18216,10 @@ impl Interpreter { _ => None, }; let Some(items) = pushed else { break }; - let seq = stack.pop().expect("sequence checked above"); + drop(stack.pop().expect("sequence checked above")); stack.extend(items); last = pc; pc += 1; - if gc_trace::note_dropped_marks(&seq) { - gc_trace::mark_maybe_dead(); - drop(seq); - stop = LeafStop::Marked; - break; - } } OpCode::BuildTuple => { // Tuples are never collector-tracked at build time and @@ -19519,15 +18267,9 @@ impl Interpreter { }; let it = Object::Iter(Rc::new(RefCell::new(it))); let Some(top) = stack.last_mut() else { break }; - let v = std::mem::replace(top, it); + *top = it; last = pc; pc += 1; - if gc_trace::note_dropped_marks(&v) { - gc_trace::mark_maybe_dead(); - drop(v); - stop = LeafStop::Marked; - break; - } } OpCode::UnaryOp => { // SAFETY: as in the full handler. @@ -19588,15 +18330,8 @@ impl Interpreter { let a = std::mem::replace(&mut stack[n - 2], Object::Bool(r)); last = pc; pc += 1; - let marked = - gc_trace::note_dropped_marks(&a) | gc_trace::note_dropped_marks(&b); drop(a); drop(b); - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } OpCode::ContainsOp => { let n = stack.len(); @@ -19611,15 +18346,8 @@ impl Interpreter { let item = std::mem::replace(&mut stack[n - 2], Object::Bool(r)); last = pc; pc += 1; - let marked = gc_trace::note_dropped_marks(&item) - | gc_trace::note_dropped_marks(&container); drop(item); drop(container); - if marked { - gc_trace::mark_maybe_dead(); - stop = LeafStop::Marked; - break; - } } OpCode::LoadDeref => { let v = match cells.get(ins.arg as usize) { @@ -19638,14 +18366,7 @@ impl Interpreter { let Some(cell) = cells.get(ins.arg as usize) else { break; }; - let reap = match cell.try_borrow() { - Ok(old) => { - Self::local_needs_prompt_reap(&old) - || matches!(*old, Object::Function(_)) - } - Err(_) => true, - }; - if reap { + if cell.try_borrow().is_err() { break; } let Some(v) = stack.pop() else { break }; @@ -19653,14 +18374,58 @@ impl Interpreter { Ok(mut slot) => std::mem::replace(&mut *slot, v), Err(_) => break, }; + drop(old); last = pc; pc += 1; - if gc_trace::note_dropped_marks(&old) { - gc_trace::mark_maybe_dead(); - drop(old); - stop = LeafStop::Marked; + } + // An `except` clause's entry and exit (the full handlers' + // common shapes; see `step`): an exception instance on + // entry, class operands that are BaseException subclasses. + OpCode::PushExcInfo => { + if !matches!(stack.last(), Some(Object::Instance(_))) { break; } + let exc = stack.pop().expect("checked above"); + let prev = self + .exc_info_stack + .borrow() + .last() + .map_or(Object::None, |pe| pe.instance.clone()); + stack.push(prev); + stack.push(exc.clone()); + let pe = PyException::new(exc); + exc_handlers.push((ins.arg, pe.clone())); + self.exc_info_stack.borrow_mut().push(pe); + last = pc; + pc += 1; + } + OpCode::CheckExcMatch => { + let n = stack.len(); + if n < 2 { + break; + } + let Some(matched) = Self::leaf_exc_match(&stack[n - 2], &stack[n - 1]) else { + break; + }; + let ty = std::mem::replace(&mut stack[n - 1], Object::Bool(matched)); + last = pc; + pc += 1; + self.release(ty); + } + OpCode::PopExcept => { + let Some(prev) = stack.pop() else { break }; + let popped = exc_handlers.pop(); + let top = self.exc_info_stack.borrow_mut().pop(); + // See the full handler: the handled exception dies + // here unless something else holds it. + self.recheck_frame_observed = true; + last = pc; + pc += 1; + drop(top); + if let Some((_, pe)) = popped { + self.release(pe.instance); + } + self.release(prev); } _ => break, } @@ -19670,6 +18435,43 @@ impl Interpreter { stop } + /// `CHECK_EXC_MATCH` of the exception instance `exc` against `ty`, a + /// class or a tuple of classes, when every class derives from + /// BaseException (`None` leaves the full handler its TypeError). + #[inline] + fn leaf_exc_match(exc: &Object, ty: &Object) -> Option { + let Object::Instance(inst) = exc else { + return None; + }; + let cls = inst.cls_raw(); + let base = &builtin_types().base_exception; + match ty { + Object::Type(t) => { + if std::ptr::eq(cls, Rc::as_ptr(t)) { + return Some(true); + } + if !t.is_subclass_of(base) { + return None; + } + Some(cls.is_subclass_of(t)) + } + Object::Tuple(items) => { + let mut matched = false; + for t in items.iter() { + let Object::Type(t) = t else { + return None; + }; + if !t.is_subclass_of(base) { + return None; + } + matched = matched || cls.is_subclass_of(t); + } + Some(matched) + } + _ => None, + } + } + /// The truth of an instance when it is settled natively: a class with /// default attribute access whose `__bool__` (else `__len__`) is a /// registered leaf builtin, or neither is defined (always true). @@ -19889,20 +18691,41 @@ impl Interpreter { return None; } let ver = cls.attr_version.get(); - let fresh = !matches!(&*self.core_truth.borrow(), Some((v, _)) if *v == ver); - if fresh { - let t = self.native_truth_of(v, cls); - *self.core_truth.borrow_mut() = Some((ver, t)); - } // The cache (and the class) keep the builtin alive through the // call, whose body runs no Python. - let (b, is_len): (*const crate::object::BuiltinFn, bool) = match &*self.core_truth.borrow() - { - Some((_, NativeTruth::AlwaysTrue)) => return Some(true), - Some((_, NativeTruth::Bool(b))) => (Rc::as_ptr(b), false), - Some((_, NativeTruth::Len(b))) => (Rc::as_ptr(b), true), - _ => return None, + let lookup = + |t: &NativeTruth| -> Result<(*const crate::object::BuiltinFn, bool, u8), bool> { + match t { + NativeTruth::AlwaysTrue => Err(true), + NativeTruth::Bool(b, op) => Ok((Rc::as_ptr(b), false, *op)), + NativeTruth::Len(b) => Ok((Rc::as_ptr(b), true, 0)), + NativeTruth::Decline => Err(false), + } + }; + let cached = match &*self.core_truth.borrow() { + Some((v, t)) if *v == ver => Some(lookup(t)), + _ => None, + }; + let found = match cached { + Some(found) => found, + None => { + let t = self.native_truth_of(v, cls); + let found = lookup(&t); + *self.core_truth.borrow_mut() = Some((ver, t)); + found + } + }; + let (b, is_len, op) = match found { + Ok(found) => found, + Err(true) => return Some(true), + Err(false) => return None, }; + // A deque's `__bool__`: its direct operation, without the call. + if op == crate::stdlib::collections_native::OP_BOOL { + if let Some(Object::Bool(b)) = crate::stdlib::collections_native::fast_op(op, v, None) { + return Some(b); + } + } // SAFETY: see above. let b = unsafe { &*b }; let r = match b.call_kw.as_ref() { @@ -19935,7 +18758,8 @@ impl Interpreter { return if is_len { NativeTruth::Len(b) } else { - NativeTruth::Bool(b) + let op = leaf_builtins::jit_method_op(&b).unwrap_or(0); + NativeTruth::Bool(b, op) }; } Some(_) => return NativeTruth::Decline, @@ -19991,20 +18815,12 @@ impl Interpreter { fast } LeafKind::Opaque => { - if Self::looks_reapable_temporary(&args[0]) { - return None; - } return Some((builtin.call)(args)); } _ => return None, } } }; - // The full handler promptly reaps a consumed temporary and any - // finalizable contents. Decline before the read can do work. - if Self::looks_reapable_temporary(&args[0]) { - return None; - } let completed = fast(args)?; #[cfg(test)] NATIVE_FAST_SUBSCRIPTS.with(|calls| calls.set(calls.get() + 1)); @@ -20086,7 +18902,11 @@ impl Interpreter { if let Some(Object::Builtin(f)) = b.get(&crate::object::StrKey("next")) { next_ptr = Rc::as_ptr(f) as usize; } - for (name, kind) in [("len", LeafKind::Len), ("isinstance", LeafKind::Isinstance)] { + for (name, kind) in [ + ("len", LeafKind::Len), + ("isinstance", LeafKind::Isinstance), + ("getattr", LeafKind::GetAttr), + ] { if let Some(Object::Builtin(f)) = b.get(&crate::object::StrKey(name)) { calls.insert(Rc::as_ptr(f) as usize, kind); if name == "len" { @@ -20291,7 +19111,7 @@ impl Interpreter { _ => return None, }; let obj = Object::new_list(items); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); return Some(obj); } None @@ -20324,14 +19144,43 @@ impl Interpreter { /// lookup. #[inline] fn core_leaf_next_ptr(&self, inst: &PyInstance) -> Option<*const crate::object::BuiltinFn> { + self.core_leaf_next_op(inst).map(|(b, _)| b) + } + + /// [`Self::core_leaf_next_ptr`] with the builtin's direct operation + /// (`collections_native::fast_op`, `0` for none). + #[inline] + fn core_leaf_next_op( + &self, + inst: &PyInstance, + ) -> Option<(*const crate::object::BuiltinFn, u8)> { let ver = inst.cls_raw().attr_version.get(); - if let Some((v, b)) = &*self.core_next.borrow() { + if let Some((v, b, op)) = &*self.core_next.borrow() { if *v == ver { - return b.as_ref().map(Rc::as_ptr); + return b.as_ref().map(|b| (Rc::as_ptr(b), *op)); } } self.core_leaf_next_resolve(inst, ver) - .map(|b| Rc::as_ptr(&b)) + } + + /// A native iterator step through [`Self::core_leaf_next_op`]: the + /// direct operation when it applies, else the builtin. `None` when the + /// class's `__next__` is not a registered leaf. + #[inline] + fn leaf_next_step( + &self, + it: &Object, + inst: &PyInstance, + ) -> Option> { + let (b, op) = self.core_leaf_next_op(inst)?; + if op != 0 { + if let Some(v) = crate::stdlib::collections_native::fast_op(op, it, None) { + return Some(Ok(v)); + } + } + // SAFETY: the cache and the class keep the builtin alive; its body + // runs no Python. + Some(unsafe { ((*b).call)(std::slice::from_ref(it)) }) } #[cold] @@ -20340,7 +19189,7 @@ impl Interpreter { &self, inst: &PyInstance, ver: u64, - ) -> Option> { + ) -> Option<(*const crate::object::BuiltinFn, u8)> { // A miss is remembered too: a Python-level `__next__` class asks // again on every step of a generic iteration. let found = match inst.cls().lookup("__next__") { @@ -20352,8 +19201,13 @@ impl Interpreter { } _ => None, }; - *self.core_next.borrow_mut() = Some((ver, found.clone())); - found + let op = found + .as_ref() + .and_then(leaf_builtins::jit_method_op) + .unwrap_or(0); + let out = found.as_ref().map(|b| (Rc::as_ptr(b), op)); + *self.core_next.borrow_mut() = Some((ver, found, op)); + out } /// Which leaf builtin `b` is, if any (pointer identity). @@ -20415,11 +19269,6 @@ impl Interpreter { slot.set_non_leaf(bm); return None; } - // Collector handles and weakref-registry clones aren't program - // owners. Use the same conservative proof as prompt reaping. - if Self::looks_reapable_temporary(callable) { - return None; - } let Some(result) = self.leaf_builtin_call(kind, b, std::slice::from_ref(&bm.receiver)) else { // The same builtin can serve an exact receiver or an excluded @@ -20520,6 +19369,59 @@ impl Interpreter { Some(kind) } + /// `getattr(obj, name[, default])` on a plain instance whose lookup + /// runs no code: an instance attribute, a plain class value, or a + /// method to bind, and for a missing attribute with no `__getattr__`, + /// the default. `None` for anything else (descriptors, dunders, a + /// missing attribute's `AttributeError`), which the full call handles. + fn leaf_getattr(args: &[Object]) -> Option> { + let (Object::Instance(inst), Object::Str(name)) = (args.first()?, args.get(1)?) else { + return None; + }; + let name: &str = name; + if args.len() > 3 || name.starts_with("__") || crate::object::exotic_str_keys_possible() { + return None; + } + let cls = inst.cls_raw(); + if cls.native_kind.get() != 0 + || inst.native.get().is_some() + || !Self::default_getattribute(cls) + { + return None; + } + // Only class values without `__get__` (or a function, which binds) + // leave the instance dictionary in charge. + let on_class = cls.lookup(name); + if !matches!( + on_class, + None | Some( + Object::Function(_) + | Object::Int(_) + | Object::Float(_) + | Object::Bool(_) + | Object::None + | Object::Str(_) + | Object::Bytes(_) + | Object::Tuple(_) + ) + ) { + return None; + } + if let Some(v) = inst.attr_get_str(name) { + return Some(Ok(v)); + } + Some(Ok(match on_class { + Some(f @ Object::Function(_)) => { + Object::BoundMethod(Rc::new(BoundMethod::new(args[0].clone(), f))) + } + Some(v) => v, + None if args.len() == 3 && !cls.dunder(crate::types::Dunder::GetAttr).present() => { + args[2].clone() + } + None => return None, + })) + } + /// Call a leaf builtin if `args` (receiver first for methods) has an /// admitted shape; `None` sends the call down the full path. fn leaf_builtin_call( @@ -20554,6 +19456,7 @@ impl Interpreter { ) } K::Isinstance => return Self::core_isinstance(args), + K::GetAttr => return Self::leaf_getattr(args), // `list.append` and `list.pop` in line (as `list_append` and // `list_pop` do them). K::ListAppend => { @@ -21097,6 +20000,8 @@ impl Interpreter { K::Value(v) => Some(LeafAttr::Value(Self::clone_operand(v))), K::ValueInstance(w) => Some(LeafAttr::Value(Object::Instance(w.upgrade()?))), K::ValueType(w) => Some(LeafAttr::Value(Object::Type(w.upgrade()?))), + K::ValueDict(w) => Some(LeafAttr::Value(Object::Dict(w.upgrade()?))), + K::ValueList(w) => Some(LeafAttr::Value(Object::List(w.upgrade()?))), K::Property(w) => Some(LeafAttr::Property(w.upgrade()?)), K::Other => None, }; @@ -21150,6 +20055,8 @@ impl Interpreter { | Object::Str(_)), ) => K::Value(v), Some(Object::Type(t)) => K::ValueType(Rc::downgrade(&t)), + Some(Object::Dict(d)) => K::ValueDict(Rc::downgrade(&d)), + Some(Object::List(l)) => K::ValueList(Rc::downgrade(&l)), // A plain instance stored on the class (`timezone.utc`): // a value unless its class makes it a descriptor. Some(Object::Instance(i)) => { @@ -21170,6 +20077,8 @@ impl Interpreter { K::Value(v) => Some(LeafAttr::Value(Self::clone_operand(v))), K::ValueInstance(w) => w.upgrade().map(|i| LeafAttr::Value(Object::Instance(i))), K::ValueType(w) => w.upgrade().map(|t| LeafAttr::Value(Object::Type(t))), + K::ValueDict(w) => w.upgrade().map(|d| LeafAttr::Value(Object::Dict(d))), + K::ValueList(w) => w.upgrade().map(|l| LeafAttr::Value(Object::List(l))), K::Property(w) => w.upgrade().map(LeafAttr::Property), K::Other => None, }; @@ -21202,9 +20111,46 @@ impl Interpreter { // (A VM-native builtin class with a faithful C mirror keeps its // own dict authoritative; only an extension-defined type's C // attribute hooks are opaque here.) - if (cls.c_ext_ptr.get() != 0 && !cls.flags.is_builtin) || !Self::plain_metaclass(cls) { + if cls.c_ext_ptr.get() != 0 && !cls.flags.is_builtin { return None; } + // `type.__dict__` is a data descriptor on the metatype that answers + // ahead of everything else, unless the metaclass replaces + // `__getattribute__` (a watched type re-arms on every lookup). + if code + .names + .get(name_idx as usize) + .is_some_and(|n| n == "__dict__") + { + if crate::capi_watchers::types_active() { + return None; + } + // SAFETY: GIL-serialized raw read, as in `plain_metaclass`. + let meta = unsafe { (*cls.metaclass.as_ptr()).as_ref() }; + if meta.is_some_and(|m| { + !Rc::ptr_eq(m, &builtin_types().type_) + && (m.c_ext_ptr.get() != 0 || !Self::default_getattribute(m)) + }) { + return None; + } + return Some(Object::MappingProxy(cls.dict.clone())); + } + if !Self::plain_metaclass(cls) { + return Self::leaf_load_meta_class_attr(code, cls, name_idx); + } + // A natively served class method (`datetime.fromisoformat`; see + // `stdlib::datetime_native`), bound as `classmethod.__get__` + // binds it. + if cls.native_kind.get() != 0 { + if let Some(Object::Str(name)) = code_name_obj(code, name_idx) { + if let Some(f) = crate::stdlib::datetime_native::type_classmethod(cls, name) { + return Some(Object::BoundMethod(Rc::new(BoundMethod::py_method( + Object::Type(cls.clone()), + f, + )))); + } + } + } match Self::leaf_class_attr(code, cls, name_idx)? { LeafAttr::Method(f) => { if let Some(s) = slot { @@ -21227,6 +20173,41 @@ impl Interpreter { } } + /// [`Self::leaf_load_type_attr`] for a class whose metaclass is a + /// Python subclass of `type` (`EnumType`, `ABCMeta`) that keeps the + /// default attribute access. CPython's `type_getattro` lets only a + /// metatype attribute intercept, so when the metaclass's MRO has no + /// entry for the name at all, the class's MRO answers as for a plain + /// metaclass. Nothing is cached per site: the class's attribute + /// version doesn't follow the metaclass. + #[inline(never)] + fn leaf_load_meta_class_attr( + code: &CodeObject, + cls: &Rc, + name_idx: u32, + ) -> Option { + if crate::object::exotic_str_keys_possible() || cls.native_kind.get() != 0 { + return None; + } + // SAFETY: GIL-serialized raw read, as in `plain_metaclass`; the + // reference does not outlive the checks. + let meta = unsafe { (*cls.metaclass.as_ptr()).as_ref() }?; + if meta.c_ext_ptr.get() != 0 || !Self::default_getattribute(meta) { + return None; + } + let Some(Object::Str(name)) = code_name_obj(code, name_idx) else { + return None; + }; + if meta.lookup(name).is_some() { + return None; + } + match Self::leaf_class_attr(code, cls, name_idx)? { + LeafAttr::Method(f) => Some(Object::Function(f)), + LeafAttr::Value(v) => Some(v), + LeafAttr::InstanceOnly | LeafAttr::BuiltinMethod(_) | LeafAttr::Property(_) => None, + } + } + /// Whether a class attribute read of dunder `name` resolves in the /// class's own MRO rather than the metaclass. Deliberately a short /// list: `type` carries data descriptors for most dunders @@ -21396,11 +20377,6 @@ impl Interpreter { let displaced = match slots.get_index_mut(key_idx as usize) { Some((key, slot)) if matches!(&key.0, Object::Str(stored) if stored.as_ref() == name.as_str()) => { - if Self::local_needs_prompt_reap(slot) - && Self::looks_reapable_temporary(slot) - { - return None; - } let val = std::mem::replace(value_slot, Object::Unbound); inst.note_slot_store(&val); Some(std::mem::replace(slot, val)) @@ -21466,15 +20442,10 @@ impl Interpreter { { // SAFETY: a leaf store runs no code while the // views below are live. - let (k, old) = unsafe { inst.attr_peek_index(key_idx as usize) }?; + let (k, _) = unsafe { inst.attr_peek_index(key_idx as usize) }?; if !slot_name_matches(code, name_idx, k) { return None; } - if Self::local_needs_prompt_reap(old) - && Self::looks_reapable_temporary(old) - { - return None; - } } // As the core loop's arm: an in-place value store. // SAFETY: as above. @@ -21489,18 +20460,8 @@ impl Interpreter { let Some(Object::Str(shared)) = code_name_obj(code, name_idx) else { return None; }; - { - // SAFETY: as above. - let split = unsafe { inst.dict.split_cell().peek() }?; - if let Some(i) = split.position(shared) { - let old = &split.values()[i]; - if Self::local_needs_prompt_reap(old) - && Self::looks_reapable_temporary(old) - { - return None; - } - } - } + // SAFETY: as above. + unsafe { inst.dict.split_cell().peek() }?; let val = std::mem::replace(value_slot, Object::Unbound); match inst.split_store(shared, val) { Ok(old) => old, @@ -21543,11 +20504,6 @@ impl Interpreter { &mut **d }; if let Some((_, slot)) = found.and_then(|i| d.get_index_mut(i)) { - if Self::local_needs_prompt_reap(slot) - && Self::looks_reapable_temporary(slot) - { - return None; - } let val = std::mem::replace(value_slot, Object::Unbound); let old = std::mem::replace(slot, val); // The name is in the dict: the indexed shape @@ -21589,57 +20545,6 @@ impl Interpreter { Some(displaced) } - /// `POP_TOP` of an instance whose tracking is still deferred: it - /// holds only atomic values and has no weakref or collector handle, - /// so its last reference frees (here: recycles) it. `false` leaves - /// the discard to the full handler's prompt-reap gate. Out of line - /// to keep the burst's `POP_TOP` arm as small as it was. - #[inline(never)] - fn leaf_pop_instance(stack: &mut Vec) -> bool { - match stack.last() { - Some(Object::Instance(i)) if i.dies_by_plain_drop() => { - if let Some(Object::Instance(i)) = stack.pop() { - PyInstance::try_recycle(i); - } - true - } - _ => false, - } - } - - /// `POP_TOP` of a list held only by the stack and its collector - /// handle, with no weakref and only atomic elements (a discarded - /// comprehension, a `list(range(...))` temporary): untracked and - /// freed here, the way CPython's decref frees it, instead of the - /// prompt-reap cascade. `false` leaves the discard to the full handler. - #[inline(never)] - fn leaf_pop_dead_list(stack: &mut Vec) -> bool { - let Some(Object::List(l)) = stack.last() else { - return false; - }; - let id = Rc::as_ptr(l) as usize as u64; - if crate::weakref_registry::may_have_weakrefs(id) - || crate::stdlib::testinternalcapi_mod::reftrace_print_active() - { - return false; - } - { - let Ok(items) = l.try_borrow() else { - return false; - }; - if items.len() > 64 || !items.iter().all(Object::is_gc_atomic) { - return false; - } - } - if !gc_trace::is_tracked(id) { - return false; - } - let obj = stack.pop().expect("checked above"); - gc_trace::untrack(&obj); - drop(obj); - true - } - /// Float arithmetic the leaf burst may run inline: the operators that /// cannot raise for the given operands. Same results as the generic /// `binary_op` float arms (`fresh_float` NaN identity included). @@ -21779,7 +20684,6 @@ impl Interpreter { specialize::record_dispatch(); frame.pc = pc + 1; let v = frame.pop()?; - gc_trace::note_dropped(&v); if matches!(v, Object::None) == (ins.op == OpCode::PopJumpIfNone) { frame.pc += ins.arg; } @@ -21965,7 +20869,7 @@ impl Interpreter { // Without this, the temp stayed pinned by its weakref // slot's strong clone until the next cyclic collection. if matches!(&v, Object::Instance(_)) { - self.prompt_reap_dropped(v); + self.release(v); } Ok(()) } @@ -22067,7 +20971,7 @@ impl Interpreter { // container as STORE_SUBSCR retires its stack operand, // and its weakrefs must clear on that refcount edge. if matches!(&target, Object::Instance(_)) { - self.prompt_reap_dropped(target); + self.release(target); } Ok(()) } @@ -22373,7 +21277,7 @@ impl Interpreter { // the binding's removal from `prompt_reap_dropped`'s // refcount test. self.sync_py_locals(frame); - self.prompt_reap_dropped(old); + self.release(old); } } OpCode::StoreName => { @@ -22395,7 +21299,11 @@ impl Interpreter { // through its `__setitem__` (e.g. `enum._EnumDict`); the old // value is the namespace's to manage, so no reap here. let g = frame.globals.clone(); - self.class_ns_store(&ns_obj, &name, v, &g)?; + let key = match code_name_obj(&frame.code, ins.arg) { + Some(k @ Object::Str(s)) if **s == *name => k.clone(), + _ => Object::from_str(name), + }; + self.class_ns_store_key(&ns_obj, key, v, &g)?; } else { // Dict-subclass exec globals with an overridden // `__setitem__` observe the binding (CPython's @@ -22447,9 +21355,7 @@ impl Interpreter { } }; if let Some(old) = old { - if Self::local_needs_prompt_reap(&old) { - self.prompt_reap_dropped(old); - } + self.release(old); } } } @@ -22479,9 +21385,7 @@ impl Interpreter { // `__del__` now, matching the `DeleteGlobal` path and CPython's // decref-on-store (module-scope `handle = None`). if let Some(old) = old { - if Self::local_needs_prompt_reap(&old) { - self.prompt_reap_dropped(old); - } + self.release(old); } } OpCode::DeleteName => { @@ -22512,7 +21416,7 @@ impl Interpreter { .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); if let Some(old) = old { - self.prompt_reap_dropped(old); + self.release(old); } } OpCode::DeleteGlobal => { @@ -22522,7 +21426,7 @@ impl Interpreter { .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); if let Some(old) = old { - self.prompt_reap_dropped(old); + self.release(old); } } OpCode::LoadDeref => { @@ -22560,10 +21464,7 @@ impl Interpreter { // `_formatArray` breaks its recurser cycle this way and // test_arrayprint's test_refcount asserts the array's // refcount recovers without a collect (RFC 0076 WS1). - if Self::local_needs_prompt_reap(&old) || matches!(old, Object::Function(_)) { - self.sync_py_locals(frame); - self.prompt_reap_dropped(old); - } + self.release(old); } OpCode::DeleteDeref => { // `del NAME` for a cell/free var empties the cell WITHOUT @@ -22578,7 +21479,7 @@ impl Interpreter { return Err(Self::unbound_deref(&frame.code, ins.arg as usize)); } let old = std::mem::replace(&mut *cell.borrow_mut(), Object::Unbound); - self.prompt_reap_dropped(old); + self.release(old); } OpCode::MakeCell => { let slot = ins.arg as usize; @@ -22653,13 +21554,13 @@ impl Interpreter { | Object::Coroutine(_) | Object::AsyncGenerator(_) ) { - self.prompt_reap_dropped(r); + self.release(r); } } } Err(e) => { if let Some(r) = receiver { - self.prompt_reap_dropped(r); + self.release(r); } return Err(e); } @@ -22814,7 +21715,7 @@ impl Interpreter { frame.push(r); // Operand retirement, as for BINARY_OP. if matches!(&v, Object::Instance(_)) { - self.prompt_reap_dropped(v); + self.release(v); } } OpCode::CompareOp => { @@ -22828,8 +21729,6 @@ impl Interpreter { // RFC 0077 (WS2): audited operand drops (exempt from the // coarse stack-shrink mark). `x is None` grades one leaf and // one live binding and schedules nothing. - gc_trace::note_dropped(&a); - gc_trace::note_dropped(&b); frame.push(Object::Bool(result)); } OpCode::ContainsOp => { @@ -22849,11 +21748,7 @@ impl Interpreter { // next cyclic collection (pdb's `'__return__' in // frame.f_locals` leaked one FrameLocalsProxy per stop — // test_pdb's test_pdb_frame_refleak counts the referrers). - if Self::local_needs_prompt_reap(&container) - && Self::looks_reapable_temporary(&container) - { - self.prompt_reap_dropped(container); - } + self.release(container); let result = if ins.arg == 1 { !found } else { found }; frame.push(Object::Bool(result)); } @@ -22905,9 +21800,6 @@ impl Interpreter { // or the callee's own frame-exit sweep for moved // arguments); the raise path drops the staged operands // wholesale on unwind, so it keeps the coarse mark. - if r.is_err() { - gc_trace::mark_maybe_dead(); - } r?; } OpCode::PushNull => { @@ -23021,13 +21913,13 @@ impl Interpreter { | Object::Coroutine(_) | Object::AsyncGenerator(_) ) { - self.prompt_reap_dropped(r); + self.release(r); } } } Err(e) => { if let Some(r) = receiver { - self.prompt_reap_dropped(r); + self.release(r); } return Err(e); } @@ -23044,6 +21936,13 @@ impl Interpreter { // `co_names[arg >> 2]` from the super object, pushing a // null self slot after it when the method flag (bit 0) // is set. + if let Some(attr) = self.load_super_attr_direct(frame, ins.arg)? { + frame.push(attr); + if ins.arg & 1 != 0 { + frame.push(Object::Unbound); + } + return Ok(StepOutcome::Continue); + } let self_obj = frame.pop()?; let class_obj = frame.pop()?; // The callable (global_super) is already in position @@ -23150,7 +22049,7 @@ impl Interpreter { let callable = frame.pop()?; let r = self.run_py_exact_nofree(&f, locals)?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); return Ok(StepOutcome::Continue); } None => { @@ -23244,7 +22143,7 @@ impl Interpreter { let f = f.clone(); let r = self.call_python_owned(&f, std::mem::take(&mut pos_args), kw_pairs)?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); return Ok(StepOutcome::Continue); } let mut kw_pairs = kw_pairs; @@ -23272,7 +22171,7 @@ impl Interpreter { } let r = result?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut pos_args); for (_, v) in &mut kw_pairs { self.reap_call_args(std::slice::from_mut(v)); @@ -23315,7 +22214,7 @@ impl Interpreter { if let (Object::Instance(inst), Object::Tuple(_)) = (&callable, &args_obj) { if crate::stdlib::testcapi_call::instance_call_returns_args_tuple(inst) { frame.push(args_obj); - self.reap_call_receiver(callable); + self.release(callable); return Ok(StepOutcome::Continue); } } @@ -23406,7 +22305,7 @@ impl Interpreter { let r = self.call_c_profiled(&ctor, &call_args, &[], &frame.globals)?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut pos_args); return Ok(StepOutcome::Continue); } @@ -23431,7 +22330,7 @@ impl Interpreter { let r = self.call_c_profiled(&raw, &call_args, &[], &frame.globals)?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut pos_args); return Ok(StepOutcome::Continue); } @@ -23563,7 +22462,7 @@ impl Interpreter { } let r = result?; frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut pos_args); for (_, v) in &mut kw_pairs { self.reap_call_args(std::slice::from_mut(v)); @@ -23582,10 +22481,10 @@ impl Interpreter { // deadness guard inside leaves a still-referenced mapping // untouched, so a user-held dict passed as `**d` is safe. if let Some(kw) = kwargs_obj { - self.prompt_reap_dropped(kw); + self.release(kw); } // Same for the `*args` sequence temporary. - self.prompt_reap_dropped(args_obj); + self.release(args_obj); } OpCode::ReturnValue => { return Ok(StepOutcome::Return(frame.pop()?)); @@ -23598,14 +22497,12 @@ impl Interpreter { } OpCode::PopJumpIfNone => { let v = frame.pop()?; - gc_trace::note_dropped(&v); if matches!(v, Object::None) { frame.pc += ins.arg; } } OpCode::PopJumpIfNotNone => { let v = frame.pop()?; - gc_trace::note_dropped(&v); if !matches!(v, Object::None) { frame.pc += ins.arg; } @@ -23654,37 +22551,6 @@ impl Interpreter { // CPython frees a *temporary* iterable's elements by refcount // the instant the exhausted FOR_ITER pops the iterator (the // iterator held the last reference to the list). Our - // `next_value` detaches the backing list on exhaustion with a - // plain `Rc` drop, which skips finalizers — so a - // `for x in f():` loop over a list of `__del__`-bearing - // objects never ran them promptly (test_interpreters - // test_channels.test_list_all_closed: ChannelID handles from - // `_channels.list_all()` must decref their channels at loop - // end). Harvest a uniquely-held backing list's elements - // before the detach and run them through the reap cascade. - let harvested: Option> = match &it_obj { - Object::Iter(it) => { - let guard = it.borrow(); - if let crate::object::PyIterator::List { - items, - index, - owner, - } = &*guard - { - if owner.is_none() - && *index >= items.borrow().len() - && Self::iter_backing_list_dead(items) - { - Some(std::mem::take(&mut *items.borrow_mut())) - } else { - None - } - } else { - None - } - } - _ => None, - }; let next = match &it_obj { Object::Iter(it) => it.borrow_mut().next_value_checked()?, Object::LazyIter(l) => { @@ -23706,7 +22572,11 @@ impl Interpreter { } Err(e) => return Err(e), }, - Object::Instance(_) => { + Object::Instance(_) => 'next: { + // `map` and `filter` step natively when they can. + if let Some(r) = self.seqtools_step(&it_obj, &frame.globals) { + break 'next r?; + } // Call __next__; treat StopIteration as exhaustion. let next = match instance_native_dunder(&it_obj, "__next__", None) { Some(r) => Some(r), @@ -23758,9 +22628,6 @@ impl Interpreter { frame.pop()?; frame.pc += ins.arg; frame.skip_end_for(); - if let Some(items) = harvested { - self.reap_iter_backing(items); - } } } } @@ -23780,7 +22647,7 @@ impl Interpreter { // missed entirely. Track unconditionally so the collector both // reports it via `gc.get_objects()`/`is_tracked` and can break // the cycle when it becomes unreachable. - gc_trace::track(obj.clone()); + gc_trace::track(&obj); frame.push(obj); if gc_trace::maybe_auto_collect() { self.run_pending_finalizers(); @@ -23837,7 +22704,7 @@ impl Interpreter { self.record_alloc(&obj); // CPython tracks every set (`gc.is_tracked(set())` is True); // track unconditionally like lists. - gc_trace::track(obj.clone()); + gc_trace::track(&obj); frame.push(obj); if gc_trace::maybe_auto_collect() { self.run_pending_finalizers(); @@ -23869,7 +22736,7 @@ impl Interpreter { self.record_alloc(&obj); // CPython tracks dicts at creation; track unconditionally so a // dict that becomes cyclic (`d = {}; d[0] = d`) is collectable. - gc_trace::track(obj.clone()); + gc_trace::track(&obj); frame.push(obj); if gc_trace::maybe_auto_collect() { self.run_pending_finalizers(); @@ -24382,73 +23249,15 @@ impl Interpreter { defaults = items.iter().cloned().collect(); } } - let name = code.name.clone(); - // Function getset slots live *outside* `__dict__` - // (`f.__dict__` starts empty in CPython; only genuine - // user attributes land there). - let slots = RefCell::new(DictData::default()); - // Stamp __module__ from globals['__name__'] (mirrors CPython's - // function dispatch). Pickle relies on this to serialise the - // function by qualified name. - let keys = fn_slot_keys(); - if let Some(name_obj) = frame - .globals - .borrow() - .get(&DictKey(keys[1].clone())) - .cloned() - { - slots - .borrow_mut() - .insert(DictKey(keys[0].clone()), name_obj); - } - // Pin __name__ and __qualname__ as stable objects so - // repeated `func.__name__` reads (and delegated reads - // through classmethod/staticmethod wrappers) return the - // *same* object — CPython exposes these as slots with - // stable identity, which `assertIs(wrapper.__name__, - // func.__name__)` in test_decorators relies on. - // Both come from the code object (see - // `CodeConstObjects::fn_names`), so two functions made - // from one `def` share them exactly as CPython's do. - let (name_obj, qualname_obj) = ( - Object::from_str(name.clone()), - Object::from_str(code.qualname.clone()), - ); - { - let mut sl = slots.borrow_mut(); - sl.insert(DictKey(keys[1].clone()), name_obj); - // `__qualname__` is the code object's PEP 3155 dotted - // name (computed at compile time from lexical - // nesting), not the bare `__name__`. - sl.insert(DictKey(keys[2].clone()), qualname_obj); - if let Some(ann) = annotations_obj { - sl.insert(DictKey(keys[3].clone()), ann); - } - } - let f = PyFunction { - name, - code: RefCell::new(code), - globals: frame.globals.clone(), - // The defining frame's builtins, not a fresh - // resolution: same mapping CPython's - // `PyFunction_New` snapshots (framestate's - // f_builtins came from the same globals). - builtins: frame.builtins.clone(), + let obj = new_function( + code, + &frame.globals, + &frame.builtins, defaults, kw_defaults, closure, - attrs: RefCell::new(None), - slots, - closure_cells: std::sync::OnceLock::new(), - defaults_override: crate::object::OverrideFlag::new(false), - }; - // A function participates in cycles through its globals - // (`exec(src, d)` builds `d -> f -> d`), its `__dict__`, and - // any closure cell that closes over the function. Track it so - // `gc.collect()` can reclaim those cycles and `gc.is_tracked(f)` - // reports True, matching CPython (functions are GC objects). - let obj = Object::Function(Rc::new(f)); - gc_trace::track(obj.clone()); + annotations_obj, + ); // PEP 590-era function watchers: PyFunction_EVENT_CREATE // fires from `PyFunction_New*` (test_capi.test_watchers). if crate::capi_watchers::funcs_active() { @@ -24495,7 +23304,7 @@ impl Interpreter { } } 0x04 => { - f.slots + f.slots() .borrow_mut() .insert(DictKey(Object::from_static("__annotations__")), value); } @@ -24520,7 +23329,7 @@ impl Interpreter { Object::from_str(format!("{owner_qualname}.__annotate__")), ); } - f.slots + f.slots() .borrow_mut() .insert(DictKey(Object::from_static("__annotate__")), value); } @@ -24532,7 +23341,7 @@ impl Interpreter { } } let obj = Object::Function(pf); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); frame.push(obj); } // RFC 0068 — CPython prologue unit; WeavePy's frame setup @@ -24825,13 +23634,7 @@ impl Interpreter { .get(raised_at as usize) .copied() .unwrap_or(0); - let tb = Rc::new(PyTraceback { - frame: py_frame, - lineno, - lasti: raised_at, - raw_lasti: None, - next: RefCell::new(None), - }); + let tb = Rc::new(PyTraceback::new(py_frame, lineno, raised_at, None)); inst.slot_set("__traceback__", Object::Traceback(tb)); } (wrapper, Object::None) @@ -24894,104 +23697,14 @@ impl Interpreter { // discarded). let _prev = frame.pop()?; let popped = frame.exc_handlers.pop(); - // Drop the interpreter-wide `sys.exc_info()` entry first so - // its clone of the instance is gone before we test liveness. drop(self.exc_info_stack.borrow_mut().pop()); // CPython clears the just-handled exception at the end of an // `except` block (the implicit `del` of the bound name plus - // the per-frame exc-state pop). When that exception held the - // last reference to a finalizable object — `AttributeError.obj`, - // or an exception wrapping a file/process/lock in its args — - // its `__del__` must run *now*, matching CPython's refcount - // timing rather than waiting for the next cyclic collection - // (RFC 0040 deterministic-finalization arc: `test_io` / - // `test_subprocess` destructor-timing cases). + // the per-frame exc-state pop): it dies here unless something + // else holds it. self.recheck_frame_observed = true; if let Some((_, pe)) = popped { - // Deconstruct the handler's `PyException` *first*: its - // `context`/`cause` boxes and traceback entries hold - // their own strong clones of the chained exceptions and - // frames. Held across the reap below, they make the - // `__context__` chain look externally referenced and the - // cascade stops at the outer instance — leaving e.g. the - // `CancelledError` behind asyncio's `TimeoutError` (and - // every frame its traceback pins) to the next full - // collection (test_ssl.test_handshake_timeout_handler_leak). - let PyException { - instance, - traceback, - context, - cause, - .. - } = pe; - drop(traceback); - drop(context); - drop(cause); - let pe_instance = instance; - // A handled exception is itself an untracked, - // non-finalizable leaf, so it slips past the fast-path - // early-return in `prompt_reap_dropped`; when it anchors a - // finalizable object (`AttributeError.obj`, an exception - // wrapping a file/process) drive the cascade directly so - // that `__del__` runs at handler exit like CPython. - // - // A *weakref* to the exception (or, equivalently, to a - // value reachable only through it) must likewise observe - // the death now: CPython frees the just-handled exception - // by refcount the instant the `except` block ends, so - // `weakref.ref(e)()` is already `None` on the next line — - // even with the cycle GC disabled. `test_multiprocessing`'s - // `TestManagerExceptions` (bpo-106558, "Manager exceptions - // avoid creating cyclic references") asserts exactly this - // under `gc.disable()` with a *non-finalizable* `Empty()` - // /`RemoteError`, which the finalizable-only pre-check would - // leave pinned on its GC handle until the next collection. - // - // Both pre-checks keep the hot path cheap: the - // overwhelmingly common scalar-arg exception that nothing - // weakly references and has nothing to finalize skips the - // allocating cascade entirely (the O(1) `count_for` probe - // short-circuits before the depth-bounded - // `exc_has_finalizable` walk). Binding the exception to an - // outer name (`except E as e: saved = e`) or a pending - // re-raise keeps its refcount above the dead threshold, and - // every cascade node is independently refcount-guarded, so - // the reap is a no-op in those cases too. - // A GC-*tracked* exception must also reap: dead, it is - // otherwise pinned by its collector handle until the - // next cycle collection — and through its traceback it - // pins every frame (and their locals) it unwound. - // `pickle.load` ends with `except _Stop: return - // stopinst.value`, so a lingering `_Stop` kept the - // whole `_Unpickler` (memo included, i.e. every - // unpickled object) alive; `concurrent.futures`' - // `_process_worker` then never finalized call-item - // arguments after `del call_item` - // (test_concurrent_futures - // `test_ressources_gced_in_workers`). - let id = crate::weakref_registry::id_of(&pe_instance); - let weakly_observed = crate::weakref_registry::count_for(id) > 0; - // `anchors_tracked_child` too: an *untracked* exception - // can be the last holder of a GC-tracked one — via - // `__cause__`/`__context__`, or via its traceback's - // frames — asyncio's `TimeoutError` raised `from` the - // tracked `CancelledError` whose traceback pins the - // whole cancelled call chain (frames, transports, the - // SSLContext). A plain `Rc` drop frees the outer - // exception but leaves the tracked descendants on their - // collector handles until the next full collection - // (test_ssl.test_handshake_timeout_handler_leak). Only - // probed when the exception is actually about to die, - // so an `except E as e: saved = e` pays nothing. - if weakly_observed - || gc_trace::is_tracked(id) - || (!Self::exc_plainly_inert(&pe_instance) - && (Self::exc_has_finalizable(&pe_instance, 6) - || (Self::is_refcount_dead(&pe_instance, 1) - && Self::anchors_tracked_child(&pe_instance, 5)))) - { - self.reap_dead_subgraph(pe_instance); - } + self.release(pe.instance); } } OpCode::PrepReraiseStar => { @@ -25023,8 +23736,8 @@ impl Interpreter { // test_exception_refcycles_errors asserts // `gc.get_referrers(exc) == []` without a collect). drop(items); - self.prompt_reap_dropped(excs); - self.prompt_reap_dropped(orig); + self.release(excs); + self.release(orig); } OpCode::Reraise => { // CPython 3.13 `RERAISE n`: the exception is TOS; when @@ -25785,6 +24498,166 @@ impl Interpreter { // ---------- exception handling ---------- + /// The raise-site half of the general loop's exception path, for an + /// exception arriving at a frame: CPython's `_PyErr_SetObject` chains + /// *every* fresh exception, including ones raised from C (here: Rust + /// opcodes and builtins), to the currently handled exception. + /// `RAISE_VARARGS` already did this at the raise site + /// (`context_settled`); a fresh Rust-raised error has empty traceback + /// and unset context/cause. + fn chain_fresh_exception(&self, exc: &mut PyException) { + let fresh = !exc.context_settled + && exc.context.is_none() + && exc.cause.is_none() + && exc.traceback.is_empty() + && !exc.suppress_tb_once; + // Nothing is being handled: there is no context to chain (the + // decision `attach_implicit_context` would settle the same way), + // so the instance's own `__context__` need not be probed. + if fresh && self.exc_info_len() == 0 { + exc.context_settled = true; + return; + } + if fresh && !instance_has_nonnull_attr(&exc.instance, "__context__") { + self.attach_implicit_context(exc); + if exc.context.is_some() { + Self::sync_exc_attrs(exc); + } + } else if exc.traceback.is_empty() && (exc.cause.is_some() || exc.context.is_some()) { + // A fresh Rust-raised error that already carries an + // explicit `cause`/`context` (e.g. `_io` chaining a + // misbehaving raw `readinto`'s `TypeError` into an + // `OSError`, `test_bad_readinto_type`) must still + // mirror those onto the instance dict so Python's + // `except` sees `e.__cause__`. + Self::sync_exc_attrs(exc); + } + } + + /// The quiet loop's exception path: `err`, raised by the instruction + /// before `frame.pc` in the running activation, goes to this frame's + /// handler without leaving the loop, exactly as the general loop's + /// path does when nothing observes the frame (no trace or monitoring + /// callback to fire, so no `'exception'`/RERAISE/EXCEPTION_HANDLED + /// event either). Hands `err` back when the frame has no handler for + /// it: the activation then finishes, and the exception propagates, + /// through the general loop as before. + #[inline(never)] + fn quiet_catch( + &mut self, + frame: &mut Frame, + shell: &mut QuietShell<'_>, + err: RuntimeError, + ) -> Result<(), RuntimeError> { + let RuntimeError::PyException(mut exc) = err else { + return Err(err); + }; + let raise_pc = frame.pc.wrapping_sub(1); + if find_handler(&frame.code.exception_table, raise_pc).is_none() + || crate::trace::any_observers_active() + { + return Err(RuntimeError::PyException(exc)); + } + self.chain_fresh_exception(&mut exc); + // The traceback entry records this activation's shell, and the + // handler's instructions run through `step`, which needs it. + self.flush_lean(frame, shell) + .lasti + .store(raise_pc, std::sync::atomic::Ordering::Relaxed); + match self.handle_exception(frame, exc) { + Ok(_) => Ok(()), + Err(e) => Err(e), + } + } + + /// The general loop's unwind of an inline activation that `exc` + /// leaves, when nothing observes the frame and no handler in it covers + /// the raise (see `quiet_catch`): this frame's traceback entry, then + /// `run_activation`'s epilogue for an exceptional exit. Returns the + /// propagating error. + #[inline(never)] + fn quiet_unwind( + &mut self, + frame: &mut Frame, + shell: Rc, + mut exc: PyException, + exc_depth: usize, + ) -> RuntimeError { + self.chain_fresh_exception(&mut exc); + let err = match self.handle_exception(frame, exc) { + Err(e) => e, + Ok(_) => RuntimeError::Internal("quiet unwind found a handler".to_owned()), + }; + self.pop_frame_shell(); + Self::settle_exited_frame_object(&shell, frame, false); + if self.exc_info_len() > exc_depth { + self.exc_info_stack.borrow_mut().truncate(exc_depth); + } + self.recycle_frame_shell(shell); + err + } + + /// Whether an inline activation returning with `shell` needs nothing + /// of the full epilogue: no frame object yet and no handled-exception + /// entries left above its entry depth (`inline_finish`'s second arm). + #[inline] + fn shell_pops_quietly(&self, shell: &crate::object::FrameShell, exc_depth: usize) -> bool { + !shell + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) + && self.exc_info_len() <= exc_depth + } + + /// Retire the shell of a returning inline activation that + /// [`Self::shell_pops_quietly`] accepted, its return at `cur_pc`. + #[cold] + #[inline(never)] + fn retire_quiet_shell( + &mut self, + shell: Rc, + frame: &Frame, + cur_pc: usize, + ) { + if shell.tb_refs.load(std::sync::atomic::Ordering::Relaxed) != 0 { + // See `sync_lazy_tb_lasti`. + shell + .lasti + .store(cur_pc as u32, std::sync::atomic::Ordering::Relaxed); + } + self.pop_frame_shell(); + Self::settle_exited_frame_object(&shell, frame, true); + self.recycle_frame_shell(shell); + } + + /// `run_activation`'s epilogue for the frame object of an activation + /// that just left the spine (`shell` popped), for the lean finishes: + /// leave it at the last executed instruction, and give a returned + /// frame's object that outlives the activation its own locals + /// (CPython's `take_ownership`), else bring its `f_locals` current. + fn settle_exited_frame_object( + shell: &crate::object::FrameShell, + frame: &Frame, + returned: bool, + ) { + if !shell + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) + { + return; + } + let Some(py) = shell.materialized.borrow().clone() else { + return; + }; + shell.refresh_materialized(&py); + // The shell's cell and `py`; see `run_activation`. + let internal = 2 + usize::from(frame.py_frame.as_ref().is_some_and(|p| Rc::ptr_eq(p, &py))); + if returned && Rc::strong_count(&py) > internal { + py.take_ownership_of_locals(); + } else { + py.invalidate_locals(); + } + } + /// Look up a handler for `exc` at the current pc. If found, /// truncate the stack and jump to the handler. Otherwise propagate. fn handle_exception( @@ -25881,60 +24754,91 @@ impl Interpreter { // out-of-range `tb_lasti` values that made `traceback`'s // `_get_code_position` StopIterate (RFC 0054: asyncio cancellation // tracebacks crashed `format_exc` with PEP 479 RuntimeError). - let py_frame = frame - .py_frame - .clone() - .or_else(|| { - // Materialise this activation's shell — the traceback - // must reference *this* frame's snapshot, and it must be - // the *same* object the live stack holds so - // `frame.clear()` can refuse to wipe an executing frame - // (its shared locals storage!). Activation identity is - // the shared locals storage, which is exact even under - // recursion (the code object is shared). Searching the - // whole spine (not just the top) matters when a trace - // callback raises: the exception unwinds while the stack - // still holds dispatch-machinery entries above the frame - // being annotated (test_sys_settrace jump-from-'call'). - let idx = self - .frame_stack - .borrow() - .iter() - .rposition(|shell| Rc::ptr_eq(&shell.locals, &frame.locals)); - idx.and_then(|idx| crate::object::materialize_stack_at(&self.frame_stack, idx)) - }) - .unwrap_or_else(|| { - // Fall back to a fresh snapshot when neither source - // matches (e.g. a not-yet-entered generator frame - // receiving a throw, or an empty stack) so the chain - // stays non-empty. Cache it on generator-family frames - // so `gi_frame` keeps a stable identity. - let py = self.build_py_frame(frame, None); - py.lasti.set(lasti); - if frame.code.is_generator - || frame.code.is_coroutine - || frame.code.is_async_generator - { - frame.py_frame = Some(py.clone()); + let new_tb = match frame.py_frame.clone() { + Some(py_frame) => PyTraceback::new(py_frame, lineno, lasti, None), + None => { + // This activation's shell: the traceback must reference + // *this* frame's snapshot, and it must be the *same* + // object the live stack holds so `frame.clear()` can + // refuse to wipe an executing frame (its shared locals + // storage!). Activation identity is the shared locals + // storage, which is exact even under recursion (the code + // object is shared). Searching the whole spine (not just + // the top) matters when a trace callback raises: the + // exception unwinds while the stack still holds + // dispatch-machinery entries above the frame being + // annotated (test_sys_settrace jump-from-'call'). + let found = { + let stack = self.frame_stack.borrow(); + stack + .iter() + .rposition(|shell| Rc::ptr_eq(&shell.locals, &frame.locals)) + .map(|idx| (idx, stack[idx].clone())) + }; + match found { + // An ordinary activation nobody has looked at yet + // records its shell: the frame object is built only + // if something reads `tb_frame` or the entry outlives + // the activation (see `PyTraceback::new_lazy`). + Some((_, shell)) + if !shell.is_gen + && !shell + .has_materialized + .load(std::sync::atomic::Ordering::Relaxed) => + { + PyTraceback::new_lazy(shell, lineno, lasti) + } + found => { + let py_frame = found + .and_then(|(idx, _)| { + crate::object::materialize_stack_at(&self.frame_stack, idx) + }) + .unwrap_or_else(|| { + // Fall back to a fresh snapshot when neither + // source matches (e.g. a not-yet-entered + // generator frame receiving a throw, or an + // empty stack) so the chain stays non-empty. + // Cache it on generator-family frames so + // `gi_frame` keeps a stable identity. + let py = self.build_py_frame(frame, None); + py.lasti.set(lasti); + if frame.code.is_generator + || frame.code.is_coroutine + || frame.code.is_async_generator + { + frame.py_frame = Some(py.clone()); + } + py + }); + PyTraceback::new(py_frame, lineno, lasti, None) + } } - py - }); - let new_tb = Rc::new(PyTraceback { - frame: py_frame, - lineno, - lasti, - raw_lasti: None, - next: RefCell::new(None), - }); + } + }; + let new_tb = Rc::new(new_tb); // CPython chains outward: the catching frame ends up at the // *head*; `tb_next` walks toward the raise site. Each // propagation prepends the current frame's `tb` to the // existing chain. + // One probe of the slot table: replace an existing chain's head in + // place, else add the slot. if let Object::Instance(inst) = &exc.instance { - if let Some(Object::Traceback(prev_tb)) = inst.slot_get("__traceback__") { - *new_tb.next.borrow_mut() = Some(prev_tb); + let new_tb = Object::Traceback(new_tb); + inst.note_slot_store(&new_tb); + let mut slots = inst.slots.borrow_mut(); + match slots.get_mut("__traceback__") { + Some(slot) => { + let prev = std::mem::replace(slot, new_tb); + if let (Object::Traceback(prev_tb), Object::Traceback(head)) = (&prev, &*slot) { + *head.next.borrow_mut() = Some(prev_tb.clone()); + } + drop(slots); + drop(prev); + } + None => { + slots.insert("__traceback__", new_tb); + } } - inst.slot_set("__traceback__", Object::Traceback(new_tb)); } } @@ -25964,13 +24868,12 @@ impl Interpreter { let Some(py_frame) = self.materialize_top_py_frame() else { return; }; - let new_tb = Rc::new(PyTraceback { - lineno: py_frame.last_line.get().unwrap_or(1), - lasti: py_frame.lasti.get(), - frame: py_frame, - raw_lasti: None, - next: RefCell::new(None), - }); + let new_tb = Rc::new(PyTraceback::new( + py_frame.clone(), + py_frame.last_line.get().unwrap_or(1), + py_frame.lasti.get(), + None, + )); inst.slot_set("__traceback__", Object::Traceback(new_tb)); } @@ -26225,6 +25128,14 @@ impl Interpreter { } fn exception_matches(&self, exc: &Object, ty: &Object) -> Result { + // The commonest handler names the raised exception's own class, + // which (being raised) derives from BaseException. + if let (Object::Instance(inst), Object::Type(t)) = (exc, ty) { + if !crate::gil::free_threading_enabled() && std::ptr::eq(inst.cls_raw(), Rc::as_ptr(t)) + { + return Ok(true); + } + } // Every entry must be a BaseException subclass *before* any // matching happens — `except (ValueError, 42):` is a TypeError // even when the raised exception would match ValueError @@ -26539,20 +25450,9 @@ impl Interpreter { let Some(Object::Dict(d)) = inst.native.get() else { return true; }; - // The instance's own slot (+ the collector's handle, if the - // dict is tracked) is the "unreferenced" baseline for the dict. - let dict_id = crate::weakref_registry::id_of(&Object::Dict(d.clone())); - let dict_allow = 1 + usize::from(gc_trace::is_tracked(dict_id)); - if Rc::strong_count(d) > dict_allow { - return true; - } - let obj = Object::Instance(inst.clone()); - let id = crate::weakref_registry::id_of(&obj); - // This table + `obj` + the collector's handle + weakref slots. - let allow = 2 - + usize::from(gc_trace::is_tracked(id)) - + crate::weakref_registry::strong_clone_count(id); - Rc::strong_count(inst) > allow + // The instance's own slot is the "unreferenced" baseline for + // the dict, and this table's for the instance. + Rc::strong_count(d) > 1 || Rc::strong_count(inst) > 1 }); } @@ -27484,6 +26384,7 @@ impl Interpreter { // it suspends (see `pop_frame_shell`); a later resume // re-derives it here, from the live spine. if fr.back.borrow().is_none() + && fr.lazy_back.borrow().is_none() && fr.on_stack.get() > 0 && (fr.code.is_generator || fr.code.is_coroutine @@ -27500,8 +26401,8 @@ impl Interpreter { crate::object::materialize_stack_at(&self.frame_stack, idx); } } - match fr.back.borrow().as_ref() { - Some(parent) => Ok(Object::Frame(parent.clone())), + match fr.back_frame() { + Some(parent) => Ok(Object::Frame(parent)), None => Ok(Object::None), } } @@ -27564,11 +26465,11 @@ impl Interpreter { ))), }, Object::Traceback(tb) => match name { - "tb_frame" => Ok(Object::Frame(tb.frame.clone())), + "tb_frame" => Ok(Object::Frame(tb.frame())), "tb_lineno" => Ok(Object::Int(i64::from(tb.lineno))), "tb_lasti" => Ok(match tb.raw_lasti { Some(v) => Object::Int(v), - None => Object::Int(i64::from(tb.frame.code.cpython_lasti(tb.lasti))), + None => Object::Int(i64::from(tb.code().cpython_lasti(tb.lasti))), }), "tb_next" => match tb.next.borrow().as_ref() { Some(n) => Ok(Object::Traceback(n.clone())), @@ -28456,6 +27357,118 @@ impl Interpreter { } } + /// The first stage of `super_getattro` for a bound proxy over + /// `receiver` (CPython's `su->obj`), walking `obj_type`'s MRO past + /// `thisclass`: the attribute through the descriptor protocol, or the + /// receiver's native payload's when a built-in base keeps it off its + /// type dict. `None` leaves the remaining special cases to the caller. + fn super_mro_attr( + &mut self, + obj_type: &Rc, + receiver: &Object, + thisclass: &Rc, + name: &str, + ) -> Result, RuntimeError> { + // Walk `obj_type`'s MRO *after* `thisclass` (CPython's + // `super_getattro`: start one past `su->type` in + // `su->obj_type->tp_mro`). The proxy's own class (`super` or a + // user subclass) is irrelevant to the lookup target. + let found = { + let mro = obj_type.mro.borrow(); + let start = mro + .iter() + .position(|t| Rc::ptr_eq(t, thisclass)) + .map_or(mro.len(), |i| i + 1); + mro[start..].iter().find_map(|t| { + let v = t.dict.borrow().get(&crate::object::StrKey(name)).cloned()?; + // Introspection-only mirrors in builtin dicts (docs + // surface pass, RFC 0056 WS4) are invisible to super's + // walk too: `super().__repr__()` must fall through to + // the caller's non-virtual `object.__repr__` special case, + // not the virtual `slot_repr` (asyncio.Lock's repr + // chains through super and would re-enter itself). + if t.flags.is_builtin && crate::descr_registry::is_surface_only(&v) { + return None; + } + Some(v) + }) + }; + if let Some(v) = found { + // CPython passes `su->obj_type` as `owner`, and a NULL + // instance when `su->obj == su->obj_type` (the class-bound + // form `super(C, cls)` inside `__new__`/classmethods), so + // plain functions come back unbound while classmethods + // still bind to the class. + let owner = Object::Type(obj_type.clone()); + let instance_for_get = match receiver { + Object::Type(r) if Rc::ptr_eq(r, obj_type) => Object::None, + _ => receiver.clone(), + }; + return self.descriptor_get(&v, &instance_for_get, &owner).map(Some); + } + // The MRO beyond the starting class reaches a built-in base + // (`dict`, `list`, …) whose methods aren't stored on the type + // dict and so don't surface above. Resolve `name` against the + // receiver's native payload so `super().__setitem__`, + // `super().append`, … dispatch to the wrapped built-in and + // operate on the shared payload. (Only names absent from the + // MRO reach here, so this never shadows a user override.) + if let Object::Instance(recv) = receiver { + if let Some(native) = recv.native.get() { + if let Ok(v) = self.load_attr(&native.clone(), name) { + return Ok(Some(v)); + } + } + } + Ok(None) + } + + /// `LOAD_SUPER_ATTR`'s zero-argument form without the proxy: the + /// class and receiver on the stack are what `super()` would find, so + /// when the builtin `super` is the callable, nothing observes the call + /// (CPython's instrumented form reports it), and the receiver passes + /// the basic `supercheck`, the attribute is the MRO walk's. On a hit + /// the three operands leave the stack; `None` leaves them for the + /// general path, which also serves the walk's special cases. + fn load_super_attr_direct( + &mut self, + frame: &mut Frame, + arg: u32, + ) -> Result, RuntimeError> { + if arg & 2 != 0 || crate::trace::any_observers_active() { + return Ok(None); + } + let n = frame.stack.len(); + if n < 3 || !is_super_callable(&frame.stack[n - 3]) { + return Ok(None); + } + let Object::Type(cls) = &frame.stack[n - 2] else { + return Ok(None); + }; + let receiver = &frame.stack[n - 1]; + let obj_type = match receiver { + Object::Type(t) if Rc::ptr_eq(t, cls) || t.is_subclass_of(cls) => t.clone(), + _ => { + let oc = crate::builtins::class_of(receiver); + if !oc.is_subclass_of(cls) { + return Ok(None); + } + oc + } + }; + let (cls, receiver) = (cls.clone(), receiver.clone()); + let code = frame.code.clone(); + let Some(name) = code.names.get((arg >> 2) as usize) else { + return Ok(None); + }; + let Some(attr) = self.super_mro_attr(&obj_type, &receiver, &cls, name)? else { + return Ok(None); + }; + let len = frame.stack.len(); + frame.stack.truncate(len - 3); + Ok(Some(attr)) + } + /// The default `object.__getattribute__` body: data descriptor → instance /// dict → class attr → built-in-subclass payload, ending in /// `AttributeError`. Does **not** consult `__getattr__` (the caller does) @@ -28515,56 +27528,8 @@ impl Interpreter { None }; if let Some((obj_type, receiver, thisclass)) = super_fields { - // Walk `obj_type`'s MRO *after* `thisclass` (CPython's - // `super_getattro`: start one past `su->type` in - // `su->obj_type->tp_mro`). The proxy's own class (`super` or a - // user subclass) is irrelevant to the lookup target. - let found = { - let mro = obj_type.mro.borrow(); - let start = mro - .iter() - .position(|t| Rc::ptr_eq(t, &thisclass)) - .map_or(mro.len(), |i| i + 1); - mro[start..].iter().find_map(|t| { - let v = t.dict.borrow().get(&crate::object::StrKey(name)).cloned()?; - // Introspection-only mirrors in builtin dicts (docs - // surface pass, RFC 0056 WS4) are invisible to super's - // walk too — `super().__repr__()` must fall through to - // the non-virtual `object.__repr__` special case below, - // not the virtual `slot_repr` (asyncio.Lock's repr - // chains through super and would re-enter itself). - if t.flags.is_builtin && crate::descr_registry::is_surface_only(&v) { - return None; - } - Some(v) - }) - }; - if let Some(v) = found { - // CPython passes `su->obj_type` as `owner`, and a NULL - // instance when `su->obj == su->obj_type` — the class-bound - // form `super(C, cls)` inside `__new__`/classmethods — so - // plain functions come back unbound while classmethods - // still bind to the class. - let owner = Object::Type(obj_type.clone()); - let instance_for_get = match &receiver { - Object::Type(r) if Rc::ptr_eq(r, &obj_type) => Object::None, - _ => receiver.clone(), - }; - return self.descriptor_get(&v, &instance_for_get, &owner); - } - // The MRO beyond the starting class reaches a built-in base - // (`dict`, `list`, …) whose methods aren't stored on the type - // dict and so don't surface above. Resolve `name` against the - // receiver's native payload so `super().__setitem__`, - // `super().append`, … dispatch to the wrapped built-in and - // operate on the shared payload. (Only names absent from the - // MRO reach here, so this never shadows a user override.) - if let Object::Instance(recv) = &receiver { - if let Some(native) = recv.native.get() { - if let Ok(v) = self.load_attr(&native.clone(), name) { - return Ok(v); - } - } + if let Some(v) = self.super_mro_attr(&obj_type, &receiver, &thisclass, name)? { + return Ok(v); } // `object.__repr__` / `object.__str__` are identity slots that // WeavePy keeps as native behaviour rather than type-dict @@ -32035,7 +31000,7 @@ impl Interpreter { // literal and `b_list` paths, or a cycle closed through the // result (`x = list(...); x[0].back = x`) is never collected. let obj = Object::new_list(collected); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); Ok(obj) } else { Ok(Object::new_tuple(collected)) @@ -32056,7 +31021,7 @@ impl Interpreter { // match the `{}` literal and `b_dict` paths so a cycle through // the copy is collectable. if let Some(d) = &out { - gc_trace::track(d.clone()); + gc_trace::track(d); } Ok(out) } @@ -32302,6 +31267,13 @@ impl Interpreter { kwargs: &[(String, Object)], globals: &Rc>, ) -> Result, RuntimeError> { + // The common shape is built here (what its `__new__` would build): + // a callable over native containers or iterators, no keywords. + if kwargs.is_empty() { + if let Some(it) = self.seqtools_new(class_name, args, globals) { + return Ok(Some(it)); + } + } let module = match self.do_import("_seqtools", &Object::None, 0, globals) { Ok(m) => m, Err(_) => return Ok(None), @@ -32320,6 +31292,135 @@ impl Interpreter { } } + /// The `_seqtools` classes, importing them on first use. + fn seqtools(&mut self, globals: &Rc>) -> Option<&SeqTools> { + if self.seqtools.get().is_none() { + let module = self.do_import("_seqtools", &Object::None, 0, globals).ok(); + let class = |name: &str| match &module { + Some(Object::Module(m)) => { + match m.dict.borrow().get(&DictKey(Object::from_str(name))) { + Some(Object::Type(t)) => Some(t.clone()), + _ => None, + } + } + _ => None, + }; + let tools = class("_MapIter") + .zip(class("_FilterIter")) + .map(|(map, filter)| SeqTools { map, filter }); + let _ = self.seqtools.set(tools); + } + self.seqtools.get()?.as_ref() + } + + /// A `map(func, *iterables)` or `filter(func, iterable)` instance + /// built without running its class's `__new__`, when every iterable is + /// a list, tuple, string or range (iterated natively) or already an + /// iterator; `None` leaves the call to the class. + fn seqtools_new( + &mut self, + class_name: &str, + args: &[Object], + globals: &Rc>, + ) -> Option { + let filter = match class_name { + "_MapIter" => false, + "_FilterIter" => true, + _ => return None, + }; + let (func, sources) = args.split_first()?; + if sources.is_empty() || (filter && sources.len() != 1) { + return None; + } + let mut iters = Vec::with_capacity(sources.len()); + for s in sources { + iters.push(match s { + Object::Iter(_) | Object::Generator(_) => s.clone(), + Object::List(_) | Object::Tuple(_) | Object::Str(_) | Object::Range(..) => { + Object::Iter(Rc::new(RefCell::new(s.make_iter().ok()?))) + } + _ => return None, + }); + } + let tools = self.seqtools(globals)?; + let cls = if filter { &tools.filter } else { &tools.map }; + let inst = crate::types::PyInstance::new(cls.clone()); + inst.slot_set("_func", func.clone()); + if filter { + inst.slot_set("_it", iters.pop()?); + } else { + inst.slot_set("_iters", Object::new_tuple(iters)); + inst.slot_set("_strict", Object::Bool(false)); + } + let obj = Object::Instance(Rc::new(inst)); + gc_trace::track(&obj); + Some(obj) + } + + /// The next value of an exact `map` or `filter` over one native + /// iterator, stepped here rather than through its `__next__`: + /// `Some(Ok(None))` at exhaustion, `None` when the class's own + /// `__next__` must run. + fn seqtools_step( + &mut self, + it: &Object, + globals: &Rc>, + ) -> Option, RuntimeError>> { + let Object::Instance(inst) = it else { + return None; + }; + let tools = self.seqtools.get()?.as_ref()?; + let cls = inst.cls_raw(); + let filter = if std::ptr::eq(cls, &raw const *tools.map) { + false + } else if std::ptr::eq(cls, &raw const *tools.filter) { + true + } else { + return None; + }; + if crate::trace::any_observers_active() { + return None; + } + let func = inst.slot_get("_func")?; + let source = if filter { + inst.slot_get("_it")? + } else { + match inst.slot_get("_iters")? { + Object::Tuple(t) if t.len() == 1 => t[0].clone(), + _ => return None, + } + }; + let Object::Iter(cursor) = &source else { + return None; + }; + loop { + let item = match cursor.borrow_mut().next_value_checked() { + Ok(Some(v)) => v, + Ok(None) => return Some(Ok(None)), + Err(e) => return Some(Err(e)), + }; + if !filter { + return Some( + self.call(&func, std::slice::from_ref(&item), &[], globals) + .map(Some), + ); + } + let keep = match &func { + Object::None => self.op_truth(&item), + Object::Type(t) if Rc::ptr_eq(t, &builtin_types().bool_) => self.op_truth(&item), + f => match self.call(f, std::slice::from_ref(&item), &[], globals) { + Ok(r) => self.op_truth(&r), + Err(e) => Err(e), + }, + }; + match keep { + Ok(true) => return Some(Ok(Some(item))), + Ok(false) => {} + Err(e) => return Some(Err(e)), + } + } + } + /// `map(func, *iterables)` — VM-aware (the plain builtin can't call /// back into the interpreter). /// @@ -33034,7 +32135,7 @@ impl Interpreter { }; // The temporary result is released before hash() returns, including // an integer subclass with a finalizer. External references survive. - self.maybe_prompt_reap_replaced(result); + self.release(result); match value { Some(value) => Ok(Object::Int(if value == -1 { -2 } else { value })), None => Err(type_error("__hash__ method should return an integer")), @@ -33418,7 +32519,7 @@ impl Interpreter { .find_map(|(k, v)| (k == "key").then(|| v.clone())); self.sort_with_key(&mut items, key_fn.as_ref(), reverse, globals)?; let obj = Object::new_list(items); - gc_trace::track(obj.clone()); + gc_trace::track(&obj); Ok(obj) } @@ -33743,10 +32844,10 @@ impl Interpreter { if reverse { decorated.reverse(); } - // Undecorate, then promptly reap the dying key objects: CPython - // decrefs the keys inside `list.sort` while the list is still - // detached, so a key `__del__` that mutates the list is caught by - // the caller's mutation check (test_sort.test_key_with_mutating_del). + // Undecorate, then release the keys and run their finalizers now: + // CPython decrefs the keys inside `list.sort` while the list is + // still detached, so a key `__del__` that mutates the list is caught + // by the caller's mutation check (test_sort.test_key_with_mutating_del). let mut dead_keys: Vec = Vec::with_capacity(decorated.len()); *items = decorated .into_iter() @@ -33755,18 +32856,8 @@ impl Interpreter { v }) .collect(); - for k in dead_keys { - if matches!( - k, - Object::Instance(_) - | Object::Generator(_) - | Object::Coroutine(_) - | Object::AsyncGenerator(_) - ) && Self::is_refcount_dead(&k, 1) - { - self.reap_dead_subgraph(k); - } - } + drop(dead_keys); + self.run_pending_finalizers(); result } @@ -35188,6 +34279,9 @@ impl Interpreter { Err(e) => Err(e), }, Object::Instance(inst) => { + if let Some(r) = self.seqtools_step(iter, globals) { + return r; + } // A registered native leaf `__next__` (the class cache the // core loop's `FOR_ITER` uses): called directly. if !crate::gil::free_threading_enabled() && !crate::trace::any_observers_active() { @@ -36815,7 +35909,7 @@ impl Interpreter { if line != 0 { py.last_line.set(Some(line)); } - *py.back.borrow_mut() = self.materialize_top_py_frame(); + py.set_back(self.materialize_top_py_frame()); self.push_materialized_frame(&py); let hook_result = self .fire_call_event(&py, crate::trace::EVENT_PY_THROW, exc.instance.clone()) @@ -36892,7 +35986,7 @@ impl Interpreter { frame.py_frame = Some(py.clone()); py }); - *py.back.borrow_mut() = self.materialize_top_py_frame(); + py.set_back(self.materialize_top_py_frame()); self.push_materialized_frame(&py); let exc_obj = match &err { RuntimeError::PyException(pe) => pe.instance.clone(), @@ -36919,21 +36013,7 @@ impl Interpreter { return Ok(Object::None); } } - // Snapshot reapable locals/stack before the frame is torn down: - // CPython's refcounting runs their `__del__` (and clears their - // weakrefs) the moment `close()` drops the frame (gh-142766), - // and tests assert on that promptness. - let frame_caps = match receiver { - Object::Generator(g) | Object::Coroutine(g) | Object::AsyncGenerator(g) => { - frame_reapables(g) - } - _ => Vec::new(), - }; - let result = self.gen_method_close_inner(receiver); - for cap in frame_caps { - self.prompt_reap_dropped(cap); - } - result + self.gen_method_close_inner(receiver) } /// CPython `gen_close`'s fast paths: a never-started generator, or @@ -36986,7 +36066,6 @@ impl Interpreter { else { unreachable!("checked above"); }; - self.reap_dead_frame(&mut boxed); self.recycle_frame_allocs(&mut boxed); Self::release_finished_gen(g); drop(boxed); @@ -37055,13 +36134,12 @@ impl Interpreter { if let (Object::Instance(inst), Object::Frame(pf)) = (&pyexc.instance, &frame_obj) { - let tb = Rc::new(PyTraceback { - frame: pf.clone(), - lineno: pf.last_line.get().unwrap_or(1), - lasti: pf.lasti.get(), - raw_lasti: None, - next: RefCell::new(None), - }); + let tb = Rc::new(PyTraceback::new( + pf.clone(), + pf.last_line.get().unwrap_or(1), + pf.lasti.get(), + None, + )); inst.slot_set("__traceback__", Object::Traceback(tb)); } } @@ -37187,7 +36265,7 @@ impl Interpreter { py }); py.lasti.set(frame.pc); - *py.back.borrow_mut() = self.materialize_top_py_frame(); + py.set_back(self.materialize_top_py_frame()); self.push_materialized_frame(&py); let hook_result = self.fire_monitoring_event(&py, crate::trace::EVENT_EXCEPTION_HANDLED, inst); @@ -37450,11 +36528,6 @@ impl Interpreter { && !frame.sent_consumed { self.gil_countdown = crate::gil::GIL_CHECK_INTERVAL; - if !gc_trace::active_suspects_present() && gc_trace::has_suspects() { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } crate::gil::yield_checkpoint(); if crate::hot_gates::loop_gen() != snap_gen { break; @@ -37470,8 +36543,6 @@ impl Interpreter { fold.map(|sink| (std::ptr::from_mut(frame) as usize, sink)), ); let exc_depth_on_entry = self.exc_info_len(); - let fin_live = gc_trace::has_any_finalizable(); - let fin = fin_live || gc_trace::active_suspects_present(); let mut act = LeanAct { frame: std::ptr::from_mut(frame), shell: None, @@ -37479,24 +36550,10 @@ impl Interpreter { let mut prev_pc: Option = None; let run = if guard.depth() % 4 == 0 { stacker::maybe_grow(512 * 1024, 8 * 1024 * 1024, || { - self.quiet_run( - frame, - QuietShell::Lazy(&mut act), - snap_gen, - fin, - fin_live, - &mut prev_pc, - ) + self.quiet_run(frame, QuietShell::Lazy(&mut act), snap_gen, &mut prev_pc) }) } else { - self.quiet_run( - frame, - QuietShell::Lazy(&mut act), - snap_gen, - fin, - fin_live, - &mut prev_pc, - ) + self.quiet_run(frame, QuietShell::Lazy(&mut act), snap_gen, &mut prev_pc) }; self.sum_fold = prev_fold; match (run, act.shell) { @@ -37547,7 +36604,6 @@ impl Interpreter { Ok(FrameOutcome::Returned(v)) => { *gen.state.borrow_mut() = GeneratorState::Finished; let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(gen); Err(stop_iteration_with(v)) @@ -37562,7 +36618,6 @@ impl Interpreter { *gen.state.borrow_mut() = GeneratorState::Finished; let escaped = self.pep479_escape(gen, err); let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(gen); Err(escaped) @@ -37724,7 +36779,6 @@ impl Interpreter { // "")`. *gen.state.borrow_mut() = GeneratorState::Finished; let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); // RFC 0059 WS4: the exhausted frame's storage is dead — // donate it back to the frame pools. self.recycle_frame_allocs(frame); @@ -37741,7 +36795,6 @@ impl Interpreter { *gen.state.borrow_mut() = GeneratorState::Finished; let escaped = self.pep479_escape(gen, err); let frame: &mut Frame = &mut boxed; - self.reap_dead_frame(frame); self.recycle_frame_allocs(frame); Self::release_finished_gen(gen); Err(escaped) @@ -38557,9 +37610,11 @@ impl Interpreter { // Surrogate-aware printf: a `WStr` template (and any `WStr` // argument resolved via `%s`) bridges its lone surrogates into // the PUA window for the UTF-8 engine, then maps them back. - let template = match a { - Object::WStr(cps) => crate::builtins::bridge_encode_cps(cps), - _ => a.to_str(), + // An exact `str` template is borrowed, not copied per call. + let template: std::borrow::Cow<'_, str> = match a { + Object::Str(s) => std::borrow::Cow::Borrowed(s), + Object::WStr(cps) => crate::builtins::bridge_encode_cps(cps).into(), + _ => a.to_str().into(), }; let bridged = matches!(a, Object::WStr(_)) || percent_args_need_bridge(b); let mut resolve = |obj: &Object, kind: char| -> Result, RuntimeError> { @@ -38847,6 +37902,22 @@ impl Interpreter { if let Some(r) = native_scalar_compare(a, b, op) { return Ok(Object::Bool(r)); } + // Two instances of one class whose comparison dunder is a Python + // function: the forward call runs directly, with no bound method + // and none of the special operand shapes below. A `NotImplemented` + // answer continues the full protocol, past the forward call. + let mut forward_ran = false; + if let (Object::Instance(ia), Object::Instance(ib)) = (a, b) { + if std::ptr::eq(ia.cls_raw(), ib.cls_raw()) { + if let Some(f @ Object::Function(_)) = ia.cls_raw().lookup(cmp_dunder(op).0) { + let r = self.call(&f, &[a.clone(), b.clone()], &[], globals)?; + if !r.is_same(&crate::vm_singletons::not_implemented()) { + return Ok(r); + } + forward_ran = true; + } + } + } // An object-backed `mappingproxy` compares as the wrapped mapping // (CPython `mappingproxy_richcompare` delegates unconditionally). if matches!(a, Object::MappingProxyObj(_)) || matches!(b, Object::MappingProxyObj(_)) { @@ -38996,7 +38067,9 @@ impl Interpreter { Err(e) => return Err(e), } } - if let Some(method) = self.cmp_method(a, dunder, globals) { + if forward_ran { + a_slot_ran = true; + } else if let Some(method) = self.cmp_method(a, dunder, globals) { a_slot_ran = true; let r = self.call(&method, std::slice::from_ref(b), &[], globals)?; if !r.is_same(¬_impl) { @@ -39772,7 +38845,7 @@ impl Interpreter { normalize_index_msg(idx, xs.len(), "list assignment index out of range")?; std::mem::replace(&mut xs[i], value) }; - crate::vm_singletons::queue_container_removed(&evicted); + drop(evicted); specialize::record_hit(op_idx); Ok(true) } @@ -39785,10 +38858,7 @@ impl Interpreter { frame.stack.truncate(len - 2); let value = frame.pop()?; crate::builtins::ensure_dict_key(&key)?; - let old = crate::builtins::dict_insert(&d, key, value)?; - if let Some(old) = old { - crate::vm_singletons::queue_container_removed(&old); - } + drop(crate::builtins::dict_insert(&d, key, value)?); specialize::record_hit(op_idx); Ok(true) } @@ -40336,7 +39406,6 @@ impl Interpreter { // hits; the slot and generic paths (descriptors, `__setattr__`, // `member_set`) release values out of sight and keep the coarse // mark. - gc_trace::note_dropped(&receiver); match cache { IC::StoreAttrInstance { key_idx, ver } => { if let Object::Instance(inst) = &receiver { @@ -40359,7 +39428,7 @@ impl Interpreter { // on the instance must join the cycle collector // (see `generic_setattr_instance`). if matches!(val, Object::BoundMethod(_)) { - gc_trace::track(val.clone()); + gc_trace::track(&val); } // The slot still exists; reach in by index and // overwrite, recovering the displaced value. We @@ -40378,7 +39447,7 @@ impl Interpreter { specialize::record_hit(op_idx); // CPython decrefs the overwritten value now; // reap it if this was its last live binding. - self.maybe_prompt_reap_replaced(old); + self.release(old); return Ok(()); } } @@ -40412,7 +39481,7 @@ impl Interpreter { // method escaping into an instance attribute // joins the cycle collector. if matches!(val, Object::BoundMethod(_)) { - gc_trace::track(val.clone()); + gc_trace::track(&val); } let watch_value = if crate::capi_watchers::dicts_active() { Some(val.clone()) @@ -40424,6 +39493,9 @@ impl Interpreter { // (the core loop's `STORE_ATTR` arm serves it). let mut upgrade_idx = false; let mut val = Some(val); + // A value whose store met a stored key only a + // user `__eq__` could equate to the name. + let mut exotic = None; let old = 'store: { // The split layout (see `core_store_new_attr`). if watch_value.is_none() && inst.dict.published().is_none() { @@ -40443,12 +39515,22 @@ impl Interpreter { } else { &mut **dict }; - let probe = code_name_key(code, name_idx); - let slot = match probe { - Some(k) => dict.get_mut(&k), - None => dict.get_mut(&crate::object::StrKey(name.as_str())), + // Never run Python under the borrow: the + // generic store compares an exotic key once, + // with nothing borrowed. + let probe = match code_name_key(code, name_idx) { + Some(k) => crate::object::LeafNameProbe::new(k.s, k.hash), + None => crate::object::LeafNameProbe::new( + name.as_str(), + crate::object::py_str_hash(name.as_str()), + ), }; - if let Some(slot) = slot { + let idx = dict.get_index_of(&probe); + if probe.saw_exotic() { + exotic = Some(val); + break 'store None; + } + if let Some((_, slot)) = idx.and_then(|i| dict.get_index_mut(i)) { let old = std::mem::replace(slot, val); upgrade_idx = true; Some(old) @@ -40465,6 +39547,11 @@ impl Interpreter { None } }; + if let Some(val) = exotic { + frame.push(val); + return self + .deopt_store_attr_taken(frame, cache_pc, name_idx, receiver); + } // Watched instance `__dict__` (test_watchers // test_object_dict): same events as the slow path. if let Some(v) = watch_value { @@ -40478,11 +39565,16 @@ impl Interpreter { } specialize::record_hit(op_idx); if upgrade_idx { - use crate::specialize::DictDataExt; + let probe = crate::object::LeafNameProbe::new( + name.as_str(), + crate::object::py_str_hash(name.as_str()), + ); let idx = inst .dict_cell() .borrow() - .index_of_key_str(name.as_str()) + .get_index_of(&probe) + .filter(|_| !probe.saw_exotic()) + .and_then(|i| u32::try_from(i).ok()) .map(|i| (i, inst.class.borrow().attr_version.get())); if let Some((key_idx, ver)) = idx { frame @@ -40492,7 +39584,7 @@ impl Interpreter { } } if let Some(old) = old { - self.maybe_prompt_reap_replaced(old); + self.release(old); } return Ok(()); } @@ -40534,7 +39626,6 @@ impl Interpreter { } }; drop(old); - gc_trace::mark_maybe_dead(); specialize::record_hit(op_idx); return Ok(()); } @@ -40553,7 +39644,6 @@ impl Interpreter { specialize::record_specialize_success(op_idx); } let val = frame.pop()?; - gc_trace::mark_maybe_dead(); self.store_attr(&receiver, &name, val) } IC::Cooldown(n) => { @@ -40565,13 +39655,11 @@ impl Interpreter { frame.code.caches.set(cache_pc, next); let val = frame.pop()?; let name = self.name_at(&frame.code, name_idx)?; - gc_trace::mark_maybe_dead(); self.store_attr(&receiver, &name, val) } _ => { let val = frame.pop()?; let name = self.name_at(&frame.code, name_idx)?; - gc_trace::mark_maybe_dead(); self.store_attr(&receiver, &name, val) } } @@ -40594,7 +39682,6 @@ impl Interpreter { .set(cache_pc, weavepy_compiler::InlineCache::Cooldown(COOLDOWN)); let val = frame.pop()?; let name = self.name_at(&frame.code, name_idx)?; - gc_trace::mark_maybe_dead(); self.store_attr(&receiver, &name, val) } @@ -40624,12 +39711,7 @@ impl Interpreter { match cache { IC::ForIterList => { let mut it = it_handle.borrow_mut(); - if let crate::object::PyIterator::List { - items, - index, - owner, - } = &mut *it - { + if let crate::object::PyIterator::List { items, index, .. } = &mut *it { let next = items.borrow().get(*index).cloned(); if let Some(v) = next { *index += 1; @@ -40643,24 +39725,11 @@ impl Interpreter { // generic path detaches but this fast path leaks a // live cursor (test_list.test_exhausted_iterator, // test_tier2_invalidates_iterator). - // - // A uniquely-held backing list (the loop iterated a - // temporary) has its elements harvested for the - // prompt-reap cascade — CPython frees them by - // refcount here (see the generic FOR_ITER arm). - let harvested = if owner.is_none() && Self::iter_backing_list_dead(items) { - Some(std::mem::take(&mut *items.borrow_mut())) - } else { - None - }; *items = Rc::new(RefCell::new(Vec::new())); drop(it); frame.pop()?; frame.pc += jump_arg; frame.skip_end_for(); - if let Some(items) = harvested { - self.reap_iter_backing(items); - } } specialize::record_hit(op_idx); return Ok(true); @@ -41655,7 +40724,7 @@ impl Interpreter { // close their files) the instant the log does // (test_tempfile.test_warnings_on_cleanup). if let Some(old) = old { - self.maybe_prompt_reap_replaced(old); + self.release(old); } Ok(()) } @@ -41731,11 +40800,11 @@ impl Interpreter { // stored dict; either way the deferred `__annotate__` // is dropped so the next read doesn't recompute. "__annotations__" => { - f.slots + f.slots() .borrow_mut() .shift_remove(&DictKey(Object::from_static("__annotate__"))); if matches!(value, Object::None) { - f.slots + f.slots() .borrow_mut() .shift_remove(&DictKey(Object::from_static("__annotations__"))); return Ok(()); @@ -41754,7 +40823,7 @@ impl Interpreter { if !crate::builtins::object_is_callable(&value) { return Err(type_error("__annotate__ must be callable or None")); } - f.slots + f.slots() .borrow_mut() .shift_remove(&DictKey(Object::from_static("__annotations__"))); f.set_slot(name, value); @@ -41950,7 +41019,7 @@ impl Interpreter { // is reclaimable and the drop-time "unclosed file" // `ResourceWarning` still fires // (test_io `test_garbage_collection`). - gc_trace::track(obj.clone()); + gc_trace::track(obj); Ok(()) } }, @@ -42427,7 +41496,7 @@ impl Interpreter { // instance attribute so the cycle collector can reclaim them // (CPython's `method` type is always GC-tracked — `test_method`). if matches!(value, Object::BoundMethod(_)) { - gc_trace::track(value.clone()); + gc_trace::track(&value); } // `insert` returns the value this store displaces; reap it if the // slot held its last live binding, mirroring CPython's decref of the @@ -42478,7 +41547,7 @@ impl Interpreter { ); } if let Some(old) = old { - self.maybe_prompt_reap_replaced(old); + self.release(old); } Ok(()) } @@ -42578,7 +41647,7 @@ impl Interpreter { .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); if let Some(old) = removed { - self.maybe_prompt_reap_replaced(old); + self.release(old); Ok(()) } else { Err(attribute_error(format!( @@ -42614,7 +41683,7 @@ impl Interpreter { if let Some(old) = removed { // Same decref-at-removal semantics as the store arm // above / `DeleteGlobal`. - self.maybe_prompt_reap_replaced(old); + self.release(old); Ok(()) } else { Err(attribute_error(format!( @@ -42653,7 +41722,7 @@ impl Interpreter { // `del f.__annotations__` clears both the dict and the // deferred `__annotate__` (the setter with NULL). "__annotations__" => { - let mut slots = f.slots.borrow_mut(); + let mut slots = f.slots().borrow_mut(); slots.shift_remove(&DictKey(Object::from_static("__annotations__"))); slots.shift_remove(&DictKey(Object::from_static("__annotate__"))); return Ok(()); @@ -42673,7 +41742,7 @@ impl Interpreter { // CPython allows deleting the nullable slots // (`__doc__`, `__annotations__`, …): the value // resets so the computed default resurfaces. - f.slots + f.slots() .borrow_mut() .shift_remove(&DictKey(Object::from_str(name))); return Ok(()); @@ -42903,7 +41972,7 @@ impl Interpreter { // `del self._target, self._args, self._kwargs` releases the // thread's hold on its argument cycle without waiting for a GC // pass (`test_threading.test_no_refcycle_through_target`). - self.prompt_reap_dropped(removed); + self.release(removed); Ok(()) } @@ -43448,7 +42517,7 @@ impl Interpreter { normalize_index_msg(*i, items.len(), "list assignment index out of range")?; std::mem::replace(&mut items[idx], value) }; - crate::vm_singletons::queue_container_removed(&evicted); + drop(evicted); Ok(()) } (Object::List(items), Object::Slice(s)) => { @@ -43471,9 +42540,7 @@ impl Interpreter { let mut data = items.borrow_mut(); apply_slice_assignment(&mut data, s, replacement)? }; - for old in &evicted { - crate::vm_singletons::queue_container_removed(old); - } + drop(evicted); // The RHS was only read from — its *elements* were spliced in, // and the RHS container itself is discarded right here. A // literal RHS (`xs[-2:] = [obj]`) is GC-tracked the moment it @@ -43487,17 +42554,12 @@ impl Interpreter { // gh-106558). Elements stay alive through the target list; // an RHS still bound elsewhere fails the deadness test // untouched. - if Self::local_needs_prompt_reap(&value) && Self::looks_reapable_temporary(&value) { - self.prompt_reap_dropped(value); - } + self.release(value); Ok(()) } (Object::Dict(d), key) => { crate::builtins::ensure_dict_key(key)?; - let old = crate::builtins::dict_insert(d, key.clone(), value)?; - if let Some(old) = old { - crate::vm_singletons::queue_container_removed(&old); - } + drop(crate::builtins::dict_insert(d, key.clone(), value)?); Ok(()) } (Object::ByteArray(b), Object::Int(i)) => { @@ -43889,24 +42951,20 @@ impl Interpreter { normalize_index_msg(*i, items.len(), "list assignment index out of range")?; items.remove(idx) }; - crate::vm_singletons::queue_container_removed(&removed); + drop(removed); Ok(()) } (Object::List(items), Object::Slice(s)) => { let removed = apply_slice_deletion(&mut items.borrow_mut(), s)?; - for v in &removed { - crate::vm_singletons::queue_container_removed(v); - } + drop(removed); Ok(()) } (Object::Dict(d), key) => { crate::builtins::ensure_dict_key(key)?; let removed = crate::builtins::dict_remove(d, key)?; - let Some((k, v)) = removed else { + if removed.is_none() { return Err(key_error_object(key.clone())); - }; - crate::vm_singletons::queue_container_removed(&k); - crate::vm_singletons::queue_container_removed(&v); + } Ok(()) } (Object::ByteArray(b), Object::Int(i)) => { @@ -44603,27 +43661,6 @@ impl Interpreter { // drain the queue synchronously, matching CPython // semantics where `gc.collect()` returns *after* // every finaliser has fired. - // Heap introspection must observe CPython's *refcount* - // timing, not our deferred one: an object whose last - // program reference died a few instructions ago may still - // sit on the suspect list (its cascade was skipped while a - // Rust-side transient pinned it — see `note_suspect`). - // CPython would have freed it already, so - // `gc.get_objects()` / `gc.get_referrers()` must not - // enumerate it — worse, the returned snapshot list would - // *pin* the corpse through the caller's inspection loop, - // decaying its probe budget until it is evicted and leaks - // for good (test_taskgroups.test_exception_refcycles_* - // call `gc.get_referrers` right after an `except*` block - // and assert the handled group's ctor list is gone). - // Settle the pending reaps before taking the snapshot. - if matches!(b.name, ".gc.get_objects" | ".gc.get_referrers") - && gc_trace::has_suspects() - { - for obj in gc_trace::take_dead_suspects() { - self.reap_dead_subgraph(obj); - } - } if b.name == ".gc.collect" { // `generation` may be positional or the `generation=` // keyword (CPython accepts both). Validate it the way @@ -45909,7 +44946,8 @@ impl Interpreter { kw_defaults, closure, attrs: RefCell::new(None), - slots: RefCell::new(DictData::default()), + slots_raw: RefCell::new(DictData::default()), + slot_seed: RefCell::new(None), closure_cells: std::sync::OnceLock::new(), defaults_override: crate::object::OverrideFlag::new(false), }))) @@ -45926,10 +44964,33 @@ impl Interpreter { value: Object, globals: &Rc>, ) -> Result<(), RuntimeError> { - let key = Object::from_str(name); - if let Some(method) = instance_method(ns_obj, "__setitem__") { - self.call(&method, &[key, value], &[], globals)?; - return Ok(()); + self.class_ns_store_key(ns_obj, Object::from_str(name), value, globals) + } + + /// [`Self::class_ns_store`] with the key already an object (the code + /// object's interned name, whose hash is cached, like the `co_names` + /// entry CPython's `STORE_NAME` passes). + fn class_ns_store_key( + &mut self, + ns_obj: &Object, + key: Object, + value: Object, + globals: &Rc>, + ) -> Result<(), RuntimeError> { + if let Object::Instance(inst) = ns_obj { + if let Some(m) = inst.cls().lookup("__setitem__") { + // A plain function takes the namespace as its first + // argument, with no bound method to build; anything else + // binds at call time (a descriptor's `__get__` runs then). + if matches!(m, Object::Function(_)) { + self.call(&m, &[ns_obj.clone(), key, value], &[], globals)?; + } else { + let method = + Object::BoundMethod(Rc::new(BoundMethod::dispatch(ns_obj.clone(), m))); + self.call(&method, &[key, value], &[], globals)?; + } + return Ok(()); + } } self.store_subscr(ns_obj, &key, value, globals) } @@ -45978,7 +45039,7 @@ impl Interpreter { name: &str, globals: &Rc>, ) -> Result, RuntimeError> { - let key = DictKey(Object::from_str(name)); + let key = crate::object::StrKey(name); match ns_obj { Object::Dict(d) => Ok(d.borrow().get(&key).cloned()), Object::MappingProxy(d) => Ok(d.borrow().get(&key).cloned()), @@ -47815,13 +46876,18 @@ impl Interpreter { } if let Object::Instance(inst) = &value { if let Some(hook) = inst.cls().lookup("__set_name__") { - let bound = Object::BoundMethod(Rc::new(BoundMethod::new(value.clone(), hook))); - let res = self.call( - &bound, - &[Object::Type(ty.clone()), Object::from_str(&attr_name)], - &[], - &self.builtins.clone(), - ); + let owner = Object::Type(ty.clone()); + let name = Object::from_str(&attr_name); + let builtins = self.builtins.clone(); + // A plain function takes the descriptor as its first + // argument, with no bound method to build. + let res = if matches!(hook, Object::Function(_)) { + self.call(&hook, &[value.clone(), owner, name], &[], &builtins) + } else { + let bound = + Object::BoundMethod(Rc::new(BoundMethod::new(value.clone(), hook))); + self.call(&bound, &[owner, name], &[], &builtins) + }; // PEP 678 / CPython `type.__new__`: a failing // `__set_name__` is re-raised with a note naming the // descriptor, attribute, and owning class so the @@ -48083,7 +47149,7 @@ impl Interpreter { ) } else { let inst = Object::Instance(Rc::new(PyInstance::new(cls.clone()))); - gc_trace::track(inst.clone()); + gc_trace::track(&inst); (inst, true) } } @@ -48573,17 +47639,18 @@ impl Interpreter { "TracebackType() lasti/lineno must be ints".to_owned(), )); }; - return Ok(Object::Traceback(Rc::new(crate::object::PyTraceback { - frame: frame.clone(), - lineno: u32::try_from(*lineno).unwrap_or(0), - lasti: u32::try_from(*lasti).unwrap_or(0), + let tb = crate::object::PyTraceback::new( + frame.clone(), + u32::try_from(*lineno).unwrap_or(0), + u32::try_from(*lasti).unwrap_or(0), // Explicit construction stores the given lasti // verbatim (test_raise TestTracebackType): it's // already a CPython-style byte offset, not a // WeavePy instruction index to remap on read. - raw_lasti: Some(*lasti), - next: RefCell::new(next), - }))); + Some(*lasti), + ); + *tb.next.borrow_mut() = next; + return Ok(Object::Traceback(Rc::new(tb))); } // `types.MethodType(func, obj)` — bind `func` to `obj`, // producing a callable bound method (CPython `method`). @@ -48711,7 +47778,7 @@ impl Interpreter { })?; } let out = Object::Dict(Rc::new(RefCell::new(d))); - crate::gc_trace::track(out.clone()); + crate::gc_trace::track(&out); return Ok(out); } } @@ -48758,7 +47825,7 @@ impl Interpreter { })?; } let out = Object::Dict(Rc::new(RefCell::new(d))); - crate::gc_trace::track(out.clone()); + crate::gc_trace::track(&out); return Ok(out); } // `int(x)` / `float(x)` honour the user's `__int__` / @@ -49003,7 +48070,7 @@ impl Interpreter { // type whose `tp_traverse` is non-NULL — for us that's // every Python-defined class (they all carry a dict). if !deferred { - gc_trace::track(inst.clone()); + gc_trace::track(&inst); } // CPython's allocator runs a threshold-driven young // collection right here; without it the tracked set @@ -49206,6 +48273,23 @@ impl Interpreter { // class against its attribute version (the MRO walk compared // nine names per entry on every construction). let families = exc_family_flags(&cls); + // The common case (`ValueError("x")`, a plain user subclass): no + // family shapes construction, so `args` is the only slot, built + // in place without the per-slot probes below. + if families & !EXC_CUSTOM_INIT == 0 { + let inst = PyInstance::new(cls); + let args_tuple = Object::new_tuple(args.to_vec()); + inst.note_slot_store(&args_tuple); + let mut entries = Vec::with_capacity(4); + entries.push((crate::types::slot_key("args"), args_tuple)); + *inst.slots.borrow_mut() = crate::types::SlotStorage::from_entries(entries); + let obj = Object::Instance(Rc::new(inst)); + // See the GC note at the end. + if args.iter().any(|x| !crate::gc_trace::is_atomic(x)) { + crate::gc_trace::track(&obj); + } + return obj; + } let mro_has = |name: &str| families & exc_family_bit(name) != 0; let is_stop_iteration = mro_has("StopIteration"); let inst = PyInstance::new(cls.clone()); @@ -49224,10 +48308,9 @@ impl Interpreter { } else { Object::new_tuple(args.to_vec()) }; + // (No `message` mirror: CPython has none, and `str()` and the Rust + // display derive the text from `args`.) inst.slot_set("args", args_tuple); - if let Some(first) = args.first() { - inst.slot_set("message", first.clone()); - } // PEP 380: `StopIteration.value` is the first constructor arg // (or None). Generator `return` goes through // `stop_iteration_with`, but user code constructs @@ -49361,7 +48444,7 @@ impl Interpreter { // `KeyError('k')`, every internal `raise`) stays off the GC books, // matching the container-untracking fast path used for list/dict/set. if exception_holds_nonatomic(&obj) { - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); } obj } @@ -49396,9 +48479,40 @@ impl Interpreter { args: &[Object], kwargs: &[(String, Object)], ) -> Result { + if kwargs.is_empty() { + if let Some(v) = self.call_pure_leaf(f, args) { + return Ok(v); + } + } self.call_python_owned(f, args.to_vec(), kwargs.to_vec()) } + /// A native caller's positional call of a pure leaf (a `key=` + /// function, a `map` callable, ...), evaluated in place as the core + /// loop evaluates one (see [`Self::pure_leaf_eval`]): `None` leaves the + /// call to the full machinery, nothing having happened. + #[inline] + fn call_pure_leaf(&self, f: &PyFunction, args: &[Object]) -> Option { + const MAX_ARGS: usize = 6; + // SAFETY: a function's code is only replaced under the GIL, and + // nothing here runs code that could replace it. + let code = unsafe { &*f.code.as_ptr() }; + if args.len() > MAX_ARGS + || !code_is_pure_leaf(code) + || !pure_leaf_warm(code) + || crate::trace::any_observers_active() + || !self.inline_calls_ok() + || crate::recursion::current_depth() >= crate::recursion::recursion_limit() + { + return None; + } + let mut ptrs = [std::ptr::null::(); MAX_ARGS]; + for (p, a) in ptrs.iter_mut().zip(args) { + *p = a; + } + self.pure_leaf_eval::(code, f, &ptrs[..args.len()]) + } + /// The full argument binder for [`Self::call_python_owned`]: CPython's /// `initialize_locals` order (positional move, keyword loop with its /// unexpected-keyword / multiple-values errors, too-many-positional, @@ -49859,7 +48973,7 @@ impl Interpreter { // collection). let mut positional = positional; for v in positional.drain(..) { - self.maybe_prompt_reap_replaced(v); + self.release(v); } self.recycle_scratch(positional); return result; @@ -49939,7 +49053,7 @@ impl Interpreter { // The pinned slot objects are shared, as CPython's // `gi_name` shares the function's `__name__`. let (gen_name, gen_qualname) = { - let slots = f.slots.borrow(); + let slots = f.slots().borrow(); let attr_str = |attr: &'static str| match slots.get(&crate::object::StrKey(attr)) { Some(o @ Object::Str(_)) => Some(o.clone()), @@ -49974,7 +49088,7 @@ impl Interpreter { // RFC 0024: generator frames can participate in // reference cycles (a local that holds the generator // itself), so track them like instances. - gc_trace::track(obj.clone()); + gc_trace::track(&obj); // Threshold-driven young collection at the // allocation site, as in the instance path. if gc_trace::maybe_auto_collect() { @@ -50038,7 +49152,7 @@ impl Interpreter { } else { Object::Generator(gen) }; - gc_trace::track(obj.clone()); + gc_trace::track(&obj); if gc_trace::maybe_auto_collect() { self.run_pending_finalizers(); } @@ -50176,7 +49290,7 @@ impl Interpreter { )?; } frame.push(r); - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut args); return Ok(()); } @@ -50527,7 +49641,7 @@ impl Interpreter { // Shared tail (skipped via early `?` only on error): reap the // bound-method receiver and any dead argument temporaries now that // the call has returned and the operands are about to be dropped. - self.reap_call_receiver(callable); + self.release(callable); self.reap_call_args(&mut args); self.recycle_scratch(args); Ok(()) @@ -57288,7 +56402,8 @@ pub(crate) fn str_subscript_slice(s: &SharedStr, slc: &PySlice) -> Result Result= bstop { return Ok(Object::from_static("")); } - return Ok(Object::from_str(s[bstart..bstop].to_string())); + // The whole string is the string itself (CPython's + // `PyUnicode_Substring` returns an exact `str` unchanged). + if bstop - bstart == blen { + return Ok(Object::Str(s.clone())); + } + let piece = &s[bstart..bstop]; + return Ok(Object::Str(if ascii { + SharedStr::from_ascii(piece) + } else { + SharedStr::from(piece) + })); } } // General path: negative bounds or non-unit step, over the code points. @@ -57777,54 +56902,6 @@ fn normalize_bool_index(n: Object) -> Object { } } -/// Bit 0 of [`COARSE_DROP_CLASS`]: the opcode's handler grades every -/// reference it releases through `gc_trace::note_dropped` and is exempt -/// from the eval loop's coarse stack-shrink mark. -const COARSE_EXEMPT: u8 = 1; -/// Bit 1 of [`COARSE_DROP_CLASS`]: a stack-neutral `DELETE_*` opcode that -/// releases a binding and counts as a drop regardless of stack depth. -const COARSE_DELETE: u8 = 2; - -/// Per-opcode classification for the coarse maybe-dead mark in -/// `run_until_yield_or_return_impl` (RFC 0077 WS3): one byte load -/// replaces two `Option` match chains on every instruction. -static COARSE_DROP_CLASS: [u8; 256] = { - let mut t = [0u8; 256]; - let exempt = [ - OpCode::PopTop, - OpCode::StoreFast, - OpCode::BinaryOp, - OpCode::CompareOp, - OpCode::Call, - OpCode::ReturnValue, - OpCode::StoreAttr, - OpCode::PopJumpIfFalse, - OpCode::PopJumpIfTrue, - OpCode::PopJumpIfNone, - OpCode::PopJumpIfNotNone, - OpCode::IsOp, - // The yielded value moves to the consumer; nothing is released. - OpCode::YieldValue, - ]; - let mut i = 0; - while i < exempt.len() { - t[exempt[i] as u8 as usize] |= COARSE_EXEMPT; - i += 1; - } - let deletes = [ - OpCode::DeleteFast, - OpCode::DeleteName, - OpCode::DeleteGlobal, - OpCode::DeleteDeref, - ]; - let mut i = 0; - while i < deletes.len() { - t[deletes[i] as u8 as usize] |= COARSE_DELETE; - i += 1; - } - t -}; - /// What [`Interpreter::leaf_resolve_instance_attr`] found. enum LeafAttr { /// A plain value: the instance's own, or a non-descriptor class value. @@ -57857,6 +56934,8 @@ enum LeafKind { ObjectNew, Len, Isinstance, + /// `getattr` (see `Interpreter::leaf_getattr`). + GetAttr, ListAppend, ListPop, ListInsert, @@ -58092,6 +57171,9 @@ struct InlineAct { /// the activation runs its `__init__`, and the caller receives the /// instance instead of the (`None`) result. init_inst: Option, + /// The frame's `cells` handle is the activation's own (a callee with + /// cell variables, see [`fresh_cells`]), not borrowed from the callee. + owns_cells: bool, } impl InlineAct { @@ -58140,6 +57222,7 @@ impl InlineAct { gen_frame: std::ptr::null_mut(), exhaust_arg: 0, init_inst: None, + owns_cells: false, }); act.act.frame = std::ptr::from_mut::(&mut act.frame); // SAFETY: parked slots hold a stale code copy (see the type docs): @@ -58156,6 +57239,11 @@ unsafe impl Send for InlineAct {} impl Drop for InlineAct { fn drop(&mut self) { + if self.owns_cells { + // SAFETY: owned cells are released once (the frame's own drop + // forgets the handle). + unsafe { drop(std::ptr::read(&raw const self.frame.cells)) }; + } if self.parked { // SAFETY: the stale copies are overwritten without a drop, so // the frame's and callable's own drops release valid values. @@ -58232,7 +57320,7 @@ struct SwitchRoots { root: *mut Frame, root_shell: *mut QuietShell<'static>, root_last: *mut usize, - fin: bool, + /// This thread's pending-work flag (see `gc_trace::maybe_dead_flag`). maybe_dead: *const std::cell::Cell, } @@ -58269,7 +57357,6 @@ struct CoreSwitch { /// The last-executed-pc slot of inline activations other than the /// entry (never read back). scratch: usize, - fin: bool, maybe_dead: *const std::cell::Cell, /// This thread's recursion-depth cell (see `recursion::depth_cell`). depth_cell: *const std::cell::Cell, @@ -58303,7 +57390,6 @@ impl CoreSwitch { entry_dead: false, pending: None, scratch: usize::MAX, - fin: roots.fin, maybe_dead: roots.maybe_dead, depth_cell: crate::recursion::depth_cell(), } @@ -58541,7 +57627,9 @@ struct LeanKwShape { /// handles are *borrowed* from the callee function — which the caller /// keeps alive for the whole activation, and whose handles are /// immutable (a lean frame has no cell variables, so nothing rebinds -/// `cells`) — so building one costs no reference-count traffic. The +/// `cells`; an inline activation with them owns its `cells`, see +/// `InlineAct::owns_cells`) — so building one costs no reference-count +/// traffic. The /// borrowed handles are forgotten, never dropped; the rest is released /// on drop, or recycled by [`Interpreter::retire_lean_frame`]. Anything /// that keeps one of these handles beyond the activation (a shell or a @@ -58603,6 +57691,88 @@ impl Drop for LeanFrame { } } +/// Whether `f`'s closure is exactly cells for `code`'s free variables +/// (what [`fresh_cells`] appends). +#[inline] +fn closure_cells_ok(f: &PyFunction, code: &CodeObject) -> bool { + f.closure.len() == code.freevars.len() && f.closure.iter().all(|c| matches!(c, Object::Cell(_))) +} + +/// The cells of a call of `f` running `code`, which has cell variables, +/// as `make_frame` builds them: a fresh cell per cell variable, holding +/// the bound local of that name (moved out of `locals`) or nothing, then +/// `f`'s closure cells (checked by [`closure_cells_ok`]). +fn fresh_cells( + f: &PyFunction, + code: &CodeObject, + locals: &mut [Object], +) -> Rc>>> { + let mut cells: Vec>> = + Vec::with_capacity(code.cellvars.len() + code.freevars.len()); + for name in &code.cellvars { + let initial = code + .varnames + .iter() + .position(|n| n == name) + .and_then(|i| locals.get_mut(i)) + .map_or(Object::Unbound, |slot| { + std::mem::replace(slot, Object::Unbound) + }); + cells.push(Rc::new(RefCell::new(initial))); + } + if crate::stdlib::tracemalloc_real::is_tracking() { + crate::stdlib::tracemalloc_real::track_new_cells(&cells); + } + cells.extend(f.closure.iter().filter_map(|c| match c { + Object::Cell(c) => Some(c.clone()), + _ => None, + })); + Rc::new(cells) +} + +/// `v.push(x)` with the room check in line (the call paths push one +/// activation, slot or result per call, and the out-of-line `push` showed +/// up in their profiles). +#[inline(always)] +fn push_fast(v: &mut Vec, x: T) { + let n = v.len(); + if n < v.capacity() { + // SAFETY: `n < capacity`: the slot is allocated and unused. + unsafe { + v.as_mut_ptr().add(n).write(x); + v.set_len(n + 1); + } + } else { + push_grow(v, x); + } +} + +#[cold] +#[inline(never)] +fn push_grow(v: &mut Vec, x: T) { + v.push(x); +} + +/// Empty a finished activation's `locals`, releasing each value (an +/// object that dies queues its finalizer, as any release does). +/// +/// # Safety +/// +/// Nothing else borrows `locals`. +#[inline(always)] +unsafe fn release_locals(locals: &mut Vec) { + let n = locals.len(); + let base = locals.as_ptr(); + // The vector forgets its values first: each is read out exactly once. + // SAFETY: the first `n` values are initialized. + unsafe { + locals.set_len(0); + for k in 0..n { + drop_hot(base.add(k).read()); + } + } +} + /// An `Rc` sharing `r`'s allocation *without* a count of its own: it must /// be forgotten, never dropped, and must not outlive `r`'s owner. #[inline] @@ -58619,9 +57789,50 @@ enum QuietShell<'a> { Lazy(&'a mut LeanAct), } +thread_local! { + /// The `**kwargs` dictionary of the last keyword leaf call, emptied, + /// when nothing kept it (see [`recycle_kw_dict`]). + static SPARE_KW_DICT: std::cell::Cell>>> = + const { std::cell::Cell::new(None) }; +} + +/// An empty `**kwargs` dictionary with room for `n` keys: the spare one, +/// if any, else a fresh one. +fn spare_kw_dict(n: usize) -> Rc> { + SPARE_KW_DICT + .try_with(std::cell::Cell::take) + .ok() + .flatten() + .unwrap_or_else(|| { + Rc::new(RefCell::new(DictData::with_capacity_and_hasher( + n, + crate::fasthash::FxBuildHasher, + ))) + }) +} + +/// Keep a keyword leaf call's `**kwargs` dictionary for the next call +/// when nothing else holds it. A call per iteration otherwise allocates +/// and frees the same blocks, and when they are alone on their allocator +/// page the page is released and fetched again every time (on Windows +/// that doubled `call_overhead`). Clearing restamps the dictionary, so +/// the next call sees a new one. +fn recycle_kw_dict(mut d: Rc>) { + const MAX_KEPT_KEYS: usize = 16; + let Some(cell) = Rc::get_mut(&mut d) else { + return; + }; + let data = cell.get_mut(); + if data.capacity() > MAX_KEPT_KEYS { + return; + } + data.clear(); + let _ = SPARE_KW_DICT.try_with(|spare| spare.set(Some(d))); +} + #[cfg(test)] thread_local! { - static NATIVE_FAST_SUBSCRIPTS: std::cell::Cell = const { std::cell::Cell::new(0) }; + pub(crate) static NATIVE_FAST_SUBSCRIPTS: std::cell::Cell = const { std::cell::Cell::new(0) }; static LEAF_SET_REMOVALS: std::cell::Cell<[u64; 2]> = const { std::cell::Cell::new([0; 2]) }; static SAVED_NATIVE_LEAF_CALLS: std::cell::Cell = const { std::cell::Cell::new(0) }; static SAVED_NATIVE_REJECTED_CALLS: std::cell::Cell = const { std::cell::Cell::new(0) }; @@ -58631,6 +57842,8 @@ thread_local! { static SCALAR_FIELD_UPDATE_BOXED_RETURNS: std::cell::Cell = const { std::cell::Cell::new(0) }; #[cfg(feature = "jit")] static SCALAR_FIELD_UPDATE_NATIVE_CALLS: std::cell::Cell = const { std::cell::Cell::new(0) }; + #[cfg(feature = "jit")] + static SCALAR_FIELD_UPDATE_ARMED: std::cell::Cell = const { std::cell::Cell::new(0) }; static PURE_LITERAL_ARGUMENT_CALLS: std::cell::Cell<[u64; 4]> = const { std::cell::Cell::new([0; 4]) }; static PURE_SLOT_FIELD_READS: std::cell::Cell<[u64; 3]> = const { std::cell::Cell::new([0; 3]) }; @@ -58736,6 +57949,35 @@ pub(crate) mod leaf_builtins { GENERATION.load(Ordering::Acquire) } + /// Builtins opted into the JIT's native method lane (see + /// `tier2::MethodCallee::Native`), each with its direct operation. + static JIT_METHODS: parking_lot::Mutex, u8)>> = + parking_lot::Mutex::new(Vec::new()); + + /// Opt `b` (already registered as a leaf, binding its instance) into + /// the JIT's native method lane: a compiled caller may call its body + /// directly on a receiver whose class resolves to it, with no + /// interpreter frame. The body must also be safe to call with a + /// receiver that is a subclass instance or any other object (it + /// validates its receiver). A nonzero `op` names the operation's + /// direct form (`collections_native::fast_op`), which the caller + /// tries first. + pub(crate) fn register_jit_method(b: &Rc, op: u8) { + let mut reg = JIT_METHODS.lock(); + reg.retain(|(w, _)| w.strong_count() > 0); + reg.push((Rc::downgrade(b), op)); + } + + /// `b`'s direct operation when it opted into the JIT's native method + /// lane (by identity; probed at compile time only), `0` for none. + pub(crate) fn jit_method_op(b: &Rc) -> Option { + JIT_METHODS + .lock() + .iter() + .find(|(w, _)| std::ptr::eq(w.as_ptr(), Rc::as_ptr(b)) && w.strong_count() > 0) + .map(|&(_, op)| op) + } + /// The live registrations, keyed by address. pub(crate) fn snapshot() -> LeafMap { REGISTRY @@ -58978,7 +58220,8 @@ enum LeafStop { /// A back edge found the GIL countdown spent or the loop generation /// changed: the outer prologue must run. Breaker, - /// The last leaf instruction's discard scheduled a finalization sweep. + /// A dying object queued work (a finalizer, a weakref callback) for + /// the next safe point. Marked, /// A leaf builtin raised; `frame.pc` is past the call. Raised(RuntimeError), @@ -59006,14 +58249,17 @@ enum CoreExit { /// (Inside [`Interpreter::leaf_core`] only.) An in-loop call or /// return synced and moved the running activation: reload. Reload, + /// The running activation (synced) has native code and the loop is + /// its interpreted copy, or the other way round: run the other copy + /// (see `frame_jit`). + #[cfg(feature = "jit")] + Switch, } /// How [`Interpreter::leaf_core_attr`] ran its instruction. enum CoreAttr { /// Executed. Done, - /// Executed, and a discard scheduled a finalization sweep. - Marked, /// Not a cache hit: nothing touched. Decline, /// The site's first execution: the full handler must run (and @@ -59036,6 +58282,7 @@ static SLOW_LEAF_OPS: [bool; 256] = { OpCode::BuildString, OpCode::BuildTuple, OpCode::Call, + OpCode::CheckExcMatch, OpCode::CompareOp, OpCode::CopyFreeVars, OpCode::CopyTop, @@ -59048,6 +58295,8 @@ static SLOW_LEAF_OPS: [bool; 256] = { OpCode::JumpForward, OpCode::ListAppend, OpCode::LoadAttr, + OpCode::LoadClosure, + OpCode::LoadClosureBorrow, OpCode::LoadConst, OpCode::LoadDeref, OpCode::LoadFast, @@ -59056,13 +58305,17 @@ static SLOW_LEAF_OPS: [bool; 256] = { OpCode::LoadMethodAttr, OpCode::LoadSmallInt, OpCode::LoadSuperAttr, + OpCode::MakeCell, + OpCode::MakeFunction, OpCode::Nop, OpCode::NotTaken, OpCode::PopJumpIfFalse, OpCode::PopJumpIfNone, OpCode::PopJumpIfNotNone, + OpCode::PopExcept, OpCode::PopJumpIfTrue, OpCode::PopTop, + OpCode::PushExcInfo, OpCode::PushNull, OpCode::Resume, OpCode::StoreAttr, @@ -59111,6 +58364,7 @@ static CORE_LEAF_OPS: [bool; 256] = { OpCode::LoadAttr, OpCode::StoreAttr, OpCode::LoadMethodAttr, + OpCode::LoadSpecial, OpCode::CopyTop, OpCode::Swap, OpCode::GetIter, @@ -59120,6 +58374,11 @@ static CORE_LEAF_OPS: [bool; 256] = { OpCode::PopJumpIfNone, OpCode::PopJumpIfNotNone, OpCode::CallEx, + OpCode::Call, + OpCode::ReturnValue, + OpCode::LoadDeref, + OpCode::BuildList, + OpCode::ContainsOp, ]; let mut i = 0; while i < ops.len() { @@ -59874,12 +59133,11 @@ fn unbound_gen_method_sentinel(ty: &Rc, name: &str) -> Option<&'stat /// overwrite it. fn instance_has_nonnull_attr(instance: &Object, name: &str) -> bool { match instance { + // Read without publishing a `__dict__` the instance never needed + // (every fresh exception asks). Object::Instance(i) => { matches!(i.slot_get(name), Some(v) if !matches!(v, Object::None)) - || matches!( - i.dict_cell().borrow().get(&crate::object::StrKey(name)), - Some(v) if !matches!(v, Object::None) - ) + || matches!(i.attr_get_str(name), Some(v) if !matches!(v, Object::None)) } _ => false, } @@ -61100,9 +60358,12 @@ pub(crate) fn percent_format_with( return Err(value_error("incomplete format")); } } - // Codepoint index of the conversion char, for error messages. - let kind_index = template[..i].chars().count(); - let kind = bytes[i] as char; + // Byte offset of the conversion char; its code-point index (for + // error messages only) is counted when one is raised. + let kind_at = i; + // The whole character, so a non-ASCII one is reported by its + // code point rather than its first UTF-8 byte. + let kind = template[i..].chars().next().unwrap_or(char::from(bytes[i])); i += 1; // `%%` is a literal percent only when the two `%` are adjacent; // any intervening flag/width/precision/mapping makes the second @@ -61120,7 +60381,10 @@ pub(crate) fn percent_format_with( if mapping_key.is_none() && idx >= positional.len() { return Err(type_error("not enough arguments for format string")); } - return Err(unsupported_format_char('%', kind_index)); + return Err(unsupported_format_char( + '%', + template[..kind_at].chars().count(), + )); } out.push('%'); continue; @@ -61497,12 +60761,18 @@ pub(crate) fn percent_format_with( &spec.replace('c', "s"), )? } - _ => return Err(unsupported_format_char(kind, kind_index)), + _ => { + return Err(unsupported_format_char( + kind, + template[..kind_at].chars().count(), + )) + } }; out.push_str(&rendered); } else { - let ch_len = utf8_seq_len(bytes[i]); - let end = (i + ch_len).min(bytes.len()); + // The literal run up to the next `%` (ASCII, so always a + // character boundary), copied at once. + let end = memchr::memchr(b'%', &bytes[i..]).map_or(bytes.len(), |k| i + k); out.push_str(&template[i..end]); i = end; } @@ -63424,6 +62694,10 @@ struct CodeConstObjects { /// Inline-call shapes per `CALL` site (see [`CallSlot`]); allocated /// on the first inline call in this code object. call_slots: std::sync::OnceLock>, + /// The getset slots of a function made from this code (see + /// [`function_slots`]): the module name they were built for, and the + /// entries, copied into each new function. + fn_slots: std::sync::OnceLock<(Option, Rc)>, /// Whether this code is a *pure leaf* (see [`code_is_pure_leaf`]): /// `0` not yet decided, `1` no, `2` general leaf, `3` argument return, /// `4` constant return, `5` small-int return, `6` attribute return, @@ -63442,6 +62716,11 @@ struct CodeConstObjects { /// Split-layout attribute shortcuts per `LOAD_ATTR` site (see /// [`FieldSlot`]); allocated on the first recorded one. field_slots: std::sync::OnceLock>, + /// The `BINARY_OP` and `COMPARE_OP` sites the core loop has run on a + /// natively served operand (see `stdlib::datetime_native`), for the + /// frame compiler to send through its helpers instead of the scalar + /// arms; allocated on the first recorded one. + native_sites: std::sync::OnceLock>, /// Verified frameless method calls per method-load site (see /// [`LeafSite`]); allocated on the first recorded one. leaf_sites: std::sync::OnceLock>, @@ -63451,6 +62730,9 @@ struct CodeConstObjects { /// Whether every return is `return None` (see /// [`code_returns_only_none`]): `0` not yet decided, `1` no, `2` yes. returns_none: std::sync::atomic::AtomicU8, + /// The code's native form for the core loop (see [`frame_jit`]). + #[cfg(feature = "jit")] + frame_jit: frame_jit::Slot, } /// A `CALL` site's inline-call shape (see `Interpreter::core_call`): the @@ -63494,14 +62776,18 @@ unsafe impl Send for CallSlot {} unsafe impl Sync for CallSlot {} impl CallShape { + /// The shape for a call of `code`: it depends only on the code and + /// the site (every caller still checks the function's own defaults), + /// so a fresh function made from the same `def` or `lambda` hits too. + /// (The weak handle keeps the code's allocation reserved, so an equal + /// address is the same code object.) #[inline] fn matches( &self, - func: *const crate::object::PyFunction, + _func: *const crate::object::PyFunction, code: *const CodeObject, ) -> Option<(u32, bool)> { - (std::ptr::eq(self.func.as_ptr(), func) && std::ptr::eq(self.code.as_ptr(), code)) - .then_some((self.missing, self.has_self)) + std::ptr::eq(self.code.as_ptr(), code).then_some((self.missing, self.has_self)) } } @@ -63656,6 +62942,7 @@ fn code_attr_poly(code: &CodeObject, cache_pc: u32) -> Option<&AttrPoly> { /// and (for a builtin hit) the builtins dict's stamp. While all three /// match, the dicts are exactly as they were when the site last ran its /// full guards, so the cached index can be read straight away. +#[repr(transparent)] struct StampSlot(std::cell::UnsafeCell<[u64; 3]>); // SAFETY: as `MethodSlot`: read and written only from the dispatch loop @@ -63689,8 +62976,21 @@ impl StampSlot { /// is process-unique, so while both match, the value at that position of /// a split instance *is* the attribute: no inline-cache decode and no /// name comparison. +#[repr(transparent)] struct FieldSlot(std::cell::UnsafeCell<(u64, u32)>); +/// Where a [`FieldSlot`] keeps its class version (`u64`) and split +/// position (`u32`), for native code that reads it in line. +pub(crate) const FIELD_SLOT_VER: usize = std::mem::offset_of!((u64, u32), 0); +pub(crate) const FIELD_SLOT_IDX: usize = std::mem::offset_of!((u64, u32), 1); + +/// The [`FieldSlot`] of the attribute site at `pc` of `code`, if the code +/// has any (native code reads it in line). +pub(crate) fn code_field_slot(code: &CodeObject, pc: usize) -> Option<*const u8> { + let slot = code_vm_ext(code)?.field_slots.get()?.get(pc)?; + Some(std::ptr::from_ref(slot).cast()) +} + // SAFETY: as `StampSlot`. unsafe impl Send for FieldSlot {} unsafe impl Sync for FieldSlot {} @@ -63719,8 +63019,23 @@ impl FieldSlot { /// `ver`, the method is `func`, whose code was `code`, a pure (or, with /// `effect`, an effect) leaf taking exactly the site's arguments. The /// class holds the function at that version; `func` is weak all the same, -/// as [`MethodSlot`]'s are. -struct LeafSite(std::cell::UnsafeCell>); +/// as [`MethodSlot`]'s are. A site that sees several classes (a loop over +/// a list of mixed constraint kinds) keeps the ones it saw before in its +/// second cell, as [`MethodSlot`] keeps earlier resolutions. +struct LeafSite( + std::cell::UnsafeCell>, + std::cell::UnsafeCell>>, +); + +/// How many earlier callees a polymorphic [`LeafSite`] keeps. +const POLY_LEAF_SITES: usize = 3; + +/// A [`LeafSite`]'s earlier callees, by class version, replaced +/// round-robin. +struct PolyLeafSites { + entries: [Option; POLY_LEAF_SITES], + next: usize, +} struct LeafSiteData { ver: u64, @@ -63735,7 +63050,63 @@ unsafe impl Sync for LeafSite {} impl LeafSite { const fn empty() -> Self { - Self(std::cell::UnsafeCell::new(None)) + Self( + std::cell::UnsafeCell::new(None), + std::cell::UnsafeCell::new(None), + ) + } + + /// An earlier callee verified under `ver` (see `leaf_site_hit`). + #[cold] + #[inline(never)] + fn poly_hit( + &self, + ver: u64, + ) -> Option<(*const crate::object::PyFunction, *const CodeObject, bool)> { + // SAFETY: GIL-serialized; the borrow ends before any refill. + let poly = unsafe { &*self.1.get() }.as_deref()?; + poly.entries + .iter() + .flatten() + .find(|d| d.ver == ver && d.func.strong_count() > 0) + .map(|d| (d.func.as_ptr(), d.code, d.effect)) + } + + /// Record `data` as the site's callee, keeping the one it replaces + /// (a live callee under another class version) among the earlier + /// ones; `None` forgets them all. + fn set(&self, data: Option) { + // SAFETY: GIL-serialized; no reference into the site is live. + let (primary, poly) = unsafe { (&mut *self.0.get(), &mut *self.1.get()) }; + let Some(data) = data else { + *primary = None; + *poly = None; + return; + }; + if let Some(old) = primary.take() { + if old.ver != data.ver && old.func.strong_count() > 0 { + let poly = poly.get_or_insert_with(|| { + Box::new(PolyLeafSites { + entries: [const { None }; POLY_LEAF_SITES], + next: 0, + }) + }); + // The new callee's own entry, or the oldest, makes room. + let i = poly + .entries + .iter() + .position(|e| { + e.as_ref() + .is_some_and(|e| e.ver == old.ver || e.ver == data.ver) + }) + .unwrap_or(poly.next); + if i == poly.next { + poly.next = (i + 1) % POLY_LEAF_SITES; + } + poly.entries[i] = Some(old); + } + } + *primary = Some(data); } } @@ -63747,10 +63118,41 @@ fn leaf_site_hit( pc: usize, ver: u64, ) -> Option<(*const crate::object::PyFunction, *const CodeObject, bool)> { + let site = ext.leaf_sites.get()?.get(pc)?; // SAFETY: GIL-serialized; the borrow ends before any refill. - let site = unsafe { &*ext.leaf_sites.get()?.get(pc)?.0.get() }.as_ref()?; - (site.ver == ver && site.func.strong_count() > 0) - .then(|| (site.func.as_ptr(), site.code, site.effect)) + match unsafe { &*site.0.get() } { + Some(d) if d.ver == ver && d.func.strong_count() > 0 => { + Some((d.func.as_ptr(), d.code, d.effect)) + } + Some(_) => site.poly_hit(ver), + None => None, + } +} + +/// Whether the method call whose `LOAD_ATTR` is at `pc` last resolved to +/// a callee the core loop runs in place (a builtin, or a leaf function), +/// without an activation of its own. +#[cfg(feature = "jit")] +pub(crate) fn method_site_in_place(ext: &CodeConstObjects, pc: usize) -> bool { + // SAFETY: GIL-serialized; the borrows end before any refill. + let leaf = ext + .leaf_sites + .get() + .and_then(|s| s.get(pc)) + .and_then(|s| unsafe { &*s.0.get() }.as_ref()) + .is_some_and(|site| site.func.strong_count() > 0); + leaf || ext + .method_slots + .get() + .and_then(|s| s.get(pc)) + .is_some_and(|slot| match unsafe { &(*slot.0.get()).1 } { + MethodSlotFn::Builtin(_) | MethodSlotFn::Leaf(..) => true, + MethodSlotFn::Py(w) | MethodSlotFn::Static(w) => { + // SAFETY: the function is alive while the upgrade holds it. + w.upgrade().is_some_and(|f| fn_is_leaf(&f)) + } + _ => false, + }) } /// Record (or, with `data` `None`, forget) the site at `pc`'s callee. @@ -63763,8 +63165,7 @@ fn leaf_site_set(ext: &CodeConstObjects, ninstrs: usize, pc: usize, data: Option .leaf_sites .get_or_init(|| (0..ninstrs).map(|_| LeafSite::empty()).collect()); if let Some(site) = sites.get(pc) { - // SAFETY: GIL-serialized; no reference into the site is live. - unsafe { *site.0.get() = data }; + site.set(data); } } @@ -63819,6 +63220,42 @@ fn field_slot_note(ext: &CodeConstObjects, ninstrs: usize, pc: usize, inst: &PyI } } +/// Record that the operator at `pc` ran on a natively served operand (see +/// [`CodeConstObjects::native_sites`]). +#[inline] +fn native_site_note(ext: &CodeConstObjects, ninstrs: usize, pc: usize) { + use std::sync::atomic::{AtomicBool, Ordering}; + if let Some(site) = ext.native_sites.get().and_then(|s| s.get(pc)) { + if !site.load(Ordering::Relaxed) { + site.store(true, Ordering::Relaxed); + } + return; + } + let sites = ext + .native_sites + .get_or_init(|| (0..ninstrs).map(|_| AtomicBool::new(false)).collect()); + if let Some(site) = sites.get(pc) { + site.store(true, Ordering::Relaxed); + } +} + +/// Whether the `LOAD_ATTR` at `pc` has recorded a split-layout field (see +/// [`FieldSlot`]). +pub(crate) fn field_site(ext: &CodeConstObjects, pc: usize) -> bool { + ext.field_slots + .get() + .and_then(|s| s.get(pc)) + .is_some_and(|s| s.get().0 != 0) +} + +/// Whether the operator at `pc` has run on a natively served operand. +pub(crate) fn native_site(ext: &CodeConstObjects, pc: usize) -> bool { + ext.native_sites + .get() + .and_then(|s| s.get(pc)) + .is_some_and(|s| s.load(std::sync::atomic::Ordering::Relaxed)) +} + /// `v.clone()` with the scalars copied and the common heap variants' /// increments in line (the enum's `Clone` is an out-of-line call and a /// variant switch per clone). @@ -63990,6 +63427,14 @@ enum MethodSlotFn { /// A rejected native bound-call body, weakly held so a call site /// doesn't extend its lifetime. The key is the registry generation. NonLeaf(crate::sync::Weak), + /// A `LOAD_ATTR` site's `property` getter for instances of the class + /// at the key's version, and the code it was checked against (see + /// `Interpreter::try_inline_property`). + Getter { + prop: crate::sync::Weak, + func: crate::sync::Weak, + code: crate::sync::Weak, + }, } // SAFETY: read and written only from the dispatch loop with the GIL @@ -64155,6 +63600,16 @@ impl MethodSlot { } } + /// The plain function the slot resolved last, if it's still alive: a + /// compiler's guess at what the site will call next. + pub(crate) fn last_fn(&self) -> Option> { + // SAFETY: GIL-serialized; no `&mut` escapes `set`. + match unsafe { &*self.0.get() } { + (_, MethodSlotFn::Py(w)) => w.upgrade(), + _ => None, + } + } + /// [`Self::peek_fn`] for a class receiver: a plain function or a /// static method's function, either called with an empty self slot. #[inline] @@ -64300,6 +63755,52 @@ impl MethodSlot { } } + /// The property getter the slot names for instances of the class at + /// `ver`, while the property still holds that function and the + /// function that code. The class holds the property for as long as it + /// reads `ver` (see `get_held`), so no upgrade is needed. + #[inline] + fn getter(&self, ver: u64) -> Option<(Rc, Rc)> { + // SAFETY: GIL-serialized; no `&mut` escapes `set_getter`. + let (v, MethodSlotFn::Getter { prop, func, code }) = (unsafe { &*self.0.get() }) else { + return None; + }; + if *v != ver || prop.strong_count() == 0 { + return None; + } + // SAFETY: the class holds the property (see above); a raw read of + // its getter cell, compared before anything runs. + let Object::Function(f) = (unsafe { &*(*prop.as_ptr()).fget.as_ptr() }) else { + return None; + }; + if Rc::as_ptr(f) != func.as_ptr() { + return None; + } + let c = f.code(); + (Rc::as_ptr(&c) == code.as_ptr()).then(|| (f.clone(), c)) + } + + #[inline] + fn set_getter( + &self, + ver: u64, + prop: &Rc, + func: &Rc, + code: &Rc, + ) { + // SAFETY: as `set`. + unsafe { + *self.0.get() = ( + ver, + MethodSlotFn::Getter { + prop: Rc::downgrade(prop), + func: Rc::downgrade(func), + code: Rc::downgrade(code), + }, + ); + }; + } + #[inline] fn set_native_subscript(&self, ver: u64, generation: u64, fast: leaf_builtins::Fast) { // SAFETY: as `set`; neither value owns Python objects. @@ -64444,8 +63945,9 @@ fn code_pure_leaf_decide(code: &CodeObject, ext: &CodeConstObjects) -> bool { && !code.is_async_generator && !code.is_iterable_coroutine && !code.is_class_body + // (Free variables are read through the function's closure; cell + // variables would need cells of the evaluation's own.) && code.cellvars.is_empty() - && code.freevars.is_empty() && !code.has_varargs && code.kwonly_count == 0 && leaf_arity(code) <= 8 @@ -64463,6 +63965,8 @@ fn code_pure_leaf_decide(code: &CodeObject, ext: &CodeConstObjects) -> bool { OpCode::Resume | OpCode::Nop | OpCode::NotTaken + | OpCode::CopyFreeVars + | OpCode::LoadDeref | OpCode::LoadFast | OpCode::LoadFastBorrow | OpCode::LoadFastCheck @@ -64489,6 +63993,7 @@ fn code_pure_leaf_decide(code: &CodeObject, ext: &CodeConstObjects) -> bool { | OpCode::PopJumpIfNotNone | OpCode::JumpForward | OpCode::BinaryOp + | OpCode::BinarySubscr | OpCode::CopyTop | OpCode::Swap | OpCode::PopTop @@ -64801,13 +64306,17 @@ fn code_vm_ext_build( stamp_slots: std::sync::OnceLock::new(), attr_poly: std::sync::OnceLock::new(), call_slots: std::sync::OnceLock::new(), + fn_slots: std::sync::OnceLock::new(), pure_leaf: std::sync::atomic::AtomicU8::new(0), fast_pairs: std::sync::OnceLock::new(), gen_fast: std::sync::atomic::AtomicU8::new(0), field_slots: std::sync::OnceLock::new(), + native_sites: std::sync::OnceLock::new(), leaf_sites: std::sync::OnceLock::new(), leaf_plan: std::sync::OnceLock::new(), returns_none: std::sync::atomic::AtomicU8::new(0), + #[cfg(feature = "jit")] + frame_jit: frame_jit::Slot::default(), }) }) } @@ -64865,6 +64374,109 @@ fn code_names_anywhere(code: &CodeObject, name: &str) -> bool { /// The function getset-slot keys, interned once: a fresh /// `Object::from_static` would allocate the text and recompute its hash /// at every `MAKE_FUNCTION`. +/// `MAKE_FUNCTION`'s function: `code` with the defining frame's globals +/// and builtins, and the operands the instruction's flags popped. +#[allow(clippy::too_many_arguments)] +fn new_function( + code: Rc, + globals: &Rc>, + builtins: &Rc>, + defaults: Vec, + kw_defaults: Vec<(String, Object)>, + closure: Vec, + annotations: Option, +) -> Object { + // Function getset slots live *outside* `__dict__` (`f.__dict__` + // starts empty in CPython; only genuine user attributes land there): + // `__module__` (pickle serialises a function by qualified name), + // and `__name__` and `__qualname__` as stable objects, so repeated + // `func.__name__` reads (and delegated reads through classmethod or + // staticmethod wrappers) return the *same* object, which + // `assertIs(wrapper.__name__, func.__name__)` in test_decorators + // relies on. + let (mut slots, seed) = function_slots(&code, globals, annotations.is_none()); + if let Some(ann) = annotations { + slots.insert(DictKey(fn_slot_keys()[3].clone()), ann); + } + let f = PyFunction { + name: code.name.clone(), + code: RefCell::new(code), + globals: globals.clone(), + // The defining frame's builtins, not a fresh resolution: the same + // mapping CPython's `PyFunction_New` snapshots (the frame's + // builtins came from the same globals). + builtins: builtins.clone(), + defaults, + kw_defaults, + closure, + attrs: RefCell::new(None), + slots_raw: RefCell::new(slots), + slot_seed: RefCell::new(seed), + closure_cells: std::sync::OnceLock::new(), + defaults_override: crate::object::OverrideFlag::new(false), + }; + // A function participates in cycles through its globals (`exec(src, + // d)` builds `d -> f -> d`), its `__dict__`, and any closure cell that + // closes over the function. Tracked so `gc.collect()` can reclaim + // those cycles and `gc.is_tracked(f)` reports True, matching CPython + // (functions are GC objects). + let obj = Object::Function(Rc::new(f)); + gc_trace::track(&obj); + obj +} + +/// The getset slots a new function made from `code` in `globals` starts +/// with: `__module__` (the globals' `__name__`, when set), `__name__` and +/// `__qualname__`. Built once per code object (for the module name it +/// first sees) and shared by each function after, so two functions made +/// from one `def` share their name objects, as CPython's do: with +/// `seeded`, as the function's seed (see `PyFunction::slot_seed`) beside +/// empty slots, otherwise copied. +fn function_slots( + code: &CodeObject, + globals: &Rc>, + seeded: bool, +) -> (DictData, Option>) { + let keys = fn_slot_keys(); + let module = globals.borrow().get(&DictKey(keys[1].clone())).cloned(); + let build = |module: Option| { + let mut map = + crate::object::DictMap::with_capacity_and_hasher(3, crate::fasthash::FxBuildHasher); + if let Some(m) = module { + map.insert(DictKey(keys[0].clone()), m); + } + map.insert( + DictKey(keys[1].clone()), + Object::from_str(code.name.clone()), + ); + // `__qualname__` is the code object's PEP 3155 dotted name + // (computed at compile time from lexical nesting), not the bare + // `__name__`. + map.insert( + DictKey(keys[2].clone()), + Object::from_str(code.qualname.clone()), + ); + map + }; + let same = |a: &Option, b: &Option| match (a, b) { + (Some(a), Some(b)) => a.is_same(b), + (None, None) => true, + _ => false, + }; + if let Some(ext) = code_vm_ext(code) { + let (built_for, map) = ext + .fn_slots + .get_or_init(|| (module.clone(), Rc::new(build(module.clone())))); + if same(built_for, &module) { + if seeded { + return (DictData::default(), Some(map.clone())); + } + return (DictData::from((**map).clone()), None); + } + } + (DictData::from(build(module)), None) +} + fn fn_slot_keys() -> &'static [Object; 4] { static KEYS: std::sync::OnceLock<[Object; 4]> = std::sync::OnceLock::new(); KEYS.get_or_init(|| { @@ -64891,6 +64503,11 @@ const EXC_FAMILIES: [&str; 9] = [ "UnicodeTranslateError", ]; +/// [`EXC_FAMILIES`] bits that Rust-side raise sites test directly. +pub(crate) const EXC_FAM_STOP_ITERATION: u16 = 1 << 0; +pub(crate) const EXC_FAM_SYNTAX_ERROR: u16 = 1 << 4; +pub(crate) const EXC_FAM_IMPORT_ERROR: u16 = 1 << 5; + /// `name`'s bit in an [`exc_family_flags`] set (`0` for a name that is /// not a family — never asked in practice). #[inline] @@ -64903,7 +64520,7 @@ fn exc_family_bit(name: &str) -> u16 { /// Which exception families `cls` inherits from, memoised on the type /// against its attribute version (an MRO reshape bumps it). -fn exc_family_flags(cls: &crate::types::TypeObject) -> u16 { +pub(crate) fn exc_family_flags(cls: &crate::types::TypeObject) -> u16 { let ver = cls.attr_version.get(); let memo = cls.exc_families.get(); if memo & 1 != 0 && memo >> 11 == ver { @@ -64934,8 +64551,9 @@ fn code_name_obj(code: &CodeObject, name_idx: u32) -> Option<&Object> { enum NativeTruth { /// Neither `__bool__` nor `__len__`: always true. AlwaysTrue, - /// A registered native `__bool__`. - Bool(Rc), + /// A registered native `__bool__`, with its direct operation + /// (`collections_native::fast_op`, `0` for none). + Bool(Rc, u8), /// A registered native `__len__`. Len(Rc), /// Anything else: the full path decides. @@ -65349,7 +64967,7 @@ fn binary_op(a: &Object, b: &Object, op: BinOpKind) -> Result f64 { /// Bignum-aware integer arithmetic for `int`-flavoured operands. /// Both inputs are guaranteed `int`/`long`/`bool` by the caller; the /// fast path stays in `i64` until an overflow forces promotion. +/// [`bignum_op`] for the core loop, kept out of its body. +#[inline(never)] +fn core_bignum_op(a: &Object, b: &Object, op: BinOpKind) -> Result { + bignum_op(a, b, op) +} + fn bignum_op(a: &Object, b: &Object, op: BinOpKind) -> Result { use BinOpKind as B; @@ -67390,6 +67014,12 @@ fn compare_str(a: &str, b: &str, op: CompareKind) -> bool { pub use object::Object as Value; +/// The `_seqtools` classes behind `map` and `filter`. +struct SeqTools { + map: Rc, + filter: Rc, +} + #[cfg(test)] mod tests { use super::*; @@ -67406,7 +67036,7 @@ mod tests { static HEAP_SNAPSHOT_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); #[test] - fn finalizer_queue_claims_once_and_preserves_drop_fallback() { + fn finalizer_queue_claims_once_and_keeps_identity_on_last_release() { let code = compile_module( &parse_module( r" @@ -67466,15 +67096,17 @@ next(gen) ); inst }; - // An untracked last-Arc drop must still queue and run its copy once. + // The last release moves the reference itself into the queue, so + // the finalizer sees the same object, which dies after it runs. let dropped = drop_instance("drop"); let weak = Rc::downgrade(&dropped); drop(dropped); - assert!(weak.upgrade().is_none()); + assert!(weak.upgrade().is_some()); assert_eq!(interp.run_pending_finalizers(), 1); + assert!(weak.upgrade().is_none()); assert_eq!(events.borrow().len(), 3); - // A queue borrow that prevents Drop from scheduling must release the - // unscheduled copy without recursively resurrecting it again. + // A queue borrow that prevents Drop from scheduling releases the + // object without its finalizer. let rejected = drop_instance("rejected"); let weak = Rc::downgrade(&rejected); crate::vm_singletons::PENDING_FINALIZERS.with(|queue| { @@ -68360,7 +67992,9 @@ assert loop(2000) == 1999000 ); let native = SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(std::cell::Cell::get); let attempts = SCALAR_FIELD_UPDATE_ATTEMPTS.with(std::cell::Cell::get); + let armed = SCALAR_FIELD_UPDATE_ARMED.with(std::cell::Cell::get); run(&mut interp, &source); + let armed = SCALAR_FIELD_UPDATE_ARMED.with(std::cell::Cell::get) - armed; let native = SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(std::cell::Cell::get) - native; eprintln!("Completed scalar field update {kind} calls: native {native}"); @@ -68369,7 +68003,15 @@ assert loop(2000) == 1999000 if crate::tier2::jit_off_for_process() { assert_eq!(native, 0, "native entry requires JIT"); assert_eq!(attempts, 0, "interpreted calls use their ordinary path"); - } else if matches!(kind, "parameter" | "constant" | "default" | "alias") { + assert_eq!(armed, 0, "in-line updates require JIT"); + } else if matches!(kind, "parameter" | "constant") { + // The helper runs the update until it arms the + // method entry; compiled code runs the rest in line. + assert!( + native > 100 || (armed > 0 && native > 0), + "compiled {kind} native update coverage: {native} (armed {armed})" + ); + } else if matches!(kind, "default" | "slots" | "alias") { assert!( native > 100, "compiled {kind} native update coverage: {native}" @@ -68397,15 +68039,22 @@ assert loop(2000) == 1999000 ), ] { let native = SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(std::cell::Cell::get); + let armed = SCALAR_FIELD_UPDATE_ARMED.with(std::cell::Cell::get); run( &mut interp, &format!("__name__ = 'update_coverage'\n{fixture}\nbench({work})\n"), ); let native = SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(std::cell::Cell::get) - native; - eprintln!("Application {name} completed updates: native {native}"); + let armed = SCALAR_FIELD_UPDATE_ARMED.with(std::cell::Cell::get) - armed; + eprintln!( + "Application {name} completed updates: native {native}, armed {armed}" + ); if name == "richards" && !crate::tier2::jit_off_for_process() { - assert!(native > 100, "Richards native update coverage: {native}"); + assert!( + native > 100 || (armed > 0 && native > 0), + "Richards native update coverage: {native} (armed {armed})" + ); } if crate::tier2::jit_off_for_process() { assert_eq!(native, 0, "{name} ordinary interpreter dispatch"); @@ -72858,9 +72507,26 @@ for _ in range(10): fn jit_native_string_arguments_validate_and_preserve_values() { std::thread::spawn(|| { crate::tier2::force_enable_for_test(2); - let out = run(include_str!( + // The fixture's case functions are loop-free, so calls from a + // quiet loop run them as inline activations, which never enter + // compiled code. Dispatch every call through the general path + // (its exact-arity `CALL` enters compiled callees directly), as + // a frame the quiet loops leave does: the driver loop's frame + // used to, once its first caught exception materialized it. + let module = parse_module(include_str!( "../../../tests/regrtest/test_native_string_arguments.py" - )); + )) + .expect("parse"); + let code = compile_module(&module).expect("compile"); + let mut interp = Interpreter::new(); + interp.quiet_off = true; + let buf: Rc>> = Rc::new(RefCell::new(Vec::new())); + let writer: Stdout = crate::rc_unsize!(buf.clone() => RefCell); + interp.set_stdout(writer); + interp + .run_module(&code) + .unwrap_or_else(|error| panic!("run: {error}")); + let out = String::from_utf8(buf.borrow().clone()).expect("utf-8"); assert_eq!(out, "ok\n"); assert!( crate::tier2::direct_call_count_for_test() >= 1, diff --git a/crates/weavepy-vm/src/object.rs b/crates/weavepy-vm/src/object.rs index 22b4bbbe..c8860e45 100644 --- a/crates/weavepy-vm/src/object.rs +++ b/crates/weavepy-vm/src/object.rs @@ -42,50 +42,22 @@ const _: () = { assert_sync::(); }; -/// Number of slots in the per-thread code-point-length cache. -const STR_LEN_CACHE_CAP: usize = 1024; - -thread_local! { - /// Direct-mapped cache of `SharedStr` heap identity -> code-point length. - /// - /// WeavePy stores `str` as UTF-8, so counting code points — needed by - /// `len(s)`, index/slice bounds, and `re` span clamping — is O(n). - /// Scanners and tokenisers touch the *same* large string repeatedly, so - /// without memoisation an otherwise-linear pass degrades to O(n^2) (the - /// cause of `test_json`'s deep-recursion cases hanging). Each slot holds - /// the source `Rc` so a freed allocation cannot be reused at the same - /// address and alias a stale length; collisions simply recompute and - /// overwrite, so the cache is always correct and bounded to CAP entries. - static STR_LEN_CACHE: std::cell::RefCell>> = - std::cell::RefCell::new(vec![None; STR_LEN_CACHE_CAP]); -} - -/// Code-point length of a UTF-8 `str`, memoised by heap identity. O(1) on a -/// cache hit; O(n) (and caches the result) on a miss. Strings of 0 or 1 -/// bytes are counted directly — they are necessarily 0 or 1 code points, so -/// caching them would only evict useful entries. +/// Code-point length of a `str`: O(1) from the count memoised in the +/// string itself (see [`SharedStr::char_count`]). WeavePy stores `str` as +/// UTF-8, so the first count is a scan; scanners and tokenisers that ask +/// for the length of the same large string repeatedly stay linear. +#[inline] pub(crate) fn str_char_len(s: &SharedStr) -> usize { - let bytes = s.len(); - if bytes <= 1 { - return bytes; - } - let ptr = SharedStr::as_ptr(s).cast::() as usize; - let slot = (ptr / 8) % STR_LEN_CACHE_CAP; - STR_LEN_CACHE.with(|c| { - let mut cache = c.borrow_mut(); - if let Some((p, _, n)) = &cache[slot] { - if *p == ptr { - return *n; - } - } - let n = s.chars().count(); - cache[slot] = Some((ptr, s.clone(), n)); - n - }) + SharedStr::char_count(s) } /// A Python value as seen by the interpreter. +/// +/// `repr(u8)` fixes the layout native code relies on (see `frame_jit`): +/// the variant's index in a tag byte at offset 0, and each payload at its +/// natural alignment after it (every payload is one word or smaller). #[derive(Clone)] +#[repr(u8)] pub enum Object { None, /// The "no value" marker for local-variable slots — CPython's NULL @@ -478,6 +450,35 @@ pub struct PyFrame { /// CPython clearing the cell slots out of `f_localsplus` while /// the cell objects live on in the closures that captured them. pub cleared: Cell, + /// `f_back` while it is still a live activation nobody has looked at: + /// a frame object materialized as its activation leaves the spine + /// (for a traceback entry that outlives it) links its caller lazily, + /// so the caller's own frame object is built only if `f_back` is read + /// or the caller leaves the spine while this link lives. Resolved + /// into [`Self::back`] by [`Self::back_frame`]. + pub(crate) lazy_back: RefCell>, +} + +impl PyFrame { + /// `f_back`, materializing a lazy link (see [`Self::lazy_back`]). + pub fn back_frame(&self) -> Option> { + let shell = match self.lazy_back.borrow().as_ref() { + None => return self.back.borrow().clone(), + Some(lazy) => lazy.0.clone(), + }; + let py = materialize_shell(&shell); + *self.back.borrow_mut() = Some(py.clone()); + *self.lazy_back.borrow_mut() = None; + Some(py) + } + + /// Replace `f_back` (dropping any lazy link). + pub fn set_back(&self, back: Option>) { + *self.back.borrow_mut() = back; + if self.lazy_back.borrow().is_some() { + *self.lazy_back.borrow_mut() = None; + } + } } impl fmt::Debug for PyFrame { @@ -493,6 +494,25 @@ impl fmt::Debug for PyFrame { impl Drop for PyFrame { fn drop(&mut self) { + // A dead frame's locals die with its frame object, and CPython's + // `frame_dealloc` releases them from the top of the value stack + // down, before `f_back`: finalizers observe that order (the + // innermost frame of a dropped traceback frees its last local + // first). + if let Some(locals) = self.locals_mirror.get_mut().take() { + if Rc::strong_count(&locals) == 1 { + if let Ok(mut values) = locals.try_borrow_mut() { + while let Some(value) = values.pop() { + drop(values); + drop(value); + values = match locals.try_borrow_mut() { + Ok(values) => values, + Err(_) => break, + }; + } + } + } + } // Iteratively tear down the `back` chain. A plain recursive drop // would drop this frame's `back` Arc, whose own drop drops the // *next* frame's `back`, and so on — one native stack frame per @@ -949,6 +969,10 @@ pub struct FrameShell { pub has_materialized: std::sync::atomic::AtomicBool, /// The real Python frame object, once someone asked for it. pub materialized: RefCell>>, + /// Live traceback entries that recorded this activation without + /// materializing it (see `PyTraceback::new_lazy`). The pop + /// materializes a shell that still has some. Zero for a parked shell. + pub tb_refs: std::sync::atomic::AtomicU32, } /// Metadata in a live frame shell. Recycled shells leave these slots @@ -1072,6 +1096,7 @@ impl FrameShell { lasti: std::sync::atomic::AtomicU32::new(py.lasti.get()), has_materialized: std::sync::atomic::AtomicBool::new(true), materialized: RefCell::new(Some(py.clone())), + tb_refs: std::sync::atomic::AtomicU32::new(0), } } @@ -1092,11 +1117,23 @@ impl FrameShell { } pub fn materialize(&self, back: Option>) -> Rc { + let py = self.materialize_departing(back); + // RFC 0065 (WS1): a freshly materialized frame must kick its + // dispatch loop off the quiet path so the per-instruction + // `lasti`-cell sync resumes. + crate::hot_gates::bump_loop_gen(); + py + } + + /// [`Self::materialize`] for an activation that is leaving the spine: + /// no dispatch loop will run it again, so none needs kicking off its + /// quiet path. + pub fn materialize_departing(&self, back: Option>) -> Rc { if let Some(existing) = self.materialized.borrow().as_ref() { + // (`materialize` still bumps: the new holder may keep it, and + // a quiet loop running this activation re-derives, finding it + // observed if so.) self.refresh_materialized(existing); - // The new holder may keep it: a quiet loop running this - // activation re-derives, and finds it observed if so. - crate::hot_gates::bump_loop_gen(); return existing.clone(); } let py = Rc::new(PyFrame { @@ -1122,14 +1159,11 @@ impl FrameShell { on_stack: Cell::new(1), extra_locals: RefCell::new(None), cleared: Cell::new(false), + lazy_back: RefCell::new(None), }); *self.materialized.borrow_mut() = Some(py.clone()); self.has_materialized .store(true, std::sync::atomic::Ordering::Release); - // RFC 0065 (WS1): a freshly materialized frame must kick its - // dispatch loop off the quiet path so the per-instruction - // `lasti`-cell sync resumes. - crate::hot_gates::bump_loop_gen(); py } @@ -1200,7 +1234,7 @@ pub fn materialize_stack_at(stack: &FrameStack, idx: usize) -> Option { shell.refresh_materialized(&py); refreshed = true; - *py.back.borrow_mut() = back; + py.set_back(back); py } None => { @@ -1233,7 +1267,10 @@ pub fn materialize_stack_top(stack: &FrameStack) -> Option> { /// unwind machinery and chained outward through [`Self::next`]. #[derive(Debug)] pub struct PyTraceback { - pub frame: Rc, + /// The frame this entry records: the frame object, or the live + /// activation's spine shell until something asks for the object + /// (see [`TbFrame`]). Read through [`Self::frame`]. + frame: RefCell, pub lineno: u32, pub lasti: u32, /// `types.TracebackType(…)` stores the caller-supplied `tb_lasti` @@ -1243,6 +1280,143 @@ pub struct PyTraceback { pub next: RefCell>>, } +/// A traceback entry's frame. +/// +/// CPython creates the frame object for every traceback entry, but its +/// frame objects are cheap; WeavePy's materialization links the whole +/// `f_back` chain and takes the running activation off its quiet path. +/// An exception caught and dropped in the frame that raised it (the +/// `try: d[k] except KeyError:` idiom) never shows anyone that frame, +/// so an entry recorded while its activation runs keeps the spine +/// shell instead and materializes from it on first use. While the +/// activation is live, materializing then is what materializing at the +/// raise would have produced by now (the object tracks the running +/// frame either way); when the activation leaves the spine with lazy +/// entries still alive, `pop_frame_shell` materializes it on the way +/// out, so the object exists exactly as CPython's would. +#[derive(Debug)] +enum TbFrame { + Ready(Rc), + Lazy(LazyShellRef), +} + +/// A hold on a live activation's shell standing for its frame object +/// until someone asks for it: a lazy traceback entry (see [`TbFrame`]) or +/// a frame's lazy `f_back` (see [`PyFrame::lazy_back`]). Counted in +/// [`FrameShell::tb_refs`] so the activation's pop knows to materialize. +#[derive(Debug)] +pub(crate) struct LazyShellRef(Rc); + +impl LazyShellRef { + pub(crate) fn new(shell: Rc) -> Self { + shell + .tb_refs + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + Self(shell) + } +} + +impl Drop for LazyShellRef { + fn drop(&mut self) { + self.0 + .tb_refs + .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + } +} + +impl PyTraceback { + /// An entry for an existing frame object. + pub fn new(frame: Rc, lineno: u32, lasti: u32, raw_lasti: Option) -> Self { + Self { + frame: RefCell::new(TbFrame::Ready(frame)), + lineno, + lasti, + raw_lasti, + next: RefCell::new(None), + } + } + + /// An entry for the activation `shell` stands for, which must be + /// live on this thread's spine and not yet materialized (see + /// [`TbFrame`]). + pub fn new_lazy(shell: Rc, lineno: u32, lasti: u32) -> Self { + Self { + frame: RefCell::new(TbFrame::Lazy(LazyShellRef::new(shell))), + lineno, + lasti, + raw_lasti: None, + next: RefCell::new(None), + } + } + + /// `tb_frame`: the frame object, materialized on first use. + pub fn frame(&self) -> Rc { + let shell = match &*self.frame.borrow() { + TbFrame::Ready(py) => return py.clone(), + TbFrame::Lazy(lazy) => lazy.0.clone(), + }; + let py = materialize_shell(&shell); + *self.frame.borrow_mut() = TbFrame::Ready(py.clone()); + py + } + + /// The frame object if one exists already (no materialization). + pub fn frame_if_ready(&self) -> Option> { + match &*self.frame.borrow() { + TbFrame::Ready(py) => Some(py.clone()), + TbFrame::Lazy(lazy) => lazy.0.materialized.borrow().clone(), + } + } + + /// The entry's code object, without materializing its frame. + pub fn code(&self) -> Rc { + match &*self.frame.borrow() { + TbFrame::Ready(py) => py.code.clone(), + TbFrame::Lazy(lazy) => (*lazy.0.code).clone(), + } + } +} + +/// The frame object for `shell`, a lazy traceback entry's activation: +/// the one it already has, else materialized where it sits on this +/// thread's spine (linking `f_back` like any other materialization). +pub(crate) fn materialize_shell(shell: &Rc) -> Rc { + if let Some(handles) = crate::vm_singletons::current_thread_handles() { + let idx = handles + .frame_stack + .borrow() + .iter() + .rposition(|s| Rc::ptr_eq(s, shell)); + if let Some(idx) = idx { + if let Some(py) = materialize_stack_at(&handles.frame_stack, idx) { + return py; + } + } + } + if let Some(py) = shell.materialized.borrow().as_ref() { + shell.refresh_materialized(py); + return py.clone(); + } + // Live on another thread's spine (an exception from + // `sys._current_exceptions()`): materialize it there, as + // `sys._current_frames()` does; peer threads are parked under the GIL. + for (_, stack, _) in crate::stdlib::faulthandler_mod::thread_snapshots() { + let idx = stack + .try_borrow() + .ok() + .and_then(|s| s.iter().rposition(|s| Rc::ptr_eq(s, shell))); + if let Some(idx) = idx { + if let Some(py) = materialize_stack_at(&stack, idx) { + return py; + } + } + } + // A greenlet's parked spine: the shell's slots are valid while the + // activation lives, but the `f_back` chain belongs to that stack. Its + // pop decrements `on_stack` as for any materialization. + shell.materialize(None) +} + impl Drop for PyTraceback { fn drop(&mut self) { // Same unbounded-recursion hazard as `PyFrame::drop`: a traceback @@ -2026,6 +2200,7 @@ pub fn c_contiguous_strides(shape: &[usize], itemsize: usize) -> Vec { impl Drop for PyMemoryView { fn drop(&mut self) { self.release(); + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); } } @@ -2880,6 +3055,13 @@ pub struct PyModule { pub dict: Rc>, } +/// A dying PyModule clears the weak references watching it. +impl Drop for PyModule { + fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); + } +} + impl fmt::Debug for PyModule { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "", self.name) @@ -3064,12 +3246,19 @@ pub struct LeafProbe<'a> { } impl<'a> LeafProbe<'a> { - /// `None` unless `key` is a `str` or a machine `int`. + /// `None` unless `key` is a `str`, a machine `int`, or a plain + /// instance hashed by identity (its `__hash__` is `object`'s). #[inline] pub fn new(key: &'a Object) -> Option { let hash = match key { Object::Str(s) => SharedStr::hash_cached(s), Object::Int(_) => py_hash_value(key)?, + Object::Instance(i) + if i.native.get().is_none() + && i.class_dunder(crate::types::Dunder::Hash).object_owner() => + { + identity_hash(key) + } _ => return None, }; Some(Self { @@ -3103,6 +3292,9 @@ impl indexmap::Equivalent for LeafProbe<'_> { match (self.key, &key.0) { (Object::Str(a), Object::Str(b)) => a.as_bytes() == b.as_bytes(), (Object::Int(a), Object::Int(b)) => a == b, + // Identity settles an instance probe (CPython compares keys + // with `is` first); any other pairing may need `__eq__`. + (Object::Instance(a), Object::Instance(b)) if Rc::ptr_eq(a, b) => true, _ => { self.foreign.set(true); false @@ -3137,6 +3329,12 @@ pub fn exotic_str_keys_possible() -> bool { EXOTIC_CLASS_KEYS.load(std::sync::atomic::Ordering::Acquire) != 0 } +/// The counter [`exotic_str_keys_possible`] reads, for native code that +/// reads it in line (a `usize`: nonzero once exotic keys are possible). +pub(crate) fn exotic_str_keys_flag() -> *const usize { + EXOTIC_CLASS_KEYS.as_ptr() +} + /// Reach for the running interpreter to compute a user instance's Python /// `__hash__`. `DictKey`'s `Hash`/`Eq` impls have no interpreter handle, so /// they borrow the thread's published interpreter pointer — the same bridge @@ -3200,6 +3398,13 @@ pub(crate) fn instance_has_custom_dunder(obj: &Object, dunder: crate::types::Dun inst.native.get().is_none() && !info.object_owner() } +/// A plain instance compared by identity: no native payload, and its +/// `__eq__` is `object`'s. +#[inline] +pub(crate) fn plain_identity_eq(inst: &crate::types::PyInstance) -> bool { + inst.native.get().is_none() && inst.class_dunder(crate::types::Dunder::Eq).object_owner() +} + /// [`instance_has_custom_dunder`] for `__eq__`: membership tests and /// `list.remove`/`index`/`count` ask it for every element they compare. pub(crate) fn instance_has_custom_eq(obj: &Object) -> bool { @@ -3282,6 +3487,14 @@ pub(crate) fn member_eq(a: &Object, b: &Object) -> Result { if a.is_same(b) { return Ok(true); } + // Two distinct plain instances (`object.__eq__` on both sides) are + // unequal: both comparisons return `NotImplemented`. Settled before + // the general path, which resolves each side's `__eq__` twice. + if let (Object::Instance(x), Object::Instance(y)) = (a, b) { + if plain_identity_eq(x) && plain_identity_eq(y) { + return Ok(false); + } + } // Dispatch the full Python comparison whenever an operand can carry // custom `__eq__` semantics: a pure-Python instance with a user dunder, // OR a foreign (C-extension) object whose `tp_richcompare` the native @@ -4047,6 +4260,11 @@ pub fn global_value_epoch() -> u64 { GLOBAL_VALUE_EPOCH.load(std::sync::atomic::Ordering::Relaxed) } +/// Where [`GLOBAL_VALUE_EPOCH`] lives (native code reads it in line). +pub(crate) fn global_value_epoch_ptr() -> *const u64 { + GLOBAL_VALUE_EPOCH.as_ptr() +} + /// Advance [`GLOBAL_VALUE_EPOCH`]. #[inline] pub fn bump_global_value_epoch() { @@ -4179,6 +4397,9 @@ impl DictData { crate::gc_trace::track_deferred_owner(owner); } + /// Where the stamp sits in a dict (native code reads it in line). + pub(crate) const STAMP_OFFSET: usize = std::mem::offset_of!(DictData, stamp); + /// The stamp of the dict's current state (see the type docs). #[inline] pub fn mutation_stamp(&self) -> u64 { @@ -4364,8 +4585,13 @@ pub struct PyFunction { /// data descriptors on the `function` type, so `f.__name__ = x` /// must never appear in `f.__dict__` (functools.update_wrapper /// copies `__dict__` and asserts the wrapper's annotations are - /// untouched by the wrapped function's slots). - pub slots: RefCell, + /// untouched by the wrapped function's slots). Read through + /// [`Self::slots`], which first copies in [`Self::slot_seed`]. + pub slots_raw: RefCell, + /// The slots a new function starts with, shared by every function + /// one `def` makes (see `new_function`), until the first access to + /// [`Self::slots`] copies them in: most functions never read theirs. + pub slot_seed: RefCell>>, /// The frame `cells` vector for a function whose code has free /// variables but no cell variables: exactly `closure`'s cells, built /// once (the closure is immutable) for the lean call paths. @@ -4377,6 +4603,13 @@ pub struct PyFunction { pub defaults_override: OverrideFlag, } +/// A dying PyFunction clears the weak references watching it. +impl Drop for PyFunction { + fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); + } +} + /// A clonable set-once flag (see [`PyFunction::defaults_override`]). #[derive(Debug, Default)] pub struct OverrideFlag(std::sync::atomic::AtomicBool); @@ -4494,11 +4727,35 @@ impl PyFunction { .cloned() } + /// The function's getset slots (see [`Self::slots_raw`]), with the + /// seed it was made with copied in. + #[inline] + pub fn slots(&self) -> &RefCell { + // SAFETY: a read of the seed's presence with nothing running. + if unsafe { (*self.slot_seed.as_ptr()).is_some() } { + self.plant_slot_seed(); + } + &self.slots_raw + } + + #[cold] + #[inline(never)] + fn plant_slot_seed(&self) { + let Some(seed) = self.slot_seed.borrow_mut().take() else { + return; + }; + let mut slots = self.slots_raw.borrow_mut(); + // Every store goes through `slots()`, which plants the seed + // first: nothing is there yet to keep. + debug_assert!(slots.is_empty()); + *slots = DictData::from((*seed).clone()); + } + /// Read a slot value if one has been stored (explicitly assigned or /// stamped at definition time). Computed fallbacks live at the /// attribute-access sites. pub fn slot(&self, name: &str) -> Option { - self.slots + self.slots() .borrow() .get(&crate::object::StrKey(name)) .cloned() @@ -4510,7 +4767,7 @@ impl PyFunction { .0 .store(true, std::sync::atomic::Ordering::Relaxed); } - self.slots + self.slots() .borrow_mut() .insert(DictKey(Object::from_str(name)), value); } @@ -4553,6 +4810,13 @@ pub struct BoundMethod { pub py_method: bool, } +/// A dying BoundMethod clears the weak references watching it. +impl Drop for BoundMethod { + fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); + } +} + impl BoundMethod { /// A fully-bound method (CPython `method`): calling it prepends /// `receiver` and calls `function` directly. @@ -4802,54 +5066,24 @@ impl fmt::Debug for PyGenerator { impl Drop for PyGenerator { fn drop(&mut self) { - // CPython finalizes a generator the moment its refcount dies: - // a *suspended* frame gets `GeneratorExit` thrown in so - // `finally:`/`with` cleanup runs. We can't run Python from - // `Drop`, so resurrect the live frame into the VM's - // pending-finalizer queue; `gc.collect()` and the module-exit - // path drain it. Created-but-never-started frames have run no - // user code, so (like CPython's `gen_close`) they are simply - // marked completed. - let Ok(mut state) = self.state.try_borrow_mut() else { - return; - }; - let prev = std::mem::replace(&mut *state, GeneratorState::Finished); - drop(state); - match prev { - GeneratorState::Suspended(frame) => { - // Finalized once already (CPython's `_PyGC_FINALIZED` bit): - // drop the frame for real instead of looping forever - // through resurrection → finalize → drop. - if self.finalize_ran.get() { - defer_generator_state_drop(GeneratorState::Suspended(frame)); - return; - } - let resurrected = Rc::new(PyGenerator { - name: RefCell::new(self.name.borrow().clone()), - qualname: RefCell::new(self.qualname.borrow().clone()), - kind: self.kind, - code: self.code.clone(), - state: RefCell::new(GeneratorState::Suspended(frame)), - origin: RefCell::new(self.origin.borrow().clone()), - hooks_inited: crate::sync::Cell::new(self.hooks_inited.get()), - finalizer: RefCell::new(self.finalizer.borrow().clone()), - finalize_ran: crate::sync::Cell::new(self.finalize_ran.get()), - }); - let obj = match self.kind { - CoroutineKind::Generator => Object::Generator(resurrected), - CoroutineKind::Coroutine => Object::Coroutine(resurrected), - CoroutineKind::AsyncGenerator => Object::AsyncGenerator(resurrected), - }; - crate::vm_singletons::try_push_pending_finalizer(obj); + // A suspended generator's `close()` (and a never-awaited + // coroutine's warning) already ran: `rc::Rc`'s `Drop` queued the + // generator for it at its last release. A created or suspended + // frame still owns locals that can hold the *next* generator of a + // pipeline (`chain(chain(chain(…)))`); dropping it inline recurses + // one native stack frame per link and overflows on long chains, so + // route it through the iterative trampoline below. + if let Ok(mut state) = self.state.try_borrow_mut() { + let prev = std::mem::replace(&mut *state, GeneratorState::Finished); + drop(state); + if matches!( + prev, + GeneratorState::Suspended(_) | GeneratorState::Created(_) + ) { + defer_generator_state_drop(prev); } - // A never-started frame still owns locals that can hold the - // *next* generator of a pipeline (`chain(chain(chain(…)))`). - // Dropping it inline recurses one native stack frame per - // link and overflows on long chains, so route it through - // the iterative trampoline below. - state @ GeneratorState::Created(_) => defer_generator_state_drop(state), - GeneratorState::Finished | GeneratorState::Running => {} } + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); } } @@ -7670,6 +7904,7 @@ impl fmt::Debug for PyFile { impl Drop for PyFile { fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); // CPython's fileio/buffered/textio deallocators emit a // `ResourceWarning("unclosed file %R")` when an *open*, fd-backed // stream is reclaimed without an explicit `close()`/`with` @@ -8517,25 +8752,10 @@ impl PyIterator { } } - /// Like [`next_value`], but enforces CPython's "container changed - /// size during iteration" invariant before yielding. Used on the - /// *user-visible* `__next__` boundaries (`FOR_ITER`, the `next()` - /// builtin, `iter.__next__()`); the many internal consumers that - /// drain a freshly-built iterator without mutating its source can - /// keep calling the cheaper [`next_value`]. - pub fn next_value_checked(&mut self) -> Result, RuntimeError> { - if let PyIterator::Set { set, len, index } = self { - // `setiter_iternext`: a size change since creation is fatal, - // even on the step that would otherwise raise StopIteration. - if set.borrow().len() != *len { - // Sticky, like CPython's `si_used = -1`: every further - // `next()` keeps raising, and `__length_hint__` reports 0 - // (test_iterlen.test_immutable_during_iteration). - *len = usize::MAX; - *index = usize::MAX; - return Err(runtime_error("Set changed size during iteration")); - } - } + /// A dict iterator's mutation guards, before each checked step: + /// `Ok(true)` when the step must end silently; an error once the dict + /// changed under it. Nothing for any other iterator. + fn dict_iter_guard(&mut self) -> Result { if let PyIterator::DictKeys { dict: Some(d), len, @@ -8567,7 +8787,7 @@ impl PyIterator { // array leaves `di_pos` past `dk_nentries`, which // is a silent StopIteration, not an error. *index = 0; - return Ok(None); + return Ok(true); } return Err(runtime_error("dictionary keys changed during iteration")); } @@ -8575,6 +8795,73 @@ impl PyIterator { _ => {} } } + Ok(false) + } + + /// The checked step of a forward `dict.items()` iterator without its + /// `(key, value)` tuple, for a consumer that unpacks it at once (a + /// native tuple-target loop). `None` for any other iterator. + pub fn next_item_pair(&mut self) -> Option, RuntimeError>> { + if !matches!( + self, + PyIterator::DictKeys { + kind: DictViewKind::Items, + reverse: false, + .. + } + ) { + return None; + } + match self.dict_iter_guard() { + Err(e) => return Some(Err(e)), + Ok(true) => return Some(Ok(None)), + Ok(false) => {} + } + let PyIterator::DictKeys { + index, dict, owner, .. + } = self + else { + return None; + }; + let Some(d) = dict.as_ref() else { + return Some(Ok(None)); + }; + let entry = d + .borrow() + .get_index(*index) + .map(|(k, v)| (k.0.clone(), v.clone())); + if entry.is_some() { + *index += 1; + } else { + // Exhausted: detached, as `next_value` leaves it. + *dict = None; + *owner = None; + } + Some(Ok(entry)) + } + + /// Like [`next_value`], but enforces CPython's "container changed + /// size during iteration" invariant before yielding. Used on the + /// *user-visible* `__next__` boundaries (`FOR_ITER`, the `next()` + /// builtin, `iter.__next__()`); the many internal consumers that + /// drain a freshly-built iterator without mutating its source can + /// keep calling the cheaper [`next_value`]. + pub fn next_value_checked(&mut self) -> Result, RuntimeError> { + if let PyIterator::Set { set, len, index } = self { + // `setiter_iternext`: a size change since creation is fatal, + // even on the step that would otherwise raise StopIteration. + if set.borrow().len() != *len { + // Sticky, like CPython's `si_used = -1`: every further + // `next()` keeps raising, and `__length_hint__` reports 0 + // (test_iterlen.test_immutable_during_iteration). + *len = usize::MAX; + *index = usize::MAX; + return Err(runtime_error("Set changed size during iteration")); + } + } + if self.dict_iter_guard()? { + return Ok(None); + } if let PyIterator::File { file } = self { // Surface read errors (OSError, decode errors) at the // user-visible `__next__` boundary instead of swallowing them. @@ -11967,6 +12254,13 @@ pub(crate) fn py_hash_value(obj: &Object) -> Option { }))) } Object::Instance(inst) => { + // The common case first: a plain instance whose `__hash__` is + // `object`'s hashes by identity, with no interpreter round trip. + if inst.native.get().is_none() + && inst.class_dunder(crate::types::Dunder::Hash).object_owner() + { + return Some(identity_hash(obj)); + } // A `weakref.ref` hashes as its referent (CPython `weakref_hash`), // computed natively and memoised so the hot `DictKey` path never // pays a reentrant `__hash__` dispatch — see `weakref_native_hash`. @@ -12690,6 +12984,10 @@ impl Object { } } +// Every payload is one word or smaller, so a value is a tag and a word. +const _: () = assert!(std::mem::size_of::() == 16); +const _: () = assert!(std::mem::size_of::>() == 16); + #[cfg(test)] mod tests { use super::*; diff --git a/crates/weavepy-vm/src/rc.rs b/crates/weavepy-vm/src/rc.rs index 34308d73..38f45863 100644 --- a/crates/weavepy-vm/src/rc.rs +++ b/crates/weavepy-vm/src/rc.rs @@ -119,6 +119,55 @@ pub(crate) unsafe fn try_release_shared(value: *const T) -> bool { true } +/// Free the `Arc` allocation holding `value` when this is its last owner +/// and no weak reference exists, with plain loads instead of `Arc`'s two +/// locked decrements (strong, then the implicit weak). Only for payloads +/// with no destructor, so nothing runs while the counts still read one. +/// Returns `false`, having changed nothing, when another owner or a weak +/// reference exists (or the bias is off); the caller must then drop an +/// `Arc` normally. +/// +/// While the bias holds, this thread is the only one that can touch the +/// counts, and with no weak reference nobody else can reach the payload. +/// +/// # Safety +/// +/// `value` must point at the payload of a live `Arc` allocation, and the +/// caller must own one of its strong references, which this consumes on +/// success. +#[inline(always)] +pub(crate) unsafe fn try_free_last(value: *const T) -> bool { + debug_assert!(!std::mem::needs_drop::()); + if !refcounts_biased() { + return false; + } + // SAFETY: live allocation, per the caller; the weak count is the + // word after the strong count (`ArcInner` is `repr(C)`). + let (strong, weak) = unsafe { + let strong = strong_word(value); + (&*strong, &*strong.add(1)) + }; + if strong.load(Ordering::Relaxed) != 1 || weak.load(Ordering::Relaxed) != 1 { + return false; + } + // `Arc` sizes its allocation as the two counters followed by the + // payload's layout, padded (`arcinner_layout_for_value_layout`). + // SAFETY: as above; the payload is live until the deallocation. + let payload = std::alloc::Layout::for_value(unsafe { &*value }); + let (inner, _) = std::alloc::Layout::new::<[usize; 2]>() + .extend(payload) + .expect("Arc layout"); + // SAFETY: the sole owner frees the allocation it owns, with the layout + // `Arc` allocated it with; the payload needs no drop. + unsafe { + std::alloc::dealloc( + ptr::from_ref(strong).cast::().cast_mut(), + inner.pad_to_align(), + ); + } + true +} + /// Convert an [`Rc`] to one of an unsized type, such as a trait object: /// `rc_unsize!(rc => dyn Trait)`. Stable Rust only coerces its own smart /// pointers, so the conversion passes through `Arc`. @@ -132,13 +181,13 @@ macro_rules! rc_unsize { /// A reference-counted shared pointer with biased counting. See the module /// documentation. #[repr(transparent)] -pub struct Rc(ManuallyDrop>); +pub struct Rc(ManuallyDrop>); /// A weak reference to an [`Rc`] allocation. #[repr(transparent)] pub struct Weak(std::sync::Weak); -impl Rc { +impl Rc { #[inline] pub fn new(value: T) -> Self { Self(ManuallyDrop::new(Arc::new(value))) @@ -167,7 +216,7 @@ impl Rc { } } -impl Rc { +impl Rc { #[inline] pub fn from_arc(arc: Arc) -> Self { Self(ManuallyDrop::new(arc)) @@ -248,14 +297,14 @@ impl Rc { } } -impl Rc { +impl Rc { #[inline] pub fn make_mut(this: &mut Self) -> &mut T { Arc::make_mut(&mut this.0) } } -impl Clone for Rc { +impl Clone for Rc { #[inline(always)] fn clone(&self) -> Self { // SAFETY: `self` keeps the allocation alive, and the new owner @@ -267,20 +316,119 @@ impl Clone for Rc { } } -impl Drop for Rc { +impl Drop for Rc { #[inline(always)] fn drop(&mut self) { // SAFETY: this owner holds one strong reference, released exactly - // once: either here, or by Arc's own drop. + // once: either here, or by Arc's own drop, or by handing it on. unsafe { - if !try_release_shared(Arc::as_ptr(&self.0)) { - ManuallyDrop::drop(&mut self.0); + if try_release_shared(Arc::as_ptr(&self.0)) { + return; } + // The last reference to an object that owes a finalizer moves + // to the finalizer queue instead: the object stays alive, at + // its address, until its `__del__` (or generator close) has + // run, as CPython's `tp_finalize` before `tp_dealloc`. + if finalizable_kind::() && Arc::strong_count(&self.0) == 1 { + let arc = ManuallyDrop::take(&mut self.0); + crate::vm_singletons::finalize_on_last_release(arc); + return; + } + if nesting_kind::() { + release_nested(ManuallyDrop::take(&mut self.0)); + return; + } + ManuallyDrop::drop(&mut self.0); } } } -impl Deref for Rc { +/// Whether `T` is a container payload, whose release can release more +/// containers in turn. Constant per instantiation. +#[inline(always)] +fn nesting_kind() -> bool { + use crate::object::{ + BoundMethod, DictData, FrozenSetObj, PyFrame, PyFunction, PyGenerator, PyIterator, + PyTraceback, SetData, + }; + use crate::sync::RefCell; + use std::any::TypeId; + let t = TypeId::of::(); + t == TypeId::of::() + || t == TypeId::of::>>() + || t == TypeId::of::>() + || t == TypeId::of::>() + || t == TypeId::of::() + || t == TypeId::of::>() + || t == TypeId::of::>() + || t == TypeId::of::() + || t == TypeId::of::() + || t == TypeId::of::() + || t == TypeId::of::() + || t == TypeId::of::() +} + +/// How deep releases may nest before the trashcan takes over. +const TRASHCAN_DEPTH: u32 = 50; + +/// CPython's trashcan: releasing a deeply nested structure (a 500,000-deep +/// element tree, a long linked list of instances) recurses through every +/// level's drop glue, which would overflow the native stack. Past +/// [`TRASHCAN_DEPTH`] nested releases a container's last reference is +/// parked instead, and the outermost release frees the parked ones one at +/// a time. +struct Trashcan { + depth: std::cell::Cell, + parked: std::cell::RefCell>>, +} + +thread_local! { + static TRASHCAN: Trashcan = const { + Trashcan { + depth: std::cell::Cell::new(0), + parked: std::cell::RefCell::new(Vec::new()), + } + }; +} + +/// Release the last reference to a container (see [`Trashcan`]). +pub(crate) fn release_nested(last: R) { + let mut last = Some(last); + let _ = TRASHCAN.try_with(|t| { + let depth = t.depth.get(); + if depth >= TRASHCAN_DEPTH { + if let Ok(mut parked) = t.parked.try_borrow_mut() { + parked.push(Box::new(last.take())); + } + return; + } + t.depth.set(depth + 1); + drop(last.take()); + if depth == 0 { + loop { + let next = t.parked.try_borrow_mut().ok().and_then(|mut p| p.pop()); + match next { + Some(next) => drop(next), + None => break, + } + } + } + t.depth.set(depth); + }); + // Thread teardown, with the trashcan gone: release in place. + drop(last); +} + +/// Whether `T` is a payload whose last release may owe a finalizer (see +/// `vm_singletons::finalize_on_last_release`). Constant per instantiation. +#[inline(always)] +fn finalizable_kind() -> bool { + use std::any::TypeId; + let t = TypeId::of::(); + t == TypeId::of::() || t == TypeId::of::() +} + +impl Deref for Rc { type Target = T; #[inline(always)] fn deref(&self) -> &T { @@ -288,81 +436,81 @@ impl Deref for Rc { } } -impl AsRef for Rc { +impl AsRef for Rc { fn as_ref(&self) -> &T { self } } -impl Borrow for Rc { +impl Borrow for Rc { fn borrow(&self) -> &T { self } } -impl fmt::Debug for Rc { +impl fmt::Debug for Rc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&**self, f) } } -impl fmt::Display for Rc { +impl fmt::Display for Rc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(&**self, f) } } -impl fmt::Pointer for Rc { +impl fmt::Pointer for Rc { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Pointer::fmt(&Self::as_ptr(self), f) } } -impl PartialEq for Rc { +impl PartialEq for Rc { fn eq(&self, other: &Self) -> bool { **self == **other } } -impl Eq for Rc {} +impl Eq for Rc {} -impl PartialOrd for Rc { +impl PartialOrd for Rc { fn partial_cmp(&self, other: &Self) -> Option { (**self).partial_cmp(&**other) } } -impl Ord for Rc { +impl Ord for Rc { fn cmp(&self, other: &Self) -> std::cmp::Ordering { (**self).cmp(&**other) } } -impl Hash for Rc { +impl Hash for Rc { fn hash(&self, state: &mut H) { (**self).hash(state) } } -impl Default for Rc { +impl Default for Rc { fn default() -> Self { Self::new(T::default()) } } -impl From for Rc { +impl From for Rc { fn from(value: T) -> Self { Self::new(value) } } -impl From> for Rc { +impl From> for Rc { fn from(value: Box) -> Self { Self::from_arc(Arc::from(value)) } } -impl From> for Rc<[T]> { +impl From> for Rc<[T]> { fn from(value: Vec) -> Self { Self::from_arc(Arc::from(value)) } @@ -380,19 +528,19 @@ impl From for Rc { } } -impl From> for Rc { +impl From> for Rc { fn from(value: Arc) -> Self { Self::from_arc(value) } } -impl Weak { +impl Weak { pub const fn new() -> Self { Self(std::sync::Weak::new()) } } -impl Weak { +impl Weak { #[inline] pub fn upgrade(&self) -> Option> { self.0.upgrade().map(Rc::from_arc) @@ -421,7 +569,7 @@ impl Clone for Weak { } } -impl Default for Weak { +impl Default for Weak { fn default() -> Self { Self::new() } diff --git a/crates/weavepy-vm/src/shared_value.rs b/crates/weavepy-vm/src/shared_value.rs index 78b01224..dd2719c3 100644 --- a/crates/weavepy-vm/src/shared_value.rs +++ b/crates/weavepy-vm/src/shared_value.rs @@ -8,11 +8,11 @@ use std::borrow::Borrow; use std::fmt; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; -use std::mem::{align_of, size_of, ManuallyDrop}; +use std::mem::{align_of, size_of, ManuallyDrop, MaybeUninit}; use std::ops::Deref; use std::ptr::{self, NonNull}; use std::sync::{ - atomic::{AtomicI64, AtomicU64, Ordering}, + atomic::{AtomicI64, AtomicU64, AtomicUsize, Ordering}, Arc, Weak, }; @@ -30,6 +30,11 @@ mod sealed { /// Constructors must preserve this invariant for every live strong reference. pub unsafe trait ThinPayload: sealed::Sealed { type View: ?Sized; + /// Release the last reference to a payload. A payload whose release + /// can release more containers goes through `rc::release_nested`. + fn release_last(last: Arc) { + drop(last); + } fn pointer(data: *const usize, len: usize) -> *const Self; fn view(&self) -> &Self::View; } @@ -129,9 +134,15 @@ impl Drop for ThinArc { // SAFETY: release this owner's one strong reference exactly once. // Arc drops the payload and handles remaining weak references normally. unsafe { - if !crate::rc::try_release_shared(self.raw()) { - drop(Arc::from_raw(self.raw())) + if crate::rc::try_release_shared(self.raw()) { + return; + } + // Text and byte payloads have no destructor: their last owner + // can free them without `Arc`'s locked decrements. + if !std::mem::needs_drop::() && crate::rc::try_free_last(self.raw()) { + return; } + T::release_last(Arc::from_raw(self.raw())) } } } @@ -332,44 +343,148 @@ impl SharedSlice { } } } -impl SharedSlice { - /// A byte slice of `len` bytes written by `fill` (which receives a - /// zeroed buffer of exactly that length): one allocation, no - /// intermediate `Vec`. - // The backing allocation comes from `Arc::<[usize]>::new_zeroed_slice`, +impl From> for SharedSlice { + fn from(items: Vec) -> Self { + Self::from_vec_owned(items) + } +} +impl From> for SharedSlice { + fn from(items: Box<[T]>) -> Self { + Self::from_vec_owned(items.into_vec()) + } +} +impl From<&[T]> for SharedSlice { + fn from(items: &[T]) -> Self { + Self::from_copy_slice(items) + } +} +impl From<&[T; N]> for SharedSlice { + fn from(items: &[T; N]) -> Self { + Self::from_copy_slice(items) + } +} + +/// The code-point count of a [`StrStorage`] not yet counted. +const CHARS_UNKNOWN: usize = usize::MAX; + +/// UTF-8 text payload: [`SliceStorage`]'s length and memoised hash, plus +/// the memoised code-point count (`CHARS_UNKNOWN` until first needed). +/// CPython's PEP 393 strings know their length in code points, and with +/// it whether they are ASCII; this word gives `len(s)`, index and slice +/// bounds, and every ASCII fast path the same O(1) answer, and lets a +/// string derived from ASCII text (a split field, a case mapping, a join +/// of ASCII parts) be born knowing it. +#[repr(C)] +#[derive(Debug)] +pub struct StrStorage { + len: usize, + hash: AtomicI64, + chars: AtomicUsize, + bytes: [u8], +} +impl sealed::Sealed for StrStorage {} +// SAFETY: repr(C) places the immutable usize first; the byte tail carries +// len. The constructors below initialize the header and exactly len bytes. +unsafe impl ThinPayload for StrStorage { + type View = [u8]; + // `data` is the word-aligned header the byte view was cast from. + #[allow(clippy::cast_ptr_alignment)] + fn pointer(data: *const usize, len: usize) -> *const Self { + ptr::slice_from_raw_parts(data.cast::(), len) as *const Self + } + fn view(&self) -> &[u8] { + &self.bytes + } +} + +/// Copy `len` bytes from `src` to `dst`. Most strings are a few words +/// long (a split field, a dict key), and for them two overlapping word +/// moves beat a call to the platform `memcpy` through its stub. +/// +/// # Safety +/// +/// As [`ptr::copy_nonoverlapping`]. +#[inline(always)] +unsafe fn copy_text(src: *const u8, dst: *mut u8, len: usize) { + // SAFETY: every access below lies within the first `len` bytes of + // both buffers (the caller's contract). + unsafe { + if len >= 16 { + ptr::copy_nonoverlapping(src, dst, len); + } else if len >= 8 { + let head = src.cast::().read_unaligned(); + let tail = src.add(len - 8).cast::().read_unaligned(); + dst.cast::().write_unaligned(head); + dst.add(len - 8).cast::().write_unaligned(tail); + } else if len >= 4 { + let head = src.cast::().read_unaligned(); + let tail = src.add(len - 4).cast::().read_unaligned(); + dst.cast::().write_unaligned(head); + dst.add(len - 4).cast::().write_unaligned(tail); + } else if len > 0 { + *dst = *src; + *dst.add(len / 2) = *src.add(len / 2); + *dst.add(len - 1) = *src.add(len - 1); + } + } +} + +impl StrStorage { + /// The padded payload layout for `len` bytes of text: three header + /// words, then the bytes, rounded up to whole words. + fn layout(len: usize) -> Layout { + Layout::new::<[usize; 3]>() + .extend(Layout::array::(len).expect("string exceeds layout limit")) + .expect("string exceeds layout limit") + .0 + .pad_to_align() + } + + /// A payload of `len` bytes whose contents `init` writes, given the + /// (uninitialized) destination. One allocation, no intermediate `Vec`. + /// + /// # Safety + /// + /// `init` must initialize all `len` bytes with UTF-8 text, and + /// `chars`, when not `CHARS_UNKNOWN`, must be its code-point count. + // The backing allocation comes from `Arc::<[usize]>::new_uninit_slice`, // so it carries `usize` alignment by construction; the casts below // reinterpret it as the `usize`-headed DST it was sized for. #[allow(clippy::cast_ptr_alignment)] - pub fn build(len: usize, fill: impl FnOnce(&mut [u8])) -> Self { - let layout = Self::layout(len); - let words = layout.size() / size_of::(); - let storage = Arc::<[usize]>::new_zeroed_slice(words); - // SAFETY: an all-zero word slice is a valid `[usize]`; the header - // and bytes are (re)written below before any reader exists. - let storage = unsafe { storage.assume_init() }; + #[inline] + unsafe fn alloc(len: usize, chars: usize, init: impl FnOnce(*mut u8)) -> ThinArc { + let words = Self::layout(len).size() / size_of::(); + let storage = Arc::<[usize]>::new_uninit_slice(words); let data = Arc::into_raw(storage).cast::().cast_mut(); - let raw = ptr::slice_from_raw_parts_mut(data, len) as *mut SliceStorage; + let raw = ptr::slice_from_raw_parts_mut(data, len) as *mut Self; + // SAFETY: the word slice and the payload have identical padded + // size and alignment. The header is written here and the bytes by + // `init` (the caller's contract) before any reader can exist. unsafe { ptr::addr_of_mut!((*raw).len).write(len); ptr::addr_of_mut!((*raw).hash).write(AtomicI64::new(-1)); - let dst = ptr::addr_of_mut!((*raw).items).cast::(); - fill(std::slice::from_raw_parts_mut(dst, len)); - Self::from_arc(Arc::from_raw(raw)) + ptr::addr_of_mut!((*raw).chars).write(AtomicUsize::new(chars)); + init(ptr::addr_of_mut!((*raw).bytes).cast::()); + ThinArc::from_arc(Arc::from_raw(raw)) } } } -impl SharedSlice { - /// Append `extra` in place when this handle is the slice's only owner - /// (no other strong or weak reference): the allocation is resized - /// rather than copied, so repeated appends cost what `realloc` does - /// (CPython's `PyUnicode_Append` on a lone reference). `false` leaves - /// `this` untouched. - // The backing allocation comes from `Arc::<[usize]>::new_zeroed_slice`, - // so it carries `usize` alignment by construction; the casts below - // reinterpret it as the `usize`-headed DST it was sized for. +impl ThinArc { + #[inline] + fn storage(&self) -> &StrStorage { + // SAFETY: a live payload borrowed from `self`. + unsafe { &*self.raw() } + } + + /// Append `extra` (of `extra_chars` code points) in place when this + /// handle is the string's only owner (no other strong or weak + /// reference): the allocation is resized rather than copied, so + /// repeated appends cost what `realloc` does (CPython's + /// `PyUnicode_Append` on a lone reference). `false` leaves `this` + /// untouched. #[allow(clippy::cast_ptr_alignment)] - fn try_extend_unique(this: &mut Self, extra: &[u8]) -> bool { + fn try_extend_unique(this: &mut Self, extra: &[u8], extra_chars: usize) -> bool { if ThinArc::get_mut(this).is_none() { return false; } @@ -378,22 +493,29 @@ impl SharedSlice { let Some(new_len) = old_len.checked_add(extra.len()) else { return false; }; + let old_chars = this.storage().chars.load(Ordering::Relaxed); + let new_chars = if old_chars == CHARS_UNKNOWN { + CHARS_UNKNOWN + } else { + old_chars + extra_chars + }; // The `Arc` allocation is its two counters followed by the payload // (`ArcInner` is `repr(C)`, and std sizes it from the value layout // exactly this way), so the payload's offset and both sizes follow. let counters = Layout::new::<[usize; 2]>(); let (Ok((old_inner, offset)), Ok((new_inner, _))) = ( - counters.extend(Self::layout(old_len)), - counters.extend(Self::layout(new_len)), + counters.extend(StrStorage::layout(old_len)), + counters.extend(StrStorage::layout(new_len)), ) else { return false; }; let (old_inner, new_inner) = (old_inner.pad_to_align(), new_inner.pad_to_align()); // SAFETY: the sole owner (checked above) moves the allocation it owns // through the global allocator with the layout it was allocated - // with; the new length word and hash are written before any reader - // can exist, and the handle then points at the moved payload, whose - // eventual `Arc` drop computes the new layout from the new length. + // with; the new length, hash, and count are written before any + // reader can exist, and the handle then points at the moved + // payload, whose eventual `Arc` drop computes the new layout from + // the new length. unsafe { let base = this.data.as_ptr().cast::().sub(offset); let grown = std::alloc::realloc(base, old_inner, new_inner.size()); @@ -401,11 +523,11 @@ impl SharedSlice { return false; } let data = grown.add(offset).cast::(); - let raw = - ptr::slice_from_raw_parts_mut(data.cast::(), new_len) as *mut SliceStorage; + let raw = ptr::slice_from_raw_parts_mut(data.cast::(), new_len) as *mut StrStorage; ptr::addr_of_mut!((*raw).len).write(new_len); (*raw).hash.store(-1, Ordering::Relaxed); - let dst = ptr::addr_of_mut!((*raw).items).cast::(); + (*raw).chars.store(new_chars, Ordering::Relaxed); + let dst = ptr::addr_of_mut!((*raw).bytes).cast::(); ptr::copy_nonoverlapping(extra.as_ptr(), dst.add(old_len), extra.len()); this.data = NonNull::new_unchecked(data); } @@ -413,63 +535,11 @@ impl SharedSlice { } } -impl SharedStr { - /// Append `suffix` in place when this handle is the string's only - /// owner (see `SharedSlice::try_extend_unique`); `false` leaves it - /// untouched. - pub fn try_append(this: &mut Self, suffix: &str) -> bool { - SharedSlice::try_extend_unique(&mut this.0, suffix.as_bytes()) - } - - /// The concatenation of `parts`, in one allocation. - pub fn concat(parts: &[&str]) -> Self { - let len = parts.iter().map(|p| p.len()).sum(); - Self(SharedSlice::build(len, |buf| { - let mut at = 0; - for p in parts { - buf[at..at + p.len()].copy_from_slice(p.as_bytes()); - at += p.len(); - } - })) - } - - /// `s` repeated `times` times, in one allocation. - pub fn repeat(s: &str, times: usize) -> Self { - let len = s.len() * times; - Self(SharedSlice::build(len, |buf| { - for chunk in buf.chunks_exact_mut(s.len().max(1)) { - chunk.copy_from_slice(s.as_bytes()); - } - })) - } -} - -impl From> for SharedSlice { - fn from(items: Vec) -> Self { - Self::from_vec_owned(items) - } -} -impl From> for SharedSlice { - fn from(items: Box<[T]>) -> Self { - Self::from_vec_owned(items.into_vec()) - } -} -impl From<&[T]> for SharedSlice { - fn from(items: &[T]) -> Self { - Self::from_copy_slice(items) - } -} -impl From<&[T; N]> for SharedSlice { - fn from(items: &[T; N]) -> Self { - Self::from_copy_slice(items) - } -} - /// Shared UTF-8 text. Its private byte owner can only be constructed from str. #[derive(Clone, Eq, PartialEq, Ord, PartialOrd)] -pub struct SharedStr(SharedSlice); +pub struct SharedStr(ThinArc); #[derive(Debug)] -pub struct WeakStr(WeakSlice); +pub struct WeakStr(ThinWeak); impl SharedStr { pub fn as_ptr(this: &Self) -> *const str { ptr::from_ref(&**this) @@ -477,13 +547,43 @@ impl SharedStr { pub fn ptr_eq(this: &Self, other: &Self) -> bool { ThinArc::ptr_eq(&this.0, &other.0) } - /// The memoised `hash(s)` (see [`SharedSlice::hash_cached`]). + /// The memoised `hash(s)`, computed on first use the way CPython's + /// `str` caches its hash: a dict key hashes once per object, not once + /// per lookup. #[inline] pub fn hash_cached(this: &Self) -> i64 { - SharedSlice::hash_cached(&this.0, |bytes| { - // SAFETY: the private owner is only ever built from `str`. - crate::object::py_str_hash(unsafe { std::str::from_utf8_unchecked(bytes) }) - }) + let storage = this.0.storage(); + let h = storage.hash.load(Ordering::Relaxed); + if h != -1 { + return h; + } + let h = crate::object::py_str_hash(this); + storage.hash.store(h, Ordering::Relaxed); + h + } + /// `len(s)` in code points, memoised in the string itself. + #[inline] + pub fn char_count(this: &Self) -> usize { + let storage = this.0.storage(); + let n = storage.chars.load(Ordering::Relaxed); + if n != CHARS_UNKNOWN { + return n; + } + let n = this.chars().count(); + storage.chars.store(n, Ordering::Relaxed); + n + } + /// The code-point count if it is already known, without counting. + #[inline] + pub fn known_char_count(this: &Self) -> Option { + let n = this.0.storage().chars.load(Ordering::Relaxed); + (n != CHARS_UNKNOWN).then_some(n) + } + /// Whether the text is pure ASCII (one byte per code point), from the + /// memoised count. + #[inline] + pub fn is_ascii(this: &Self) -> bool { + Self::char_count(this) == this.len() } pub fn strong_count(this: &Self) -> usize { ThinArc::strong_count(&this.0) @@ -494,23 +594,178 @@ impl SharedStr { pub fn downgrade(this: &Self) -> WeakStr { WeakStr(ThinArc::downgrade(&this.0)) } + + /// A copy of `text`, known to be pure ASCII (`debug_assert`ed): it is + /// born with its code-point count. + #[inline] + pub fn from_ascii(text: &str) -> Self { + debug_assert!(text.is_ascii()); + Self::with_count(text, text.len()) + } + + /// A copy of `text`, which the caller knows to hold `chars` code points. + #[inline] + pub fn with_count(text: &str, chars: usize) -> Self { + debug_assert_eq!(text.chars().count(), chars); + let len = text.len(); + // SAFETY: exactly `len` bytes of UTF-8 are copied, and `chars` + // counts them (the caller's word, checked in debug builds). + Self(unsafe { + StrStorage::alloc(len, chars, |dst| { + copy_text(text.as_ptr(), dst, len); + }) + }) + } + + /// A string of `len` bytes that `fill` writes, in order, through a + /// [`TextWriter`] over its final allocation: one allocation, no + /// intermediate `String`, no zeroing. `chars` is its code-point count + /// when the caller knows it. Panics unless `fill` writes exactly `len` + /// bytes. + /// + /// # Safety + /// + /// The bytes `fill` writes must be valid UTF-8, of `chars` code points + /// when `chars` is given. + #[inline] + pub unsafe fn build_unchecked( + len: usize, + chars: Option, + fill: impl FnOnce(&mut TextWriter<'_>), + ) -> Self { + // SAFETY: the writer initializes every byte it accounts for, and + // the assertion checks that it accounted for all `len`; the + // caller vouches for the text and its count. + let out = Self(unsafe { + StrStorage::alloc(len, chars.unwrap_or(CHARS_UNKNOWN), |dst| { + let mut writer = TextWriter { + buf: std::slice::from_raw_parts_mut(dst.cast::>(), len), + at: 0, + }; + fill(&mut writer); + assert_eq!(writer.at, len, "string builder wrote the wrong length"); + }) + }); + debug_assert!(std::str::from_utf8(out.0.as_ref()).is_ok()); + debug_assert!(chars.is_none_or(|n| n == out.chars().count())); + out + } + + /// Append `suffix` in place when this handle is the string's only + /// owner (see `ThinArc::::try_extend_unique`); `false` + /// leaves it untouched. + pub fn try_append(this: &mut Self, suffix: &str) -> bool { + // Keep a known count known: `len(s)` after `s += t` in a loop would + // otherwise recount the whole string each time. + let extra_chars = if this.0.storage().chars.load(Ordering::Relaxed) == CHARS_UNKNOWN { + 0 + } else { + suffix.chars().count() + }; + ThinArc::::try_extend_unique(&mut this.0, suffix.as_bytes(), extra_chars) + } + + /// The concatenation of `parts`, in one allocation. + pub fn concat(parts: &[&str]) -> Self { + let len = parts.iter().map(|p| p.len()).sum(); + // SAFETY: a concatenation of `str`s is UTF-8. + unsafe { + Self::build_unchecked(len, None, |out| { + for p in parts { + out.push(p.as_bytes()); + } + }) + } + } + + /// `s` repeated `times` times, in one allocation. + pub fn repeat(s: &str, times: usize) -> Self { + let len = s.len() * times; + // SAFETY: a repetition of a `str` is UTF-8. + unsafe { + Self::build_unchecked(len, None, |out| { + if !s.is_empty() { + for _ in 0..times { + out.push(s.as_bytes()); + } + } + }) + } + } +} + +/// The sequential writer [`SharedStr::build_unchecked`] hands its `fill`: +/// each write lands after the last, and the builder checks that the +/// writes covered the whole string. +#[derive(Debug)] +pub struct TextWriter<'a> { + buf: &'a mut [MaybeUninit], + at: usize, +} +impl TextWriter<'_> { + /// Append `bytes`. Panics past the end of the string. + #[inline] + pub fn push(&mut self, bytes: &[u8]) { + let end = self.at + bytes.len(); + assert!(end <= self.buf.len(), "string builder overflow"); + // SAFETY: `at..end` lies within the buffer (checked above), which + // can't overlap the borrowed `bytes`. + unsafe { + copy_text( + bytes.as_ptr(), + self.buf.as_mut_ptr().add(self.at).cast(), + bytes.len(), + ) + }; + self.at = end; + } + + /// Append one byte. Panics past the end of the string. + #[inline] + pub fn push_byte(&mut self, byte: u8) { + self.buf[self.at] = MaybeUninit::new(byte); + self.at += 1; + } + + /// Append `n` bytes that `write` initializes, given exactly those + /// bytes. Panics past the end of the string. + /// + /// # Safety + /// + /// `write` must initialize all `n` bytes it is given. + #[inline] + pub unsafe fn push_with(&mut self, n: usize, write: impl FnOnce(&mut [MaybeUninit])) { + let end = self.at + n; + write(&mut self.buf[self.at..end]); + self.at = end; + } } impl Deref for SharedStr { type Target = str; + #[inline] fn deref(&self) -> &str { - // SAFETY: only From can create the private byte owner. - // No mutable byte view is exposed; clones and weak upgrades preserve it. + // SAFETY: only the UTF-8 constructors above create the private + // byte owner. No mutable byte view is exposed; clones and weak + // upgrades preserve it. unsafe { std::str::from_utf8_unchecked(&self.0) } } } impl From<&str> for SharedStr { + #[inline] fn from(value: &str) -> Self { - Self(SharedSlice::from(value.as_bytes())) + let len = value.len(); + // SAFETY: exactly `len` bytes of UTF-8 are copied; the count is + // left for first use. + Self(unsafe { + StrStorage::alloc(len, if len == 0 { 0 } else { CHARS_UNKNOWN }, |dst| { + copy_text(value.as_ptr(), dst, len); + }) + }) } } impl From for SharedStr { fn from(value: String) -> Self { - Self(SharedSlice::from(value.into_bytes())) + Self::from(value.as_str()) } } impl From<&String> for SharedStr { @@ -520,7 +775,7 @@ impl From<&String> for SharedStr { } impl From> for SharedStr { fn from(value: Box) -> Self { - Self::from(String::from(value)) + Self::from(&*value) } } impl AsRef for SharedStr { @@ -567,6 +822,9 @@ impl sealed::Sealed for crate::tuple_storage::TupleStorage {} // initialized [Object] tail. Only its constructors create these Arc payloads; // unique mutation changes elements/hash, never slice length or metadata. unsafe impl ThinPayload for crate::tuple_storage::TupleStorage { + fn release_last(last: Arc) { + crate::rc::release_nested(last); + } type View = Self; fn pointer(data: *const usize, len: usize) -> *const Self { ptr::slice_from_raw_parts(data.cast::(), len) as *const Self @@ -657,6 +915,58 @@ mod tests { } } + #[test] + fn strings_memoise_code_point_counts_through_appends() { + for text in [ + "", + "a", + "plain ascii", + "mañana", + "日本語", + "🧶🙂", + "e\u{301}", + ] { + let value = SharedStr::from(text); + assert_eq!( + SharedStr::known_char_count(&value), + text.is_empty().then_some(0) + ); + assert_eq!(SharedStr::char_count(&value), text.chars().count()); + assert_eq!( + SharedStr::known_char_count(&value), + Some(text.chars().count()) + ); + assert_eq!(SharedStr::is_ascii(&value), text.is_ascii()); + let mut grown = value.clone(); + drop(value); + assert!(SharedStr::try_append(&mut grown, "ñx")); + let expected = format!("{text}ñx"); + assert_eq!(&*grown, expected); + assert_eq!( + SharedStr::known_char_count(&grown), + Some(expected.chars().count()) + ); + assert_eq!( + SharedStr::hash_cached(&grown), + crate::object::py_str_hash(&expected) + ); + } + let ascii = SharedStr::from_ascii("abc"); + assert_eq!(SharedStr::known_char_count(&ascii), Some(3)); + // SAFETY: ASCII parts and their count. + let built = unsafe { + SharedStr::build_unchecked(7, Some(7), |out| { + out.push(b"ab"); + out.push_byte(b'-'); + out.push(b"cdef"); + }) + }; + assert_eq!(&*built, "ab-cdef"); + assert_eq!(SharedStr::concat(&["x", "ñ", ""]).chars().count(), 2); + assert_eq!(&*SharedStr::repeat("ab", 3), "ababab"); + assert_eq!(&*SharedStr::repeat("", 3), ""); + } + #[test] fn unique_access_rejects_strong_and_weak_aliases() { let mut value = SharedSlice::from(&[11_u8, 22, 33]); diff --git a/crates/weavepy-vm/src/stdlib/ast_build.rs b/crates/weavepy-vm/src/stdlib/ast_build.rs index 012b7797..9c973913 100644 --- a/crates/weavepy-vm/src/stdlib/ast_build.rs +++ b/crates/weavepy-vm/src/stdlib/ast_build.rs @@ -82,11 +82,7 @@ impl Builder<'_> { ) -> Result { let builder = self.recursive.resolve(function)?; let result = self.walk(vm, spec, &builder, &function.globals); - if result.is_ok() { - vm.reap_call_receiver(builder); - } else { - crate::gc_trace::mark_maybe_dead(); - } + vm.release(builder); result } @@ -178,11 +174,10 @@ impl Builder<'_> { let node = vm.call(&new, std::slice::from_ref(&cls), &[], globals); let node = match node { Ok(node) => { - vm.reap_call_receiver(new); + vm.release(new); node } Err(error) => { - crate::gc_trace::mark_maybe_dead(); return Err(error); } }; @@ -219,24 +214,20 @@ impl Builder<'_> { // setter's unused result. An ignored child can die here. let result = match result { Ok(result) => { - vm.reap_call_receiver(setter); + vm.release(setter); vm.reap_call_args(&mut args); result } Err(error) => { - crate::gc_trace::mark_maybe_dead(); return Err(error); } }; - crate::gc_trace::note_dropped(&result); - if Interpreter::local_needs_prompt_reap(&result) { - vm.prompt_reap_dropped(result); - } + vm.release(result); } Ok(()) })(); if let Err(error) = outcome { - vm.prompt_reap_dropped(node); + vm.release(node); return Err(error); } Ok(node) @@ -258,7 +249,7 @@ impl Builder<'_> { Ok(()) })(); if let Err(error) = outcome { - vm.prompt_reap_dropped(result); + vm.release(result); return Err(error); } Ok(result) diff --git a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs index 6206fe96..6524b66f 100644 --- a/crates/weavepy-vm/src/stdlib/asyncio_mod.rs +++ b/crates/weavepy-vm/src/stdlib/asyncio_mod.rs @@ -1340,11 +1340,8 @@ fn future_iter_new(fut: &Object) -> Object { // Built outside the ordinary instantiation path, so enrol with the // collector by hand: a suspended `await` parks this iterator on the // coroutine's value stack, and its strong `fut` edge must be - // subtractable for task↔future cycles to collapse. Prompt-reclaim - // enrollment keeps its death by "refcount" observable — a lingering - // iterator pins the awaited Task, its coroutine frame, and every local - // in it (test_ssl's weakref leak tests watch exactly that chain). - crate::gc_trace::track_prompt_reclaim(out.clone()); + // subtractable for task↔future cycles to collapse. + crate::gc_trace::track(&out); out } diff --git a/crates/weavepy-vm/src/stdlib/collections_native.rs b/crates/weavepy-vm/src/stdlib/collections_native.rs index e811d625..4f3105c2 100644 --- a/crates/weavepy-vm/src/stdlib/collections_native.rs +++ b/crates/weavepy-vm/src/stdlib/collections_native.rs @@ -148,6 +148,7 @@ impl DequeState<'_> { // unguarded (`peek_mut` returns `None` if any guard is live) and neither // reference outlives the native call, which runs no Python. #[allow(clippy::mut_from_ref)] +#[inline(always)] fn fast_parts(args: &[Object]) -> Option> { let Object::Instance(inst) = args.first()? else { return None; @@ -156,7 +157,8 @@ fn fast_parts(args: &[Object]) -> Option> { // checks), and neither reference outlives the native call. let slots = unsafe { inst.slots.peek_mut() }?; let n = names(); - let [data, head, maxlen, state] = slots.leading_mut([&n.data, &n.head, &n.maxlen, &n.state])?; + let [data, head, maxlen, state] = + slots.leading_interned_mut([&n.data, &n.head, &n.maxlen, &n.state])?; let Object::List(data) = data else { return None; }; @@ -216,12 +218,14 @@ impl Fast<'_> { } } - /// [`popleft_locked`] over the unguarded parts. - fn popleft(&mut self) -> Result { + /// [`popleft_locked`] over the unguarded parts; `None` (nothing + /// touched) on an empty deque. + #[inline] + fn popleft(&mut self) -> Option { let mut h = self.head(); let d = &mut *self.d; if h >= d.len() { - return Err(index_error("pop from an empty deque")); + return None; } let x = std::mem::replace(&mut d[h], Object::None); h += 1; @@ -234,7 +238,101 @@ impl Fast<'_> { } self.bump_state(); self.set_head(h); - Ok(x) + Some(x) + } +} + +/// The deque operations a compiled caller may run directly (see +/// [`fast_op`]); `0` names none. The JIT's method registry carries one +/// per builtin (`leaf_builtins::register_jit_method`). +pub(crate) const OP_APPEND: u8 = 1; +pub(crate) const OP_APPENDLEFT: u8 = 2; +pub(crate) const OP_POP: u8 = 3; +pub(crate) const OP_POPLEFT: u8 = 4; +pub(crate) const OP_LEN: u8 = 5; +pub(crate) const OP_BOOL: u8 = 6; +pub(crate) const OP_GETITEM: u8 = 7; +/// A forward (`_deque_iterator`) or reverse iterator step; the receiver +/// is the iterator. +pub(crate) const OP_NEXT: u8 = 8; +pub(crate) const OP_RNEXT: u8 = 9; + +/// One deque operation's common case on `recv` (and its one argument), +/// over the unguarded views [`fast_parts`] takes: the whole operation, +/// or `None` with nothing touched when it needs the full body (a guarded +/// cell, a foreign receiver, an empty deque's `IndexError`, an index out +/// of range or not an exact `int`). The builtins' fast paths and the +/// JIT's direct calls share it; it runs no Python code. Inlined, so a +/// builtin's constant `op` folds the dispatch away. +#[inline(always)] +pub(crate) fn fast_op(op: u8, recv: &Object, arg: Option<&Object>) -> Option { + if op >= OP_NEXT { + let Object::Instance(iterator) = recv else { + return None; + }; + return deque_next_fast(iterator, op == OP_RNEXT); + } + let mut f = fast_parts(std::slice::from_ref(recv))?; + match op { + OP_APPEND => { + let x = arg?; + f.bump_state(); + f.d.push(x.clone()); + let trimmed = match f.maxlen() { + Some(m) if f.d.len() - f.head() > m => f.popleft(), + _ => None, + }; + drop(trimmed); + Some(Object::None) + } + OP_APPENDLEFT => { + let x = arg?; + f.bump_state(); + let mut h = f.head().min(f.d.len()); + if h == 0 { + h = std::cmp::max(8, f.d.len() / 2); + f.d.splice(0..0, std::iter::repeat_n(Object::None, h)); + } + h -= 1; + f.d[h] = x.clone(); + f.set_head(h); + let trimmed = match f.maxlen() { + Some(m) if f.d.len() - h > m => f.d.pop(), + _ => None, + }; + drop(trimmed); + Some(Object::None) + } + OP_POP => { + let h = f.head(); + if f.d.len() <= h { + return None; + } + f.bump_state(); + let x = f.d.pop().expect("len checked"); + // An emptied deque keeps a short free prefix for the next + // `appendleft` instead of splicing a new one. + if f.d.len() == h && h > 32 { + f.d.clear(); + f.set_head(0); + } + Some(x) + } + OP_POPLEFT => f.popleft(), + OP_LEN => Some(Object::Int(f.d.len().saturating_sub(f.head()) as i64)), + OP_BOOL => Some(Object::Bool(f.d.len() > f.head())), + OP_GETITEM => { + let Some(&Object::Int(i)) = arg else { + return None; + }; + let h = f.head(); + let n = f.d.len().saturating_sub(h) as i64; + let index = if i < 0 { i + n } else { i }; + (0..n) + .contains(&index) + .then(|| f.d[h + index as usize].clone()) + } + _ => None, } } @@ -277,19 +375,9 @@ fn popleft_locked(st: &mut DequeState<'_>, d: &mut Vec) -> Result Result { - if let [_, x] = args { - if let Some(mut f) = fast_parts(args) { - f.bump_state(); - f.d.push(x.clone()); - let trimmed = match f.maxlen() { - Some(m) if f.d.len() - f.head() > m => Some(f.popleft()?), - _ => None, - }; - if trimmed.is_some() { - crate::gc_trace::mark_maybe_dead(); - } - drop(trimmed); - return Ok(Object::None); + if let [recv, x] = args { + if let Some(v) = fast_op(OP_APPEND, recv, Some(x)) { + return Ok(v); } } let mut st = receiver(args, "append")?; @@ -319,35 +407,14 @@ fn deque_append(args: &[Object]) -> Result { }; } drop(st); - if trimmed.is_some() { - // A bounded deque released its oldest item (leaf contract). - crate::gc_trace::mark_maybe_dead(); - } drop(trimmed); Ok(Object::None) } fn deque_appendleft(args: &[Object]) -> Result { - if let [_, x] = args { - if let Some(mut f) = fast_parts(args) { - f.bump_state(); - let mut h = f.head().min(f.d.len()); - if h == 0 { - h = std::cmp::max(8, f.d.len() / 2); - f.d.splice(0..0, std::iter::repeat_n(Object::None, h)); - } - h -= 1; - f.d[h] = x.clone(); - f.set_head(h); - let trimmed = match f.maxlen() { - Some(m) if f.d.len() - h > m => f.d.pop(), - _ => None, - }; - if trimmed.is_some() { - crate::gc_trace::mark_maybe_dead(); - } - drop(trimmed); - return Ok(Object::None); + if let [recv, x] = args { + if let Some(v) = fast_op(OP_APPENDLEFT, recv, Some(x)) { + return Ok(v); } } let mut st = receiver(args, "appendleft")?; @@ -379,28 +446,14 @@ fn deque_appendleft(args: &[Object]) -> Result { }; } drop(st); - if trimmed.is_some() { - crate::gc_trace::mark_maybe_dead(); - } drop(trimmed); Ok(Object::None) } fn deque_pop(args: &[Object]) -> Result { - if args.len() == 1 { - if let Some(mut f) = fast_parts(args) { - let h = f.head(); - if f.d.len() > h { - f.bump_state(); - let x = f.d.pop().expect("len checked"); - // An emptied deque keeps a short free prefix for the next - // `appendleft` instead of splicing a new one. - if f.d.len() == h && h > 32 { - f.d.clear(); - f.set_head(0); - } - return Ok(x); - } + if let [recv] = args { + if let Some(v) = fast_op(OP_POP, recv, None) { + return Ok(v); } } let mut st = receiver(args, "pop")?; @@ -426,11 +479,9 @@ fn deque_pop(args: &[Object]) -> Result { } fn deque_popleft(args: &[Object]) -> Result { - if args.len() == 1 { - if let Some(mut f) = fast_parts(args) { - if f.head() < f.d.len() { - return f.popleft(); - } + if let [recv] = args { + if let Some(v) = fast_op(OP_POPLEFT, recv, None) { + return Ok(v); } } let mut st = receiver(args, "popleft")?; @@ -446,9 +497,9 @@ fn deque_popleft(args: &[Object]) -> Result { } fn deque_len(args: &[Object]) -> Result { - if args.len() == 1 { - if let Some(f) = fast_parts(args) { - return Ok(Object::Int(f.d.len().saturating_sub(f.head()) as i64)); + if let [recv] = args { + if let Some(v) = fast_op(OP_LEN, recv, None) { + return Ok(v); } } let st = receiver(args, "__len__")?; @@ -460,9 +511,9 @@ fn deque_len(args: &[Object]) -> Result { } fn deque_bool(args: &[Object]) -> Result { - if args.len() == 1 { - if let Some(f) = fast_parts(args) { - return Ok(Object::Bool(f.d.len() > f.head())); + if let [recv] = args { + if let Some(v) = fast_op(OP_BOOL, recv, None) { + return Ok(v); } } let st = receiver(args, "__bool__")?; @@ -474,14 +525,9 @@ fn deque_bool(args: &[Object]) -> Result { } fn deque_getitem(args: &[Object]) -> Result { - if let [_, Object::Int(i)] = args { - if let Some(f) = fast_parts(args) { - let h = f.head(); - let n = f.d.len().saturating_sub(h) as i64; - let index = if *i < 0 { *i + n } else { *i }; - if (0..n).contains(&index) { - return Ok(f.d[h + index as usize].clone()); - } + if let [recv, index] = args { + if let Some(v) = fast_op(OP_GETITEM, recv, Some(index)) { + return Ok(v); } } let [_, index] = args else { @@ -555,6 +601,50 @@ fn deque_rotate(args: &[Object]) -> Result { Ok(Object::None) } +/// `iter(d)` / `reversed(d)`: a fresh `_deque_iterator` (or reverse +/// iterator) over `d` at index 0, snapshotting the mutation counter, +/// exactly what the iterator classes' `__init__` builds, but with no +/// Python code running. The classes come from the deque class's +/// `_iter_types` pair (set by `_collections.py`), so a subclass and each +/// copy of the module use their own. +fn deque_make_iter(args: &[Object], reverse: bool) -> Result { + let method = if reverse { "__reversed__" } else { "__iter__" }; + let state = { + let st = receiver(args, method)?; + st.slots + .get_hinted(SLOT_STATE, &names().state) + .cloned() + .unwrap_or(Object::Int(0)) + }; + let [recv @ Object::Instance(inst)] = args else { + return Err(type_error(format!( + "deque.{method}() takes no arguments ({} given)", + args.len().saturating_sub(1) + ))); + }; + let it_cls = match inst.cls().lookup("_iter_types") { + Some(Object::Tuple(types)) => match types.get(usize::from(reverse)) { + Some(Object::Type(cls)) => cls.clone(), + _ => return Err(type_error("deque iterator types are not set")), + }, + _ => return Err(type_error("deque iterator types are not set")), + }; + let it = crate::types::PyInstance::new_deferred(it_cls); + // The iterator classes' slot order (see `deque_next_fast`). + it.slot_set("_deq", recv.clone()); + it.slot_set("_index", Object::Int(0)); + it.slot_set("_deq_state", state); + Ok(Object::Instance(it)) +} + +fn deque_iter(args: &[Object]) -> Result { + deque_make_iter(args, false) +} + +fn deque_reversed(args: &[Object]) -> Result { + deque_make_iter(args, true) +} + fn deque_iterator_index(args: &[Object]) -> Result { receiver(args, "__iter__")?; let [_, index] = args else { @@ -586,13 +676,14 @@ const IT_STATE: usize = 2; /// [`fast_parts`]): a live iterator over an unmutated deque with an item /// left. `None` (nothing touched) leaves every other case to the full /// body. +#[inline(always)] fn deque_next_fast(iterator: &crate::types::PyInstance, reverse: bool) -> Option { // SAFETY: no guard is live on the cells (`peek`/`peek_mut` check), no // Python runs before the last use, and the iterator and its deque are // distinct objects. let its = unsafe { iterator.slots.peek_mut() }?; let n = names(); - let [deque, index, it_state] = its.leading_mut([&n.deq, &n.index, &n.deq_state])?; + let [deque, index, it_state] = its.leading_interned_mut([&n.deq, &n.index, &n.deq_state])?; let Object::Instance(deque) = deque else { return None; }; @@ -603,12 +694,14 @@ fn deque_next_fast(iterator: &crate::types::PyInstance, reverse: bool) -> Option }; // SAFETY: as above. let ds = unsafe { (*deque).slots.peek() }?; - let [data, head, _, state] = ds.leading([&n.data, &n.head, &n.maxlen, &n.state])?; + let [data, head, _, state] = ds.leading_interned([&n.data, &n.head, &n.maxlen, &n.state])?; let Object::List(data) = data else { return None; }; - if state.as_i64() != it_state.as_i64() { - return None; + // The counters are plain ints (any other value takes the full body). + match (state, &*it_state) { + (Object::Int(a), Object::Int(b)) if a == b => {} + _ => return None, } let h = match *head { Object::Int(h) if h >= 0 => h as usize, @@ -635,7 +728,7 @@ fn deque_next(args: &[Object], reverse: bool) -> Result { let [Object::Instance(iterator)] = args else { return Err(type_error("deque iterator __next__ requires one iterator")); }; - if let Some(v) = deque_next_fast(iterator, reverse) { + if let Some(v) = fast_op(if reverse { OP_RNEXT } else { OP_NEXT }, &args[0], None) { return Ok(v); } let (deque, index, it_state) = { @@ -746,6 +839,8 @@ pub fn build(_cache: &ModuleCache) -> Rc { reg("__bool__", "__bool__", deque_bool); reg("__getitem__", "__getitem__", deque_getitem); reg("rotate", "rotate", deque_rotate); + reg("__iter__", "__iter__", deque_iter); + reg("__reversed__", "__reversed__", deque_reversed); reg("iterator_index", "iterator_index", deque_iterator_index); reg("iterator_next", "__next__", deque_iterator_next); reg( @@ -753,9 +848,10 @@ pub fn build(_cache: &ModuleCache) -> Rc { "__next__", deque_reverse_iterator_next, ); - // The end operations, length, truth, and iterator steps run no - // Python code. Indexing and rotation require an argument guard - // because index coercion can invoke Python's __index__ protocol. + // The end operations, length, truth, iterator construction, and + // iterator steps run no Python code. Indexing and rotation require + // an argument guard because index coercion can invoke Python's + // __index__ protocol. for name in [ "append", "appendleft", @@ -763,6 +859,8 @@ pub fn build(_cache: &ModuleCache) -> Rc { "popleft", "__len__", "__bool__", + "__iter__", + "__reversed__", "iterator_next", "reverse_iterator_next", ] { @@ -781,6 +879,24 @@ pub fn build(_cache: &ModuleCache) -> Rc { crate::leaf_builtins::register_fast(b, fast); } } + // The deque's own methods ride the JIT's native method lane (the + // receiver checks above reject a foreign receiver exactly). + for (name, op) in [ + ("append", OP_APPEND), + ("appendleft", OP_APPENDLEFT), + ("pop", OP_POP), + ("popleft", OP_POPLEFT), + ("__len__", OP_LEN), + ("__bool__", OP_BOOL), + ("__getitem__", OP_GETITEM), + ("rotate", 0), + ("iterator_next", OP_NEXT), + ("reverse_iterator_next", OP_RNEXT), + ] { + if let Some(Object::Builtin(b)) = d.get(&DictKey(Object::from_static(name))) { + crate::leaf_builtins::register_jit_method(b, op); + } + } } Rc::new(PyModule { name: "_weave_collections".to_owned(), diff --git a/crates/weavepy-vm/src/stdlib/datetime_native.rs b/crates/weavepy-vm/src/stdlib/datetime_native.rs index df69bda3..1337dd6b 100644 --- a/crates/weavepy-vm/src/stdlib/datetime_native.rs +++ b/crates/weavepy-vm/src/stdlib/datetime_native.rs @@ -22,13 +22,22 @@ //! ([`leaf_binop`], [`leaf_compare`], [`leaf_field`]) apply only while a //! class is in the state [`install`] left it in. //! +//! The natives build exact `timedelta`, `date` and `datetime` instances +//! with packed storage ([`PackedSlots`]): the fields as two machine words +//! instead of a vector of objects, as CPython's C structs hold them. The +//! exact classes' constructors (keywords included), arithmetic, and +//! `fromisoformat` produce such values, and a dead one returns to its +//! class set's [`Pool`], so a value's life costs no allocation. Python +//! code reading a slot (`self._year`) sees field objects built on first +//! use; writing one turns the storage back into a plain field vector. +//! //! Exact-class instances are never cycle-collector tracked: CPython's C //! datetime types are not GC types either (a cycle through `tzinfo` is //! uncollectable there too), and their fields are ints, `None` and the //! `tzinfo` reference. use std::collections::HashMap; -use std::fmt::Write as _; +use std::sync::atomic::{AtomicUsize, Ordering}; use crate::error::{overflow_error, type_error, RuntimeError}; use crate::object::{BuiltinFn, DictKey, Object}; @@ -63,15 +72,374 @@ const DT_TZINFO: usize = 7; const DT_FOLD: usize = 9; const TZ_OFFSET: usize = 0; +// --------------------------------------------------------------------- +// Packed storage. + +/// A natively built `timedelta`, `date` or `datetime`, packed into two +/// words: the instance's slot storage (see `SlotData::Packed`) instead +/// of a vector of field objects. CPython's C types store these as C +/// struct members; packing gets WeavePy the same allocation-free +/// representation, and the natives read a field with a shift. +/// +/// - `datetime` and `date`: `word` holds the fields in chronological +/// significance (see [`pack_dt`]), so two naive values compare as +/// `word >> 1`; `aux` owns the `tzinfo` (an `Rc` turned +/// into a raw pointer), or is `0` for `None`. +/// - `timedelta`: `word` is the days (as `i64`), `aux` the seconds and +/// microseconds (`seconds << 20 | microseconds`). +/// +/// Code that reads a slot by reference (a slot descriptor serving +/// `self._year` in a Python method, `copy`, the collector) gets a copy +/// of the field objects built on first use and kept until the storage +/// dies or is written; every write first converts the storage to a +/// field vector (`SlotStorage::unpack`). +pub(crate) struct PackedSlots { + word: u64, + aux: usize, + /// The kind (low three bits), and the lazily built field objects (a + /// thin pointer to `layout().len()` objects; `0` until built). + meta: AtomicUsize, +} + +const META_KIND: usize = 7; + +// Bit positions in a `datetime`/`date` word. +const W_FOLD: u32 = 0; +const W_US: u32 = 1; +const W_SS: u32 = 21; +const W_MM: u32 = 27; +const W_HH: u32 = 33; +const W_DAY: u32 = 38; +const W_MONTH: u32 = 43; +const W_YEAR: u32 = 47; + +/// A validated `datetime`'s fields as a packed word. +#[inline] +fn pack_dt(f: &Dt) -> u64 { + ((f.y as u64) << W_YEAR) + | ((f.m as u64) << W_MONTH) + | ((f.d as u64) << W_DAY) + | ((f.hh as u64) << W_HH) + | ((f.mm as u64) << W_MM) + | ((f.ss as u64) << W_SS) + | ((f.us as u64) << W_US) + | ((f.fold as u64) << W_FOLD) +} + +#[inline] +fn bits(word: u64, shift: u32, width: u32) -> i64 { + ((word >> shift) & ((1 << width) - 1)) as i64 +} + +/// `(year, month, day)` of a packed `date`/`datetime` word. +#[inline] +fn word_ymd(w: u64) -> (i64, i64, i64) { + (bits(w, W_YEAR, 14), bits(w, W_MONTH, 4), bits(w, W_DAY, 5)) +} + +/// `(hour, minute, second, microsecond, fold)` of a packed word. +#[inline] +fn word_time(w: u64) -> (i64, i64, i64, i64, i64) { + ( + bits(w, W_HH, 5), + bits(w, W_MM, 6), + bits(w, W_SS, 6), + bits(w, W_US, 20), + bits(w, W_FOLD, 1), + ) +} + +/// The slot layouts packed values describe themselves with, shared by +/// every interpreter: `[timedelta, date, datetime]`, in the order the +/// classes' `__new__` assigns the slots. +fn layouts() -> &'static [SharedSlice; 3] { + static LAYOUTS: std::sync::OnceLock<[SharedSlice; 3]> = std::sync::OnceLock::new(); + LAYOUTS.get_or_init(|| { + let layout = |names: &[&str]| -> SharedSlice { + names + .iter() + .map(|n| DictKey(Object::Str(interned(n)))) + .collect::>() + .into() + }; + [ + layout(&[ + "_days", + "_seconds", + "_microseconds", + "_hashcode", + "days", + "seconds", + "microseconds", + ]), + layout(&["_year", "_month", "_day", "_hashcode"]), + layout(&[ + "_year", + "_month", + "_day", + "_hour", + "_minute", + "_second", + "_microsecond", + "_tzinfo", + "_hashcode", + "_fold", + ]), + ] + }) +} + +#[inline] +fn layout_of(kind: u8) -> &'static SharedSlice { + let l = layouts(); + match kind { + KIND_TIMEDELTA => &l[0], + KIND_DATE => &l[1], + _ => &l[2], + } +} + +impl PackedSlots { + #[inline] + fn new(kind: u8, word: u64, aux: usize) -> Self { + Self { + word, + aux, + meta: AtomicUsize::new(kind as usize), + } + } + + #[inline] + pub(crate) fn kind(&self) -> u8 { + (self.meta.load(Ordering::Relaxed) & META_KIND) as u8 + } + + #[inline] + fn has_tz(&self) -> bool { + self.kind() == KIND_DATETIME + } + + /// The owned `tzinfo` pointer (`null` for `None` and other kinds). + #[inline] + fn tz_ptr(&self) -> *const PyInstance { + if self.has_tz() { + self.aux as *const PyInstance + } else { + std::ptr::null() + } + } + + /// A new reference to the `tzinfo`. + #[inline] + fn tz(&self) -> Object { + let p = self.tz_ptr(); + if p.is_null() { + return Object::None; + } + // SAFETY: `aux` owns one strong reference; this adds another. + unsafe { + Rc::increment_strong_count(p); + Object::Instance(Rc::from_raw(p)) + } + } + + pub(crate) fn layout(&self) -> &'static SharedSlice { + layout_of(self.kind()) + } + + /// The field objects (built on first use; see the type docs). + pub(crate) fn values(&self) -> &[Object] { + let len = self.layout().len(); + let m = self.meta.load(Ordering::Acquire); + let ptr = if m & !META_KIND != 0 { + (m & !META_KIND) as *const Object + } else { + self.materialize() + }; + // SAFETY: a built copy holds exactly `len` objects and lives until + // the storage is dropped, cleared or unpacked (all `&mut self`). + unsafe { std::slice::from_raw_parts(ptr, len) } + } + + #[cold] + #[inline(never)] + fn materialize(&self) -> *const Object { + let vals = self.build_values().into_boxed_slice(); + let len = vals.len(); + let ptr = Box::into_raw(vals).cast::(); + let kind = self.kind() as usize; + match self.meta.compare_exchange( + kind, + ptr as usize | kind, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => ptr, + Err(cur) => { + // Another reader built a copy first. + // SAFETY: `ptr` came from `Box::into_raw` just above. + drop(unsafe { Box::from_raw(std::ptr::slice_from_raw_parts_mut(ptr, len)) }); + (cur & !META_KIND) as *const Object + } + } + } + + fn build_values(&self) -> Vec { + let w = self.word; + match self.kind() { + KIND_TIMEDELTA => { + let (d, s, us) = self.td(); + vec![ + Object::Int(d), + Object::Int(s), + Object::Int(us), + Object::Int(-1), + Object::Int(d), + Object::Int(s), + Object::Int(us), + ] + } + KIND_DATE => { + let (y, m, d) = word_ymd(w); + vec![ + Object::Int(y), + Object::Int(m), + Object::Int(d), + Object::Int(-1), + ] + } + _ => { + let (y, m, d) = word_ymd(w); + let (hh, mm, ss, us, fold) = word_time(w); + vec![ + Object::Int(y), + Object::Int(m), + Object::Int(d), + Object::Int(hh), + Object::Int(mm), + Object::Int(ss), + Object::Int(us), + self.tz(), + Object::Int(-1), + Object::Int(fold), + ] + } + } + } + + /// `(days, seconds, microseconds)` of a packed `timedelta`. + #[inline] + fn td(&self) -> (i64, i64, i64) { + ( + self.word as i64, + (self.aux >> 20) as i64, + (self.aux & 0xF_FFFF) as i64, + ) + } + + /// Take the field objects, releasing the packed form. + pub(crate) fn into_values(self) -> Box<[Object]> { + let this = std::mem::ManuallyDrop::new(self); + let len = this.layout().len(); + let m = this.meta.load(Ordering::Acquire); + let vals = if m & !META_KIND != 0 { + // SAFETY: a built copy of `len` objects (see `materialize`), + // owned by the storage being consumed. + unsafe { + Box::from_raw(std::ptr::slice_from_raw_parts_mut( + (m & !META_KIND) as *mut Object, + len, + )) + } + } else { + this.build_values().into_boxed_slice() + }; + this.release_aux(); + vals + } + + /// Release the owned `tzinfo` reference (the caller forgets `aux`). + #[inline] + fn release_aux(&self) { + let p = self.tz_ptr(); + if !p.is_null() { + // SAFETY: `aux` owns one strong reference. + drop(unsafe { Rc::from_raw(p) }); + } + } + + /// Release the built copy, if any. + #[inline] + fn free_values(&mut self) { + let m = *self.meta.get_mut(); + if m & !META_KIND != 0 { + let len = self.layout().len(); + *self.meta.get_mut() = m & META_KIND; + // SAFETY: see `into_values`. + drop(unsafe { + Box::from_raw(std::ptr::slice_from_raw_parts_mut( + (m & !META_KIND) as *mut Object, + len, + )) + }); + } + } + + /// Release everything the value owns, keeping the kind (a pooled + /// instance's storage; see [`recycle`]). + fn clear(&mut self) { + self.release_aux(); + self.aux = 0; + self.free_values(); + } +} + +impl Drop for PackedSlots { + fn drop(&mut self) { + self.release_aux(); + self.free_values(); + } +} + +impl Clone for PackedSlots { + fn clone(&self) -> Self { + let p = self.tz_ptr(); + if !p.is_null() { + // SAFETY: `aux` keeps the allocation alive; the clone owns the + // added reference. + unsafe { Rc::increment_strong_count(p) }; + } + Self::new(self.kind(), self.word, self.aux) + } +} + +impl std::fmt::Debug for PackedSlots { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PackedSlots") + .field("kind", &self.kind()) + .field("word", &self.word) + .field("aux", &self.aux) + .finish_non_exhaustive() + } +} + +/// The instance's slot storage for a read (no Python code may run while +/// `f` reads). +#[inline(always)] +fn with_slots(i: &PyInstance, f: impl FnOnce(&SlotStorage) -> Option) -> Option { + // SAFETY: the view is dropped when `f` returns, and `f` runs no code. + if let Some(s) = unsafe { i.slots.peek() } { + return f(s); + } + let s = i.slots.try_borrow().ok()?; + f(&s) +} + /// Interned slot and field names. struct Names { days: SharedStr, seconds: SharedStr, microseconds: SharedStr, - hashcode: SharedStr, - pub_days: SharedStr, - pub_seconds: SharedStr, - pub_microseconds: SharedStr, year: SharedStr, month: SharedStr, day: SharedStr, @@ -93,6 +461,10 @@ struct Names { f_microsecond: SharedStr, f_tzinfo: SharedStr, f_fold: SharedStr, + f_days: SharedStr, + f_seconds: SharedStr, + f_microseconds: SharedStr, + f_fromisoformat: SharedStr, } fn interned(s: &str) -> SharedStr { @@ -108,10 +480,6 @@ impl Names { days: interned("_days"), seconds: interned("_seconds"), microseconds: interned("_microseconds"), - hashcode: interned("_hashcode"), - pub_days: interned("days"), - pub_seconds: interned("seconds"), - pub_microseconds: interned("microseconds"), year: interned("_year"), month: interned("_month"), day: interned("_day"), @@ -132,6 +500,10 @@ impl Names { f_microsecond: interned("microsecond"), f_tzinfo: interned("tzinfo"), f_fold: interned("fold"), + f_days: interned("days"), + f_seconds: interned("seconds"), + f_microseconds: interned("microseconds"), + f_fromisoformat: interned("fromisoformat"), } } } @@ -160,6 +532,11 @@ struct State { /// `(kind, name)` → what the class resolved the name to after /// [`install`]: the natives, and the field properties. expect: std::sync::OnceLock>, + /// Dead instances kept for reuse (see [`Pool`]). + pool: Pool, + /// The native `datetime.fromisoformat` (the function its + /// `classmethod` wraps). + fromiso: std::sync::OnceLock, } /// The dunders and fields the dispatch-loop shortcuts serve without a @@ -174,6 +551,11 @@ const SHORTCUT_NAMES: &[(u8, &str)] = &[ (KIND_TIMEDELTA, "__le__"), (KIND_TIMEDELTA, "__gt__"), (KIND_TIMEDELTA, "__ge__"), + (KIND_TIMEDELTA, "__new__"), + (KIND_TIMEDELTA, "__init__"), + (KIND_TIMEDELTA, "days"), + (KIND_TIMEDELTA, "seconds"), + (KIND_TIMEDELTA, "microseconds"), (KIND_DATE, "__add__"), (KIND_DATE, "__sub__"), (KIND_DATE, "__eq__"), @@ -206,6 +588,7 @@ const SHORTCUT_NAMES: &[(u8, &str)] = &[ (KIND_DATETIME, "microsecond"), (KIND_DATETIME, "tzinfo"), (KIND_DATETIME, "fold"), + (KIND_DATETIME, "fromisoformat"), ]; /// Whether `cls` (of `kind`) still resolves every shortcut name to what @@ -238,10 +621,17 @@ fn verify_slow(st: &State, cls: &TypeObject, kind: u8, ver: u64) -> bool { ok } -// SAFETY-free: `State` is reached from an exact class through its -// `native_ext` slot (see `state_of`). +/// The [`State`] a class marked by [`install`] carries in its +/// `native_ext` slot. +#[inline] +#[allow(clippy::cast_ptr_alignment)] fn state_of_cls(cls: &TypeObject) -> Option<&State> { - cls.native_ext.get()?.downcast_ref::() + let ext = cls.native_ext.get()?; + debug_assert!(ext.downcast_ref::().is_some()); + // SAFETY: only `install` fills `native_ext`, always with a `State` + // (checked above in debug builds), so the payload pointer is a + // `State`'s, aligned for it; the cast drops the vtable. + Some(unsafe { &*Rc::as_ptr(ext).cast::() }) } #[inline] @@ -281,9 +671,20 @@ fn as_int(o: &Object) -> Option { // from the shared layout; any other storage is read by name. #[allow(clippy::index_refutable_slice)] +#[inline] fn td_fields(i: &PyInstance, st: &State) -> Option<(i64, i64, i64)> { + with_slots(i, |s| { + if let Some(p) = s.as_packed() { + return (p.kind() == KIND_TIMEDELTA).then(|| p.td()); + } + td_fields_slow(s, st) + }) +} + +#[allow(clippy::index_refutable_slice)] +#[inline(never)] +fn td_fields_slow(s: &SlotStorage, st: &State) -> Option<(i64, i64, i64)> { let n = &st.names; - let s = i.slots.try_borrow().ok()?; if let Some(v) = s.values_for_layout(&st.td_layout) { return Some(( as_int(&v[TD_DAYS])?, @@ -302,10 +703,20 @@ fn td_us(f: (i64, i64, i64)) -> i128 { (i128::from(f.0) * 86_400 + i128::from(f.1)) * 1_000_000 + i128::from(f.2) } -#[allow(clippy::index_refutable_slice)] +#[inline] fn date_fields(i: &PyInstance, st: &State) -> Option<(i64, i64, i64)> { + with_slots(i, |s| { + if let Some(p) = s.as_packed() { + return (p.kind() != KIND_TIMEDELTA).then(|| word_ymd(p.word)); + } + date_fields_slow(s, st) + }) +} + +#[allow(clippy::index_refutable_slice)] +#[inline(never)] +fn date_fields_slow(s: &SlotStorage, st: &State) -> Option<(i64, i64, i64)> { let n = &st.names; - let s = i.slots.try_borrow().ok()?; if let Some(v) = s .values_for_layout(&st.date_layout) .or_else(|| s.values_for_layout(&st.dt_layout)) @@ -335,9 +746,49 @@ struct Dt { fold: i64, } +impl Dt { + /// The fields of a packed `datetime` word, with `tz`. + #[inline] + fn from_word(w: u64, tz: Object) -> Self { + let (y, m, d) = word_ymd(w); + let (hh, mm, ss, us, fold) = word_time(w); + Dt { + y, + m, + d, + hh, + mm, + ss, + us, + tz, + fold, + } + } +} + +#[inline] fn dt_fields(i: &PyInstance, st: &State) -> Option
{ + with_slots(i, |s| { + if let Some(p) = s.as_packed() { + return (p.kind() == KIND_DATETIME).then(|| Dt::from_word(p.word, p.tz())); + } + dt_fields_slow(s, st) + }) +} + +/// A packed `datetime`'s word and borrowed `tzinfo` pointer (`null` for +/// `None`): the comparison and difference paths, which keep no field. +#[inline] +fn dt_packed(i: &PyInstance) -> Option<(u64, *const PyInstance)> { + with_slots(i, |s| { + let p = s.as_packed()?; + (p.kind() == KIND_DATETIME).then(|| (p.word, p.tz_ptr())) + }) +} + +#[inline(never)] +fn dt_fields_slow(s: &SlotStorage, st: &State) -> Option
{ let n = &st.names; - let s = i.slots.try_borrow().ok()?; if let Some(v) = s.values_for_layout(&st.dt_layout) { return Some(Dt { y: as_int(&v[D_YEAR])?, @@ -371,10 +822,25 @@ enum Tz { Fixed(i128), } +/// [`tz_of`] for a packed `tzinfo` pointer (`null` for `None`). +#[inline] +fn tz_of_ptr(p: *const PyInstance, st: &State) -> Option { + if p.is_null() { + return Some(Tz::Naive); + } + if std::ptr::eq(p, st.utc.as_ptr()) { + return Some(Tz::Fixed(0)); + } + // SAFETY: a borrowed view of the owner's reference, never dropped. + let o = std::mem::ManuallyDrop::new(Object::Instance(unsafe { Rc::from_raw(p) })); + tz_of(&o, st) +} + fn tz_of(tz: &Object, st: &State) -> Option { let n = &st.names; match tz { Object::None => Some(Tz::Naive), + Object::Instance(i) if std::ptr::eq(Rc::as_ptr(i), st.utc.as_ptr()) => Some(Tz::Fixed(0)), o => { let i = inst(o, KIND_TIMEZONE)?; let off = { @@ -466,72 +932,154 @@ fn instance_fixed( Object::Instance(Rc::new(i)) } -/// `date(y, m, d)` and `datetime(y, m, d[, hh[, mm[, ss[, us[, tz]]]]])` -/// (`tzinfo=` by keyword too) of the exact classes, with in-range `int` -/// fields and a naive or fixed-offset zone: the instance the replaced -/// `__new__` builds, built natively. `None` for every other shape, which -/// the Python constructor serves (and diagnoses). +/// `timedelta(...)`, `date(...)` and `datetime(...)` of the exact classes +/// with plain `int` fields (by position or keyword) and, for `datetime`, a +/// naive or fixed-offset zone: the instance the replaced `__new__` builds, +/// built natively. `None` for every other shape, which the Python +/// constructor serves (and diagnoses). pub(crate) fn construct( cls: &Rc, args: &[Object], kwargs: &[(String, Object)], +) -> Option> { + construct_with(cls, args, kwargs.len(), |i| { + (kwargs[i].0.as_str(), &kwargs[i].1) + }) +} + +/// [`construct`] with the keyword names as the call's names tuple +/// (`names[i]` names `values[i]`). +pub(crate) fn construct_names( + cls: &TypeObject, + args: &[Object], + names: &[Object], + values: &[Object], +) -> Option> { + if names.len() != values.len() { + return None; + } + construct_with(cls, args, names.len(), |i| match &names[i] { + Object::Str(s) => (s.as_ref(), &values[i]), + _ => ("", &values[i]), + }) +} + +/// The constructors' parameter names, by kind, in positional order. +const TD_PARAMS: [&str; 7] = [ + "days", + "seconds", + "microseconds", + "milliseconds", + "minutes", + "hours", + "weeks", +]; +const DATE_PARAMS: [&str; 3] = ["year", "month", "day"]; +const DT_PARAMS: [&str; 9] = [ + "year", + "month", + "day", + "hour", + "minute", + "second", + "microsecond", + "tzinfo", + "fold", +]; + +fn construct_with<'a>( + cls: &TypeObject, + args: &'a [Object], + nkw: usize, + kw: impl Fn(usize) -> (&'a str, &'a Object), ) -> Option> { let st = state_of_cls(cls)?; let kind = cls.native_kind.get(); - let exact = match kind { - KIND_DATE => st.date.upgrade(), - KIND_DATETIME => st.datetime.upgrade(), - _ => None, + let (exact, params): (*const TypeObject, &[&str]) = match kind { + KIND_TIMEDELTA => (st.timedelta.as_ptr(), &TD_PARAMS), + KIND_DATE => (st.date.as_ptr(), &DATE_PARAMS), + // `fold` is keyword-only. + KIND_DATETIME => (st.datetime.as_ptr(), &DT_PARAMS), + _ => return None, }; - if !exact.is_some_and(|c| Rc::ptr_eq(&c, cls)) || !verified(st, cls, kind) { + if !std::ptr::eq(cls, exact) || !verified(st, cls, kind) { return None; } - let int = |o: &Object| match o { - Object::Int(v) => Some(*v), - _ => None, + let npos = if kind == KIND_DATETIME { + 8 + } else { + params.len() }; - if kind == KIND_DATE { - let ([y, m, d], true) = (args, kwargs.is_empty()) else { - return None; - }; - let (y, m, d) = (int(y)?, int(m)?, int(d)?); - return valid_date(y, m, d) - .then(|| new_date(st, y, m, d)) - .flatten() - .map(Ok); - } - if !(3..=8).contains(&args.len()) { + if args.len() > npos { return None; } - let mut tz = args.get(7).cloned().unwrap_or(Object::None); - for (name, v) in kwargs { - if name != "tzinfo" || args.len() == 8 { + // Each parameter's argument, by position then keyword. + let mut slots: [Option<&'a Object>; 9] = [None; 9]; + for (slot, a) in slots.iter_mut().zip(args) { + *slot = Some(a); + } + for i in 0..nkw { + let (name, v) = kw(i); + let ix = params.iter().position(|p| *p == name)?; + if slots[ix].is_some() { + // A duplicate: the Python constructor raises. return None; } - tz = v.clone(); - } - let field = |ix: usize| args.get(ix).map_or(Some(0), int); - let f = Dt { - y: int(&args[0])?, - m: int(&args[1])?, - d: int(&args[2])?, - hh: field(3)?, - mm: field(4)?, - ss: field(5)?, - us: field(6)?, - tz, - fold: 0, + slots[ix] = Some(v); + } + let int = |ix: usize, default: Option| -> Option { + match slots[ix] { + Some(Object::Int(v)) => Some(*v), + None => default, + _ => None, + } }; - let ok = valid_date(f.y, f.m, f.d) - && (0..24).contains(&f.hh) - && (0..60).contains(&f.mm) - && (0..60).contains(&f.ss) - && (0..1_000_000).contains(&f.us); - if !ok { - return None; + match kind { + KIND_TIMEDELTA => { + let mut v = [0i128; 7]; + for (ix, slot) in v.iter_mut().enumerate() { + *slot = i128::from(int(ix, Some(0))?); + } + let [days, seconds, us, ms, minutes, hours, weeks] = v; + new_td( + st, + days + weeks * 7, + seconds + minutes * 60 + hours * 3600, + us + ms * 1000, + ) + } + KIND_DATE => { + let (y, m, d) = (int(0, None)?, int(1, None)?, int(2, None)?); + valid_date(y, m, d) + .then(|| new_date(st, y, m, d)) + .flatten() + .map(Ok) + } + _ => { + let f = Dt { + y: int(0, None)?, + m: int(1, None)?, + d: int(2, None)?, + hh: int(3, Some(0))?, + mm: int(4, Some(0))?, + ss: int(5, Some(0))?, + us: int(6, Some(0))?, + tz: slots[7].cloned().unwrap_or(Object::None), + fold: int(8, Some(0))?, + }; + let ok = valid_date(f.y, f.m, f.d) + && (0..24).contains(&f.hh) + && (0..60).contains(&f.mm) + && (0..60).contains(&f.ss) + && (0..1_000_000).contains(&f.us) + && (0..=1).contains(&f.fold); + if !ok { + return None; + } + tz_of(&f.tz, st)?; + new_dt(st, f).map(Ok) + } } - tz_of(&f.tz, st)?; - new_dt(st, &f).map(Ok) } /// A normalized exact `timedelta` from unnormalized components. @@ -558,38 +1106,57 @@ fn new_td(st: &State, d: i128, s: i128, us: i128) -> Option Option { - let cls = st.date.upgrade()?; - Some(instance_fixed( - cls, - &st.date_layout, - vec![ - Object::Int(y), - Object::Int(m), - Object::Int(d), - Object::Int(-1), - ], - )) + if !valid_date(y, m, d) { + let cls = st.date.upgrade()?; + return Some(instance_fixed( + cls, + &st.date_layout, + vec![ + Object::Int(y), + Object::Int(m), + Object::Int(d), + Object::Int(-1), + ], + )); + } + let word = ((y as u64) << W_YEAR) | ((m as u64) << W_MONTH) | ((d as u64) << W_DAY); + new_packed(st, &st.date, KIND_DATE, word, 0) } -#[allow(clippy::too_many_arguments)] -fn new_dt(st: &State, f: &Dt) -> Option { +fn new_dt(st: &State, f: Dt) -> Option { + let ok = valid_date(f.y, f.m, f.d) + && (0..24).contains(&f.hh) + && (0..60).contains(&f.mm) + && (0..60).contains(&f.ss) + && (0..1_000_000).contains(&f.us) + && (0..=1).contains(&f.fold); + if ok { + let word = pack_dt(&f); + match f.tz { + Object::None => return new_packed(st, &st.datetime, KIND_DATETIME, word, 0), + Object::Instance(tz) => { + let aux = Rc::into_raw(tz) as usize; + let r = new_packed(st, &st.datetime, KIND_DATETIME, word, aux); + if r.is_none() { + // SAFETY: not adopted; release the reference taken above. + drop(unsafe { Rc::from_raw(aux as *const PyInstance) }); + } + return r; + } + _ => {} + } + } let cls = st.datetime.upgrade()?; Some(instance_fixed( cls, @@ -602,16 +1169,171 @@ fn new_dt(st: &State, f: &Dt) -> Option { Object::Int(f.mm), Object::Int(f.ss), Object::Int(f.us), - f.tz.clone(), + f.tz, Object::Int(-1), Object::Int(f.fold), ], )) } +// --------------------------------------------------------------------- +// The instance pool: CPython's C types keep per-type freelists; a dead +// natively served instance keeps its allocation, class and packed +// storage here for the next value of its kind. + +const POOL_CAP: usize = 64; + +/// One class set's pooled instances, per kind (`timedelta`, `date`, +/// `datetime`). A pooled instance keeps its class, so the pool (in the +/// classes' shared [`State`]) and the classes keep each other alive, as +/// CPython's static types live for the whole process. +#[derive(Default)] +struct Pool(std::cell::UnsafeCell<[Vec>; 3]>); + +// SAFETY: the pool is touched only while `pool_ok()`: then every thread +// that reaches VM objects holds the GIL, which serializes the accesses +// (each is a push or a pop that runs no other code). +unsafe impl Sync for Pool {} +unsafe impl Send for Pool {} + +/// Whether the pools may be used: under the GIL with no thread reaching +/// objects outside it (see `sync::cells_unguarded`). Debug builds (the +/// unit-test binary runs interpreters on concurrent threads with no GIL +/// between them) never pool. +#[inline] +fn pool_ok() -> bool { + #[cfg(debug_assertions)] + { + false + } + #[cfg(not(debug_assertions))] + { + !crate::sync::cells_unguarded() + } +} + +impl Pool { + #[inline(always)] + fn with(&self, kind: u8, f: impl FnOnce(&mut Vec>) -> R) -> Option { + if !pool_ok() { + return None; + } + let ix = match kind { + KIND_TIMEDELTA => 0, + KIND_DATE => 1, + KIND_DATETIME => 2, + _ => return None, + }; + // SAFETY: see the `Sync` impl; `f` only pushes or pops. + Some(f(unsafe { &mut (*self.0.get())[ix] })) + } +} + +/// An instance of `cls` (the exact class of `kind`) holding the packed +/// value `(word, aux)`; `aux` is adopted on success. +#[inline] +fn new_packed( + st: &State, + cls: &Weak, + kind: u8, + word: u64, + aux: usize, +) -> Option { + if let Some(inst) = st.pool.with(kind, Vec::pop).flatten() { + // SAFETY: pooled instances are unique (see `recycle`): nothing + // else can observe the class or storage while they change. + let m = unsafe { &mut *Rc::as_ptr(&inst).cast_mut() }; + if std::ptr::eq(Rc::as_ptr(m.class.get_mut()), cls.as_ptr()) { + if let Some(p) = m.slots.get_mut().as_packed_mut() { + p.word = word; + p.aux = aux; + return Some(Object::Instance(inst)); + } + } + // Not this class's (a `__class__` assignment cannot reach a pooled + // instance, so this does not happen): free it outright. + drop(Rc::into_arc(inst)); + } + let cls = cls.upgrade()?; + let mut i = PyInstance::new(cls); + *i.slots.get_mut() = SlotStorage::from_packed(PackedSlots::new(kind, word, aux)); + Some(Object::Instance(Rc::new(i))) +} + +/// Whether `i`'s class keeps a pool (see [`recycle`]). +#[inline] +pub(crate) fn pooled_kind(i: &PyInstance) -> bool { + // SAFETY: a read between two instructions (see `GilCell::peek`). + unsafe { i.class.peek() }.is_some_and(|c| { + matches!( + c.native_kind.get(), + KIND_TIMEDELTA | KIND_DATE | KIND_DATETIME + ) + }) +} + +/// Retire a dying natively served instance into its class set's pool +/// (see above). `Err` hands back an instance that must be freed normally +/// (dropped as an `Arc`, not through the finalizing `Rc` drop that may +/// have called here). +pub(crate) fn recycle(mut inst: Rc) -> Result<(), Rc> { + if !pool_ok() { + return Err(inst); + } + // While the refcount bias holds, this thread owns every count. + let unique = if crate::rc::refcounts_biased() { + Rc::strong_count(&inst) == 1 && Rc::weak_count(&inst) == 0 + } else { + Rc::get_mut(&mut inst).is_some() + }; + if !unique { + return Err(inst); + } + // SAFETY: `inst` is the only reference (checked above). + let m = unsafe { &mut *Rc::as_ptr(&inst).cast_mut() }; + let cls = m.class.get_mut(); + let kind = cls.native_kind.get(); + if !matches!(kind, KIND_TIMEDELTA | KIND_DATE | KIND_DATETIME) || cls.instances_need_finalize() + { + return Err(inst); + } + let Some(st) = state_of_cls(cls) else { + return Err(inst); + }; + if m.dict.published().is_some() + || !m.dict.split_mut().is_empty() + || m.native.get().is_some() + || m.c_body.get() != 0 + || m.finalize_ran.get() + { + return Err(inst); + } + match m.slots.get_mut().as_packed_mut() { + Some(p) if p.kind() == kind => { + // Releasing the `tzinfo` runs no code (a dying one is queued + // for finalization, not finalized here). + p.clear(); + } + _ => return Err(inst), + } + m.hash_cache = crate::sync::CachedHash::new(None); + // SAFETY: `st` lives in the class `inst` holds. + let st: &State = unsafe { &*std::ptr::from_ref(st) }; + let mut slot = Some(inst); + st.pool.with(kind, |v| { + if v.len() < POOL_CAP { + v.extend(slot.take()); + } + }); + match slot { + None => Ok(()), + Some(inst) => Err(inst), + } +} + /// `datetime` fields shifted by `delta_us` (fold cleared, `tzinfo` /// kept), or `None` past the representable range. -fn dt_shift(f: &Dt, delta_us: i128) -> Option
{ +fn dt_shift(f: Dt, delta_us: i128) -> Option
{ // Every representable datetime is under 2^59 microseconds, so a // delta that fits no `i64` lands out of range (the Python path // raises); the rest is 64-bit arithmetic over in-range fields. @@ -643,11 +1365,19 @@ fn dt_shift(f: &Dt, delta_us: i128) -> Option
{ mm: secs % 3600 / 60, ss: secs % 60, us: (rest % 1_000_000) as i64, - tz: f.tz.clone(), + tz: f.tz, fold: 0, }) } +/// Microseconds since 0001-01-01T00:00 of a packed (so valid) word. +#[inline] +fn word_us(w: u64) -> i64 { + let (y, m, d) = word_ymd(w); + let (hh, mm, ss, us, _) = word_time(w); + (ymd2ord(y, m, d) * 86_400 + hh * 3600 + mm * 60 + ss) * 1_000_000 + us +} + /// Microseconds since 0001-01-01T00:00, naive. fn dt_us(f: &Dt) -> i128 { (i128::from(ymd2ord(f.y, f.m, f.d)) * 86_400 + i128::from(f.hh * 3600 + f.mm * 60 + f.ss)) @@ -714,6 +1444,73 @@ fn td_bool(a: &[Object]) -> Option> { Some(Ok(Object::Bool(p != (0, 0, 0)))) } +/// `hash((days, seconds, microseconds))`: the replaced `__hash__`'s +/// value for a `timedelta` (`hash(self._getstate())`). +fn td_tuple_hash(d: i64, s: i64, us: i64) -> Option { + let h = |v: i64| crate::object::numeric_hash(&Object::Int(v)); + Some(crate::object::combine_tuple_hash(&[h(d)?, h(s)?, h(us)?])) +} + +fn td_hash(a: &[Object]) -> Option> { + let [x] = a else { return None }; + let st = state_of(x)?; + let (d, s, us) = td_fields(inst(x, KIND_TIMEDELTA)?, st)?; + Some(Ok(Object::Int(td_tuple_hash(d, s, us)?))) +} + +fn date_hash(a: &[Object]) -> Option> { + let [x] = a else { return None }; + let st = state_of(x)?; + let (y, m, d) = date_fields(inst(x, KIND_DATE)?, st)?; + if !valid_date(y, m, d) { + return None; + } + // `hash(self._getstate())`: a one-tuple of the state bytes. + let state = [(y / 256) as u8, (y % 256) as u8, m as u8, d as u8]; + let h = crate::object::py_bytes_hash(&state); + Some(Ok(Object::Int(crate::object::combine_tuple_hash(&[h])))) +} + +/// The replaced `datetime.__hash__` for a naive or fixed-offset value: +/// the state bytes' hash when naive, else the hash of the UTC value as a +/// `timedelta` since 0001-01-01 (`fold` never changes a fixed offset). +fn dt_hash(a: &[Object]) -> Option> { + let [x] = a else { return None }; + let st = state_of(x)?; + let f = dt_fields(inst(x, KIND_DATETIME)?, st)?; + if !valid_date(f.y, f.m, f.d) { + return None; + } + let h = match tz_of(&f.tz, st)? { + Tz::Naive => { + let state = [ + (f.y / 256) as u8, + (f.y % 256) as u8, + f.m as u8, + f.d as u8, + f.hh as u8, + f.mm as u8, + f.ss as u8, + (f.us >> 16) as u8, + (f.us >> 8 & 0xff) as u8, + (f.us & 0xff) as u8, + ]; + crate::object::py_bytes_hash(&state) + } + Tz::Fixed(off) => { + let total = dt_us(&f) - off; + let days = total.div_euclid(US_PER_DAY); + let rest = total.rem_euclid(US_PER_DAY); + td_tuple_hash( + i64::try_from(days).ok()?, + (rest / 1_000_000) as i64, + (rest % 1_000_000) as i64, + )? + } + }; + Some(Ok(Object::Int(h))) +} + fn td_cmp(a: &[Object]) -> Option { let [x, y] = a else { return None }; let st = state_of(x)?; @@ -833,7 +1630,75 @@ fn date_isoformat(a: &[Object]) -> Option> { if !valid_date(y, m, d) { return None; } - Some(Ok(Object::from_str(format!("{y:04}-{m:02}-{d:02}")))) + let mut out = Vec::with_capacity(10); + push_ymd(&mut out, y, m, d); + Some(Ok(text_object(&out))) +} + +/// Append `v` in decimal, zero-padded to `width` digits: +/// `format!("{v:0width$}")` without the formatting machinery. +#[inline] +fn push_num(out: &mut Vec, v: i64, width: usize) { + const LIMIT: [i64; 7] = [1, 10, 100, 1_000, 10_000, 100_000, 1_000_000]; + if width < LIMIT.len() && (0..LIMIT[width]).contains(&v) { + // Exactly `width` digits. + let start = out.len(); + out.resize(start + width, b'0'); + let mut n = v as u32; + for b in out[start..].iter_mut().rev() { + *b = b'0' + (n % 10) as u8; + n /= 10; + } + return; + } + let mut buf = [b'0'; 20]; + let mut n = v.unsigned_abs(); + let mut i = buf.len(); + loop { + i -= 1; + buf[i] = b'0' + (n % 10) as u8; + n /= 10; + if n == 0 { + break; + } + } + let start = i.min(buf.len() - width.min(buf.len())); + if v < 0 { + out.push(b'-'); + } + out.extend_from_slice(&buf[start..]); +} + +/// `YYYY-MM-DD`. +#[inline] +fn push_ymd(out: &mut Vec, y: i64, m: i64, d: i64) { + push_num(out, y, 4); + out.push(b'-'); + push_num(out, m, 2); + out.push(b'-'); + push_num(out, d, 2); +} + +/// `HH:MM:SS`. +#[inline] +fn push_hms(out: &mut Vec, hh: i64, mm: i64, ss: i64) { + push_num(out, hh, 2); + out.push(b':'); + push_num(out, mm, 2); + out.push(b':'); + push_num(out, ss, 2); +} + +/// A `str` of `text` (UTF-8: ASCII fields around the caller's text). +#[inline] +fn text_object(text: &[u8]) -> Object { + // SAFETY: the formatters write ASCII and copy whole UTF-8 sequences. + let text = unsafe { std::str::from_utf8_unchecked(text) }; + if text.is_ascii() { + Object::Str(SharedStr::from_ascii(text)) + } else { + Object::Str(SharedStr::from(text)) + } } /// `date.replace(year=None, month=None, day=None)`, positional form. @@ -861,16 +1726,52 @@ fn date_replace(a: &[Object]) -> Option> { Some(Ok(new_date(st, y, m, d)?)) } +/// A packed `datetime` shifted by `delta_us` within its day: the same +/// date, new time fields, fold cleared, the `tzinfo` shared. `None` when +/// `xi` is not packed or the result leaves the day. +#[inline] +fn dt_shift_in_day(st: &State, xi: &PyInstance, delta_us: i128) -> Option { + let (w, tz) = dt_packed(xi)?; + if delta_us.unsigned_abs() >= US_PER_DAY as u128 { + return None; + } + let (hh, mm, ss, us, _) = word_time(w); + let n = ((hh * 3600 + mm * 60 + ss) * 1_000_000 + us) + delta_us as i64; + if !(0..US_PER_DAY as i64).contains(&n) { + return None; + } + let (secs, us) = (n / 1_000_000, n % 1_000_000); + let word = (w & !((1u64 << W_DAY) - 1)) + | (((secs / 3600) as u64) << W_HH) + | (((secs % 3600 / 60) as u64) << W_MM) + | (((secs % 60) as u64) << W_SS) + | ((us as u64) << W_US); + if !tz.is_null() { + // SAFETY: `xi` holds a reference; the new value owns this one. + unsafe { Rc::increment_strong_count(tz) }; + } + let r = new_packed(st, &st.datetime, KIND_DATETIME, word, tz as usize); + if r.is_none() && !tz.is_null() { + // SAFETY: not adopted; release the reference taken above. + drop(unsafe { Rc::from_raw(tz) }); + } + r +} + fn dt_add(a: &[Object]) -> Option> { let [x, y] = a else { return None }; let st = state_of(x)?; - let f = dt_fields(inst(x, KIND_DATETIME)?, st)?; + let xi = inst(x, KIND_DATETIME)?; let delta = td_us(td_fields(inst(y, KIND_TIMEDELTA)?, st)?); + if let Some(r) = dt_shift_in_day(st, xi, delta) { + return Some(Ok(r)); + } + let f = dt_fields(xi, st)?; if !valid_date(f.y, f.m, f.d) { return None; } - match dt_shift(&f, delta) { - Some(g) => Some(Ok(new_dt(st, &g)?)), + match dt_shift(f, delta) { + Some(g) => Some(Ok(new_dt(st, g)?)), None => Some(Err(overflow_error("date value out of range"))), } } @@ -878,20 +1779,40 @@ fn dt_add(a: &[Object]) -> Option> { fn dt_sub(a: &[Object]) -> Option> { let [x, y] = a else { return None }; let st = state_of(x)?; - let f = dt_fields(inst(x, KIND_DATETIME)?, st)?; - if !valid_date(f.y, f.m, f.d) { - return None; - } + let xi = inst(x, KIND_DATETIME)?; match kind_of(y) { KIND_TIMEDELTA => { let delta = td_us(td_fields(inst(y, KIND_TIMEDELTA)?, st)?); - match dt_shift(&f, -delta) { - Some(g) => Some(Ok(new_dt(st, &g)?)), + if let Some(r) = dt_shift_in_day(st, xi, -delta) { + return Some(Ok(r)); + } + let f = dt_fields(xi, st)?; + if !valid_date(f.y, f.m, f.d) { + return None; + } + match dt_shift(f, -delta) { + Some(g) => Some(Ok(new_dt(st, g)?)), None => Some(Err(overflow_error("date value out of range"))), } } KIND_DATETIME => { - let g = dt_fields(inst(y, KIND_DATETIME)?, st)?; + let j = inst(y, KIND_DATETIME)?; + if let (Some((wf, tf)), Some((wg, tg))) = (dt_packed(xi), dt_packed(j)) { + let mut diff = i128::from(word_us(wf) - word_us(wg)); + if !std::ptr::eq(tf, tg) { + match (tz_of_ptr(tf, st)?, tz_of_ptr(tg, st)?) { + (Tz::Naive, Tz::Naive) => {} + (Tz::Fixed(a), Tz::Fixed(b)) => diff += b - a, + _ => return None, + } + } + return new_td(st, 0, 0, diff); + } + let f = dt_fields(xi, st)?; + if !valid_date(f.y, f.m, f.d) { + return None; + } + let g = dt_fields(j, st)?; if !valid_date(g.y, g.m, g.d) { return None; } @@ -915,8 +1836,23 @@ fn dt_sub(a: &[Object]) -> Option> { fn dt_cmp_raw(a: &[Object]) -> Option<(std::cmp::Ordering, bool)> { let [x, y] = a else { return None }; let st = state_of(x)?; - let f = dt_fields(inst(x, KIND_DATETIME)?, st)?; - let g = dt_fields(inst(y, KIND_DATETIME)?, st)?; + let (xi, yi) = (inst(x, KIND_DATETIME)?, inst(y, KIND_DATETIME)?); + if let (Some((wf, tf)), Some((wg, tg))) = (dt_packed(xi), dt_packed(yi)) { + // The fields sit in significance order above the fold bit. + if std::ptr::eq(tf, tg) { + return Some(((wf >> 1).cmp(&(wg >> 1)), true)); + } + return match (tz_of_ptr(tf, st)?, tz_of_ptr(tg, st)?) { + (Tz::Naive, Tz::Naive) => Some(((wf >> 1).cmp(&(wg >> 1)), true)), + (Tz::Fixed(a), Tz::Fixed(b)) => Some(( + (i128::from(word_us(wf)) - a).cmp(&(i128::from(word_us(wg)) - b)), + true, + )), + _ => Some((std::cmp::Ordering::Equal, false)), + }; + } + let f = dt_fields(xi, st)?; + let g = dt_fields(yi, st)?; if !valid_date(f.y, f.m, f.d) || !valid_date(g.y, g.m, g.d) { return None; } @@ -965,17 +1901,22 @@ fn dt_date(a: &[Object]) -> Option> { } /// `_format_offset(off, sep)` for a fixed offset in microseconds. -fn format_offset(out: &mut String, off: i128, sep: &str) { - let (sign, mut v) = if off < 0 { ('-', -off) } else { ('+', off) }; - let hh = v / 3_600_000_000; +fn format_offset(out: &mut Vec, off: i128, sep: &[u8]) { + let (sign, mut v) = if off < 0 { (b'-', -off) } else { (b'+', off) }; + let hh = (v / 3_600_000_000) as i64; v %= 3_600_000_000; - let mm = v / 60_000_000; + let mm = (v / 60_000_000) as i64; v %= 60_000_000; - let _ = write!(out, "{sign}{hh:02}{sep}{mm:02}"); + out.push(sign); + push_num(out, hh, 2); + out.extend_from_slice(sep); + push_num(out, mm, 2); if v != 0 { - let _ = write!(out, "{sep}{:02}", v / 1_000_000); + out.extend_from_slice(sep); + push_num(out, (v / 1_000_000) as i64, 2); if v % 1_000_000 != 0 { - let _ = write!(out, ".{:06}", v % 1_000_000); + out.push(b'.'); + push_num(out, (v % 1_000_000) as i64, 6); } } } @@ -1017,47 +1958,44 @@ fn dt_isoformat(a: &[Object]) -> Option> { return None; } let tz = tz_of(&f.tz, st)?; - let mut out = String::with_capacity(32); - let _ = write!(out, "{:04}-{:02}-{:02}{sep}", f.y, f.m, f.d); + let mut out = Vec::with_capacity(40); + push_ymd(&mut out, f.y, f.m, f.d); + out.extend_from_slice(sep.encode_utf8(&mut [0; 4]).as_bytes()); match spec { - "hours" => { - let _ = write!(out, "{:02}", f.hh); - } + "hours" => push_num(&mut out, f.hh, 2), "minutes" => { - let _ = write!(out, "{:02}:{:02}", f.hh, f.mm); + push_num(&mut out, f.hh, 2); + out.push(b':'); + push_num(&mut out, f.mm, 2); } "milliseconds" => { - let _ = write!( - out, - "{:02}:{:02}:{:02}.{:03}", - f.hh, - f.mm, - f.ss, - f.us / 1000 - ); + push_hms(&mut out, f.hh, f.mm, f.ss); + out.push(b'.'); + push_num(&mut out, f.us / 1000, 3); } "microseconds" => { - let _ = write!(out, "{:02}:{:02}:{:02}.{:06}", f.hh, f.mm, f.ss, f.us); - } - "seconds" => { - let _ = write!(out, "{:02}:{:02}:{:02}", f.hh, f.mm, f.ss); + push_hms(&mut out, f.hh, f.mm, f.ss); + out.push(b'.'); + push_num(&mut out, f.us, 6); } + "seconds" => push_hms(&mut out, f.hh, f.mm, f.ss), _ => { + push_hms(&mut out, f.hh, f.mm, f.ss); if f.us != 0 { - let _ = write!(out, "{:02}:{:02}:{:02}.{:06}", f.hh, f.mm, f.ss, f.us); - } else { - let _ = write!(out, "{:02}:{:02}:{:02}", f.hh, f.mm, f.ss); + out.push(b'.'); + push_num(&mut out, f.us, 6); } } } if let Tz::Fixed(off) = tz { - format_offset(&mut out, off, ":"); + format_offset(&mut out, off, b":"); } - Some(Ok(Object::from_str(out))) + Some(Ok(text_object(&out))) } /// `strftime` for the numeric directives (no locale involvement); any -/// other directive declines. +/// other directive declines. The result is UTF-8: literal text is copied +/// byte for byte, and a directive is ASCII. fn strftime_numeric( y: i64, m: i64, @@ -1065,69 +2003,55 @@ fn strftime_numeric( time: Option<(i64, i64, i64, i64)>, tz: Option<&Tz>, fmt: &str, -) -> Option { +) -> Option> { let (hh, mm, ss, us) = time.unwrap_or((0, 0, 0, 0)); - let mut out = String::with_capacity(fmt.len() + 16); - let mut chars = fmt.chars(); - while let Some(c) = chars.next() { - if c != '%' { + let fmt = fmt.as_bytes(); + let mut out = Vec::with_capacity(fmt.len() + 16); + let mut i = 0; + while i < fmt.len() { + let c = fmt[i]; + i += 1; + if c != b'%' { out.push(c); continue; } - match chars.next()? { - 'Y' => { - let _ = write!(out, "{y:04}"); - } - 'm' => { - let _ = write!(out, "{m:02}"); - } - 'd' => { - let _ = write!(out, "{d:02}"); - } - 'H' => { - let _ = write!(out, "{hh:02}"); - } - 'M' => { - let _ = write!(out, "{mm:02}"); - } - 'S' => { - let _ = write!(out, "{ss:02}"); - } - 'f' => { - let _ = write!(out, "{us:06}"); - } - 'y' => { - let _ = write!(out, "{:02}", y % 100); - } - 'j' => { + let k = *fmt.get(i)?; + i += 1; + match k { + b'Y' => push_num(&mut out, y, 4), + b'm' => push_num(&mut out, m, 2), + b'd' => push_num(&mut out, d, 2), + b'H' => push_num(&mut out, hh, 2), + b'M' => push_num(&mut out, mm, 2), + b'S' => push_num(&mut out, ss, 2), + b'f' => push_num(&mut out, us, 6), + b'y' => push_num(&mut out, y % 100, 2), + b'j' => { let doy = DAYS_BEFORE_MONTH[(m - 1) as usize] + i64::from(m > 2 && is_leap(y)) + d; - let _ = write!(out, "{doy:03}"); + push_num(&mut out, doy, 3); } - 'I' => { + b'I' => { let h12 = match hh % 12 { 0 => 12, h => h, }; - let _ = write!(out, "{h12:02}"); + push_num(&mut out, h12, 2); } - 'F' => { - let _ = write!(out, "{y:04}-{m:02}-{d:02}"); - } - 'T' => { - let _ = write!(out, "{hh:02}:{mm:02}:{ss:02}"); - } - '%' => out.push('%'), - 'z' => match tz? { + b'F' => push_ymd(&mut out, y, m, d), + b'T' => push_hms(&mut out, hh, mm, ss), + b'%' => out.push(b'%'), + b'z' => match tz? { Tz::Naive => {} - Tz::Fixed(off) => format_offset(&mut out, *off, ""), + Tz::Fixed(off) => format_offset(&mut out, *off, b""), }, - ':' => { - if chars.next()? != 'z' { + b':' => { + if *fmt.get(i)? != b'z' { return None; } + i += 1; match tz? { Tz::Naive => {} - Tz::Fixed(off) => format_offset(&mut out, *off, ":"), + Tz::Fixed(off) => format_offset(&mut out, *off, b":"), } } _ => return None, @@ -1154,7 +2078,7 @@ fn dt_strftime(a: &[Object]) -> Option> { Some(&tz), fmt, )?; - Some(Ok(Object::from_str(s))) + Some(Ok(text_object(&s))) } fn date_strftime(a: &[Object]) -> Option> { @@ -1168,7 +2092,7 @@ fn date_strftime(a: &[Object]) -> Option> { } // A date has no time or offset: `%z` formats as empty, like CPython. let s = strftime_numeric(y, m, d, None, Some(&Tz::Naive), fmt)?; - Some(Ok(Object::from_str(s))) + Some(Ok(text_object(&s))) } fn digits(b: &[u8]) -> Option { @@ -1254,7 +2178,7 @@ fn dt_fromisoformat(a: &[Object]) -> Option> { }; } } - Some(Ok(new_dt(st, &f)?)) + Some(Ok(new_dt(st, f)?)) } /// `HH[:MM[:SS[.f{1,6}]]]` (colon-separated only). @@ -1315,21 +2239,22 @@ fn new_timezone(st: &State, us: i128) -> Option { /// keeps the careful path, which frees that object promptly. pub(crate) fn plain_drop_ok(i: &PyInstance) -> bool { match i.cls_raw().native_kind.get() { - KIND_DATETIME | KIND_TIME => { - let Ok(s) = i.slots.try_borrow() else { - return false; - }; - let idx = if i.cls_raw().native_kind.get() == KIND_DATETIME { - DT_TZINFO - } else { - 4 - }; - match s.get_hinted(idx, "_tzinfo") { + kind @ (KIND_DATETIME | KIND_TIME) => with_slots(i, |s| { + if let Some(p) = s.as_packed() { + let tz = p.tz_ptr(); + // SAFETY: `tz` is owned by the packed value being read. + return Some( + tz.is_null() || unsafe { &*tz }.cls_raw().native_kind.get() == KIND_TIMEZONE, + ); + } + let idx = if kind == KIND_DATETIME { DT_TZINFO } else { 4 }; + Some(match s.get_hinted(idx, "_tzinfo") { Some(Object::None) => true, Some(o @ Object::Instance(_)) => kind_of(o) == KIND_TIMEZONE, _ => false, - } - } + }) + }) + .unwrap_or(false), KIND_TIMEDELTA | KIND_DATE | KIND_TIMEZONE => true, _ => false, } @@ -1364,15 +2289,16 @@ pub(crate) fn leaf_binop( return None; } } - let args = [a.clone(), b.clone()]; + // SAFETY: a shallow copy the fast halves only read; never dropped. + let args = std::mem::ManuallyDrop::new(unsafe { [std::ptr::read(a), std::ptr::read(b)] }); match (kind, op) { - (KIND_TIMEDELTA, B::Add) => td_add(&args), - (KIND_TIMEDELTA, B::Sub) => td_sub(&args), - (KIND_TIMEDELTA, B::Mult) => td_mul(&args), - (KIND_DATE, B::Add) => date_add(&args), - (KIND_DATE, B::Sub) => date_sub(&args), - (KIND_DATETIME, B::Add) => dt_add(&args), - (KIND_DATETIME, B::Sub) => dt_sub(&args), + (KIND_TIMEDELTA, B::Add) => td_add(&args[..]), + (KIND_TIMEDELTA, B::Sub) => td_sub(&args[..]), + (KIND_TIMEDELTA, B::Mult) => td_mul(&args[..]), + (KIND_DATE, B::Add) => date_add(&args[..]), + (KIND_DATE, B::Sub) => date_sub(&args[..]), + (KIND_DATETIME, B::Add) => dt_add(&args[..]), + (KIND_DATETIME, B::Sub) => dt_sub(&args[..]), _ => None, } } @@ -1396,11 +2322,12 @@ pub(crate) fn leaf_compare( if !verified(st, ci, kind) || !verified(st, cj, kind) { return None; } - let args = [a.clone(), b.clone()]; + // SAFETY: a shallow copy the fast halves only read; never dropped. + let args = std::mem::ManuallyDrop::new(unsafe { [std::ptr::read(a), std::ptr::read(b)] }); let r = match kind { - KIND_TIMEDELTA => td_cmp(&args).map(|o| (o, true)), - KIND_DATE => date_cmp(&args).map(|o| (o, true)), - KIND_DATETIME => dt_cmp_raw(&args), + KIND_TIMEDELTA => td_cmp(&args[..]).map(|o| (o, true)), + KIND_DATE => date_cmp(&args[..]).map(|o| (o, true)), + KIND_DATETIME => dt_cmp_raw(&args[..]), _ => None, }?; let (o, comparable) = r; @@ -1415,12 +2342,46 @@ pub(crate) fn leaf_compare( }))) } +/// The function under the native class method `name` of the exact class +/// `cls` while the class still has it (`datetime.fromisoformat`), for +/// the dispatch loop to bind to `cls` without the descriptor protocol. +pub(crate) fn type_classmethod(cls: &TypeObject, name: &SharedStr) -> Option { + let kind = cls.native_kind.get(); + if kind != KIND_DATETIME { + return None; + } + let st = state_of_cls(cls)?; + if !SharedStr::ptr_eq(name, &st.names.f_fromisoformat) + || !std::ptr::eq(cls, st.datetime.as_ptr()) + || !verified(st, cls, kind) + { + return None; + } + st.fromiso.get().cloned() +} + +/// The fast half of `function` bound to `receiver` when that is a native +/// class method of its exact class (see [`type_classmethod`]); the +/// caller passes the receiver first. +pub(crate) fn bound_fast(function: &Object, receiver: &Object) -> Option { + let (Object::Builtin(f), Object::Type(cls)) = (function, receiver) else { + return None; + }; + if cls.native_kind.get() != KIND_DATETIME { + return None; + } + match state_of_cls(cls)?.fromiso.get()? { + Object::Builtin(g) if Rc::ptr_eq(f, g) => Some(dt_fromisoformat), + _ => None, + } +} + /// A public field (`dt.hour`, `d.year`, …) of a natively served instance /// whose class still has its original property. pub(crate) fn leaf_field(i: &PyInstance, name: &SharedStr) -> Option { let cls = i.cls_raw(); let kind = cls.native_kind.get(); - if kind != KIND_DATE && kind != KIND_DATETIME { + if kind != KIND_DATE && kind != KIND_DATETIME && kind != KIND_TIMEDELTA { return None; } let st = state_of_cls(cls)?; @@ -1428,6 +2389,27 @@ pub(crate) fn leaf_field(i: &PyInstance, name: &SharedStr) -> Option { return None; } let n = &st.names; + if kind == KIND_TIMEDELTA { + let ix = if SharedStr::ptr_eq(name, &n.f_days) { + 0 + } else if SharedStr::ptr_eq(name, &n.f_seconds) { + 1 + } else if SharedStr::ptr_eq(name, &n.f_microseconds) { + 2 + } else { + return None; + }; + // The public slots themselves (a Python-built value's are written + // apart from the private ones). + return with_slots(i, |s| match s.as_packed() { + Some(p) if p.kind() == KIND_TIMEDELTA => { + let (d, s, us) = p.td(); + Some(Object::Int([d, s, us][ix])) + } + Some(_) => None, + None => Some(s.get_hinted(4 + ix, name)?.clone()), + }); + } let (idx, slot) = if SharedStr::ptr_eq(name, &n.f_year) { (D_YEAR, &n.year) } else if SharedStr::ptr_eq(name, &n.f_month) { @@ -1451,8 +2433,23 @@ pub(crate) fn leaf_field(i: &PyInstance, name: &SharedStr) -> Option { } else { return None; }; - let s = i.slots.try_borrow().ok()?; - Some(s.get_hinted(idx, slot)?.clone()) + with_slots(i, |s| { + if let Some(p) = s.as_packed() { + let w = p.word; + return Some(match idx { + D_YEAR => Object::Int(bits(w, W_YEAR, 14)), + D_MONTH => Object::Int(bits(w, W_MONTH, 4)), + D_DAY => Object::Int(bits(w, W_DAY, 5)), + DT_HOUR => Object::Int(bits(w, W_HH, 5)), + DT_MINUTE => Object::Int(bits(w, W_MM, 6)), + DT_SECOND => Object::Int(bits(w, W_SS, 6)), + DT_US => Object::Int(bits(w, W_US, 20)), + DT_FOLD => Object::Int(bits(w, W_FOLD, 1)), + _ => p.tz(), + }); + } + Some(s.get_hinted(idx, slot)?.clone()) + }) } // --------------------------------------------------------------------- @@ -1469,6 +2466,42 @@ fn with_interp( /// One native method: its name, the class kind it lives on, and the /// pure fast half. +/// A keyword call of a native method whose fast half takes its optional +/// parameters by position, `None` standing for an omitted one: the +/// positional argument list (receiver first), or `None` for the replaced +/// Python method (unknown or repeated names, other methods). +fn kw_positional( + key: (u8, &'static str), + a: &[Object], + kw: &[(String, Object)], +) -> Option> { + let params: &[&str] = match key { + (KIND_DATE, "replace") => &["year", "month", "day"], + _ => return None, + }; + let (recv, pos) = a.split_first()?; + if pos.len() > params.len() { + return None; + } + let mut out = Vec::with_capacity(params.len() + 1); + out.push(recv.clone()); + out.extend(pos.iter().cloned()); + out.resize(params.len() + 1, Object::Unbound); + for (name, v) in kw { + let ix = params.iter().position(|p| p == name)? + 1; + if !matches!(out[ix], Object::Unbound) { + return None; + } + out[ix] = v.clone(); + } + for o in &mut out { + if matches!(o, Object::Unbound) { + *o = Object::None; + } + } + Some(out) +} + struct Spec { kind: u8, name: &'static str, @@ -1520,6 +2553,24 @@ const SPECS: &[Spec] = &[ fast: td_bool, classmethod: false, }, + Spec { + kind: KIND_TIMEDELTA, + name: "__hash__", + fast: td_hash, + classmethod: false, + }, + Spec { + kind: KIND_DATE, + name: "__hash__", + fast: date_hash, + classmethod: false, + }, + Spec { + kind: KIND_DATETIME, + name: "__hash__", + fast: dt_hash, + classmethod: false, + }, Spec { kind: KIND_TIMEDELTA, name: "__eq__", @@ -1761,34 +2812,7 @@ pub(crate) fn install(args: &[Object]) -> Result { orig.insert((spec.kind, spec.name), v); } let names = Names::new(); - let layout = |keys: &[&SharedStr]| -> SharedSlice { - keys.iter() - .map(|k| DictKey(Object::Str((*k).clone()))) - .collect::>() - .into() - }; - let td_layout = layout(&[ - &names.days, - &names.seconds, - &names.microseconds, - &names.hashcode, - &names.pub_days, - &names.pub_seconds, - &names.pub_microseconds, - ]); - let date_layout = layout(&[&names.year, &names.month, &names.day, &names.hashcode]); - let dt_layout = layout(&[ - &names.year, - &names.month, - &names.day, - &names.hour, - &names.minute, - &names.second, - &names.microsecond, - &names.tzinfo, - &names.hashcode, - &names.fold, - ]); + let [td_layout, date_layout, dt_layout] = layouts().clone(); let state = Rc::new(State { timedelta: Rc::downgrade(td), date: Rc::downgrade(date), @@ -1802,6 +2826,8 @@ pub(crate) fn install(args: &[Object]) -> Result { orig, sealed: Default::default(), expect: std::sync::OnceLock::new(), + pool: Pool::default(), + fromiso: std::sync::OnceLock::new(), }); // Each class carries the state (a class in `native_ext` resolves to // it from any of its instances). @@ -1836,6 +2862,8 @@ pub(crate) fn install(args: &[Object]) -> Result { if let Some(r) = fast(a) { return r; } + } else if let Some(r) = kw_positional(key, a, kw).and_then(|a| fast(&a)) { + return r; } let f = st_kw.orig.get(&key).cloned().unwrap_or(Object::None); with_interp(|i| i.call_object(f, a, kw)) @@ -1843,6 +2871,9 @@ pub(crate) fn install(args: &[Object]) -> Result { }); crate::leaf_builtins::register_fast(&b, fast); let value = if spec.classmethod { + if spec.name == "fromisoformat" { + let _ = state.fromiso.set(Object::Builtin(b.clone())); + } Object::ClassMethod(crate::object::MethodWrapper::new(Object::Builtin(b))) } else { Object::Builtin(b) diff --git a/crates/weavepy-vm/src/stdlib/functools_mod.rs b/crates/weavepy-vm/src/stdlib/functools_mod.rs index e4fcc0f7..948bd333 100644 --- a/crates/weavepy-vm/src/stdlib/functools_mod.rs +++ b/crates/weavepy-vm/src/stdlib/functools_mod.rs @@ -311,7 +311,7 @@ fn lru_cache_wrapper_new(args: &[Object]) -> Result { // wrapper counted as an external root that made that namespace // immortal. let wrapper = Object::Instance(inst); - crate::gc_trace::track(wrapper.clone()); + crate::gc_trace::track(&wrapper); Ok(wrapper) } @@ -632,11 +632,8 @@ fn scalar_lru_operation( data.insert(DictKey(key.clone()), value.clone()); order.push(limit).ok_or_else(invalid_lru_order)?; } - // Publish a consistent cache before releasing an evicted result. The - // prompt-reap queue preserves its finalizer's real calling frame. - if let Some((_, value)) = removed { - crate::vm_singletons::queue_container_removed(&value); - } + // Publish a consistent cache before releasing an evicted result. + drop(removed); Ok(ScalarLru::Complete(None)) } @@ -811,11 +808,7 @@ fn lru_cache_clear(args: &[Object], _kwargs: &[(String, Object)]) -> Result Result { .first() .ok_or_else(|| type_error("_strong_count() requires 1 argument"))?; let strong = gc_trace::strong_count_for(target) as i64; - let weak = crate::weakref_registry::strong_clone_count(id_of(target)) as i64; + // The registry holds its referents weakly: no strong clones. Ok(Object::new_tuple_array([ Object::Int(strong), - Object::Int(weak), + Object::Int(0), ])) } @@ -504,8 +504,10 @@ fn object_is_tracked(target: &Object) -> bool { // answers as CPython's (always tracked) and is tracked from here on. Object::Instance(inst) => { inst.ensure_gc_tracked(); - gc_trace::with_state(|s| s.is_tracked(id_of(target))) + gc_trace::with_state(|s| s.is_tracked_instance(id_of(target))) } + // A function may still be young. + Object::Function(_) => gc_trace::with_state(|s| s.is_tracked_instance(id_of(target))), // A list/dict/set born holding only scalars has its tracking // deferred; CPython tracks it from birth, so answer as CPython // does and hand it to the collector from here on. @@ -513,7 +515,7 @@ fn object_is_tracked(target: &Object) -> bool { let id = id_of(target); gc_trace::with_state(|s| { if !s.is_tracked(id) { - s.track_now(target.clone()); + s.track_now(target); } s.is_tracked(id) }) @@ -585,7 +587,7 @@ fn get_stats(_args: &[Object]) -> Result { fn track_obj(args: &[Object]) -> Result { if let Some(o) = args.first() { - gc_trace::track(o.clone()); + gc_trace::track(o); } Ok(Object::None) } diff --git a/crates/weavepy-vm/src/stdlib/io.rs b/crates/weavepy-vm/src/stdlib/io.rs index 01e39199..a3df33c3 100644 --- a/crates/weavepy-vm/src/stdlib/io.rs +++ b/crates/weavepy-vm/src/stdlib/io.rs @@ -1349,7 +1349,7 @@ fn ind_new(args: &[Object]) -> Result { } }; let inst = Object::Instance(Rc::new(crate::types::PyInstance::new(cls))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); Ok(inst) } @@ -4754,7 +4754,7 @@ fn bw_new(args: &[Object]) -> Result { _ => return Err(type_error("Buffered.__new__(X): X is not a type object")), }; let inst = Object::Instance(Rc::new(crate::types::PyInstance::new(cls))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); Ok(inst) } diff --git a/crates/weavepy-vm/src/stdlib/json_accel.rs b/crates/weavepy-vm/src/stdlib/json_accel.rs index 2609fb18..813ed6a5 100644 --- a/crates/weavepy-vm/src/stdlib/json_accel.rs +++ b/crates/weavepy-vm/src/stdlib/json_accel.rs @@ -1020,7 +1020,7 @@ fn parse_object( // CPython replaces duplicate values as it parses. Run // a displaced value's finalizer before the next field, // with no table borrow held across user code. - interp.prompt_reap_dropped(previous); + interp.release(previous); } } DecodedObject::Pairs(pairs) => pairs.push((key_obj, value)), diff --git a/crates/weavepy-vm/src/stdlib/marshal_mod.rs b/crates/weavepy-vm/src/stdlib/marshal_mod.rs index e42c43dc..495f4e2d 100644 --- a/crates/weavepy-vm/src/stdlib/marshal_mod.rs +++ b/crates/weavepy-vm/src/stdlib/marshal_mod.rs @@ -1750,7 +1750,7 @@ impl<'a> MarshalReader<'a> { // PEP-657 columns recovered from long-form location entries // (RFC 0056 WS4): traceback caret underlines survive the // `.pyc` round-trip. - coltable: decoded.coltable, + coltable: decoded.coltable.into(), arg_count, posonly_count, kwonly_count, diff --git a/crates/weavepy-vm/src/stdlib/mod.rs b/crates/weavepy-vm/src/stdlib/mod.rs index 1d96968f..b2a81217 100644 --- a/crates/weavepy-vm/src/stdlib/mod.rs +++ b/crates/weavepy-vm/src/stdlib/mod.rs @@ -103,6 +103,7 @@ pub mod signal_mod; pub mod socket_mod; pub mod sqlite3_native; pub mod sre_mod; +pub(crate) mod sre_objects; pub mod statistics_accel; pub mod struct_mod; pub mod subprocess_mod; diff --git a/crates/weavepy-vm/src/stdlib/os.rs b/crates/weavepy-vm/src/stdlib/os.rs index 962e7cdf..427a3f2d 100644 --- a/crates/weavepy-vm/src/stdlib/os.rs +++ b/crates/weavepy-vm/src/stdlib/os.rs @@ -7497,7 +7497,7 @@ fn struct_seq_replace( let obj = Object::Instance(Rc::new(new_inst)); // CPython allocates the copy through the GC heap, so it is tracked from // birth (test_structseq.test_replace_gc_tracked builds a cycle out of it). - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); Ok(obj) } diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel.rs b/crates/weavepy-vm/src/stdlib/pickle_accel.rs index 3f9d8c96..85724186 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel.rs @@ -1,9 +1,10 @@ //! Guarded protocol-4/5 encoding and decoding of built-in pickle data and of //! instances of plain classes. //! -//! A first pass checks the complete supported stream and its container graph -//! without creating Python objects. Cycles, deep graphs, custom reconstruction, -//! and unsupported or malformed input return to the existing unpickler. +//! Decoding builds the result in one pass and publishes it only once the +//! whole stream has been accepted. Cycles, deep graphs, custom +//! reconstruction, and unsupported or malformed input drop everything built +//! so far, which runs no Python code, and return to the existing unpickler. //! //! Classes are resolved from `sys.modules` by plain dictionary probes, and //! `NEWOBJ` and `BUILD` are accepted only for classes whose type metadata @@ -13,13 +14,16 @@ mod classes; mod encode; +use std::hash::BuildHasher; + use crate::shared_value::{SharedSlice, SharedStr}; +use indexmap::map::raw_entry_v1::{RawEntryApiV1, RawEntryMut}; use num_bigint::BigInt; use weavepy_compiler::CodeObject; use crate::error::RuntimeError; use crate::import::ModuleCache; -use crate::object::{BuiltinFn, DictData, DictKey, Object, PyFunction, PyModule, StrKey}; +use crate::object::{BuiltinFn, DictData, DictKey, DictMap, Object, PyFunction, PyModule, StrKey}; use crate::sync::{Rc, RefCell, Weak}; use crate::types::TypeObject; @@ -68,7 +72,7 @@ impl FunctionGuard { if !function.defaults_maybe_overridden() { return true; } - let slots = function.slots.borrow(); + let slots = function.slots().borrow(); !slots.contains_key(&StrKey("__defaults__")) && !slots.contains_key(&StrKey("__kwdefaults__")) } @@ -292,13 +296,14 @@ impl UnpicklerGuard { Rc::as_ptr(class) == self.class.as_ptr() && class.attr_version.get() == self.version } - fn holds(&self, class: &Rc, opcodes: &Opcodes) -> bool { - if !self.matches_class(class) - || !class - .lookup("__init__") - .is_some_and(|f| self.init.holds(&f)) - || !class.lookup("load").is_some_and(|f| self.load.holds(&f)) - { + /// Whether the functions that a stream of `opcodes` would run are still + /// the guarded ones, with their guarded code and defaults. Checked once + /// the stream proves supported, after [`Self::matches_class`]: an + /// unchanged class version (which every change to the class or its + /// bases advances) proves that `__init__`, `load`, and `find_class` + /// still resolve to the guarded functions. + fn holds(&self, opcodes: &Opcodes) -> bool { + if !self.init.holds_live() || !self.load.holds_live() { return false; } // STACK_GLOBAL is the only supported instruction that calls a @@ -307,7 +312,7 @@ impl UnpicklerGuard { && !self .find_class .as_ref() - .is_some_and(|guard| class.lookup("find_class").is_some_and(|f| guard.holds(&f))) + .is_some_and(FunctionGuard::holds_live) { return false; } @@ -391,15 +396,11 @@ fn make_loads(args: &[Object]) -> Result { // request to use the existing unpickler. let context = || instances.as_ref()?.context(); let decoded = match data { - Object::Bytes(data) => { - decode(data, |opcodes| guard.holds(class, opcodes), &context) - } + Object::Bytes(data) => decode(data, |opcodes| guard.holds(opcodes), &context), // Decoding runs no Python code, so the borrow is safe. - Object::ByteArray(data) => decode( - &data.borrow(), - |opcodes| guard.holds(class, opcodes), - &context, - ), + Object::ByteArray(data) => { + decode(&data.borrow(), |opcodes| guard.holds(opcodes), &context) + } _ => None, }; Ok(decoded.map_or(Object::None, |value| Object::new_tuple_array([value]))) @@ -569,44 +570,43 @@ fn make_dumps(args: &[Object]) -> Result { }))) } -#[derive(Clone, Copy, PartialEq, Eq)] -enum Container { - List, - Dict, - Tuple, -} - -enum Scalar<'a> { - None, - Bool(bool), - Int(i64), - Long(&'a [u8]), - Float(f64), - Str(&'a str), - Bytes(&'a [u8]), -} - const STACK_GLOBAL: u8 = 0x93; const NEWOBJ: u8 = 0x81; const BUILD: u8 = b'b'; +/// The deepest container nesting that is decoded natively. +const MAX_DEPTH: u32 = 128; /// `sys.modules`, produced on demand once a stream names a class. struct DecodeContext { modules: Rc>, } -trait Sink<'a> { - type Value: Clone; - - fn opcode(&mut self, _value: u8) {} - fn scalar(&mut self, value: Scalar<'a>) -> Self::Value; - fn empty(&mut self, kind: Container) -> Option; - fn tuple(&mut self, items: Vec) -> Option; - fn extend(&mut self, target: &Self::Value, items: Batch<'_, Self::Value>) -> Option<()>; - fn setitems(&mut self, target: &Self::Value, items: Batch<'_, Self::Value>) -> Option<()>; - fn global(&mut self, module: &Self::Value, name: &Self::Value) -> Option; - fn newobj(&mut self, class: &Self::Value, args: &Self::Value) -> Option; - fn build(&mut self, target: &Self::Value, state: Self::Value) -> Option<()>; +/// Push `$value` (with node number `$node`, by default [`LEAF`]) onto the +/// decoder stack, written straight into its slot. A value built in a +/// temporary (with a byte store for its variant) and then copied with word +/// loads stalls store forwarding, and the decoder's opcode arms otherwise +/// share such a temporary. +macro_rules! push_entry { + ($stack:expr, $value:expr) => { + push_entry!($stack, $value, LEAF) + }; + ($stack:expr, $value:expr, $node:expr) => { + push_in_place!($stack, ($value, $node)) + }; +} + +/// Push `$value` onto the vector `$items`, written straight into its slot +/// (see [`push_entry`]). +macro_rules! push_in_place { + ($items:expr, $value:expr) => {{ + let items = &mut *$items; + if items.len() == items.capacity() { + items.try_reserve(1).ok()?; + } + items.spare_capacity_mut().first_mut()?.write($value); + // SAFETY: the slot past the end has just been initialized. + unsafe { items.set_len(items.len() + 1) }; + }}; } #[inline(always)] @@ -622,25 +622,65 @@ fn push(items: &mut Vec, value: T) -> Option<()> { struct Reader<'a> { data: &'a [u8], pos: usize, - frame_end: Option, + /// The end of the current frame, or of the data outside a frame. + limit: usize, + framed: bool, } impl<'a> Reader<'a> { + fn new(data: &'a [u8], pos: usize) -> Self { + Self { + data, + pos, + limit: data.len(), + framed: false, + } + } + + #[inline(always)] fn read(&mut self, size: usize) -> Option<&'a [u8]> { - if self.frame_end == Some(self.pos) { - self.frame_end = None; + if size <= self.limit - self.pos { + let value = self.data.get(self.pos..self.pos + size)?; + self.pos += size; + return Some(value); } - let end = self.pos.checked_add(size)?; - if self.frame_end.is_some_and(|limit| end > limit) { + self.read_past_limit(size) + } + + /// A read beyond the current limit, which leaves a frame that ends + /// where the read begins, and fails otherwise. + #[cold] + fn read_past_limit(&mut self, size: usize) -> Option<&'a [u8]> { + if !self.framed || self.pos != self.limit { return None; } + self.framed = false; + self.limit = self.data.len(); + let end = self.pos.checked_add(size)?; let value = self.data.get(self.pos..end)?; self.pos = end; Some(value) } + /// Whether the current frame (or the data outside one) continues with + /// `bytes`, compared byte by byte (not with a call to `memcmp`). + #[inline(always)] + fn next_is(&self, bytes: [u8; N]) -> bool { + self.limit - self.pos >= N + && bytes + .iter() + .enumerate() + .all(|(i, byte)| self.data.get(self.pos + i) == Some(byte)) + } + + #[inline(always)] fn byte(&mut self) -> Option { - Some(self.read(1)?[0]) + if self.pos < self.limit { + let value = *self.data.get(self.pos)?; + self.pos += 1; + return Some(value); + } + Some(self.read_past_limit(1)?[0]) } fn u32(&mut self) -> Option { @@ -654,7 +694,7 @@ impl<'a> Reader<'a> { fn frame(&mut self) -> Option<()> { // FRAME is valid only after the preceding frame has ended. In // particular, its argument cannot be read out of an active frame. - if self.frame_end.is_some() { + if self.framed { return None; } let size = self.size64()?; @@ -662,22 +702,12 @@ impl<'a> Reader<'a> { if end > self.data.len() { return None; } - self.frame_end = Some(end); + self.framed = true; + self.limit = end; Some(()) } } -/// The items an APPEND(S) or SETITEM(S) moves into their container. -type Batch<'s, V> = std::vec::Drain<'s, V>; - -fn take(stack: &mut Vec, count: usize, marks: &[usize]) -> Option> { - let begin = stack.len().checked_sub(count)?; - if marks.last().is_some_and(|mark| begin < *mark) { - return None; - } - Some(stack.split_off(begin)) -} - fn pop(stack: &mut Vec, marks: &[usize]) -> Option { let begin = stack.len().checked_sub(1)?; if marks.last().is_some_and(|mark| begin < *mark) { @@ -693,194 +723,82 @@ fn top<'a, V>(stack: &'a [V], marks: &[usize]) -> Option<&'a V> { stack.last() } -/// The container below the items from `begin` on, and those items. Both -/// must lie above the innermost mark. -fn batch<'s, V: Clone>( - stack: &'s mut Vec, - begin: usize, - marks: &[usize], -) -> Option<(V, Batch<'s, V>)> { - if begin == 0 || begin > stack.len() || marks.last().is_some_and(|mark| begin <= *mark) { - return None; - } - let target = stack[begin - 1].clone(); - Some((target, stack.drain(begin..))) +/// Whether the items from `begin` on and the container below them all lie +/// above the innermost mark. +fn is_batch(stack: &[V], begin: usize, marks: &[usize]) -> bool { + begin != 0 && begin <= stack.len() && marks.last().is_none_or(|mark| begin > *mark) } -#[allow(clippy::too_many_lines)] -fn parse<'a, S: Sink<'a>>(data: &'a [u8], sink: &mut S) -> Option { - if data.len() < 3 || data[0] != 0x80 || !matches!(data[1], 4 | 5) { - return None; - } - sink.opcode(0x80); - let mut reader = Reader { - data, - pos: 2, - frame_end: None, - }; - let mut stack = Vec::new(); - stack.try_reserve(64).ok()?; - let mut marks = Vec::new(); - let mut memo: Vec = Vec::new(); - memo.try_reserve(data.len() / 16).ok()?; - loop { - let opcode = reader.byte()?; - sink.opcode(opcode); - let value = match opcode { - b'.' => { - return (marks.is_empty() && stack.len() == 1) - .then(|| stack.pop()) - .flatten(); - } - 0x95 => { - reader.frame()?; - continue; - } - b'(' => { - push(&mut marks, stack.len())?; - continue; - } - 0x94 => { - push(&mut memo, top(&stack, &marks)?.clone())?; - continue; - } - b'h' => memo.get(usize::from(reader.byte()?))?.clone(), - b'j' => memo.get(usize::try_from(reader.u32()?).ok()?)?.clone(), - b'N' => sink.scalar(Scalar::None), - 0x88 => sink.scalar(Scalar::Bool(true)), - 0x89 => sink.scalar(Scalar::Bool(false)), - b'K' => sink.scalar(Scalar::Int(i64::from(reader.byte()?))), - b'M' => sink.scalar(Scalar::Int(i64::from(u16::from_le_bytes( - reader.read(2)?.try_into().ok()?, - )))), - b'J' => sink.scalar(Scalar::Int(i64::from(i32::from_le_bytes( - reader.read(4)?.try_into().ok()?, - )))), - 0x8a | 0x8b => { - let size = if data[reader.pos - 1] == 0x8a { - usize::from(reader.byte()?) - } else { - let size = i32::from_le_bytes(reader.read(4)?.try_into().ok()?); - usize::try_from(size).ok()? - }; - sink.scalar(Scalar::Long(reader.read(size)?)) - } - b'G' => sink.scalar(Scalar::Float(f64::from_be_bytes( - reader.read(8)?.try_into().ok()?, - ))), - opcode @ (0x8c | b'X' | 0x8d | b'C' | b'B' | 0x8e) => { - let size = match opcode { - 0x8c | b'C' => usize::from(reader.byte()?), - b'X' | b'B' => usize::try_from(reader.u32()?).ok()?, - _ => reader.size64()?, - }; - let bytes = reader.read(size)?; - sink.scalar(if matches!(opcode, 0x8c | b'X' | 0x8d) { - // Surrogate-pass strings stay on the existing WStr path. - Scalar::Str(std::str::from_utf8(bytes).ok()?) - } else { - Scalar::Bytes(bytes) - }) - } - b']' => sink.empty(Container::List)?, - b'}' => sink.empty(Container::Dict)?, - b')' => sink.empty(Container::Tuple)?, - 0x85..=0x87 => { - let count = usize::from(data[reader.pos - 1] - 0x84); - sink.tuple(take(&mut stack, count, &marks)?)? - } - b't' => { - let mark = marks.pop()?; - sink.tuple(stack.split_off(mark))? - } - b'a' => { - let begin = stack.len().checked_sub(1)?; - let (target, items) = batch(&mut stack, begin, &marks)?; - sink.extend(&target, items)?; - continue; - } - b'e' => { - let mark = marks.pop()?; - let (target, items) = batch(&mut stack, mark, &marks)?; - sink.extend(&target, items)?; - continue; - } - b's' => { - let begin = stack.len().checked_sub(2)?; - let (target, items) = batch(&mut stack, begin, &marks)?; - sink.setitems(&target, items)?; - continue; - } - b'u' => { - let mark = marks.pop()?; - let (target, items) = batch(&mut stack, mark, &marks)?; - sink.setitems(&target, items)?; - continue; - } - STACK_GLOBAL => { - let name = pop(&mut stack, &marks)?; - let module = pop(&mut stack, &marks)?; - sink.global(&module, &name)? - } - NEWOBJ => { - let args = pop(&mut stack, &marks)?; - let class = pop(&mut stack, &marks)?; - sink.newobj(&class, &args)? - } - BUILD => { - let state = pop(&mut stack, &marks)?; - sink.build(top(&stack, &marks)?, state)?; - continue; - } - // GLOBAL, REDUCE, NEWOBJ_EX, persistent IDs, external buffers, - // legacy protocols, and all other operations retain the full - // unpickler. - _ => return None, - }; - push(&mut stack, value)?; - } -} +/// A value on the stack or in the memo, with the number of the [`Node`] +/// that describes it, [`LEAF`], or [`CLASS`]. +type Entry = (Object, NodeId); -#[derive(Clone, Copy)] -enum Kind<'a> { - None, - Key, - Str(&'a str), - Float, - /// An index into [`Probe::classes`]. Only NEWOBJ may consume it. - Class(u32), - /// An index into [`Probe::nodes`]: a container or an instance. - Container(u32), -} +/// A node number. It is word-sized, like the rest of an [`Entry`], so an +/// entry is copied with the same word stores that wrote it. +type NodeId = u64; -/// What BUILD needs to know about one item of a two-item state tuple. -#[derive(Clone, Copy)] -enum StateItem { - None, - Node(u32), - Other, +/// `value.clone()`, in line for the kinds that streams memoize, as a copy +/// of the existing value's words: a value rebuilt in a temporary would be +/// copied with word loads that stall on the byte store of its variant. +#[inline(always)] +fn clone_value(value: &Object) -> Object { + match value { + Object::Str(text) => std::mem::forget(text.clone()), + Object::Tuple(items) => std::mem::forget(items.clone()), + Object::List(items) => std::mem::forget(items.clone()), + Object::Dict(items) => std::mem::forget(items.clone()), + Object::Instance(instance) => std::mem::forget(instance.clone()), + Object::Type(class) => std::mem::forget(class.clone()), + Object::None | Object::Bool(_) | Object::Int(_) | Object::Float(_) => {} + _ => return value.clone(), + } + // SAFETY: each kind above is a plain value or a single counted + // reference, whose clone is a copy of the same words holding one more + // reference: the one just taken and forgotten, which the copy owns. + unsafe { std::ptr::read(value) } } -enum Shape<'a> { - List, - Tuple { - len: usize, - pair: [StateItem; 2], - }, - /// `keys` lists the string keys; `only_str` reports that there are no - /// others, so the keys are complete. - Dict { - keys: Vec<&'a str>, - only_str: bool, - }, - Instance { - class: u32, - }, +/// The dictionary (or None) and the slot dictionary of a BUILD state pair. +#[allow(clippy::type_complexity)] +fn state_dicts<'s>( + first: &'s Object, + second: &'s Object, +) -> Option<(Option<&'s Rc>>, &'s Rc>)> { + let Object::Dict(slots) = second else { + return None; + }; + match first { + Object::None => Some((None, slots)), + Object::Dict(dict) => Some((Some(dict), slots)), + _ => None, + } } -struct Node<'a> { - shape: Shape<'a>, - children: Vec, +/// The node number of a value that isn't a container. +const LEAF: NodeId = 0; +/// The node number of a class, which only NEWOBJ may consume. +const CLASS: NodeId = NodeId::MAX; +/// The node number of a memo slot whose container's storage moved into an +/// instance. Reading it returns to the full unpickler. +const SPENT: NodeId = NodeId::MAX - 1; +/// [`Node::memo`] for a container that no memo slot holds. +const UNMEMOIZED: u32 = u32::MAX; +/// [`Node::memo`] for a container that more than one memo slot holds. +const MEMOIZED_TWICE: u32 = u32::MAX - 1; + +/// What the decoder knows about one container or instance it built. +struct Node { + /// One more than the height of the tallest container inside it. + height: u32, + /// Whether a container, tuple, or BUILD has taken it. Only a container + /// that nothing holds yet may receive items, so an item can never reach + /// its own container (a cycle), and no height ever has to propagate + /// to an enclosing container. + held: bool, + /// The memo slot that holds it, or one of the markers above. + memo: u32, + /// For a two-item tuple, the node numbers of its items. + pair: [NodeId; 2], } /// A class that NEWOBJ and BUILD can handle without calling Python code. @@ -911,7 +829,7 @@ struct Opcodes([bool; 256]); impl Opcodes { fn insert(&mut self, opcode: u8) { // One indexed store avoids a shift and a read-modify-write on each - // decoded instruction. This scratch is local to one preflight. + // decoded instruction. This scratch is local to one decode. self.0[usize::from(opcode)] = true; } @@ -920,302 +838,334 @@ impl Opcodes { } } -/// One BUILD's state containers: the `__dict__` state and the slot state. -#[derive(Clone, Copy)] -struct StateNodes { - dict: Option, - slots: Option, -} - -/// What the first pass tells the second, all in stream order. -struct Plan { - classes: Vec>, - /// Per node: whether the result can reach it. An unreachable container - /// can never be mutated, so it needs no cycle-collector registration. - reachable: Vec, - /// Per BUILD: which state containers only that BUILD consumes, so that - /// their storage can move into the instance. - builds: Vec, -} - -struct Probe<'a, 'c, const RECORD_OPCODES: bool> { - nodes: Vec>, +/// Builds the result in one pass over the stream. The objects stay private +/// until the whole stream has been accepted: any reason to return to the +/// full unpickler drops them, and dropping them runs no Python code (an +/// instance's class is proven to have no finalizer). +struct Decoder<'c> { opcodes: Opcodes, - /// Every class that STACK_GLOBAL resolved, in stream order. The second - /// pass consumes the same list instead of repeating the lookups. + nodes: Vec, + memo: Vec, + /// Every class that STACK_GLOBAL resolved. classes: Vec, - /// The state of every BUILD, in stream order, with the pair tuple. - builds: Vec<(StateNodes, Option)>, context: &'c dyn Fn() -> Option, resolved: Option, /// The `__slots__` members already verified in this stream (see /// `classes::is_member_slot`): every instance of a class names the /// same ones. - member_slots: Vec<(*const TypeObject, &'a str)>, + member_slots: Vec<(*const TypeObject, SharedStr)>, + /// State keys and their interned strings (see [`Self::intern`]). + interned: Vec<(SharedStr, SharedStr)>, + /// Emptied state dictionaries that nothing else can reach, with their + /// tables' capacity, for the next EMPTY_DICT. + spare_dicts: Vec>>, + /// Every list and dictionary, and every instance tracked from birth, in + /// creation order. Those the result can reach join the cycle collector + /// once the stream is accepted. + containers: Vec, +} + +/// A decoder's vectors, kept empty between calls with their capacity, so a +/// `loads` loop neither reallocates nor regrows them. +#[derive(Default)] +struct Scratch { + stack: Vec, + marks: Vec, + memo: Vec, + nodes: Vec, + containers: Vec, } -impl<'a, 'c, const RECORD_OPCODES: bool> Probe<'a, 'c, RECORD_OPCODES> { - fn new(context: &'c dyn Fn() -> Option) -> Self { +impl Scratch { + /// The largest vector kept between calls. + const MAX_KEPT: usize = 1 << 14; + + fn take() -> Self { + SPARE_SCRATCH + .with(std::cell::Cell::take) + .unwrap_or_default() + } + + /// Empty the vectors (releasing what they hold, which runs no Python + /// code) and keep them for the next call. + fn keep(mut self) { + self.stack.clear(); + self.marks.clear(); + self.memo.clear(); + self.nodes.clear(); + self.containers.clear(); + if self.stack.capacity().max(self.marks.capacity()) <= Self::MAX_KEPT + && self.memo.capacity().max(self.nodes.capacity()) <= Self::MAX_KEPT + && self.containers.capacity() <= Self::MAX_KEPT + { + SPARE_SCRATCH.with(|spare| spare.set(Some(self))); + } + } +} + +thread_local! { + static SPARE_SCRATCH: std::cell::Cell> = const { std::cell::Cell::new(None) }; +} + +impl<'c> Decoder<'c> { + fn new(context: &'c dyn Fn() -> Option, scratch: &mut Scratch) -> Self { Self { - nodes: Vec::new(), - opcodes: Opcodes([!RECORD_OPCODES; 256]), + opcodes: Opcodes([false; 256]), + nodes: std::mem::take(&mut scratch.nodes), + memo: std::mem::take(&mut scratch.memo), classes: Vec::new(), - builds: Vec::new(), context, resolved: None, member_slots: Vec::new(), + interned: Vec::new(), + spare_dicts: Vec::new(), + containers: std::mem::take(&mut scratch.containers), } } - /// [`classes::is_member_slot`], remembered for this stream. (No Python - /// runs during the probe, so the class can't change in between.) - fn is_member_slot(&mut self, class: &Rc, name: &'a str) -> bool { - let key = Rc::as_ptr(class); - if self - .member_slots - .iter() - .any(|&(c, n)| c == key && n == name) - { - return true; - } - let yes = classes::is_member_slot(class, name); - if yes && self.member_slots.len() < 64 { - self.member_slots.push((key, name)); - } - yes + /// Return the vectors to `scratch`. + fn recycle(self, scratch: &mut Scratch) { + scratch.nodes = self.nodes; + scratch.memo = self.memo; + scratch.containers = self.containers; } - /// Count the references to each node that the result keeps: the root, - /// container items, and the state pair's items when the pair itself is - /// kept. BUILD copies its state, so its edge keeps nothing. - fn plan(self, root: Kind<'_>) -> Option { - let mut references = Vec::new(); - references.try_reserve(self.nodes.len()).ok()?; - references.resize(self.nodes.len(), 0u32); - if let Kind::Container(root) = root { - references[root as usize] = 1; - } - let mut is_pair = Vec::new(); - is_pair.try_reserve(self.nodes.len()).ok()?; - is_pair.resize(self.nodes.len(), false); - for pair in self.builds.iter().filter_map(|(_, pair)| *pair) { - is_pair[pair as usize] = true; - } - for (index, node) in self.nodes.iter().enumerate() { - if matches!(node.shape, Shape::Instance { .. }) || is_pair[index] { - continue; - } - for &child in &node.children { - references[child as usize] += 1; - } - } - // A tuple's items precede it, so a pair inside a kept pair is seen - // after the reference to it has been counted. - for (index, node) in self.nodes.iter().enumerate().rev() { - if is_pair[index] && references[index] > 0 { - for &child in &node.children { - references[child as usize] += 1; - } - } + #[inline(always)] + fn opcode(&mut self, value: u8) { + self.opcodes.insert(value); + } + + fn node(&mut self, height: u32, pair: [NodeId; 2]) -> Option { + if height > MAX_DEPTH { + return None; } - let mut uses = Vec::new(); - uses.try_reserve(self.nodes.len()).ok()?; - uses.resize(self.nodes.len(), 0u32); - for (state, _) in &self.builds { - for index in [state.dict, state.slots].into_iter().flatten() { - uses[index as usize] += 1; + let number = NodeId::try_from(self.nodes.len()) + .ok()? + .checked_add(1) + .filter(|&number| number < SPENT)?; + push( + &mut self.nodes, + Node { + height, + held: false, + memo: UNMEMOIZED, + pair, + }, + )?; + Some(number) + } + + /// Push a new, empty dictionary, reusing a moved state's when one waits. + fn empty_dict(&mut self, stack: &mut Vec) -> Option<()> { + let node = self.node(1, [LEAF; 2])?; + let dict = match self.spare_dicts.pop() { + // (A spare is already in `containers`.) + Some(dict) => dict, + None => { + let dict = Rc::new(RefCell::new(DictData::default())); + push_in_place!(&mut self.containers, Object::Dict(dict.clone())); + dict } - } - let movable = |index: Option| { - index.filter(|&i| references[i as usize] == 0 && uses[i as usize] == 1) }; - let mut builds = Vec::new(); - builds.try_reserve(self.builds.len()).ok()?; - builds.extend(self.builds.iter().map(|(state, _)| StateNodes { - dict: movable(state.dict), - slots: movable(state.slots), - })); - Some(Plan { - classes: self.classes.into_iter().map(|c| c.class).collect(), - reachable: references.into_iter().map(|n| n > 0).collect(), - builds, - }) + push_entry!(stack, Object::Dict(dict), node); + Some(()) } - fn node(&mut self, shape: Shape<'a>, items: &[Kind<'a>]) -> Option> { - let index = u32::try_from(self.nodes.len()).ok()?; - let mut children = Vec::new(); - for item in items { - match item { - Kind::Container(child) => push(&mut children, *child)?, - // A class is only ever the operand of NEWOBJ. - Kind::Class(_) => return None, - _ => {} - } + /// Push a new, empty list. + fn empty_list(&mut self, stack: &mut Vec) -> Option<()> { + let node = self.node(1, [LEAF; 2])?; + let list = Rc::new(RefCell::new(Vec::new())); + push_in_place!(&mut self.containers, Object::List(list.clone())); + push_entry!(stack, Object::List(list), node); + Some(()) + } + + /// Mark the value of node `node` as taken by a container, and return its + /// height. A class can't be taken. + #[inline] + fn hold(&mut self, node: NodeId) -> Option { + if node == LEAF { + return Some(0); } - push(&mut self.nodes, Node { shape, children })?; - Some(Kind::Container(index)) - } - - /// The string keys of a state dictionary, or `None` when the node is - /// not a dictionary keyed only by strings. - fn state_keys(&self, index: u32) -> Option<&[&'a str]> { - match &self.nodes.get(index as usize)?.shape { - Shape::Dict { - keys, - only_str: true, - } => Some(keys), - _ => None, + // (CLASS lies beyond every node.) + let node = self.nodes.get_mut(usize::try_from(node - 1).ok()?)?; + node.held = true; + Some(node.height) + } + + /// Let the container of node `node`, which nothing may hold yet, take + /// items no taller than `height`. + fn grow(&mut self, node: NodeId, height: u32) -> Option<()> { + let node = self + .nodes + .get_mut(usize::try_from(node.checked_sub(1)?).ok()?)?; + if node.held { + return None; } + node.height = node.height.max(height + 1); + (node.height <= MAX_DEPTH).then_some(()) } - fn acyclic_and_bounded(&self) -> Option<()> { - const MAX_DEPTH: usize = 128; - let mut colors = Vec::new(); - colors.try_reserve(self.nodes.len()).ok()?; - colors.resize(self.nodes.len(), 0u8); - let mut heights = Vec::new(); - heights.try_reserve(self.nodes.len()).ok()?; - heights.resize(self.nodes.len(), 0usize); - let mut stack = Vec::new(); - for root in 0..self.nodes.len() { - if colors[root] != 0 { - continue; - } - colors[root] = 1; - push(&mut stack, (root, 0usize))?; - while let Some((index, next)) = stack.last_mut() { - let node = &self.nodes[*index]; - if let Some(&child) = node.children.get(*next) { - *next += 1; - let child = child as usize; - match colors[child] { - 1 => return None, - 0 => { - if stack.len() >= MAX_DEPTH { - return None; - } - colors[child] = 1; - push(&mut stack, (child, 0))?; - } - _ => {} - } - } else { - let height = 1 + node - .children - .iter() - .map(|&child| heights[child as usize]) - .max() - .unwrap_or(0); - if height > MAX_DEPTH { - return None; - } - heights[*index] = height; - colors[*index] = 2; - stack.pop(); - } + fn memoize(&mut self, entry: &Entry) -> Option<()> { + let slot = u32::try_from(self.memo.len()).ok()?; + if let Some(node) = (entry.1.checked_sub(1)) + .and_then(|index| usize::try_from(index).ok()) + .and_then(|index| self.nodes.get_mut(index)) + { + node.memo = if node.memo == UNMEMOIZED && slot < MEMOIZED_TWICE { + slot + } else { + MEMOIZED_TWICE + }; + } + // The value usually was just pushed, its variant with a byte store: + // each kind is rebuilt from its fields (not copied by words) into + // the memo's slot. + let node = entry.1; + let memo = &mut self.memo; + match &entry.0 { + Object::Str(text) => push_entry!(memo, Object::Str(text.clone()), node), + Object::Tuple(items) => push_entry!(memo, Object::Tuple(items.clone()), node), + Object::List(items) => push_entry!(memo, Object::List(items.clone()), node), + Object::Dict(items) => push_entry!(memo, Object::Dict(items.clone()), node), + Object::Instance(instance) => { + push_entry!(memo, Object::Instance(instance.clone()), node); } + Object::Type(class) => push_entry!(memo, Object::Type(class.clone()), node), + other => push_entry!(memo, other.clone(), node), } Some(()) } -} - -impl<'a, const RECORD_OPCODES: bool> Sink<'a> for Probe<'a, '_, RECORD_OPCODES> { - type Value = Kind<'a>; - fn opcode(&mut self, value: u8) { - if RECORD_OPCODES { - self.opcodes.insert(value); + /// A MEMOIZE that follows the instruction that pushed the value, taken + /// without a dispatch: the pickler memoizes nearly every value it + /// writes. + #[inline(always)] + fn memoize_next(&mut self, reader: &mut Reader<'_>, stack: &[Entry]) -> Option<()> { + if reader.next_is([0x94]) { + reader.pos += 1; + self.opcode(0x94); + self.memoize(stack.last()?)?; } + Some(()) } - fn scalar(&mut self, value: Scalar<'a>) -> Kind<'a> { - match value { - Scalar::None => Kind::None, - Scalar::Float(_) => Kind::Float, - Scalar::Str(text) => Kind::Str(text), - _ => Kind::Key, + /// GET: push the value of memo slot `slot`. + #[inline] + fn push_memo(&self, stack: &mut Vec, slot: usize) -> Option<()> { + let (value, node) = self.memo.get(slot)?; + if *node == SPENT { + return None; } + push_entry!(stack, clone_value(value), *node); + Some(()) } - fn empty(&mut self, kind: Container) -> Option> { - let shape = match kind { - Container::List => Shape::List, - Container::Dict => Shape::Dict { - keys: Vec::new(), - only_str: true, - }, - Container::Tuple => Shape::Tuple { - len: 0, - pair: [StateItem::Other; 2], - }, - }; - self.node(shape, &[]) - } - - fn tuple(&mut self, items: Vec>) -> Option> { - let item = |kind: &Kind<'a>| match kind { - Kind::None => StateItem::None, - Kind::Container(index) => StateItem::Node(*index), - _ => StateItem::Other, - }; - let pair = match items.as_slice() { - [first, second] => [item(first), item(second)], - _ => [StateItem::Other; 2], - }; - let len = items.len(); - self.node(Shape::Tuple { len, pair }, &items) + /// The tallest of `items`, after marking them taken. + fn hold_all(&mut self, items: &[Entry]) -> Option { + let mut height = 0; + for &(_, node) in items { + height = height.max(self.hold(node)?); + } + Some(height) } - fn extend(&mut self, target: &Kind<'a>, items: Batch<'_, Kind<'a>>) -> Option<()> { - let Kind::Container(index) = target else { - return None; + /// Replace the items from `begin` on with a tuple of them. + fn tuple(&mut self, stack: &mut Vec, begin: usize) -> Option<()> { + let height = self.hold_all(&stack[begin..])?; + let pair = match &stack[begin..] { + [(_, first), (_, second)] => [*first, *second], + _ => [LEAF; 2], }; - let node = self.nodes.get_mut(*index as usize)?; - if !matches!(node.shape, Shape::List) { - return None; - } - for item in items { - match item { - Kind::Container(child) => push(&mut node.children, child)?, - Kind::Class(_) => return None, - _ => {} - } + let node = self.node(height + 1, pair)?; + if begin == stack.len() { + push_entry!(stack, Object::new_tuple(Vec::new()), node); + } else { + let items = crate::tuple_storage::TupleStorage::from_exact_iter( + stack.drain(begin..).map(|(value, _)| value), + ); + push_entry!(stack, Object::Tuple(items), node); } Some(()) } - fn setitems(&mut self, target: &Kind<'a>, mut items: Batch<'_, Kind<'a>>) -> Option<()> { - let Kind::Container(index) = target else { + /// APPEND(S): the items from `begin` on join the list below them. + fn extend(&mut self, stack: &mut Vec, begin: usize, marks: &[usize]) -> Option<()> { + if !is_batch(stack, begin, marks) { return None; - }; - let node = self.nodes.get_mut(*index as usize)?; - let Shape::Dict { keys, only_str } = &mut node.shape else { + } + // Taking the items first rejects a list that receives itself. + let height = self.hold_all(&stack[begin..])?; + let (Object::List(list), target) = &stack[begin - 1] else { return None; }; - if !items.len().is_multiple_of(2) { + let list = list.clone(); + self.grow(*target, height)?; + let mut list = list.borrow_mut(); + list.try_reserve(stack.len() - begin).ok()?; + list.extend(stack.drain(begin..).map(|(value, _)| value)); + Some(()) + } + + /// SETITEM(S): the pairs from `begin` on join the dictionary below them. + fn setitems(&mut self, stack: &mut Vec, begin: usize, marks: &[usize]) -> Option<()> { + if !is_batch(stack, begin, marks) || !(stack.len() - begin).is_multiple_of(2) { return None; } - while let Some(key) = items.next() { + let mut height = 0; + for pair in stack[begin..].as_chunks::<2>().0 { // Tuple and float keys retain the full unpickler, including // arbitrary nesting and NaN key-identity behavior. So do // instances, whose hashing can call Python code. - match key { - Kind::Str(key) => push(keys, key)?, - Kind::None | Kind::Key => *only_str = false, - _ => return None, + if pair[0].1 != LEAF + || !matches!( + pair[0].0, + Object::Str(_) + | Object::None + | Object::Bool(_) + | Object::Int(_) + | Object::Long(_) + | Object::Bytes(_) + ) + { + return None; } - match items.next()? { - Kind::Container(child) => push(&mut node.children, child)?, - Kind::Class(_) => return None, - _ => {} + height = height.max(self.hold(pair[1].1)?); + } + let (Object::Dict(dict), target) = &stack[begin - 1] else { + return None; + }; + let dict = dict.clone(); + self.grow(*target, height)?; + let mut dict = dict.borrow_mut(); + let map: &mut DictMap = &mut dict; + map.try_reserve((stack.len() - begin) / 2).ok()?; + let mut items = stack.drain(begin..); + while let Some((key, _)) = items.next() { + let (value, _) = items.next()?; + let Object::Str(name) = key else { + map.insert(DictKey(key), value); + continue; + }; + // `insert` for a string key, which equals only another string: + // the cached hash and a direct comparison, in line. + let hash = crate::fasthash::FxBuildHasher.hash_one(SharedStr::hash_cached(&name)); + match map.raw_entry_mut_v1().from_hash(hash, |key| { + matches!(&key.0, Object::Str(other) if SharedStr::ptr_eq(other, &name) || *other == name) + }) { + RawEntryMut::Occupied(mut entry) => { + entry.insert(value); + } + RawEntryMut::Vacant(entry) => { + entry.insert_hashed_nocheck(hash, DictKey(Object::Str(name)), value); + } } } Some(()) } - fn global(&mut self, module: &Kind<'a>, name: &Kind<'a>) -> Option> { - let (Kind::Str(module), Kind::Str(name)) = (module, name) else { + fn global(&mut self, module: &Object, name: &Object) -> Option { + let (Object::Str(module), Object::Str(name)) = (module, name) else { return None; }; if self.resolved.is_none() { @@ -1223,143 +1173,122 @@ impl<'a, const RECORD_OPCODES: bool> Sink<'a> for Probe<'a, '_, RECORD_OPCODES> } // An absent module keeps the full unpickler, which imports it. let class = PlainClass::resolve(self.resolved.as_ref()?, module, name)?; - let index = u32::try_from(self.classes.len()).ok()?; + let value = Object::Type(class.class.clone()); push(&mut self.classes, class)?; - Some(Kind::Class(index)) + Some(value) } - fn newobj(&mut self, class: &Kind<'a>, args: &Kind<'a>) -> Option> { - let (Kind::Class(class), Kind::Container(args)) = (class, args) else { + fn newobj(&mut self, stack: &mut Vec, class: Entry, args: Entry) -> Option<()> { + let ((Object::Type(class), CLASS), (Object::Tuple(args), _)) = (class, args) else { return None; }; // `object.__new__(cls)` accepts no further arguments. - if !matches!( - self.nodes.get(*args as usize)?.shape, - Shape::Tuple { len: 0, .. } - ) { + if !args.is_empty() { return None; } - self.node(Shape::Instance { class: *class }, &[]) + let node = self.node(1, [LEAF; 2])?; + // `object.__new__(cls)`, including its cycle-collector registration: + // deferred, as for a plain class's ordinary construction, until the + // instance could hold a non-atomic value (the state paths below + // track it then). + let instance = if class.native_kind.get() == 0 + && !class.flags.is_builtin + && !class.instances_need_finalize() + { + crate::types::PyInstance::new_deferred(class) + } else { + let instance = Rc::new(crate::types::PyInstance::new(class)); + push_in_place!(&mut self.containers, Object::Instance(instance.clone())); + instance + }; + push_entry!(stack, Object::Instance(instance), node); + Some(()) } - fn build(&mut self, target: &Kind<'a>, state: Kind<'a>) -> Option<()> { - let (Kind::Container(target), Kind::Container(state)) = (*target, state) else { - return None; - }; - let Shape::Instance { class } = &self.nodes.get(target as usize)?.shape else { - return None; - }; - let class = self.classes.get(*class as usize)?; - // The default state: a dictionary for `__dict__`, or the pair - // `(dictionary or None, {slot: value})`. - let (nodes, pair) = match &self.nodes.get(state as usize)?.shape { - Shape::Dict { .. } => ( - StateNodes { - dict: Some(state), - slots: None, - }, - None, - ), - Shape::Tuple { - len: 2, - pair: [first, StateItem::Node(slots)], - } => { - let dict = match first { - StateItem::None => None, - StateItem::Node(dict) => Some(*dict), - StateItem::Other => return None, - }; - ( - StateNodes { - dict, - slots: Some(*slots), - }, - Some(state), - ) - } - _ => return None, - }; - if let Some(dict) = nodes.dict { - if !self.state_keys(dict)?.is_empty() && !class.has_dict { - return None; - } + /// [`classes::is_member_slot`], remembered for this stream. (No Python + /// runs during decoding, so the class can't change in between.) + fn is_member_slot(&mut self, class: &Rc, name: &SharedStr) -> bool { + let key = Rc::as_ptr(class); + if self + .member_slots + .iter() + .any(|(c, n)| *c == key && n.as_ref() == name.as_ref()) + { + return true; } - if let Some(slots) = nodes.slots { - // `setattr` must reach a member descriptor of `__slots__`. - let names = self.state_keys(slots)?.to_vec(); - let class = class.class.clone(); - if !names - .into_iter() - .all(|name| self.is_member_slot(&class, name)) - { - return None; - } + let yes = classes::is_member_slot(class, name); + if yes && self.member_slots.len() < 64 { + self.member_slots.push((key, name.clone())); } - push(&mut self.builds, (nodes, pair))?; - push(&mut self.nodes.get_mut(target as usize)?.children, state) + yes } -} -/// The second pass, driven by the first pass's [`Plan`]. -struct Objects { - classes: std::vec::IntoIter>, - reachable: Vec, - builds: std::vec::IntoIter, - /// The index of the next node, in the first pass's numbering. - next_node: usize, -} + /// Whether a container of node `node`, with `count` strong references, + /// is reachable only through the `holders` references the caller + /// accounts for and its one memo slot. Then its storage may move, and + /// the memo slot (returned, when there is one) must be spent. + fn movable(&self, count: usize, node: NodeId, holders: usize) -> Option> { + let node = self + .nodes + .get(usize::try_from(node.checked_sub(1)?).ok()?)?; + match node.memo { + UNMEMOIZED => (count == holders).then_some(None), + MEMOIZED_TWICE => None, + slot => (count == holders + 1).then_some(Some(slot)), + } + } -impl Objects { - /// Register a container with the cycle collector when the result can - /// reach it. Even an acyclic decoded container can acquire a cycle - /// through later mutation, so every reachable mutable container is - /// registered, including those nested below the returned object. - fn created(&mut self, value: &Object) { - let index = self.next_node; - self.next_node += 1; - if self.reachable.get(index).copied().unwrap_or(true) - && matches!( - value, - Object::List(_) | Object::Dict(_) | Object::Instance(_) - ) + /// `sys.intern(name)`, remembered for this stream: every instance's + /// state usually names the same memoized strings. + fn intern(&mut self, name: &SharedStr) -> SharedStr { + if let Some((_, interned)) = self + .interned + .iter() + .find(|(seen, _)| SharedStr::ptr_eq(seen, name)) { - crate::gc_trace::track(value.clone()); + return interned.clone(); + } + let interned = crate::stdlib::sys::intern_shared(name); + if self.interned.len() < 64 { + self.interned.push((name.clone(), interned.clone())); } + interned } - /// `inst.__dict__[sys.intern(key)] = value` for every item, moving - /// the state's storage when nothing else can reach it. + /// `inst.__dict__[sys.intern(key)] = value` for every item, moving the + /// state's storage when nothing else can reach it. fn apply_dict_state( + &mut self, instance: &crate::types::PyInstance, state: &RefCell, movable: bool, ) -> Option<()> { - if movable && instance.dict.get().is_none() { + if movable { // The unpickled state holds arbitrary objects, so the // instance can no longer be left untracked; track it before - // publishing a dictionary that would bypass the write - // barrier (the record is retired with it). + // publishing a table that bypasses the write barrier (the + // record is retired with it). instance.ensure_gc_tracked(); - let data = std::mem::take(&mut *state.borrow_mut()); - let interned = data.keys().all(|key| match &key.0 { - Object::Str(name) => { - SharedStr::ptr_eq(&crate::stdlib::sys::intern_shared(name), name) - } - _ => false, - }); - if interned { - instance.dict.get_or_init(|| Rc::new(RefCell::new(data))); - return Some(()); - } - let mut attributes = instance.dict_cell().borrow_mut(); - attributes.try_reserve(data.len()).ok()?; - for (key, value) in data { + let mut state = state.borrow_mut(); + let table: &mut DictMap = &mut state; + // An interned key is equal to the key it replaces and has the + // same hash, so the table stays valid. + for (key, _) in indexmap::map::MutableKeys::iter_mut2(&mut *table) { let Object::Str(name) = &key.0 else { return None; }; - let name = crate::stdlib::sys::intern_shared(name); - attributes.insert(DictKey(Object::Str(name)), value); + key.0 = Object::Str(self.intern(name)); + } + let mut attributes = instance.dict_cell().borrow_mut(); + let attributes: &mut DictMap = &mut attributes; + if attributes.is_empty() { + // The state keeps the instance's empty table and its + // capacity, for reuse (see `spare_dicts`). + std::mem::swap(attributes, table); + return Some(()); } + attributes.try_reserve(table.len()).ok()?; + attributes.extend(table.drain(..)); return Some(()); } let state = state.borrow(); @@ -1369,118 +1298,120 @@ impl Objects { let Object::Str(name) = &key.0 else { return None; }; - let name = crate::stdlib::sys::intern_shared(name); + let name = self.intern(name); attributes.insert(DictKey(Object::Str(name)), value.clone()); } Some(()) } -} -impl<'a> Sink<'a> for Objects { - type Value = Object; - - fn scalar(&mut self, value: Scalar<'a>) -> Object { - match value { - Scalar::None => Object::None, - Scalar::Bool(value) => Object::Bool(value), - Scalar::Int(value) => Object::Int(value), - Scalar::Long(bytes) => Object::int_from_bigint(BigInt::from_signed_bytes_le(bytes)), - Scalar::Float(value) => Object::Float(value), - Scalar::Str(value) => Object::Str(SharedStr::from(value)), - Scalar::Bytes(value) => Object::new_bytes(value), + fn build(&mut self, target: &Entry, state: Entry) -> Option<()> { + let (state, state_node) = state; + // The instance takes the state's items, so it must be free to + // change, and the state can't reach it. + let height = self.hold(state_node)?; + match &state { + Object::Dict(dict) => { + // Held by `state` and by `containers`. + let movable = self.movable(Rc::strong_count(dict), state_node, 2); + self.build_from(target, height, Some(dict), None, [movable, None], None) + } + Object::Tuple(pair) if pair.len() == 2 => { + let [first, second] = self + .nodes + .get(usize::try_from(state_node.checked_sub(1)?).ok()?)? + .pair; + let (dict, slots) = state_dicts(&pair[0], &pair[1])?; + // The items move only when nothing else can reach the pair. + let count = crate::shared_value::ThinArc::strong_count(pair); + let Some(pair_slot) = self.movable(count, state_node, 1) else { + return self.build_from(target, height, dict, Some(slots), [None, None], None); + }; + // Each item is held by the pair and by `containers`. + let moves = [ + dict.and_then(|d| self.movable(Rc::strong_count(d), first, 2)), + self.movable(Rc::strong_count(slots), second, 2), + ]; + self.build_from(target, height, dict, Some(slots), moves, pair_slot) + } + _ => None, } } - fn empty(&mut self, kind: Container) -> Option { - let value = match kind { - Container::List => Object::new_list(Vec::new()), - Container::Dict => Object::new_dict(), - Container::Tuple => Object::new_tuple(Vec::new()), - }; - // Tuples are traced through their owners. - self.created(&value); - Some(value) - } - - fn tuple(&mut self, items: Vec) -> Option { - let value = Object::new_tuple(items); - self.created(&value); - Some(value) - } - - fn extend(&mut self, target: &Object, items: Batch<'_, Object>) -> Option<()> { - let Object::List(list) = target else { + /// TUPLE2, MEMOIZE, and BUILD, the way the pickler writes a state with + /// `__slots__`, without creating the pair. Its memo slot is spent from + /// the start, so reading it returns to the full unpickler. + fn build_pair(&mut self, stack: &mut Vec, marks: &[usize]) -> Option<()> { + let second = pop(stack, marks)?; + let first = pop(stack, marks)?; + // The pair takes its items. + let height = 1 + self.hold(first.1)?.max(self.hold(second.1)?); + if height > MAX_DEPTH { return None; - }; - let mut list = list.borrow_mut(); - list.try_reserve(items.len()).ok()?; - list.extend(items); - Some(()) - } - - fn setitems(&mut self, target: &Object, mut items: Batch<'_, Object>) -> Option<()> { - let Object::Dict(dict) = target else { - return None; - }; - if !items.len().is_multiple_of(2) { - return None; - } - let mut dict = dict.borrow_mut(); - dict.try_reserve(items.len() / 2).ok()?; - while let Some(key) = items.next() { - dict.insert(DictKey(key), items.next()?); } - Some(()) - } - - fn global(&mut self, _module: &Object, _name: &Object) -> Option { - Some(Object::Type(self.classes.next()?)) - } - - fn newobj(&mut self, class: &Object, _args: &Object) -> Option { - let Object::Type(class) = class else { + push(&mut self.memo, (Object::None, SPENT))?; + let (dict, slots) = state_dicts(&first.0, &second.0)?; + // Each item is held by `first` or `second` and by `containers`. + let moves = [ + dict.and_then(|d| self.movable(Rc::strong_count(d), first.1, 2)), + self.movable(Rc::strong_count(slots), second.1, 2), + ]; + self.build_from(top(stack, marks)?, height, dict, Some(slots), moves, None) + } + + /// BUILD for the state dictionaries `dict` and `slots`, whose state is + /// `height` high. `moves` tells, for each, whether its storage may move + /// (and its memo slot, which is then spent, as is `pair_slot`). + fn build_from( + &mut self, + target: &Entry, + height: u32, + dict: Option<&Rc>>, + slots: Option<&Rc>>, + moves: [Option>; 2], + pair_slot: Option, + ) -> Option<()> { + let (Object::Instance(instance), target) = target else { return None; }; - // `object.__new__(cls)`, including its cycle-collector registration: - // deferred, as for a plain class's ordinary construction, until the - // instance could hold a non-atomic value (the state paths below - // track it then). - if class.native_kind.get() == 0 - && !class.flags.is_builtin - && !class.instances_need_finalize() - { - self.next_node += 1; - return Some(Object::Instance(crate::types::PyInstance::new_deferred( - class.clone(), - ))); + let class = instance.cls(); + let has_dict = self + .classes + .iter() + .find(|plain| Rc::ptr_eq(&plain.class, &class))? + .has_dict; + if let Some(dict) = dict { + let dict = dict.borrow(); + if !dict.keys().all(|key| matches!(key.0, Object::Str(_))) + || (!dict.is_empty() && !has_dict) + { + return None; + } } - let instance = Object::Instance(Rc::new(crate::types::PyInstance::new(class.clone()))); - self.created(&instance); - Some(instance) - } - - fn build(&mut self, target: &Object, state: Object) -> Option<()> { - let Object::Instance(instance) = target else { - return None; - }; - let (dict, slots) = match &state { - Object::Dict(dict) => (Some(dict), None), - Object::Tuple(items) if items.len() == 2 => { - let Object::Dict(slots) = &items[1] else { + if let Some(slots) = slots { + // `setattr` must reach a member descriptor of `__slots__`. + for key in slots.borrow().keys() { + let Object::Str(name) = &key.0 else { return None; }; - match &items[0] { - Object::None => (None, Some(slots)), - Object::Dict(dict) => (Some(dict), Some(slots)), - _ => return None, + if !self.is_member_slot(&class, name) { + return None; } } - _ => return None, + } + self.grow(*target, height)?; + + // A state container that nothing else can reach gives its storage + // to the instance. Its memo slot is spent, and so is the pair's: + // reading either would see the emptied container. + let [move_dict, move_slots] = moves.map(|movable| movable.is_some()); + let spent = if move_dict || move_slots { + [pair_slot, moves[0].flatten(), moves[1].flatten()] + } else { + [None; 3] }; - let movable = self.builds.next()?; if let Some(dict) = dict { if !dict.borrow().is_empty() { - Self::apply_dict_state(instance, dict, movable.dict.is_some())?; + self.apply_dict_state(instance, dict, move_dict)?; } } if let Some(slots) = slots { @@ -1491,7 +1422,17 @@ impl<'a> Sink<'a> for Objects { instance.ensure_gc_tracked(); } let mut storage = instance.slots.borrow_mut(); - if movable.slots.is_some() { + if move_slots && storage.get_index(0).is_none() { + // Into empty storage, the distinct keys in order: the + // result of setting each in turn, built in one step. The + // emptied dictionary keeps its capacity for reuse. + let mut data = slots.borrow_mut(); + let data: &mut DictMap = &mut data; + let mut entries = Vec::new(); + entries.try_reserve_exact(data.len()).ok()?; + entries.extend(data.drain(..)); + *storage = crate::types::SlotStorage::from_entries(entries); + } else if move_slots { let data = std::mem::take(&mut *slots.borrow_mut()); for (key, value) in data { let Object::Str(name) = &key.0 else { @@ -1500,8 +1441,7 @@ impl<'a> Sink<'a> for Objects { storage.insert_shared(name, value); } } else { - let slots = slots.borrow(); - for (key, value) in slots.iter() { + for (key, value) in slots.borrow().iter() { let Object::Str(name) = &key.0 else { return None; }; @@ -1509,63 +1449,227 @@ impl<'a> Sink<'a> for Objects { } } } + for slot in spent.into_iter().flatten() { + *self.memo.get_mut(slot as usize)? = (Object::None, SPENT); + } + // A moved state dictionary is now empty and out of reach, so the + // next EMPTY_DICT can reuse it. (It remains in `containers`.) + for (moved, dict) in [(move_dict, dict), (move_slots, slots)] { + if let (true, Some(dict)) = (moved, dict) { + if self.spare_dicts.len() < 64 { + self.spare_dicts.push(dict.clone()); + } + } + } Some(()) } -} -/// Decode only after the preflight succeeds without creating Python objects. -fn decode( - data: &[u8], - validate: impl FnOnce(&Opcodes) -> bool, - context: &dyn Fn() -> Option, -) -> Option { - if data.len() < 3 || data[0] != 0x80 || !matches!(data[1], 4 | 5) { - return None; + #[allow(clippy::too_many_lines)] + fn parse( + &mut self, + data: &[u8], + stack: &mut Vec, + marks: &mut Vec, + ) -> Option { + self.opcode(0x80); + let mut reader = Reader::new(data, 2); + stack.try_reserve(64).ok()?; + self.memo.try_reserve(data.len() / 16).ok()?; + self.nodes.try_reserve(data.len() / 16).ok()?; + loop { + let opcode = reader.byte()?; + self.opcode(opcode); + // Every arm pushes its own result: a value that passes through a + // shared temporary (or an `Option`) is copied with unaligned + // loads that stall on the stores that just wrote it. + match opcode { + b'.' => { + return (marks.is_empty() && stack.len() == 1) + .then(|| stack.pop()) + .flatten() + // A class is only ever the operand of NEWOBJ. + .filter(|(_, node)| *node != CLASS); + } + 0x95 => reader.frame()?, + b'(' => push(&mut *marks, stack.len())?, + 0x94 => self.memoize(top(&*stack, &*marks)?)?, + b'h' => self.push_memo(stack, usize::from(reader.byte()?))?, + b'j' => self.push_memo(stack, usize::try_from(reader.u32()?).ok()?)?, + b'N' => push_entry!(stack, Object::None), + 0x88 => push_entry!(stack, Object::Bool(true)), + 0x89 => push_entry!(stack, Object::Bool(false)), + b'K' => { + let value = i64::from(reader.byte()?); + push_entry!(stack, Object::Int(value)); + } + b'M' => { + let value = u16::from_le_bytes(reader.read(2)?.try_into().ok()?); + push_entry!(stack, Object::Int(i64::from(value))); + } + b'J' => { + let value = i32::from_le_bytes(reader.read(4)?.try_into().ok()?); + push_entry!(stack, Object::Int(i64::from(value))); + } + 0x8a | 0x8b => { + let size = if opcode == 0x8a { + usize::from(reader.byte()?) + } else { + let size = i32::from_le_bytes(reader.read(4)?.try_into().ok()?); + usize::try_from(size).ok()? + }; + let bytes = reader.read(size)?; + let value = Object::int_from_bigint(BigInt::from_signed_bytes_le(bytes)); + push_entry!(stack, value); + } + b'G' => { + let value = f64::from_be_bytes(reader.read(8)?.try_into().ok()?); + push_entry!(stack, Object::Float(value)); + } + 0x8c | b'X' | 0x8d => { + let size = match opcode { + 0x8c => usize::from(reader.byte()?), + b'X' => usize::try_from(reader.u32()?).ok()?, + _ => reader.size64()?, + }; + let bytes = reader.read(size)?; + // Surrogate-pass strings stay on the existing WStr path. + let text = if bytes.is_ascii() { + // SAFETY: ASCII text is valid UTF-8. + unsafe { std::str::from_utf8_unchecked(bytes) } + } else { + std::str::from_utf8(bytes).ok()? + }; + let text = SharedStr::from(text); + push_entry!(stack, Object::Str(text)); + self.memoize_next(&mut reader, stack)?; + } + b'C' | b'B' | 0x8e => { + let size = match opcode { + b'C' => usize::from(reader.byte()?), + b'B' => usize::try_from(reader.u32()?).ok()?, + _ => reader.size64()?, + }; + let value = Object::new_bytes(reader.read(size)?); + push_entry!(stack, value); + self.memoize_next(&mut reader, stack)?; + } + b']' => { + self.empty_list(stack)?; + self.memoize_next(&mut reader, stack)?; + } + b'}' => { + self.empty_dict(stack)?; + self.memoize_next(&mut reader, stack)?; + } + b')' => { + let node = self.node(1, [LEAF; 2])?; + push(stack, (Object::new_tuple(Vec::new()), node))?; + } + // The pickler writes a state with `__slots__` this way. + 0x86 if reader.next_is([0x94, BUILD]) => { + reader.pos += 2; + self.opcode(0x94); + self.opcode(BUILD); + self.build_pair(stack, &*marks)?; + } + 0x85..=0x87 => { + let count = usize::from(opcode - 0x84); + let begin = stack.len().checked_sub(count)?; + if marks.last().is_some_and(|mark| begin < *mark) { + return None; + } + self.tuple(stack, begin)?; + self.memoize_next(&mut reader, stack)?; + } + b't' => { + let mark = marks.pop()?; + self.tuple(stack, mark)?; + self.memoize_next(&mut reader, stack)?; + } + b'a' => { + let begin = stack.len().checked_sub(1)?; + self.extend(stack, begin, &*marks)?; + } + b'e' => { + let mark = marks.pop()?; + self.extend(stack, mark, &*marks)?; + } + b's' => { + let begin = stack.len().checked_sub(2)?; + self.setitems(stack, begin, &*marks)?; + } + b'u' => { + let mark = marks.pop()?; + self.setitems(stack, mark, &*marks)?; + } + STACK_GLOBAL => { + let (name, _) = pop(&mut *stack, &*marks)?; + let (module, _) = pop(&mut *stack, &*marks)?; + let class = self.global(&module, &name)?; + push(stack, (class, CLASS))?; + self.memoize_next(&mut reader, stack)?; + } + NEWOBJ => { + let args = pop(&mut *stack, &*marks)?; + let class = pop(&mut *stack, &*marks)?; + self.newobj(stack, class, args)?; + self.memoize_next(&mut reader, stack)?; + } + BUILD => { + let state = pop(&mut *stack, &*marks)?; + self.build(top(&*stack, &*marks)?, state)?; + } + // GLOBAL, REDUCE, NEWOBJ_EX, persistent IDs, external buffers, + // legacy protocols, and all other operations retain the full + // unpickler. + _ => return None, + } + } } - // Checking the whole dispatch table has a fixed cost. For longer - // streams, avoid recording every opcode and pay that fixed cost once. - // Both specializations still validate the entire supported stream. - if data.len() <= 512 { - decode_with_probe::(data, validate, context) - } else { - // A replaced reader may ignore the input or reject it immediately. - // Check the full table before scanning a large stream so that such - // readers keep their existing fallback cost. - if !validate(&Opcodes([true; 256])) { - return None; + + /// Publish the accepted result. Every container that the result can + /// reach is registered with the cycle collector: even an acyclic decoded + /// container can acquire a cycle through later mutation. Releasing the + /// memo and the spare dictionaries first leaves the result's references + /// as the only ones beyond `containers`' own. + fn finish(&mut self, root: Object) -> Object { + self.memo.clear(); + self.spare_dicts.clear(); + for value in self.containers.drain(..) { + let reachable = match &value { + Object::List(list) => Rc::strong_count(list) > 1, + Object::Dict(dict) => Rc::strong_count(dict) > 1, + Object::Instance(instance) => Rc::strong_count(instance) > 1, + _ => false, + }; + if reachable { + crate::gc_trace::track_built(&value); + } } - decode_with_probe::(data, |_| true, context) + root } } -fn decode_with_probe( +fn decode( data: &[u8], validate: impl FnOnce(&Opcodes) -> bool, context: &dyn Fn() -> Option, ) -> Option { - let mut probe = Probe::::new(context); - let root = parse(data, &mut probe)?; - // A class is only ever the operand of NEWOBJ, never a result. - if matches!(root, Kind::Class(_)) { - return None; - } - probe.acyclic_and_bounded()?; - // Unsupported inputs return before checking Python dispatch functions. - // Guard supported instructions before any Python objects are constructed. - if !validate(&probe.opcodes) { + if data.len() < 3 || data[0] != 0x80 || !matches!(data[1], 4 | 5) { return None; } - // Drop the graph scratch before allocating the returned Python objects. - let plan = probe.plan(root)?; - parse( - data, - &mut Objects { - classes: plan.classes.into_iter(), - reachable: plan.reachable, - builds: plan.builds.into_iter(), - next_node: 0, - }, - ) + let mut scratch = Scratch::take(); + let mut decoder = Decoder::new(context, &mut scratch); + let result = decoder + .parse(data, &mut scratch.stack, &mut scratch.marks) + // The code of the Python functions that the stream's instructions + // would run is checked once the stream proves supported. Rejecting + // it then discards what was built. + .filter(|_| validate(&decoder.opcodes)) + .map(|(root, _)| decoder.finish(root)); + decoder.recycle(&mut scratch); + scratch.keep(); + result } #[cfg(test)] @@ -1615,6 +1719,9 @@ mod tests { &b"\x80\x05]\x94(h\x00\x85e."[..], &b"\x80\x05}]K\x01s."[..], &b"\x80\x05})K\x01s."[..], + // A list that another list holds can't change (conservatively, + // although this one would not form a cycle). + &b"\x80\x05]\x94]\x94ah\x01K\x01a\x86."[..], ] { assert!(decode(data).is_none(), "{data:?}"); } diff --git a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs index 92847caa..9159f658 100644 --- a/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs +++ b/crates/weavepy-vm/src/stdlib/pickle_accel/encode.rs @@ -40,25 +40,36 @@ struct Framer { } impl Framer { - #[inline] + #[inline(always)] fn write(&mut self, data: &[u8]) -> Option<()> { if self.frame_start.is_none() { - let start = self.output.len(); - extend(&mut self.output, &[0x95, 0, 0, 0, 0, 0, 0, 0, 0])?; - self.frame_start = Some(start); + self.open()?; } extend(&mut self.output, data) } + #[cold] + fn open(&mut self) -> Option<()> { + let start = self.output.len(); + extend(&mut self.output, &[0x95, 0, 0, 0, 0, 0, 0, 0, 0])?; + self.frame_start = Some(start); + Some(()) + } + + #[inline] fn commit(&mut self, force: bool) { let Some(start) = self.frame_start else { return; }; + if force || self.output.len() - (start + 9) >= FRAME_TARGET { + self.end_frame(start); + } + } + + #[cold] + fn end_frame(&mut self, start: usize) { let body = start + 9; let len = self.output.len() - body; - if !force && len < FRAME_TARGET { - return; - } if len >= 4 { self.output[start + 1..body].copy_from_slice(&(len as u64).to_le_bytes()); } else { @@ -113,9 +124,13 @@ struct ClassPlan { struct Encoder<'a> { writer: Framer, + // Memo positions by object identity. + memo: MemoTable, // Pins prevent another thread's container mutation from releasing an // already memoized value and reusing its address during this operation. - memo: FxHashMap, + // While one thread owns every object, nothing else can mutate the graph, + // so the values stay alive and unpinned. + pins: Option>, // Memo positions taken by the default state's temporary tuple and // dictionary, to which no later object can refer. anonymous: u32, @@ -126,23 +141,205 @@ struct Encoder<'a> { resolved: Option, classes: FxHashMap>, slot_name_caches: Vec<(Rc, Vec)>, + // An empty vector for the next instance's slot state. + spare_slots: Vec<(Object, Object)>, +} + +/// The memo: memo positions by object identity (an address, never 0), in an +/// open-addressing table with linear probing. Each entry records the +/// generation of the call that wrote it, so the next call clears the table +/// by starting a new generation. +#[derive(Default)] +struct MemoTable { + entries: Vec, + len: usize, + generation: u32, +} + +#[derive(Clone, Copy, Default)] +struct MemoEntry { + id: i64, + index: u32, + generation: u32, +} + +impl MemoTable { + const MIN_CAPACITY: usize = 64; + + /// The first slot to probe for `id`, in a table of `capacity` slots (a + /// power of two): a multiplicative hash spreads aligned addresses. + #[inline] + fn home(id: i64, capacity: usize) -> usize { + let hash = (id as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15); + (hash >> (64 - capacity.trailing_zeros())) as usize + } + + #[inline] + fn get(&self, id: i64) -> Option { + if self.entries.is_empty() { + return None; + } + let mask = self.entries.len() - 1; + let mut slot = Self::home(id, self.entries.len()); + loop { + let entry = &self.entries[slot]; + if entry.generation != self.generation { + return None; + } + if entry.id == id { + return Some(entry.index); + } + slot = (slot + 1) & mask; + } + } + + /// `id`'s memo position, or else the slot at which to insert it. Makes + /// room for one more entry first; `None` only when that fails. + #[inline] + fn find(&mut self, id: i64) -> Option> { + if (self.len + 1) * 2 > self.entries.len() { + self.grow()?; + } + let mask = self.entries.len() - 1; + let mut slot = Self::home(id, self.entries.len()); + loop { + let entry = &self.entries[slot]; + if entry.generation != self.generation { + return Some(Err(slot)); + } + if entry.id == id { + return Some(Ok(entry.index)); + } + slot = (slot + 1) & mask; + } + } + + /// Insert at a slot that [`Self::find`] returned. + #[inline] + fn insert(&mut self, slot: usize, id: i64, index: u32) { + self.entries[slot] = MemoEntry { + id, + index, + generation: self.generation, + }; + self.len += 1; + } + + #[cold] + fn grow(&mut self) -> Option<()> { + if self.generation == 0 { + // A fresh table: generation 0 marks every slot empty. + self.generation = 1; + } + let capacity = (self.entries.len() * 2).max(Self::MIN_CAPACITY); + let mut entries = Vec::new(); + entries.try_reserve_exact(capacity).ok()?; + entries.resize(capacity, MemoEntry::default()); + let old = std::mem::replace(&mut self.entries, entries); + let mask = capacity - 1; + for entry in old.into_iter().filter(|e| e.generation == self.generation) { + let mut slot = Self::home(entry.id, capacity); + while self.entries[slot].generation == self.generation { + slot = (slot + 1) & mask; + } + self.entries[slot] = entry; + } + Some(()) + } + + /// Empty the table, keeping its storage. + fn clear(&mut self) { + self.len = 0; + self.generation = self.generation.wrapping_add(1); + if self.generation == 0 { + // The generations wrapped: every stale entry must go. + self.entries.fill(MemoEntry::default()); + self.generation = 1; + } + } +} + +/// `id(value)` for the kinds of value the encoder memoizes. +#[inline] +fn identity(value: &Object) -> i64 { + match value { + Object::Str(text) => text.as_ptr() as usize as i64, + Object::Bytes(bytes) => bytes.as_ptr() as usize as i64, + Object::Tuple(items) => { + crate::shared_value::ThinArc::as_ptr(items).cast::<()>() as usize as i64 + } + Object::List(items) => Rc::as_ptr(items) as usize as i64, + Object::Dict(items) => Rc::as_ptr(items) as usize as i64, + Object::Instance(instance) => Rc::as_ptr(instance) as usize as i64, + Object::Type(class) => Rc::as_ptr(class) as usize as i64, + _ => object_identity(value), + } } impl Encoder<'_> { fn next_index(&self) -> Option { - u32::try_from(self.memo.len()) + u32::try_from(self.memo.len) .ok()? .checked_add(self.anonymous) } - fn memoize(&mut self, value: &Object) -> Option<()> { + #[inline] + fn pin(&mut self, value: &Object) -> Option<()> { + if let Some(pins) = &mut self.pins { + pins.try_reserve(1).ok()?; + pins.push(value.clone()); + } + Some(()) + } + + /// MEMOIZE for `value`, just written, whose identity is `id`. + fn memoize(&mut self, value: &Object, id: i64) -> Option<()> { let index = self.next_index()?; - self.memo.try_reserve(1).ok()?; - self.memo - .insert(object_identity(value), (index, value.clone())); + if let Err(slot) = self.memo.find(id)? { + self.memo.insert(slot, id, index); + } + self.pin(value)?; self.writer.write(&[0x94]) } + /// GET for a memoized `value` (reporting `true`), or else `write` it + /// and memoize it, with one table probe for both. + #[inline] + fn save_memoized( + &mut self, + value: &Object, + id: i64, + write: impl FnOnce(&mut Framer) -> Option<()>, + ) -> Option { + let next = self.next_index()?; + match self.memo.find(id)? { + Ok(index) => { + self.get(index)?; + Some(true) + } + Err(slot) => { + write(&mut self.writer)?; + self.memo.insert(slot, id, next); + self.pin(value)?; + self.writer.write(&[0x94])?; + Some(false) + } + } + } + + /// The checks and the frame commit with which every `save` starts. + #[inline] + fn enter(&mut self, depth: usize) -> Option<()> { + // Leave ample room for the Python pickler's constructor/descriptor + // calls and save/handler/batch expansion at every graph level. A + // low recursion budget retains its existing fallback behavior. + if depth > MAX_DEPTH || depth.saturating_mul(8).saturating_add(32) >= self.python_headroom { + return None; + } + self.writer.commit(false); + Some(()) + } + fn memoize_anonymous(&mut self) -> Option<()> { self.next_index()?; self.anonymous += 1; @@ -191,34 +388,22 @@ impl Encoder<'_> { } } - fn save(&mut self, value: &Object, depth: usize) -> Option<()> { - // Leave ample room for the Python pickler's constructor/descriptor - // calls and save/handler/batch expansion at every graph level. A - // low recursion budget retains its existing fallback behavior. - if depth > MAX_DEPTH || depth.saturating_mul(8).saturating_add(32) >= self.python_headroom { + /// Whether `id`, a container's identity, is memoized (written as GET) + /// or must be written. A container that is still being written means + /// a cycle: recursive tuples require a POP/GET repair, so every cyclic + /// graph retains the existing pickler. + fn memoized_container(&mut self, id: i64) -> Option { + if self.active.contains(&id) { return None; } - self.writer.commit(false); - let memoized = matches!( - value, - Object::Str(_) - | Object::Bytes(_) - | Object::List(_) - | Object::Dict(_) - | Object::Tuple(_) - | Object::Instance(_) - ); - let id = if memoized { object_identity(value) } else { 0 }; - if memoized { - // Recursive tuples require a POP/GET repair. Retain the existing - // pickler for every cyclic graph in this first implementation. - if !matches!(value, Object::Str(_) | Object::Bytes(_)) && self.active.contains(&id) { - return None; - } - if let Some(&(index, _)) = self.memo.get(&id) { - return self.get(index); - } + match self.memo.get(id) { + Some(index) => self.get(index).map(|()| true), + None => Some(false), } + } + + fn save(&mut self, value: &Object, depth: usize) -> Option<()> { + self.enter(depth)?; match value { Object::None => self.writer.write(b"N"), Object::Bool(value) => self.writer.write(&[if *value { 0x88 } else { 0x89 }]), @@ -231,16 +416,21 @@ impl Encoder<'_> { self.writer.write(b"G")?; self.writer.write(&value.to_bits().to_be_bytes()) } - Object::Str(text) => { - self.writer.data(text.as_bytes(), true)?; - self.memoize(value) - } - Object::Bytes(bytes) => { - self.writer.data(bytes, false)?; - self.memoize(value) - } + Object::Str(text) => self + .save_memoized(value, identity(value), |writer| { + writer.data(text.as_bytes(), true) + }) + .map(drop), + Object::Bytes(bytes) => self + .save_memoized(value, identity(value), |writer| writer.data(bytes, false)) + .map(drop), + // The empty tuple is never memoized. Object::Tuple(items) if items.is_empty() => self.writer.write(b")"), Object::Tuple(items) => { + let id = identity(value); + if self.memoized_container(id)? { + return Some(()); + } self.active.try_reserve(1).ok()?; self.active.push(id); if items.len() > 3 { @@ -255,70 +445,84 @@ impl Encoder<'_> { 3 => 0x87, _ => b't', }])?; - self.memoize(value)?; + self.memoize(value, id)?; self.active.pop(); Some(()) } Object::List(items) => { - // A bounded snapshot releases the container lock before - // recursion without duplicating the entire list's storage. + // (See `memoized_container`.) + let id = identity(value); + if self.active.contains(&id) { + return None; + } + if self.save_memoized(value, id, |writer| writer.write(b"]"))? { + return Some(()); + } // Keep the original length as a bound; observed resizing // returns to the Python pickler before any callbacks run. let length = items.borrow().len(); - let mut batch = Vec::new(); - batch.try_reserve_exact(length.min(1000)).ok()?; self.active.try_reserve(1).ok()?; self.active.push(id); - self.writer.write(b"]")?; - self.memoize(value)?; for start in (0..length).step_by(1000) { - batch.clear(); - { - let items = items.borrow(); - if items.len() != length { - return None; - } - let end = start.saturating_add(1000).min(length); - batch.extend(items[start..end].iter().cloned()); - } - if batch.len() != 1 { + let end = start.saturating_add(1000).min(length); + if end - start != 1 { self.writer.write(b"(")?; } - for item in &batch { - self.save(item, depth + 1)?; + let mut index = start; + while index < end { + // Leaves are written under one borrow. The lock is + // released before recursing into a container, which + // a clone keeps alive meanwhile. + let next = { + let items = items.borrow(); + if items.len() != length { + return None; + } + loop { + let Some(item) = items[..end].get(index) else { + break None; + }; + index += 1; + if !self.save_leaf(item, depth + 1)? { + break Some(item.clone()); + } + } + }; + let Some(item) = next else { + break; + }; + self.save(&item, depth + 1)?; } self.writer - .write(if batch.len() == 1 { b"a" } else { b"e" })?; + .write(if end - start == 1 { b"a" } else { b"e" })?; } self.active.pop(); Some(()) } Object::Dict(items) => { - let items = { - let items = items.borrow(); - let mut snapshot = Vec::new(); - snapshot.try_reserve_exact(items.len()).ok()?; - snapshot.extend( - items - .iter() - .map(|(key, value)| (key.0.clone(), value.clone())), - ); - snapshot - }; + let id = identity(value); + if self.active.contains(&id) { + return None; + } + if self.save_memoized(value, id, |writer| writer.write(b"}"))? { + return Some(()); + } self.active.try_reserve(1).ok()?; self.active.push(id); - self.writer.write(b"}")?; - self.memoize(value)?; - self.save_items(&items, depth + 1)?; + self.save_dict_items(items, depth + 1)?; self.active.pop(); Some(()) } Object::Instance(instance) => { + let id = identity(value); + if self.memoized_container(id)? { + return Some(()); + } // The instance is memoized before its state, but a graph // that returns to it is cyclic and keeps the full pickler. self.active.try_reserve(1).ok()?; self.active.push(id); - self.save_instance(value, instance, depth)?; + self.save_instance(value, id, instance, depth)?; self.active.pop(); Some(()) } @@ -326,6 +530,73 @@ impl Encoder<'_> { } } + /// `save` for a value that needs no recursion. `Some(false)` reports a + /// value that isn't such a leaf, leaving it unsaved. + #[inline] + fn save_leaf(&mut self, value: &Object, depth: usize) -> Option { + match value { + Object::None + | Object::Bool(_) + | Object::Int(_) + | Object::Long(_) + | Object::Float(_) + | Object::Str(_) + | Object::Bytes(_) => { + self.save(value, depth)?; + Some(true) + } + _ => Some(false), + } + } + + /// `_batch_setitems` over a live dictionary, batched as lists are in + /// [`Self::save`]. Any mutable access to the dictionary meanwhile, + /// which draws a new stamp, returns to the Python pickler. + fn save_dict_items(&mut self, dict: &RefCell, depth: usize) -> Option<()> { + let (length, stamp) = { + let dict = dict.borrow(); + (dict.len(), dict.mutation_stamp()) + }; + for start in (0..length).step_by(1000) { + let end = start.saturating_add(1000).min(length); + if end - start != 1 { + self.writer.write(b"(")?; + } + let mut index = start; + while index < end { + let next = { + let dict = dict.borrow(); + if dict.mutation_stamp() != stamp || dict.len() != length { + return None; + } + loop { + if index == end { + break None; + } + let (key, value) = dict.get_index(index)?; + index += 1; + if !self.save_leaf(&key.0, depth)? { + break Some((Some(key.0.clone()), value.clone())); + } + if !self.save_leaf(value, depth)? { + break Some((None, value.clone())); + } + } + }; + let Some((key, value)) = next else { + break; + }; + if let Some(key) = key { + self.save(&key, depth)?; + } + self.save(&value, depth)?; + } + self.writer + .write(if end - start == 1 { b"s" } else { b"u" })?; + } + Some(()) + } + fn save_items(&mut self, items: &[(Object, Object)], depth: usize) -> Option<()> { for batch in items.chunks(1000) { if batch.len() != 1 { @@ -346,6 +617,7 @@ impl Encoder<'_> { fn save_instance( &mut self, value: &Object, + id: i64, instance: &Rc, depth: usize, ) -> Option<()> { @@ -375,29 +647,43 @@ impl Encoder<'_> { if dict.is_some() && !plan.has_dict { return None; } - let mut slots: Vec<(Object, Object)> = Vec::new(); + // (Nested instances take a new vector.) + let mut slots = std::mem::take(&mut self.spare_slots); if !plan.slot_names.is_empty() { let storage = instance.slots.borrow(); slots.try_reserve_exact(plan.slot_names.len()).ok()?; - for name in &plan.slot_names { + for (index, name) in plan.slot_names.iter().enumerate() { let Object::Str(text) = name else { return None; }; let repeated = slots .iter() .any(|(seen, _)| matches!(seen, Object::Str(seen) if seen == text)); - if let (false, Some(value)) = (repeated, storage.get(text)) { + if repeated { + continue; + } + // Storage usually holds the slots in declaration order. + let value = match storage.get_index(index) { + Some((DictKey(Object::Str(stored)), value)) if stored == text => Some(value), + _ => storage.get(text), + }; + if let Some(value) = value { slots.push((name.clone(), value.clone())); } } } self.save_class(&class, &plan, depth + 1)?; - self.save(&Object::new_tuple(Vec::new()), depth + 1)?; + // `save(())` + self.enter(depth + 1)?; + self.writer.write(b")")?; self.writer.write(&[0x81])?; - self.memoize(value)?; + self.memoize(value, id)?; match (dict, slots.is_empty()) { - (None, true) => return Some(()), + (None, true) => { + self.spare_slots = slots; + return Some(()); + } (Some(dict), true) => self.save(&Object::Dict(dict), depth + 1)?, (dict, false) => { // The state is the temporary `(dict or None, {slot: value})`. @@ -411,6 +697,8 @@ impl Encoder<'_> { self.memoize_anonymous()?; } } + slots.clear(); + self.spare_slots = slots; self.writer.write(b"b") } @@ -418,13 +706,14 @@ impl Encoder<'_> { fn save_class(&mut self, class: &Rc, plan: &ClassPlan, depth: usize) -> Option<()> { self.writer.commit(false); let value = Object::Type(class.clone()); - if let Some(&(index, _)) = self.memo.get(&object_identity(&value)) { + let id = identity(&value); + if let Some(index) = self.memo.get(id) { return self.get(index); } self.save(&plan.module, depth + 1)?; self.save(&plan.qualname, depth + 1)?; self.writer.write(&[0x93])?; - self.memoize(&value) + self.memoize(&value, id) } fn class_plan(&mut self, class: &Rc) -> Option> { @@ -520,7 +809,7 @@ impl Encoder<'_> { continue; } let names = Object::new_list(names); - crate::gc_trace::track(names.clone()); + crate::gc_trace::track(&names); dict.insert(DictKey(Object::from_static("__slotnames__")), names); drop(dict); class.bump_attr_version(); @@ -555,17 +844,20 @@ fn encode_with_context( if !matches!(protocol, 4 | 5) { return None; } - // The previous call's memo table (emptied, its capacity kept) and - // output size: a `dumps` loop then neither regrows the table nor - // copies the output as it doubles. + // The previous call's memo table (emptied, its capacity kept) and the + // last two output sizes: a `dumps` loop, even one that alternates + // between two values, then neither regrows the table nor copies the + // output as it doubles. let memo = SPARE_MEMO.with(std::cell::Cell::take).unwrap_or_default(); - let hint = OUTPUT_HINT.with(std::cell::Cell::get); + let (last, before) = OUTPUT_HINT.with(std::cell::Cell::get); + let hint = last.max(before); let mut encoder = Encoder { writer: Framer { output: Vec::new(), frame_start: None, }, memo, + pins: (!crate::sync::bias_held()).then(Vec::new), anonymous: 0, active: Vec::new(), python_headroom, @@ -573,6 +865,7 @@ fn encode_with_context( resolved: None, classes: FxHashMap::default(), slot_name_caches: Vec::new(), + spare_slots: Vec::new(), }; let done = (|| { encoder @@ -588,12 +881,12 @@ fn encode_with_context( Some(()) })(); let mut memo = std::mem::take(&mut encoder.memo); - if memo.capacity() <= MAX_SPARE_MEMO { + if memo.entries.len() <= MAX_SPARE_MEMO { memo.clear(); SPARE_MEMO.with(|spare| spare.set(Some(memo))); } done?; - OUTPUT_HINT.with(|h| h.set(encoder.writer.output.len())); + OUTPUT_HINT.with(|h| h.set((encoder.writer.output.len(), last))); Some(encoder.writer.output) } @@ -602,9 +895,9 @@ const MAX_SPARE_MEMO: usize = 1 << 14; const MAX_OUTPUT_HINT: usize = 1 << 20; thread_local! { - static SPARE_MEMO: std::cell::Cell>> = + static SPARE_MEMO: std::cell::Cell> = const { std::cell::Cell::new(None) }; - static OUTPUT_HINT: std::cell::Cell = const { std::cell::Cell::new(0) }; + static OUTPUT_HINT: std::cell::Cell<(usize, usize)> = const { std::cell::Cell::new((0, 0)) }; } #[cfg(test)] diff --git a/crates/weavepy-vm/src/stdlib/python/_collections.py b/crates/weavepy-vm/src/stdlib/python/_collections.py index 1d250216..a5fb91af 100644 --- a/crates/weavepy-vm/src/stdlib/python/_collections.py +++ b/crates/weavepy-vm/src/stdlib/python/_collections.py @@ -223,7 +223,7 @@ def _flat(self): # `builtin_function_or_method`, like the C accelerator's. from _weave_collections import ( append, appendleft, pop, popleft, rotate, - __len__, __bool__, __getitem__, + __len__, __bool__, __getitem__, __iter__, __reversed__, ) def extend(self, iterable): @@ -314,12 +314,6 @@ def reverse(self): self._state += 1 self._flat().reverse() - def __iter__(self): - return _deque_iterator(self) - - def __reversed__(self): - return _deque_reverse_iterator(self) - def __contains__(self, x): # CPython `deque_contains`: mutation during a comparison raises. data = self._flat() @@ -941,3 +935,10 @@ def __reduce__(self): if deq is None: return type(self), (deque(),) return type(self), (deq, self._index) + + +# The native `deque.__iter__` / `__reversed__` build these iterators +# directly (no Python `__init__` runs), so a compiled loop can capture a +# deque iterator without leaving native code. The classes ride on the +# deque class itself, so each copy of this module pairs with its own. +deque._iter_types = (_deque_iterator, _deque_reverse_iterator) diff --git a/crates/weavepy-vm/src/stdlib/python/_opcode.py b/crates/weavepy-vm/src/stdlib/python/_opcode.py index 6b21ab9c..ffeda74f 100644 --- a/crates/weavepy-vm/src/stdlib/python/_opcode.py +++ b/crates/weavepy-vm/src/stdlib/python/_opcode.py @@ -284,584 +284,831 @@ _names['STORE_FAST_MAYBE_NULL']: 545, } -# `_PyOpcode_Deopt`: specialized form -> base instruction. -_DEOPT = { - _names['BINARY_OP_ADD_FLOAT']: _names['BINARY_OP'], - _names['BINARY_OP_ADD_INT']: _names['BINARY_OP'], - _names['BINARY_OP_ADD_UNICODE']: _names['BINARY_OP'], - _names['BINARY_OP_EXTEND']: _names['BINARY_OP'], - _names['BINARY_OP_INPLACE_ADD_UNICODE']: _names['BINARY_OP'], - _names['BINARY_OP_MULTIPLY_FLOAT']: _names['BINARY_OP'], - _names['BINARY_OP_MULTIPLY_INT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_DICT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_GETITEM']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_LIST_INT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_LIST_SLICE']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_STR_INT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBSCR_TUPLE_INT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBTRACT_FLOAT']: _names['BINARY_OP'], - _names['BINARY_OP_SUBTRACT_INT']: _names['BINARY_OP'], - _names['CALL_ALLOC_AND_ENTER_INIT']: _names['CALL'], - _names['CALL_BOUND_METHOD_EXACT_ARGS']: _names['CALL'], - _names['CALL_BOUND_METHOD_GENERAL']: _names['CALL'], - _names['CALL_BUILTIN_CLASS']: _names['CALL'], - _names['CALL_BUILTIN_FAST']: _names['CALL'], - _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: _names['CALL'], - _names['CALL_BUILTIN_O']: _names['CALL'], - _names['CALL_ISINSTANCE']: _names['CALL'], - _names['CALL_KW_BOUND_METHOD']: _names['CALL_KW'], - _names['CALL_KW_NON_PY']: _names['CALL_KW'], - _names['CALL_KW_PY']: _names['CALL_KW'], - _names['CALL_LEN']: _names['CALL'], - _names['CALL_LIST_APPEND']: _names['CALL'], - _names['CALL_METHOD_DESCRIPTOR_FAST']: _names['CALL'], - _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: _names['CALL'], - _names['CALL_METHOD_DESCRIPTOR_NOARGS']: _names['CALL'], - _names['CALL_METHOD_DESCRIPTOR_O']: _names['CALL'], - _names['CALL_NON_PY_GENERAL']: _names['CALL'], - _names['CALL_PY_EXACT_ARGS']: _names['CALL'], - _names['CALL_PY_GENERAL']: _names['CALL'], - _names['CALL_STR_1']: _names['CALL'], - _names['CALL_TUPLE_1']: _names['CALL'], - _names['CALL_TYPE_1']: _names['CALL'], - _names['COMPARE_OP_FLOAT']: _names['COMPARE_OP'], - _names['COMPARE_OP_INT']: _names['COMPARE_OP'], - _names['COMPARE_OP_STR']: _names['COMPARE_OP'], - _names['CONTAINS_OP_DICT']: _names['CONTAINS_OP'], - _names['CONTAINS_OP_SET']: _names['CONTAINS_OP'], - _names['FOR_ITER_GEN']: _names['FOR_ITER'], - _names['FOR_ITER_LIST']: _names['FOR_ITER'], - _names['FOR_ITER_RANGE']: _names['FOR_ITER'], - _names['FOR_ITER_TUPLE']: _names['FOR_ITER'], - _names['JUMP_BACKWARD_JIT']: _names['JUMP_BACKWARD'], - _names['JUMP_BACKWARD_NO_JIT']: _names['JUMP_BACKWARD'], - _names['LOAD_ATTR_CLASS']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_INSTANCE_VALUE']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_METHOD_LAZY_DICT']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_METHOD_NO_DICT']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_METHOD_WITH_VALUES']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_MODULE']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_PROPERTY']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_SLOT']: _names['LOAD_ATTR'], - _names['LOAD_ATTR_WITH_HINT']: _names['LOAD_ATTR'], - _names['LOAD_CONST_IMMORTAL']: _names['LOAD_CONST'], - _names['LOAD_CONST_MORTAL']: _names['LOAD_CONST'], - _names['LOAD_GLOBAL_BUILTIN']: _names['LOAD_GLOBAL'], - _names['LOAD_GLOBAL_MODULE']: _names['LOAD_GLOBAL'], - _names['LOAD_SUPER_ATTR_ATTR']: _names['LOAD_SUPER_ATTR'], - _names['LOAD_SUPER_ATTR_METHOD']: _names['LOAD_SUPER_ATTR'], - _names['RESUME_CHECK']: _names['RESUME'], - _names['SEND_GEN']: _names['SEND'], - _names['STORE_ATTR_INSTANCE_VALUE']: _names['STORE_ATTR'], - _names['STORE_ATTR_SLOT']: _names['STORE_ATTR'], - _names['STORE_ATTR_WITH_HINT']: _names['STORE_ATTR'], - _names['STORE_SUBSCR_DICT']: _names['STORE_SUBSCR'], - _names['STORE_SUBSCR_LIST_INT']: _names['STORE_SUBSCR'], - _names['TO_BOOL_ALWAYS_TRUE']: _names['TO_BOOL'], - _names['TO_BOOL_BOOL']: _names['TO_BOOL'], - _names['TO_BOOL_INT']: _names['TO_BOOL'], - _names['TO_BOOL_LIST']: _names['TO_BOOL'], - _names['TO_BOOL_NONE']: _names['TO_BOOL'], - _names['TO_BOOL_STR']: _names['TO_BOOL'], - _names['UNPACK_SEQUENCE_LIST']: _names['UNPACK_SEQUENCE'], - _names['UNPACK_SEQUENCE_TUPLE']: _names['UNPACK_SEQUENCE'], - _names['UNPACK_SEQUENCE_TWO_TUPLE']: _names['UNPACK_SEQUENCE'], -} +# The tables only `stack_effect` reads, built on its first call (most +# importers of `opcode` and `dis` never compute a stack effect). +_STACK_TABLES = None -# `_PyOpcode_num_popped(opcode, oparg)`. -_POPPED = { - _names['ANNOTATIONS_PLACEHOLDER']: lambda oparg: 0, - _names['BINARY_OP']: lambda oparg: 2, - _names['BINARY_OP_ADD_FLOAT']: lambda oparg: 2, - _names['BINARY_OP_ADD_INT']: lambda oparg: 2, - _names['BINARY_OP_ADD_UNICODE']: lambda oparg: 2, - _names['BINARY_OP_EXTEND']: lambda oparg: 2, - _names['BINARY_OP_INPLACE_ADD_UNICODE']: lambda oparg: 2, - _names['BINARY_OP_MULTIPLY_FLOAT']: lambda oparg: 2, - _names['BINARY_OP_MULTIPLY_INT']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_DICT']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_GETITEM']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_LIST_INT']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_LIST_SLICE']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_STR_INT']: lambda oparg: 2, - _names['BINARY_OP_SUBSCR_TUPLE_INT']: lambda oparg: 2, - _names['BINARY_OP_SUBTRACT_FLOAT']: lambda oparg: 2, - _names['BINARY_OP_SUBTRACT_INT']: lambda oparg: 2, - _names['BINARY_SLICE']: lambda oparg: 3, - _names['BUILD_INTERPOLATION']: lambda oparg: 2 + (oparg & 1), - _names['BUILD_LIST']: lambda oparg: oparg, - _names['BUILD_MAP']: lambda oparg: oparg*2, - _names['BUILD_SET']: lambda oparg: oparg, - _names['BUILD_SLICE']: lambda oparg: oparg, - _names['BUILD_STRING']: lambda oparg: oparg, - _names['BUILD_TEMPLATE']: lambda oparg: 2, - _names['BUILD_TUPLE']: lambda oparg: oparg, - _names['CACHE']: lambda oparg: 0, - _names['CALL']: lambda oparg: 2 + oparg, - _names['CALL_ALLOC_AND_ENTER_INIT']: lambda oparg: 2 + oparg, - _names['CALL_BOUND_METHOD_EXACT_ARGS']: lambda oparg: 2 + oparg, - _names['CALL_BOUND_METHOD_GENERAL']: lambda oparg: 2 + oparg, - _names['CALL_BUILTIN_CLASS']: lambda oparg: 2 + oparg, - _names['CALL_BUILTIN_FAST']: lambda oparg: 2 + oparg, - _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: lambda oparg: 2 + oparg, - _names['CALL_BUILTIN_O']: lambda oparg: 2 + oparg, - _names['CALL_FUNCTION_EX']: lambda oparg: 4, - _names['CALL_INTRINSIC_1']: lambda oparg: 1, - _names['CALL_INTRINSIC_2']: lambda oparg: 2, - _names['CALL_ISINSTANCE']: lambda oparg: 2 + oparg, - _names['CALL_KW']: lambda oparg: 3 + oparg, - _names['CALL_KW_BOUND_METHOD']: lambda oparg: 3 + oparg, - _names['CALL_KW_NON_PY']: lambda oparg: 3 + oparg, - _names['CALL_KW_PY']: lambda oparg: 3 + oparg, - _names['CALL_LEN']: lambda oparg: 3, - _names['CALL_LIST_APPEND']: lambda oparg: 3, - _names['CALL_METHOD_DESCRIPTOR_FAST']: lambda oparg: 2 + oparg, - _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: lambda oparg: 2 + oparg, - _names['CALL_METHOD_DESCRIPTOR_NOARGS']: lambda oparg: 2 + oparg, - _names['CALL_METHOD_DESCRIPTOR_O']: lambda oparg: 2 + oparg, - _names['CALL_NON_PY_GENERAL']: lambda oparg: 2 + oparg, - _names['CALL_PY_EXACT_ARGS']: lambda oparg: 2 + oparg, - _names['CALL_PY_GENERAL']: lambda oparg: 2 + oparg, - _names['CALL_STR_1']: lambda oparg: 3, - _names['CALL_TUPLE_1']: lambda oparg: 3, - _names['CALL_TYPE_1']: lambda oparg: 3, - _names['CHECK_EG_MATCH']: lambda oparg: 2, - _names['CHECK_EXC_MATCH']: lambda oparg: 2, - _names['CLEANUP_THROW']: lambda oparg: 3, - _names['COMPARE_OP']: lambda oparg: 2, - _names['COMPARE_OP_FLOAT']: lambda oparg: 2, - _names['COMPARE_OP_INT']: lambda oparg: 2, - _names['COMPARE_OP_STR']: lambda oparg: 2, - _names['CONTAINS_OP']: lambda oparg: 2, - _names['CONTAINS_OP_DICT']: lambda oparg: 2, - _names['CONTAINS_OP_SET']: lambda oparg: 2, - _names['CONVERT_VALUE']: lambda oparg: 1, - _names['COPY']: lambda oparg: 1 + (oparg-1), - _names['COPY_FREE_VARS']: lambda oparg: 0, - _names['DELETE_ATTR']: lambda oparg: 1, - _names['DELETE_DEREF']: lambda oparg: 0, - _names['DELETE_FAST']: lambda oparg: 0, - _names['DELETE_GLOBAL']: lambda oparg: 0, - _names['DELETE_NAME']: lambda oparg: 0, - _names['DELETE_SUBSCR']: lambda oparg: 2, - _names['DICT_MERGE']: lambda oparg: 5 + (oparg - 1), - _names['DICT_UPDATE']: lambda oparg: 2 + (oparg - 1), - _names['END_ASYNC_FOR']: lambda oparg: 2, - _names['END_FOR']: lambda oparg: 1, - _names['END_SEND']: lambda oparg: 2, - _names['ENTER_EXECUTOR']: lambda oparg: 0, - _names['EXIT_INIT_CHECK']: lambda oparg: 1, - _names['EXTENDED_ARG']: lambda oparg: 0, - _names['FORMAT_SIMPLE']: lambda oparg: 1, - _names['FORMAT_WITH_SPEC']: lambda oparg: 2, - _names['FOR_ITER']: lambda oparg: 1, - _names['FOR_ITER_GEN']: lambda oparg: 1, - _names['FOR_ITER_LIST']: lambda oparg: 1, - _names['FOR_ITER_RANGE']: lambda oparg: 1, - _names['FOR_ITER_TUPLE']: lambda oparg: 1, - _names['GET_AITER']: lambda oparg: 1, - _names['GET_ANEXT']: lambda oparg: 1, - _names['GET_AWAITABLE']: lambda oparg: 1, - _names['GET_ITER']: lambda oparg: 1, - _names['GET_LEN']: lambda oparg: 1, - _names['GET_YIELD_FROM_ITER']: lambda oparg: 1, - _names['IMPORT_FROM']: lambda oparg: 1, - _names['IMPORT_NAME']: lambda oparg: 2, - _names['INSTRUMENTED_CALL']: lambda oparg: 2 + oparg, - _names['INSTRUMENTED_CALL_FUNCTION_EX']: lambda oparg: 4, - _names['INSTRUMENTED_CALL_KW']: lambda oparg: 3 + oparg, - _names['INSTRUMENTED_END_ASYNC_FOR']: lambda oparg: 2, - _names['INSTRUMENTED_END_FOR']: lambda oparg: 2, - _names['INSTRUMENTED_END_SEND']: lambda oparg: 2, - _names['INSTRUMENTED_FOR_ITER']: lambda oparg: 1, - _names['INSTRUMENTED_INSTRUCTION']: lambda oparg: 0, - _names['INSTRUMENTED_JUMP_BACKWARD']: lambda oparg: 0, - _names['INSTRUMENTED_JUMP_FORWARD']: lambda oparg: 0, - _names['INSTRUMENTED_LINE']: lambda oparg: 0, - _names['INSTRUMENTED_LOAD_SUPER_ATTR']: lambda oparg: 3, - _names['INSTRUMENTED_NOT_TAKEN']: lambda oparg: 0, - _names['INSTRUMENTED_POP_ITER']: lambda oparg: 1, - _names['INSTRUMENTED_POP_JUMP_IF_FALSE']: lambda oparg: 1, - _names['INSTRUMENTED_POP_JUMP_IF_NONE']: lambda oparg: 1, - _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE']: lambda oparg: 1, - _names['INSTRUMENTED_POP_JUMP_IF_TRUE']: lambda oparg: 1, - _names['INSTRUMENTED_RESUME']: lambda oparg: 0, - _names['INSTRUMENTED_RETURN_VALUE']: lambda oparg: 1, - _names['INSTRUMENTED_YIELD_VALUE']: lambda oparg: 1, - _names['INTERPRETER_EXIT']: lambda oparg: 1, - _names['IS_OP']: lambda oparg: 2, - _names['JUMP']: lambda oparg: 0, - _names['JUMP_BACKWARD']: lambda oparg: 0, - _names['JUMP_BACKWARD_JIT']: lambda oparg: 0, - _names['JUMP_BACKWARD_NO_INTERRUPT']: lambda oparg: 0, - _names['JUMP_BACKWARD_NO_JIT']: lambda oparg: 0, - _names['JUMP_FORWARD']: lambda oparg: 0, - _names['JUMP_IF_FALSE']: lambda oparg: 1, - _names['JUMP_IF_TRUE']: lambda oparg: 1, - _names['JUMP_NO_INTERRUPT']: lambda oparg: 0, - _names['LIST_APPEND']: lambda oparg: 2 + (oparg-1), - _names['LIST_EXTEND']: lambda oparg: 2 + (oparg-1), - _names['LOAD_ATTR']: lambda oparg: 1, - _names['LOAD_ATTR_CLASS']: lambda oparg: 1, - _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: lambda oparg: 1, - _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: lambda oparg: 1, - _names['LOAD_ATTR_INSTANCE_VALUE']: lambda oparg: 1, - _names['LOAD_ATTR_METHOD_LAZY_DICT']: lambda oparg: 1, - _names['LOAD_ATTR_METHOD_NO_DICT']: lambda oparg: 1, - _names['LOAD_ATTR_METHOD_WITH_VALUES']: lambda oparg: 1, - _names['LOAD_ATTR_MODULE']: lambda oparg: 1, - _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: lambda oparg: 1, - _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: lambda oparg: 1, - _names['LOAD_ATTR_PROPERTY']: lambda oparg: 1, - _names['LOAD_ATTR_SLOT']: lambda oparg: 1, - _names['LOAD_ATTR_WITH_HINT']: lambda oparg: 1, - _names['LOAD_BUILD_CLASS']: lambda oparg: 0, - _names['LOAD_CLOSURE']: lambda oparg: 0, - _names['LOAD_COMMON_CONSTANT']: lambda oparg: 0, - _names['LOAD_CONST']: lambda oparg: 0, - _names['LOAD_CONST_IMMORTAL']: lambda oparg: 0, - _names['LOAD_CONST_MORTAL']: lambda oparg: 0, - _names['LOAD_DEREF']: lambda oparg: 0, - _names['LOAD_FAST']: lambda oparg: 0, - _names['LOAD_FAST_AND_CLEAR']: lambda oparg: 0, - _names['LOAD_FAST_BORROW']: lambda oparg: 0, - _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW']: lambda oparg: 0, - _names['LOAD_FAST_CHECK']: lambda oparg: 0, - _names['LOAD_FAST_LOAD_FAST']: lambda oparg: 0, - _names['LOAD_FROM_DICT_OR_DEREF']: lambda oparg: 1, - _names['LOAD_FROM_DICT_OR_GLOBALS']: lambda oparg: 1, - _names['LOAD_GLOBAL']: lambda oparg: 0, - _names['LOAD_GLOBAL_BUILTIN']: lambda oparg: 0, - _names['LOAD_GLOBAL_MODULE']: lambda oparg: 0, - _names['LOAD_LOCALS']: lambda oparg: 0, - _names['LOAD_NAME']: lambda oparg: 0, - _names['LOAD_SMALL_INT']: lambda oparg: 0, - _names['LOAD_SPECIAL']: lambda oparg: 1, - _names['LOAD_SUPER_ATTR']: lambda oparg: 3, - _names['LOAD_SUPER_ATTR_ATTR']: lambda oparg: 3, - _names['LOAD_SUPER_ATTR_METHOD']: lambda oparg: 3, - _names['MAKE_CELL']: lambda oparg: 0, - _names['MAKE_FUNCTION']: lambda oparg: 1, - _names['MAP_ADD']: lambda oparg: 3 + (oparg - 1), - _names['MATCH_CLASS']: lambda oparg: 3, - _names['MATCH_KEYS']: lambda oparg: 2, - _names['MATCH_MAPPING']: lambda oparg: 1, - _names['MATCH_SEQUENCE']: lambda oparg: 1, - _names['NOP']: lambda oparg: 0, - _names['NOT_TAKEN']: lambda oparg: 0, - _names['POP_BLOCK']: lambda oparg: 0, - _names['POP_EXCEPT']: lambda oparg: 1, - _names['POP_ITER']: lambda oparg: 1, - _names['POP_JUMP_IF_FALSE']: lambda oparg: 1, - _names['POP_JUMP_IF_NONE']: lambda oparg: 1, - _names['POP_JUMP_IF_NOT_NONE']: lambda oparg: 1, - _names['POP_JUMP_IF_TRUE']: lambda oparg: 1, - _names['POP_TOP']: lambda oparg: 1, - _names['PUSH_EXC_INFO']: lambda oparg: 1, - _names['PUSH_NULL']: lambda oparg: 0, - _names['RAISE_VARARGS']: lambda oparg: oparg, - _names['RERAISE']: lambda oparg: 1 + oparg, - _names['RESERVED']: lambda oparg: 0, - _names['RESUME']: lambda oparg: 0, - _names['RESUME_CHECK']: lambda oparg: 0, - _names['RETURN_GENERATOR']: lambda oparg: 0, - _names['RETURN_VALUE']: lambda oparg: 1, - _names['SEND']: lambda oparg: 2, - _names['SEND_GEN']: lambda oparg: 2, - _names['SETUP_ANNOTATIONS']: lambda oparg: 0, - _names['SETUP_CLEANUP']: lambda oparg: 0, - _names['SETUP_FINALLY']: lambda oparg: 0, - _names['SETUP_WITH']: lambda oparg: 0, - _names['SET_ADD']: lambda oparg: 2 + (oparg-1), - _names['SET_FUNCTION_ATTRIBUTE']: lambda oparg: 2, - _names['SET_UPDATE']: lambda oparg: 2 + (oparg-1), - _names['STORE_ATTR']: lambda oparg: 2, - _names['STORE_ATTR_INSTANCE_VALUE']: lambda oparg: 2, - _names['STORE_ATTR_SLOT']: lambda oparg: 2, - _names['STORE_ATTR_WITH_HINT']: lambda oparg: 2, - _names['STORE_DEREF']: lambda oparg: 1, - _names['STORE_FAST']: lambda oparg: 1, - _names['STORE_FAST_LOAD_FAST']: lambda oparg: 1, - _names['STORE_FAST_MAYBE_NULL']: lambda oparg: 1, - _names['STORE_FAST_STORE_FAST']: lambda oparg: 2, - _names['STORE_GLOBAL']: lambda oparg: 1, - _names['STORE_NAME']: lambda oparg: 1, - _names['STORE_SLICE']: lambda oparg: 4, - _names['STORE_SUBSCR']: lambda oparg: 3, - _names['STORE_SUBSCR_DICT']: lambda oparg: 3, - _names['STORE_SUBSCR_LIST_INT']: lambda oparg: 3, - _names['SWAP']: lambda oparg: 2 + (oparg-2), - _names['TO_BOOL']: lambda oparg: 1, - _names['TO_BOOL_ALWAYS_TRUE']: lambda oparg: 1, - _names['TO_BOOL_BOOL']: lambda oparg: 1, - _names['TO_BOOL_INT']: lambda oparg: 1, - _names['TO_BOOL_LIST']: lambda oparg: 1, - _names['TO_BOOL_NONE']: lambda oparg: 1, - _names['TO_BOOL_STR']: lambda oparg: 1, - _names['UNARY_INVERT']: lambda oparg: 1, - _names['UNARY_NEGATIVE']: lambda oparg: 1, - _names['UNARY_NOT']: lambda oparg: 1, - _names['UNPACK_EX']: lambda oparg: 1, - _names['UNPACK_SEQUENCE']: lambda oparg: 1, - _names['UNPACK_SEQUENCE_LIST']: lambda oparg: 1, - _names['UNPACK_SEQUENCE_TUPLE']: lambda oparg: 1, - _names['UNPACK_SEQUENCE_TWO_TUPLE']: lambda oparg: 1, - _names['WITH_EXCEPT_START']: lambda oparg: 5, - _names['YIELD_VALUE']: lambda oparg: 1, -} -# `_PyOpcode_num_pushed(opcode, oparg)`. -_PUSHED = { - _names['ANNOTATIONS_PLACEHOLDER']: lambda oparg: 0, - _names['BINARY_OP']: lambda oparg: 1, - _names['BINARY_OP_ADD_FLOAT']: lambda oparg: 1, - _names['BINARY_OP_ADD_INT']: lambda oparg: 1, - _names['BINARY_OP_ADD_UNICODE']: lambda oparg: 1, - _names['BINARY_OP_EXTEND']: lambda oparg: 1, - _names['BINARY_OP_INPLACE_ADD_UNICODE']: lambda oparg: 0, - _names['BINARY_OP_MULTIPLY_FLOAT']: lambda oparg: 1, - _names['BINARY_OP_MULTIPLY_INT']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_DICT']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_GETITEM']: lambda oparg: 0, - _names['BINARY_OP_SUBSCR_LIST_INT']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_LIST_SLICE']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_STR_INT']: lambda oparg: 1, - _names['BINARY_OP_SUBSCR_TUPLE_INT']: lambda oparg: 1, - _names['BINARY_OP_SUBTRACT_FLOAT']: lambda oparg: 1, - _names['BINARY_OP_SUBTRACT_INT']: lambda oparg: 1, - _names['BINARY_SLICE']: lambda oparg: 1, - _names['BUILD_INTERPOLATION']: lambda oparg: 1, - _names['BUILD_LIST']: lambda oparg: 1, - _names['BUILD_MAP']: lambda oparg: 1, - _names['BUILD_SET']: lambda oparg: 1, - _names['BUILD_SLICE']: lambda oparg: 1, - _names['BUILD_STRING']: lambda oparg: 1, - _names['BUILD_TEMPLATE']: lambda oparg: 1, - _names['BUILD_TUPLE']: lambda oparg: 1, - _names['CACHE']: lambda oparg: 0, - _names['CALL']: lambda oparg: 1, - _names['CALL_ALLOC_AND_ENTER_INIT']: lambda oparg: 0, - _names['CALL_BOUND_METHOD_EXACT_ARGS']: lambda oparg: 0, - _names['CALL_BOUND_METHOD_GENERAL']: lambda oparg: 0, - _names['CALL_BUILTIN_CLASS']: lambda oparg: 1, - _names['CALL_BUILTIN_FAST']: lambda oparg: 1, - _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: lambda oparg: 1, - _names['CALL_BUILTIN_O']: lambda oparg: 1, - _names['CALL_FUNCTION_EX']: lambda oparg: 1, - _names['CALL_INTRINSIC_1']: lambda oparg: 1, - _names['CALL_INTRINSIC_2']: lambda oparg: 1, - _names['CALL_ISINSTANCE']: lambda oparg: 1, - _names['CALL_KW']: lambda oparg: 1, - _names['CALL_KW_BOUND_METHOD']: lambda oparg: 0, - _names['CALL_KW_NON_PY']: lambda oparg: 1, - _names['CALL_KW_PY']: lambda oparg: 0, - _names['CALL_LEN']: lambda oparg: 1, - _names['CALL_LIST_APPEND']: lambda oparg: 0, - _names['CALL_METHOD_DESCRIPTOR_FAST']: lambda oparg: 1, - _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: lambda oparg: 1, - _names['CALL_METHOD_DESCRIPTOR_NOARGS']: lambda oparg: 1, - _names['CALL_METHOD_DESCRIPTOR_O']: lambda oparg: 1, - _names['CALL_NON_PY_GENERAL']: lambda oparg: 1, - _names['CALL_PY_EXACT_ARGS']: lambda oparg: 0, - _names['CALL_PY_GENERAL']: lambda oparg: 0, - _names['CALL_STR_1']: lambda oparg: 1, - _names['CALL_TUPLE_1']: lambda oparg: 1, - _names['CALL_TYPE_1']: lambda oparg: 1, - _names['CHECK_EG_MATCH']: lambda oparg: 2, - _names['CHECK_EXC_MATCH']: lambda oparg: 2, - _names['CLEANUP_THROW']: lambda oparg: 2, - _names['COMPARE_OP']: lambda oparg: 1, - _names['COMPARE_OP_FLOAT']: lambda oparg: 1, - _names['COMPARE_OP_INT']: lambda oparg: 1, - _names['COMPARE_OP_STR']: lambda oparg: 1, - _names['CONTAINS_OP']: lambda oparg: 1, - _names['CONTAINS_OP_DICT']: lambda oparg: 1, - _names['CONTAINS_OP_SET']: lambda oparg: 1, - _names['CONVERT_VALUE']: lambda oparg: 1, - _names['COPY']: lambda oparg: 2 + (oparg-1), - _names['COPY_FREE_VARS']: lambda oparg: 0, - _names['DELETE_ATTR']: lambda oparg: 0, - _names['DELETE_DEREF']: lambda oparg: 0, - _names['DELETE_FAST']: lambda oparg: 0, - _names['DELETE_GLOBAL']: lambda oparg: 0, - _names['DELETE_NAME']: lambda oparg: 0, - _names['DELETE_SUBSCR']: lambda oparg: 0, - _names['DICT_MERGE']: lambda oparg: 4 + (oparg - 1), - _names['DICT_UPDATE']: lambda oparg: 1 + (oparg - 1), - _names['END_ASYNC_FOR']: lambda oparg: 0, - _names['END_FOR']: lambda oparg: 0, - _names['END_SEND']: lambda oparg: 1, - _names['ENTER_EXECUTOR']: lambda oparg: 0, - _names['EXIT_INIT_CHECK']: lambda oparg: 0, - _names['EXTENDED_ARG']: lambda oparg: 0, - _names['FORMAT_SIMPLE']: lambda oparg: 1, - _names['FORMAT_WITH_SPEC']: lambda oparg: 1, - _names['FOR_ITER']: lambda oparg: 2, - _names['FOR_ITER_GEN']: lambda oparg: 1, - _names['FOR_ITER_LIST']: lambda oparg: 2, - _names['FOR_ITER_RANGE']: lambda oparg: 2, - _names['FOR_ITER_TUPLE']: lambda oparg: 2, - _names['GET_AITER']: lambda oparg: 1, - _names['GET_ANEXT']: lambda oparg: 2, - _names['GET_AWAITABLE']: lambda oparg: 1, - _names['GET_ITER']: lambda oparg: 1, - _names['GET_LEN']: lambda oparg: 2, - _names['GET_YIELD_FROM_ITER']: lambda oparg: 1, - _names['IMPORT_FROM']: lambda oparg: 2, - _names['IMPORT_NAME']: lambda oparg: 1, - _names['INSTRUMENTED_CALL']: lambda oparg: 1, - _names['INSTRUMENTED_CALL_FUNCTION_EX']: lambda oparg: 1, - _names['INSTRUMENTED_CALL_KW']: lambda oparg: 1, - _names['INSTRUMENTED_END_ASYNC_FOR']: lambda oparg: 0, - _names['INSTRUMENTED_END_FOR']: lambda oparg: 1, - _names['INSTRUMENTED_END_SEND']: lambda oparg: 1, - _names['INSTRUMENTED_FOR_ITER']: lambda oparg: 2, - _names['INSTRUMENTED_INSTRUCTION']: lambda oparg: 0, - _names['INSTRUMENTED_JUMP_BACKWARD']: lambda oparg: 0, - _names['INSTRUMENTED_JUMP_FORWARD']: lambda oparg: 0, - _names['INSTRUMENTED_LINE']: lambda oparg: 0, - _names['INSTRUMENTED_LOAD_SUPER_ATTR']: lambda oparg: 1 + (oparg & 1), - _names['INSTRUMENTED_NOT_TAKEN']: lambda oparg: 0, - _names['INSTRUMENTED_POP_ITER']: lambda oparg: 0, - _names['INSTRUMENTED_POP_JUMP_IF_FALSE']: lambda oparg: 0, - _names['INSTRUMENTED_POP_JUMP_IF_NONE']: lambda oparg: 0, - _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE']: lambda oparg: 0, - _names['INSTRUMENTED_POP_JUMP_IF_TRUE']: lambda oparg: 0, - _names['INSTRUMENTED_RESUME']: lambda oparg: 0, - _names['INSTRUMENTED_RETURN_VALUE']: lambda oparg: 1, - _names['INSTRUMENTED_YIELD_VALUE']: lambda oparg: 1, - _names['INTERPRETER_EXIT']: lambda oparg: 0, - _names['IS_OP']: lambda oparg: 1, - _names['JUMP']: lambda oparg: 0, - _names['JUMP_BACKWARD']: lambda oparg: 0, - _names['JUMP_BACKWARD_JIT']: lambda oparg: 0, - _names['JUMP_BACKWARD_NO_INTERRUPT']: lambda oparg: 0, - _names['JUMP_BACKWARD_NO_JIT']: lambda oparg: 0, - _names['JUMP_FORWARD']: lambda oparg: 0, - _names['JUMP_IF_FALSE']: lambda oparg: 1, - _names['JUMP_IF_TRUE']: lambda oparg: 1, - _names['JUMP_NO_INTERRUPT']: lambda oparg: 0, - _names['LIST_APPEND']: lambda oparg: 1 + (oparg-1), - _names['LIST_EXTEND']: lambda oparg: 1 + (oparg-1), - _names['LOAD_ATTR']: lambda oparg: 1 + (oparg&1), - _names['LOAD_ATTR_CLASS']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: lambda oparg: 1, - _names['LOAD_ATTR_INSTANCE_VALUE']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_METHOD_LAZY_DICT']: lambda oparg: 2, - _names['LOAD_ATTR_METHOD_NO_DICT']: lambda oparg: 2, - _names['LOAD_ATTR_METHOD_WITH_VALUES']: lambda oparg: 2, - _names['LOAD_ATTR_MODULE']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: lambda oparg: 1, - _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: lambda oparg: 1, - _names['LOAD_ATTR_PROPERTY']: lambda oparg: 0, - _names['LOAD_ATTR_SLOT']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_ATTR_WITH_HINT']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_BUILD_CLASS']: lambda oparg: 1, - _names['LOAD_CLOSURE']: lambda oparg: 1, - _names['LOAD_COMMON_CONSTANT']: lambda oparg: 1, - _names['LOAD_CONST']: lambda oparg: 1, - _names['LOAD_CONST_IMMORTAL']: lambda oparg: 1, - _names['LOAD_CONST_MORTAL']: lambda oparg: 1, - _names['LOAD_DEREF']: lambda oparg: 1, - _names['LOAD_FAST']: lambda oparg: 1, - _names['LOAD_FAST_AND_CLEAR']: lambda oparg: 1, - _names['LOAD_FAST_BORROW']: lambda oparg: 1, - _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW']: lambda oparg: 2, - _names['LOAD_FAST_CHECK']: lambda oparg: 1, - _names['LOAD_FAST_LOAD_FAST']: lambda oparg: 2, - _names['LOAD_FROM_DICT_OR_DEREF']: lambda oparg: 1, - _names['LOAD_FROM_DICT_OR_GLOBALS']: lambda oparg: 1, - _names['LOAD_GLOBAL']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_GLOBAL_BUILTIN']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_GLOBAL_MODULE']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_LOCALS']: lambda oparg: 1, - _names['LOAD_NAME']: lambda oparg: 1, - _names['LOAD_SMALL_INT']: lambda oparg: 1, - _names['LOAD_SPECIAL']: lambda oparg: 2, - _names['LOAD_SUPER_ATTR']: lambda oparg: 1 + (oparg & 1), - _names['LOAD_SUPER_ATTR_ATTR']: lambda oparg: 1, - _names['LOAD_SUPER_ATTR_METHOD']: lambda oparg: 2, - _names['MAKE_CELL']: lambda oparg: 0, - _names['MAKE_FUNCTION']: lambda oparg: 1, - _names['MAP_ADD']: lambda oparg: 1 + (oparg - 1), - _names['MATCH_CLASS']: lambda oparg: 1, - _names['MATCH_KEYS']: lambda oparg: 3, - _names['MATCH_MAPPING']: lambda oparg: 2, - _names['MATCH_SEQUENCE']: lambda oparg: 2, - _names['NOP']: lambda oparg: 0, - _names['NOT_TAKEN']: lambda oparg: 0, - _names['POP_BLOCK']: lambda oparg: 0, - _names['POP_EXCEPT']: lambda oparg: 0, - _names['POP_ITER']: lambda oparg: 0, - _names['POP_JUMP_IF_FALSE']: lambda oparg: 0, - _names['POP_JUMP_IF_NONE']: lambda oparg: 0, - _names['POP_JUMP_IF_NOT_NONE']: lambda oparg: 0, - _names['POP_JUMP_IF_TRUE']: lambda oparg: 0, - _names['POP_TOP']: lambda oparg: 0, - _names['PUSH_EXC_INFO']: lambda oparg: 2, - _names['PUSH_NULL']: lambda oparg: 1, - _names['RAISE_VARARGS']: lambda oparg: 0, - _names['RERAISE']: lambda oparg: oparg, - _names['RESERVED']: lambda oparg: 0, - _names['RESUME']: lambda oparg: 0, - _names['RESUME_CHECK']: lambda oparg: 0, - _names['RETURN_GENERATOR']: lambda oparg: 1, - _names['RETURN_VALUE']: lambda oparg: 1, - _names['SEND']: lambda oparg: 2, - _names['SEND_GEN']: lambda oparg: 1, - _names['SETUP_ANNOTATIONS']: lambda oparg: 0, - _names['SETUP_CLEANUP']: lambda oparg: 2, - _names['SETUP_FINALLY']: lambda oparg: 1, - _names['SETUP_WITH']: lambda oparg: 1, - _names['SET_ADD']: lambda oparg: 1 + (oparg-1), - _names['SET_FUNCTION_ATTRIBUTE']: lambda oparg: 1, - _names['SET_UPDATE']: lambda oparg: 1 + (oparg-1), - _names['STORE_ATTR']: lambda oparg: 0, - _names['STORE_ATTR_INSTANCE_VALUE']: lambda oparg: 0, - _names['STORE_ATTR_SLOT']: lambda oparg: 0, - _names['STORE_ATTR_WITH_HINT']: lambda oparg: 0, - _names['STORE_DEREF']: lambda oparg: 0, - _names['STORE_FAST']: lambda oparg: 0, - _names['STORE_FAST_LOAD_FAST']: lambda oparg: 1, - _names['STORE_FAST_MAYBE_NULL']: lambda oparg: 0, - _names['STORE_FAST_STORE_FAST']: lambda oparg: 0, - _names['STORE_GLOBAL']: lambda oparg: 0, - _names['STORE_NAME']: lambda oparg: 0, - _names['STORE_SLICE']: lambda oparg: 0, - _names['STORE_SUBSCR']: lambda oparg: 0, - _names['STORE_SUBSCR_DICT']: lambda oparg: 0, - _names['STORE_SUBSCR_LIST_INT']: lambda oparg: 0, - _names['SWAP']: lambda oparg: 2 + (oparg-2), - _names['TO_BOOL']: lambda oparg: 1, - _names['TO_BOOL_ALWAYS_TRUE']: lambda oparg: 1, - _names['TO_BOOL_BOOL']: lambda oparg: 1, - _names['TO_BOOL_INT']: lambda oparg: 1, - _names['TO_BOOL_LIST']: lambda oparg: 1, - _names['TO_BOOL_NONE']: lambda oparg: 1, - _names['TO_BOOL_STR']: lambda oparg: 1, - _names['UNARY_INVERT']: lambda oparg: 1, - _names['UNARY_NEGATIVE']: lambda oparg: 1, - _names['UNARY_NOT']: lambda oparg: 1, - _names['UNPACK_EX']: lambda oparg: 1 + (oparg & 0xFF) + (oparg >> 8), - _names['UNPACK_SEQUENCE']: lambda oparg: oparg, - _names['UNPACK_SEQUENCE_LIST']: lambda oparg: oparg, - _names['UNPACK_SEQUENCE_TUPLE']: lambda oparg: oparg, - _names['UNPACK_SEQUENCE_TWO_TUPLE']: lambda oparg: 2, - _names['WITH_EXCEPT_START']: lambda oparg: 6, - _names['YIELD_VALUE']: lambda oparg: 1, -} +def _stack_tables(): + global _STACK_TABLES + if _STACK_TABLES is not None: + return _STACK_TABLES + # `_PyOpcode_Deopt`: specialized form -> base instruction. + deopt = { + _names['BINARY_OP_ADD_FLOAT']: _names['BINARY_OP'], + _names['BINARY_OP_ADD_INT']: _names['BINARY_OP'], + _names['BINARY_OP_ADD_UNICODE']: _names['BINARY_OP'], + _names['BINARY_OP_EXTEND']: _names['BINARY_OP'], + _names['BINARY_OP_INPLACE_ADD_UNICODE']: _names['BINARY_OP'], + _names['BINARY_OP_MULTIPLY_FLOAT']: _names['BINARY_OP'], + _names['BINARY_OP_MULTIPLY_INT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_DICT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_GETITEM']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_LIST_INT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_LIST_SLICE']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_STR_INT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBSCR_TUPLE_INT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBTRACT_FLOAT']: _names['BINARY_OP'], + _names['BINARY_OP_SUBTRACT_INT']: _names['BINARY_OP'], + _names['CALL_ALLOC_AND_ENTER_INIT']: _names['CALL'], + _names['CALL_BOUND_METHOD_EXACT_ARGS']: _names['CALL'], + _names['CALL_BOUND_METHOD_GENERAL']: _names['CALL'], + _names['CALL_BUILTIN_CLASS']: _names['CALL'], + _names['CALL_BUILTIN_FAST']: _names['CALL'], + _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: _names['CALL'], + _names['CALL_BUILTIN_O']: _names['CALL'], + _names['CALL_ISINSTANCE']: _names['CALL'], + _names['CALL_KW_BOUND_METHOD']: _names['CALL_KW'], + _names['CALL_KW_NON_PY']: _names['CALL_KW'], + _names['CALL_KW_PY']: _names['CALL_KW'], + _names['CALL_LEN']: _names['CALL'], + _names['CALL_LIST_APPEND']: _names['CALL'], + _names['CALL_METHOD_DESCRIPTOR_FAST']: _names['CALL'], + _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: _names['CALL'], + _names['CALL_METHOD_DESCRIPTOR_NOARGS']: _names['CALL'], + _names['CALL_METHOD_DESCRIPTOR_O']: _names['CALL'], + _names['CALL_NON_PY_GENERAL']: _names['CALL'], + _names['CALL_PY_EXACT_ARGS']: _names['CALL'], + _names['CALL_PY_GENERAL']: _names['CALL'], + _names['CALL_STR_1']: _names['CALL'], + _names['CALL_TUPLE_1']: _names['CALL'], + _names['CALL_TYPE_1']: _names['CALL'], + _names['COMPARE_OP_FLOAT']: _names['COMPARE_OP'], + _names['COMPARE_OP_INT']: _names['COMPARE_OP'], + _names['COMPARE_OP_STR']: _names['COMPARE_OP'], + _names['CONTAINS_OP_DICT']: _names['CONTAINS_OP'], + _names['CONTAINS_OP_SET']: _names['CONTAINS_OP'], + _names['FOR_ITER_GEN']: _names['FOR_ITER'], + _names['FOR_ITER_LIST']: _names['FOR_ITER'], + _names['FOR_ITER_RANGE']: _names['FOR_ITER'], + _names['FOR_ITER_TUPLE']: _names['FOR_ITER'], + _names['JUMP_BACKWARD_JIT']: _names['JUMP_BACKWARD'], + _names['JUMP_BACKWARD_NO_JIT']: _names['JUMP_BACKWARD'], + _names['LOAD_ATTR_CLASS']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_INSTANCE_VALUE']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_METHOD_LAZY_DICT']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_METHOD_NO_DICT']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_METHOD_WITH_VALUES']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_MODULE']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_PROPERTY']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_SLOT']: _names['LOAD_ATTR'], + _names['LOAD_ATTR_WITH_HINT']: _names['LOAD_ATTR'], + _names['LOAD_CONST_IMMORTAL']: _names['LOAD_CONST'], + _names['LOAD_CONST_MORTAL']: _names['LOAD_CONST'], + _names['LOAD_GLOBAL_BUILTIN']: _names['LOAD_GLOBAL'], + _names['LOAD_GLOBAL_MODULE']: _names['LOAD_GLOBAL'], + _names['LOAD_SUPER_ATTR_ATTR']: _names['LOAD_SUPER_ATTR'], + _names['LOAD_SUPER_ATTR_METHOD']: _names['LOAD_SUPER_ATTR'], + _names['RESUME_CHECK']: _names['RESUME'], + _names['SEND_GEN']: _names['SEND'], + _names['STORE_ATTR_INSTANCE_VALUE']: _names['STORE_ATTR'], + _names['STORE_ATTR_SLOT']: _names['STORE_ATTR'], + _names['STORE_ATTR_WITH_HINT']: _names['STORE_ATTR'], + _names['STORE_SUBSCR_DICT']: _names['STORE_SUBSCR'], + _names['STORE_SUBSCR_LIST_INT']: _names['STORE_SUBSCR'], + _names['TO_BOOL_ALWAYS_TRUE']: _names['TO_BOOL'], + _names['TO_BOOL_BOOL']: _names['TO_BOOL'], + _names['TO_BOOL_INT']: _names['TO_BOOL'], + _names['TO_BOOL_LIST']: _names['TO_BOOL'], + _names['TO_BOOL_NONE']: _names['TO_BOOL'], + _names['TO_BOOL_STR']: _names['TO_BOOL'], + _names['UNPACK_SEQUENCE_LIST']: _names['UNPACK_SEQUENCE'], + _names['UNPACK_SEQUENCE_TUPLE']: _names['UNPACK_SEQUENCE'], + _names['UNPACK_SEQUENCE_TWO_TUPLE']: _names['UNPACK_SEQUENCE'], + } + # `_PyOpcode_num_popped(opcode, oparg)`: the count, or a function of + # `oparg` for the opcodes whose count depends on it. + popped = { + _names['ANNOTATIONS_PLACEHOLDER']: 0, + _names['BINARY_OP']: 2, + _names['BINARY_OP_ADD_FLOAT']: 2, + _names['BINARY_OP_ADD_INT']: 2, + _names['BINARY_OP_ADD_UNICODE']: 2, + _names['BINARY_OP_EXTEND']: 2, + _names['BINARY_OP_INPLACE_ADD_UNICODE']: 2, + _names['BINARY_OP_MULTIPLY_FLOAT']: 2, + _names['BINARY_OP_MULTIPLY_INT']: 2, + _names['BINARY_OP_SUBSCR_DICT']: 2, + _names['BINARY_OP_SUBSCR_GETITEM']: 2, + _names['BINARY_OP_SUBSCR_LIST_INT']: 2, + _names['BINARY_OP_SUBSCR_LIST_SLICE']: 2, + _names['BINARY_OP_SUBSCR_STR_INT']: 2, + _names['BINARY_OP_SUBSCR_TUPLE_INT']: 2, + _names['BINARY_OP_SUBTRACT_FLOAT']: 2, + _names['BINARY_OP_SUBTRACT_INT']: 2, + _names['BINARY_SLICE']: 3, + _names['BUILD_INTERPOLATION']: lambda oparg: 2 + (oparg & 1), + _names['BUILD_LIST']: lambda oparg: oparg, + _names['BUILD_MAP']: lambda oparg: oparg*2, + _names['BUILD_SET']: lambda oparg: oparg, + _names['BUILD_SLICE']: lambda oparg: oparg, + _names['BUILD_STRING']: lambda oparg: oparg, + _names['BUILD_TEMPLATE']: 2, + _names['BUILD_TUPLE']: lambda oparg: oparg, + _names['CACHE']: 0, + _names['CALL']: lambda oparg: 2 + oparg, + _names['CALL_ALLOC_AND_ENTER_INIT']: lambda oparg: 2 + oparg, + _names['CALL_BOUND_METHOD_EXACT_ARGS']: lambda oparg: 2 + oparg, + _names['CALL_BOUND_METHOD_GENERAL']: lambda oparg: 2 + oparg, + _names['CALL_BUILTIN_CLASS']: lambda oparg: 2 + oparg, + _names['CALL_BUILTIN_FAST']: lambda oparg: 2 + oparg, + _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: lambda oparg: 2 + oparg, + _names['CALL_BUILTIN_O']: lambda oparg: 2 + oparg, + _names['CALL_FUNCTION_EX']: 4, + _names['CALL_INTRINSIC_1']: 1, + _names['CALL_INTRINSIC_2']: 2, + _names['CALL_ISINSTANCE']: lambda oparg: 2 + oparg, + _names['CALL_KW']: lambda oparg: 3 + oparg, + _names['CALL_KW_BOUND_METHOD']: lambda oparg: 3 + oparg, + _names['CALL_KW_NON_PY']: lambda oparg: 3 + oparg, + _names['CALL_KW_PY']: lambda oparg: 3 + oparg, + _names['CALL_LEN']: 3, + _names['CALL_LIST_APPEND']: 3, + _names['CALL_METHOD_DESCRIPTOR_FAST']: lambda oparg: 2 + oparg, + _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: lambda oparg: 2 + oparg, + _names['CALL_METHOD_DESCRIPTOR_NOARGS']: lambda oparg: 2 + oparg, + _names['CALL_METHOD_DESCRIPTOR_O']: lambda oparg: 2 + oparg, + _names['CALL_NON_PY_GENERAL']: lambda oparg: 2 + oparg, + _names['CALL_PY_EXACT_ARGS']: lambda oparg: 2 + oparg, + _names['CALL_PY_GENERAL']: lambda oparg: 2 + oparg, + _names['CALL_STR_1']: 3, + _names['CALL_TUPLE_1']: 3, + _names['CALL_TYPE_1']: 3, + _names['CHECK_EG_MATCH']: 2, + _names['CHECK_EXC_MATCH']: 2, + _names['CLEANUP_THROW']: 3, + _names['COMPARE_OP']: 2, + _names['COMPARE_OP_FLOAT']: 2, + _names['COMPARE_OP_INT']: 2, + _names['COMPARE_OP_STR']: 2, + _names['CONTAINS_OP']: 2, + _names['CONTAINS_OP_DICT']: 2, + _names['CONTAINS_OP_SET']: 2, + _names['CONVERT_VALUE']: 1, + _names['COPY']: lambda oparg: 1 + (oparg-1), + _names['COPY_FREE_VARS']: 0, + _names['DELETE_ATTR']: 1, + _names['DELETE_DEREF']: 0, + _names['DELETE_FAST']: 0, + _names['DELETE_GLOBAL']: 0, + _names['DELETE_NAME']: 0, + _names['DELETE_SUBSCR']: 2, + _names['DICT_MERGE']: lambda oparg: 5 + (oparg - 1), + _names['DICT_UPDATE']: lambda oparg: 2 + (oparg - 1), + _names['END_ASYNC_FOR']: 2, + _names['END_FOR']: 1, + _names['END_SEND']: 2, + _names['ENTER_EXECUTOR']: 0, + _names['EXIT_INIT_CHECK']: 1, + _names['EXTENDED_ARG']: 0, + _names['FORMAT_SIMPLE']: 1, + _names['FORMAT_WITH_SPEC']: 2, + _names['FOR_ITER']: 1, + _names['FOR_ITER_GEN']: 1, + _names['FOR_ITER_LIST']: 1, + _names['FOR_ITER_RANGE']: 1, + _names['FOR_ITER_TUPLE']: 1, + _names['GET_AITER']: 1, + _names['GET_ANEXT']: 1, + _names['GET_AWAITABLE']: 1, + _names['GET_ITER']: 1, + _names['GET_LEN']: 1, + _names['GET_YIELD_FROM_ITER']: 1, + _names['IMPORT_FROM']: 1, + _names['IMPORT_NAME']: 2, + _names['INSTRUMENTED_CALL']: lambda oparg: 2 + oparg, + _names['INSTRUMENTED_CALL_FUNCTION_EX']: 4, + _names['INSTRUMENTED_CALL_KW']: lambda oparg: 3 + oparg, + _names['INSTRUMENTED_END_ASYNC_FOR']: 2, + _names['INSTRUMENTED_END_FOR']: 2, + _names['INSTRUMENTED_END_SEND']: 2, + _names['INSTRUMENTED_FOR_ITER']: 1, + _names['INSTRUMENTED_INSTRUCTION']: 0, + _names['INSTRUMENTED_JUMP_BACKWARD']: 0, + _names['INSTRUMENTED_JUMP_FORWARD']: 0, + _names['INSTRUMENTED_LINE']: 0, + _names['INSTRUMENTED_LOAD_SUPER_ATTR']: 3, + _names['INSTRUMENTED_NOT_TAKEN']: 0, + _names['INSTRUMENTED_POP_ITER']: 1, + _names['INSTRUMENTED_POP_JUMP_IF_FALSE']: 1, + _names['INSTRUMENTED_POP_JUMP_IF_NONE']: 1, + _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE']: 1, + _names['INSTRUMENTED_POP_JUMP_IF_TRUE']: 1, + _names['INSTRUMENTED_RESUME']: 0, + _names['INSTRUMENTED_RETURN_VALUE']: 1, + _names['INSTRUMENTED_YIELD_VALUE']: 1, + _names['INTERPRETER_EXIT']: 1, + _names['IS_OP']: 2, + _names['JUMP']: 0, + _names['JUMP_BACKWARD']: 0, + _names['JUMP_BACKWARD_JIT']: 0, + _names['JUMP_BACKWARD_NO_INTERRUPT']: 0, + _names['JUMP_BACKWARD_NO_JIT']: 0, + _names['JUMP_FORWARD']: 0, + _names['JUMP_IF_FALSE']: 1, + _names['JUMP_IF_TRUE']: 1, + _names['JUMP_NO_INTERRUPT']: 0, + _names['LIST_APPEND']: lambda oparg: 2 + (oparg-1), + _names['LIST_EXTEND']: lambda oparg: 2 + (oparg-1), + _names['LOAD_ATTR']: 1, + _names['LOAD_ATTR_CLASS']: 1, + _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: 1, + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: 1, + _names['LOAD_ATTR_INSTANCE_VALUE']: 1, + _names['LOAD_ATTR_METHOD_LAZY_DICT']: 1, + _names['LOAD_ATTR_METHOD_NO_DICT']: 1, + _names['LOAD_ATTR_METHOD_WITH_VALUES']: 1, + _names['LOAD_ATTR_MODULE']: 1, + _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: 1, + _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: 1, + _names['LOAD_ATTR_PROPERTY']: 1, + _names['LOAD_ATTR_SLOT']: 1, + _names['LOAD_ATTR_WITH_HINT']: 1, + _names['LOAD_BUILD_CLASS']: 0, + _names['LOAD_CLOSURE']: 0, + _names['LOAD_COMMON_CONSTANT']: 0, + _names['LOAD_CONST']: 0, + _names['LOAD_CONST_IMMORTAL']: 0, + _names['LOAD_CONST_MORTAL']: 0, + _names['LOAD_DEREF']: 0, + _names['LOAD_FAST']: 0, + _names['LOAD_FAST_AND_CLEAR']: 0, + _names['LOAD_FAST_BORROW']: 0, + _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW']: 0, + _names['LOAD_FAST_CHECK']: 0, + _names['LOAD_FAST_LOAD_FAST']: 0, + _names['LOAD_FROM_DICT_OR_DEREF']: 1, + _names['LOAD_FROM_DICT_OR_GLOBALS']: 1, + _names['LOAD_GLOBAL']: 0, + _names['LOAD_GLOBAL_BUILTIN']: 0, + _names['LOAD_GLOBAL_MODULE']: 0, + _names['LOAD_LOCALS']: 0, + _names['LOAD_NAME']: 0, + _names['LOAD_SMALL_INT']: 0, + _names['LOAD_SPECIAL']: 1, + _names['LOAD_SUPER_ATTR']: 3, + _names['LOAD_SUPER_ATTR_ATTR']: 3, + _names['LOAD_SUPER_ATTR_METHOD']: 3, + _names['MAKE_CELL']: 0, + _names['MAKE_FUNCTION']: 1, + _names['MAP_ADD']: lambda oparg: 3 + (oparg - 1), + _names['MATCH_CLASS']: 3, + _names['MATCH_KEYS']: 2, + _names['MATCH_MAPPING']: 1, + _names['MATCH_SEQUENCE']: 1, + _names['NOP']: 0, + _names['NOT_TAKEN']: 0, + _names['POP_BLOCK']: 0, + _names['POP_EXCEPT']: 1, + _names['POP_ITER']: 1, + _names['POP_JUMP_IF_FALSE']: 1, + _names['POP_JUMP_IF_NONE']: 1, + _names['POP_JUMP_IF_NOT_NONE']: 1, + _names['POP_JUMP_IF_TRUE']: 1, + _names['POP_TOP']: 1, + _names['PUSH_EXC_INFO']: 1, + _names['PUSH_NULL']: 0, + _names['RAISE_VARARGS']: lambda oparg: oparg, + _names['RERAISE']: lambda oparg: 1 + oparg, + _names['RESERVED']: 0, + _names['RESUME']: 0, + _names['RESUME_CHECK']: 0, + _names['RETURN_GENERATOR']: 0, + _names['RETURN_VALUE']: 1, + _names['SEND']: 2, + _names['SEND_GEN']: 2, + _names['SETUP_ANNOTATIONS']: 0, + _names['SETUP_CLEANUP']: 0, + _names['SETUP_FINALLY']: 0, + _names['SETUP_WITH']: 0, + _names['SET_ADD']: lambda oparg: 2 + (oparg-1), + _names['SET_FUNCTION_ATTRIBUTE']: 2, + _names['SET_UPDATE']: lambda oparg: 2 + (oparg-1), + _names['STORE_ATTR']: 2, + _names['STORE_ATTR_INSTANCE_VALUE']: 2, + _names['STORE_ATTR_SLOT']: 2, + _names['STORE_ATTR_WITH_HINT']: 2, + _names['STORE_DEREF']: 1, + _names['STORE_FAST']: 1, + _names['STORE_FAST_LOAD_FAST']: 1, + _names['STORE_FAST_MAYBE_NULL']: 1, + _names['STORE_FAST_STORE_FAST']: 2, + _names['STORE_GLOBAL']: 1, + _names['STORE_NAME']: 1, + _names['STORE_SLICE']: 4, + _names['STORE_SUBSCR']: 3, + _names['STORE_SUBSCR_DICT']: 3, + _names['STORE_SUBSCR_LIST_INT']: 3, + _names['SWAP']: lambda oparg: 2 + (oparg-2), + _names['TO_BOOL']: 1, + _names['TO_BOOL_ALWAYS_TRUE']: 1, + _names['TO_BOOL_BOOL']: 1, + _names['TO_BOOL_INT']: 1, + _names['TO_BOOL_LIST']: 1, + _names['TO_BOOL_NONE']: 1, + _names['TO_BOOL_STR']: 1, + _names['UNARY_INVERT']: 1, + _names['UNARY_NEGATIVE']: 1, + _names['UNARY_NOT']: 1, + _names['UNPACK_EX']: 1, + _names['UNPACK_SEQUENCE']: 1, + _names['UNPACK_SEQUENCE_LIST']: 1, + _names['UNPACK_SEQUENCE_TUPLE']: 1, + _names['UNPACK_SEQUENCE_TWO_TUPLE']: 1, + _names['WITH_EXCEPT_START']: 5, + _names['YIELD_VALUE']: 1, + } + # `_PyOpcode_num_pushed(opcode, oparg)`, likewise. + pushed = { + _names['ANNOTATIONS_PLACEHOLDER']: 0, + _names['BINARY_OP']: 1, + _names['BINARY_OP_ADD_FLOAT']: 1, + _names['BINARY_OP_ADD_INT']: 1, + _names['BINARY_OP_ADD_UNICODE']: 1, + _names['BINARY_OP_EXTEND']: 1, + _names['BINARY_OP_INPLACE_ADD_UNICODE']: 0, + _names['BINARY_OP_MULTIPLY_FLOAT']: 1, + _names['BINARY_OP_MULTIPLY_INT']: 1, + _names['BINARY_OP_SUBSCR_DICT']: 1, + _names['BINARY_OP_SUBSCR_GETITEM']: 0, + _names['BINARY_OP_SUBSCR_LIST_INT']: 1, + _names['BINARY_OP_SUBSCR_LIST_SLICE']: 1, + _names['BINARY_OP_SUBSCR_STR_INT']: 1, + _names['BINARY_OP_SUBSCR_TUPLE_INT']: 1, + _names['BINARY_OP_SUBTRACT_FLOAT']: 1, + _names['BINARY_OP_SUBTRACT_INT']: 1, + _names['BINARY_SLICE']: 1, + _names['BUILD_INTERPOLATION']: 1, + _names['BUILD_LIST']: 1, + _names['BUILD_MAP']: 1, + _names['BUILD_SET']: 1, + _names['BUILD_SLICE']: 1, + _names['BUILD_STRING']: 1, + _names['BUILD_TEMPLATE']: 1, + _names['BUILD_TUPLE']: 1, + _names['CACHE']: 0, + _names['CALL']: 1, + _names['CALL_ALLOC_AND_ENTER_INIT']: 0, + _names['CALL_BOUND_METHOD_EXACT_ARGS']: 0, + _names['CALL_BOUND_METHOD_GENERAL']: 0, + _names['CALL_BUILTIN_CLASS']: 1, + _names['CALL_BUILTIN_FAST']: 1, + _names['CALL_BUILTIN_FAST_WITH_KEYWORDS']: 1, + _names['CALL_BUILTIN_O']: 1, + _names['CALL_FUNCTION_EX']: 1, + _names['CALL_INTRINSIC_1']: 1, + _names['CALL_INTRINSIC_2']: 1, + _names['CALL_ISINSTANCE']: 1, + _names['CALL_KW']: 1, + _names['CALL_KW_BOUND_METHOD']: 0, + _names['CALL_KW_NON_PY']: 1, + _names['CALL_KW_PY']: 0, + _names['CALL_LEN']: 1, + _names['CALL_LIST_APPEND']: 0, + _names['CALL_METHOD_DESCRIPTOR_FAST']: 1, + _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS']: 1, + _names['CALL_METHOD_DESCRIPTOR_NOARGS']: 1, + _names['CALL_METHOD_DESCRIPTOR_O']: 1, + _names['CALL_NON_PY_GENERAL']: 1, + _names['CALL_PY_EXACT_ARGS']: 0, + _names['CALL_PY_GENERAL']: 0, + _names['CALL_STR_1']: 1, + _names['CALL_TUPLE_1']: 1, + _names['CALL_TYPE_1']: 1, + _names['CHECK_EG_MATCH']: 2, + _names['CHECK_EXC_MATCH']: 2, + _names['CLEANUP_THROW']: 2, + _names['COMPARE_OP']: 1, + _names['COMPARE_OP_FLOAT']: 1, + _names['COMPARE_OP_INT']: 1, + _names['COMPARE_OP_STR']: 1, + _names['CONTAINS_OP']: 1, + _names['CONTAINS_OP_DICT']: 1, + _names['CONTAINS_OP_SET']: 1, + _names['CONVERT_VALUE']: 1, + _names['COPY']: lambda oparg: 2 + (oparg-1), + _names['COPY_FREE_VARS']: 0, + _names['DELETE_ATTR']: 0, + _names['DELETE_DEREF']: 0, + _names['DELETE_FAST']: 0, + _names['DELETE_GLOBAL']: 0, + _names['DELETE_NAME']: 0, + _names['DELETE_SUBSCR']: 0, + _names['DICT_MERGE']: lambda oparg: 4 + (oparg - 1), + _names['DICT_UPDATE']: lambda oparg: 1 + (oparg - 1), + _names['END_ASYNC_FOR']: 0, + _names['END_FOR']: 0, + _names['END_SEND']: 1, + _names['ENTER_EXECUTOR']: 0, + _names['EXIT_INIT_CHECK']: 0, + _names['EXTENDED_ARG']: 0, + _names['FORMAT_SIMPLE']: 1, + _names['FORMAT_WITH_SPEC']: 1, + _names['FOR_ITER']: 2, + _names['FOR_ITER_GEN']: 1, + _names['FOR_ITER_LIST']: 2, + _names['FOR_ITER_RANGE']: 2, + _names['FOR_ITER_TUPLE']: 2, + _names['GET_AITER']: 1, + _names['GET_ANEXT']: 2, + _names['GET_AWAITABLE']: 1, + _names['GET_ITER']: 1, + _names['GET_LEN']: 2, + _names['GET_YIELD_FROM_ITER']: 1, + _names['IMPORT_FROM']: 2, + _names['IMPORT_NAME']: 1, + _names['INSTRUMENTED_CALL']: 1, + _names['INSTRUMENTED_CALL_FUNCTION_EX']: 1, + _names['INSTRUMENTED_CALL_KW']: 1, + _names['INSTRUMENTED_END_ASYNC_FOR']: 0, + _names['INSTRUMENTED_END_FOR']: 1, + _names['INSTRUMENTED_END_SEND']: 1, + _names['INSTRUMENTED_FOR_ITER']: 2, + _names['INSTRUMENTED_INSTRUCTION']: 0, + _names['INSTRUMENTED_JUMP_BACKWARD']: 0, + _names['INSTRUMENTED_JUMP_FORWARD']: 0, + _names['INSTRUMENTED_LINE']: 0, + _names['INSTRUMENTED_LOAD_SUPER_ATTR']: lambda oparg: 1 + (oparg & 1), + _names['INSTRUMENTED_NOT_TAKEN']: 0, + _names['INSTRUMENTED_POP_ITER']: 0, + _names['INSTRUMENTED_POP_JUMP_IF_FALSE']: 0, + _names['INSTRUMENTED_POP_JUMP_IF_NONE']: 0, + _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE']: 0, + _names['INSTRUMENTED_POP_JUMP_IF_TRUE']: 0, + _names['INSTRUMENTED_RESUME']: 0, + _names['INSTRUMENTED_RETURN_VALUE']: 1, + _names['INSTRUMENTED_YIELD_VALUE']: 1, + _names['INTERPRETER_EXIT']: 0, + _names['IS_OP']: 1, + _names['JUMP']: 0, + _names['JUMP_BACKWARD']: 0, + _names['JUMP_BACKWARD_JIT']: 0, + _names['JUMP_BACKWARD_NO_INTERRUPT']: 0, + _names['JUMP_BACKWARD_NO_JIT']: 0, + _names['JUMP_FORWARD']: 0, + _names['JUMP_IF_FALSE']: 1, + _names['JUMP_IF_TRUE']: 1, + _names['JUMP_NO_INTERRUPT']: 0, + _names['LIST_APPEND']: lambda oparg: 1 + (oparg-1), + _names['LIST_EXTEND']: lambda oparg: 1 + (oparg-1), + _names['LOAD_ATTR']: lambda oparg: 1 + (oparg&1), + _names['LOAD_ATTR_CLASS']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN']: 1, + _names['LOAD_ATTR_INSTANCE_VALUE']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_METHOD_LAZY_DICT']: 2, + _names['LOAD_ATTR_METHOD_NO_DICT']: 2, + _names['LOAD_ATTR_METHOD_WITH_VALUES']: 2, + _names['LOAD_ATTR_MODULE']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT']: 1, + _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES']: 1, + _names['LOAD_ATTR_PROPERTY']: 0, + _names['LOAD_ATTR_SLOT']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_ATTR_WITH_HINT']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_BUILD_CLASS']: 1, + _names['LOAD_CLOSURE']: 1, + _names['LOAD_COMMON_CONSTANT']: 1, + _names['LOAD_CONST']: 1, + _names['LOAD_CONST_IMMORTAL']: 1, + _names['LOAD_CONST_MORTAL']: 1, + _names['LOAD_DEREF']: 1, + _names['LOAD_FAST']: 1, + _names['LOAD_FAST_AND_CLEAR']: 1, + _names['LOAD_FAST_BORROW']: 1, + _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW']: 2, + _names['LOAD_FAST_CHECK']: 1, + _names['LOAD_FAST_LOAD_FAST']: 2, + _names['LOAD_FROM_DICT_OR_DEREF']: 1, + _names['LOAD_FROM_DICT_OR_GLOBALS']: 1, + _names['LOAD_GLOBAL']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_GLOBAL_BUILTIN']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_GLOBAL_MODULE']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_LOCALS']: 1, + _names['LOAD_NAME']: 1, + _names['LOAD_SMALL_INT']: 1, + _names['LOAD_SPECIAL']: 2, + _names['LOAD_SUPER_ATTR']: lambda oparg: 1 + (oparg & 1), + _names['LOAD_SUPER_ATTR_ATTR']: 1, + _names['LOAD_SUPER_ATTR_METHOD']: 2, + _names['MAKE_CELL']: 0, + _names['MAKE_FUNCTION']: 1, + _names['MAP_ADD']: lambda oparg: 1 + (oparg - 1), + _names['MATCH_CLASS']: 1, + _names['MATCH_KEYS']: 3, + _names['MATCH_MAPPING']: 2, + _names['MATCH_SEQUENCE']: 2, + _names['NOP']: 0, + _names['NOT_TAKEN']: 0, + _names['POP_BLOCK']: 0, + _names['POP_EXCEPT']: 0, + _names['POP_ITER']: 0, + _names['POP_JUMP_IF_FALSE']: 0, + _names['POP_JUMP_IF_NONE']: 0, + _names['POP_JUMP_IF_NOT_NONE']: 0, + _names['POP_JUMP_IF_TRUE']: 0, + _names['POP_TOP']: 0, + _names['PUSH_EXC_INFO']: 2, + _names['PUSH_NULL']: 1, + _names['RAISE_VARARGS']: 0, + _names['RERAISE']: lambda oparg: oparg, + _names['RESERVED']: 0, + _names['RESUME']: 0, + _names['RESUME_CHECK']: 0, + _names['RETURN_GENERATOR']: 1, + _names['RETURN_VALUE']: 1, + _names['SEND']: 2, + _names['SEND_GEN']: 1, + _names['SETUP_ANNOTATIONS']: 0, + _names['SETUP_CLEANUP']: 2, + _names['SETUP_FINALLY']: 1, + _names['SETUP_WITH']: 1, + _names['SET_ADD']: lambda oparg: 1 + (oparg-1), + _names['SET_FUNCTION_ATTRIBUTE']: 1, + _names['SET_UPDATE']: lambda oparg: 1 + (oparg-1), + _names['STORE_ATTR']: 0, + _names['STORE_ATTR_INSTANCE_VALUE']: 0, + _names['STORE_ATTR_SLOT']: 0, + _names['STORE_ATTR_WITH_HINT']: 0, + _names['STORE_DEREF']: 0, + _names['STORE_FAST']: 0, + _names['STORE_FAST_LOAD_FAST']: 1, + _names['STORE_FAST_MAYBE_NULL']: 0, + _names['STORE_FAST_STORE_FAST']: 0, + _names['STORE_GLOBAL']: 0, + _names['STORE_NAME']: 0, + _names['STORE_SLICE']: 0, + _names['STORE_SUBSCR']: 0, + _names['STORE_SUBSCR_DICT']: 0, + _names['STORE_SUBSCR_LIST_INT']: 0, + _names['SWAP']: lambda oparg: 2 + (oparg-2), + _names['TO_BOOL']: 1, + _names['TO_BOOL_ALWAYS_TRUE']: 1, + _names['TO_BOOL_BOOL']: 1, + _names['TO_BOOL_INT']: 1, + _names['TO_BOOL_LIST']: 1, + _names['TO_BOOL_NONE']: 1, + _names['TO_BOOL_STR']: 1, + _names['UNARY_INVERT']: 1, + _names['UNARY_NEGATIVE']: 1, + _names['UNARY_NOT']: 1, + _names['UNPACK_EX']: lambda oparg: 1 + (oparg & 0xFF) + (oparg >> 8), + _names['UNPACK_SEQUENCE']: lambda oparg: oparg, + _names['UNPACK_SEQUENCE_LIST']: lambda oparg: oparg, + _names['UNPACK_SEQUENCE_TUPLE']: lambda oparg: oparg, + _names['UNPACK_SEQUENCE_TWO_TUPLE']: 2, + _names['WITH_EXCEPT_START']: 6, + _names['YIELD_VALUE']: 1, + } + _STACK_TABLES = (deopt, popped, pushed) + return _STACK_TABLES + _BLOCK_PUSH = frozenset(_names[n] for n in ['SETUP_FINALLY', 'SETUP_WITH', 'SETUP_CLEANUP']) +# The valid opcodes, and those with each flag `has_*` reports: `opcode` +# (Lib/opcode.py) asks every predicate about every opcode at import, so +# a valid opcode is answered by one set lookup. +_VALID = frozenset(_FLAGS) +_HAS_ARG = frozenset({ + _names['BINARY_OP'], + _names['BUILD_INTERPOLATION'], + _names['BUILD_LIST'], + _names['BUILD_MAP'], + _names['BUILD_SET'], + _names['BUILD_SLICE'], + _names['BUILD_STRING'], + _names['BUILD_TUPLE'], + _names['CALL'], + _names['CALL_ALLOC_AND_ENTER_INIT'], + _names['CALL_BOUND_METHOD_EXACT_ARGS'], + _names['CALL_BOUND_METHOD_GENERAL'], + _names['CALL_BUILTIN_CLASS'], + _names['CALL_BUILTIN_FAST'], + _names['CALL_BUILTIN_FAST_WITH_KEYWORDS'], + _names['CALL_BUILTIN_O'], + _names['CALL_INTRINSIC_1'], + _names['CALL_INTRINSIC_2'], + _names['CALL_ISINSTANCE'], + _names['CALL_KW'], + _names['CALL_KW_BOUND_METHOD'], + _names['CALL_KW_NON_PY'], + _names['CALL_KW_PY'], + _names['CALL_LIST_APPEND'], + _names['CALL_METHOD_DESCRIPTOR_FAST'], + _names['CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS'], + _names['CALL_METHOD_DESCRIPTOR_NOARGS'], + _names['CALL_METHOD_DESCRIPTOR_O'], + _names['CALL_NON_PY_GENERAL'], + _names['CALL_PY_EXACT_ARGS'], + _names['CALL_PY_GENERAL'], + _names['CALL_STR_1'], + _names['CALL_TUPLE_1'], + _names['CALL_TYPE_1'], + _names['COMPARE_OP'], + _names['COMPARE_OP_FLOAT'], + _names['COMPARE_OP_INT'], + _names['COMPARE_OP_STR'], + _names['CONTAINS_OP'], + _names['CONTAINS_OP_DICT'], + _names['CONTAINS_OP_SET'], + _names['CONVERT_VALUE'], + _names['COPY'], + _names['COPY_FREE_VARS'], + _names['DELETE_ATTR'], + _names['DELETE_DEREF'], + _names['DELETE_FAST'], + _names['DELETE_GLOBAL'], + _names['DELETE_NAME'], + _names['DICT_MERGE'], + _names['DICT_UPDATE'], + _names['END_ASYNC_FOR'], + _names['ENTER_EXECUTOR'], + _names['EXTENDED_ARG'], + _names['FOR_ITER'], + _names['FOR_ITER_GEN'], + _names['FOR_ITER_LIST'], + _names['FOR_ITER_RANGE'], + _names['FOR_ITER_TUPLE'], + _names['GET_AWAITABLE'], + _names['IMPORT_FROM'], + _names['IMPORT_NAME'], + _names['INSTRUMENTED_CALL'], + _names['INSTRUMENTED_CALL_KW'], + _names['INSTRUMENTED_END_ASYNC_FOR'], + _names['INSTRUMENTED_FOR_ITER'], + _names['INSTRUMENTED_JUMP_BACKWARD'], + _names['INSTRUMENTED_JUMP_FORWARD'], + _names['INSTRUMENTED_LOAD_SUPER_ATTR'], + _names['INSTRUMENTED_POP_JUMP_IF_FALSE'], + _names['INSTRUMENTED_POP_JUMP_IF_NONE'], + _names['INSTRUMENTED_POP_JUMP_IF_NOT_NONE'], + _names['INSTRUMENTED_POP_JUMP_IF_TRUE'], + _names['INSTRUMENTED_RESUME'], + _names['INSTRUMENTED_YIELD_VALUE'], + _names['IS_OP'], + _names['JUMP_BACKWARD'], + _names['JUMP_BACKWARD_JIT'], + _names['JUMP_BACKWARD_NO_INTERRUPT'], + _names['JUMP_BACKWARD_NO_JIT'], + _names['JUMP_FORWARD'], + _names['LIST_APPEND'], + _names['LIST_EXTEND'], + _names['LOAD_ATTR'], + _names['LOAD_ATTR_CLASS'], + _names['LOAD_ATTR_CLASS_WITH_METACLASS_CHECK'], + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN'], + _names['LOAD_ATTR_INSTANCE_VALUE'], + _names['LOAD_ATTR_METHOD_LAZY_DICT'], + _names['LOAD_ATTR_METHOD_NO_DICT'], + _names['LOAD_ATTR_METHOD_WITH_VALUES'], + _names['LOAD_ATTR_MODULE'], + _names['LOAD_ATTR_NONDESCRIPTOR_NO_DICT'], + _names['LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES'], + _names['LOAD_ATTR_PROPERTY'], + _names['LOAD_ATTR_SLOT'], + _names['LOAD_ATTR_WITH_HINT'], + _names['LOAD_COMMON_CONSTANT'], + _names['LOAD_CONST'], + _names['LOAD_CONST_IMMORTAL'], + _names['LOAD_CONST_MORTAL'], + _names['LOAD_DEREF'], + _names['LOAD_FAST'], + _names['LOAD_FAST_AND_CLEAR'], + _names['LOAD_FAST_BORROW'], + _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW'], + _names['LOAD_FAST_CHECK'], + _names['LOAD_FAST_LOAD_FAST'], + _names['LOAD_FROM_DICT_OR_DEREF'], + _names['LOAD_FROM_DICT_OR_GLOBALS'], + _names['LOAD_GLOBAL'], + _names['LOAD_GLOBAL_BUILTIN'], + _names['LOAD_GLOBAL_MODULE'], + _names['LOAD_NAME'], + _names['LOAD_SMALL_INT'], + _names['LOAD_SPECIAL'], + _names['LOAD_SUPER_ATTR'], + _names['LOAD_SUPER_ATTR_ATTR'], + _names['LOAD_SUPER_ATTR_METHOD'], + _names['MAKE_CELL'], + _names['MAP_ADD'], + _names['MATCH_CLASS'], + _names['POP_JUMP_IF_FALSE'], + _names['POP_JUMP_IF_NONE'], + _names['POP_JUMP_IF_NOT_NONE'], + _names['POP_JUMP_IF_TRUE'], + _names['RAISE_VARARGS'], + _names['RERAISE'], + _names['RESUME'], + _names['SEND'], + _names['SEND_GEN'], + _names['SET_ADD'], + _names['SET_FUNCTION_ATTRIBUTE'], + _names['SET_UPDATE'], + _names['STORE_ATTR'], + _names['STORE_ATTR_WITH_HINT'], + _names['STORE_DEREF'], + _names['STORE_FAST'], + _names['STORE_FAST_LOAD_FAST'], + _names['STORE_FAST_STORE_FAST'], + _names['STORE_GLOBAL'], + _names['STORE_NAME'], + _names['SWAP'], + _names['UNPACK_EX'], + _names['UNPACK_SEQUENCE'], + _names['UNPACK_SEQUENCE_LIST'], + _names['UNPACK_SEQUENCE_TUPLE'], + _names['UNPACK_SEQUENCE_TWO_TUPLE'], + _names['YIELD_VALUE'], + _names['JUMP'], + _names['JUMP_IF_FALSE'], + _names['JUMP_IF_TRUE'], + _names['JUMP_NO_INTERRUPT'], + _names['LOAD_CLOSURE'], + _names['SETUP_CLEANUP'], + _names['SETUP_FINALLY'], + _names['SETUP_WITH'], + _names['STORE_FAST_MAYBE_NULL'], +}) +_HAS_CONST = frozenset({ + _names['LOAD_CONST'], + _names['LOAD_CONST_IMMORTAL'], + _names['LOAD_CONST_MORTAL'], +}) +_HAS_NAME = frozenset({ + _names['DELETE_ATTR'], + _names['DELETE_GLOBAL'], + _names['DELETE_NAME'], + _names['IMPORT_FROM'], + _names['IMPORT_NAME'], + _names['INSTRUMENTED_LOAD_SUPER_ATTR'], + _names['LOAD_ATTR'], + _names['LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN'], + _names['LOAD_ATTR_WITH_HINT'], + _names['LOAD_FROM_DICT_OR_GLOBALS'], + _names['LOAD_GLOBAL'], + _names['LOAD_NAME'], + _names['LOAD_SUPER_ATTR'], + _names['LOAD_SUPER_ATTR_ATTR'], + _names['LOAD_SUPER_ATTR_METHOD'], + _names['STORE_ATTR'], + _names['STORE_ATTR_WITH_HINT'], + _names['STORE_GLOBAL'], + _names['STORE_NAME'], +}) +_HAS_JUMP = frozenset({ + _names['END_ASYNC_FOR'], + _names['FOR_ITER'], + _names['FOR_ITER_LIST'], + _names['FOR_ITER_RANGE'], + _names['FOR_ITER_TUPLE'], + _names['INSTRUMENTED_END_ASYNC_FOR'], + _names['INSTRUMENTED_FOR_ITER'], + _names['JUMP_BACKWARD'], + _names['JUMP_BACKWARD_JIT'], + _names['JUMP_BACKWARD_NO_INTERRUPT'], + _names['JUMP_BACKWARD_NO_JIT'], + _names['JUMP_FORWARD'], + _names['POP_JUMP_IF_FALSE'], + _names['POP_JUMP_IF_NONE'], + _names['POP_JUMP_IF_NOT_NONE'], + _names['POP_JUMP_IF_TRUE'], + _names['SEND'], + _names['JUMP'], + _names['JUMP_IF_FALSE'], + _names['JUMP_IF_TRUE'], + _names['JUMP_NO_INTERRUPT'], +}) +_HAS_FREE = frozenset({ + _names['DELETE_DEREF'], + _names['LOAD_FROM_DICT_OR_DEREF'], + _names['MAKE_CELL'], + _names['STORE_DEREF'], +}) +_HAS_LOCAL = frozenset({ + _names['BINARY_OP_INPLACE_ADD_UNICODE'], + _names['DELETE_FAST'], + _names['LOAD_DEREF'], + _names['LOAD_FAST'], + _names['LOAD_FAST_AND_CLEAR'], + _names['LOAD_FAST_BORROW'], + _names['LOAD_FAST_BORROW_LOAD_FAST_BORROW'], + _names['LOAD_FAST_CHECK'], + _names['LOAD_FAST_LOAD_FAST'], + _names['STORE_FAST'], + _names['STORE_FAST_LOAD_FAST'], + _names['STORE_FAST_STORE_FAST'], + _names['LOAD_CLOSURE'], + _names['STORE_FAST_MAYBE_NULL'], +}) + def is_valid(opcode): """Return True if opcode is valid, False otherwise.""" - return 0 <= opcode < _NUM_OPCODES and opcode in _FLAGS + return opcode in _VALID or (0 <= opcode < _NUM_OPCODES and opcode in _FLAGS) def _has(opcode, flag): @@ -870,31 +1117,61 @@ def _has(opcode, flag): def has_arg(opcode): """Return True if the opcode uses its oparg, False otherwise.""" + if opcode in _HAS_ARG: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_ARG_FLAG) def has_const(opcode): """Return True if the opcode accesses a constant, False otherwise.""" + if opcode in _HAS_CONST: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_CONST_FLAG) def has_name(opcode): """Return True if the opcode accesses an attribute by name, False otherwise.""" + if opcode in _HAS_NAME: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_NAME_FLAG) def has_jump(opcode): """Return True if the opcode has a jump target, False otherwise.""" + if opcode in _HAS_JUMP: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_JUMP_FLAG) def has_free(opcode): """Return True if the opcode accesses a free variable, False otherwise.""" + if opcode in _HAS_FREE: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_FREE_FLAG) def has_local(opcode): """Return True if the opcode accesses a local variable, False otherwise.""" + if opcode in _HAS_LOCAL: + return True + if opcode in _VALID: + return False + # An invalid opcode, or not an int (which raises as before). return _has(opcode, HAS_LOCAL_FLAG) @@ -914,16 +1191,17 @@ def stack_effect(opcode, oparg=None, /, *, jump=None): jump_int = 0 else: raise ValueError("stack_effect: jump must be False, True or None") + deopt, popped_table, pushed_table = _stack_tables() # `get_stack_effects` (Python/flowgraph.c): specialized forms and # unknown opcodes have no effect entry. - if opcode < 0 or (opcode <= _MAX_REAL_OPCODE and _DEOPT.get(opcode, opcode) != opcode): + if opcode < 0 or (opcode <= _MAX_REAL_OPCODE and deopt.get(opcode, opcode) != opcode): raise ValueError("invalid opcode or oparg") - popped = _POPPED.get(opcode) - pushed = _PUSHED.get(opcode) + popped = popped_table.get(opcode) + pushed = pushed_table.get(opcode) if popped is None or pushed is None: raise ValueError("invalid opcode or oparg") - npop = popped(oparg_int) - npush = pushed(oparg_int) + npop = popped if popped.__class__ is int else popped(oparg_int) + npush = pushed if pushed.__class__ is int else pushed(oparg_int) if npop < 0 or npush < 0: raise ValueError("invalid opcode or oparg") if opcode in _BLOCK_PUSH and not jump_int: diff --git a/crates/weavepy-vm/src/stdlib/python/_weave_flowgraph.py b/crates/weavepy-vm/src/stdlib/python/_weave_flowgraph.py index f2668fe3..8de13ddb 100644 --- a/crates/weavepy-vm/src/stdlib/python/_weave_flowgraph.py +++ b/crates/weavepy-vm/src/stdlib/python/_weave_flowgraph.py @@ -410,22 +410,25 @@ def check_cfg(g): # ---- stack effects --------------------------------------------------------- -_POPPED = _opcode_tables._POPPED -_PUSHED = _opcode_tables._PUSHED -_DEOPT = _opcode_tables._DEOPT +# A table entry is the count, or a function of `oparg` (see `_opcode`). +_DEOPT, _POPPED, _PUSHED = _opcode_tables._stack_tables() _MAX_REAL_OPCODE = _opcode_tables._MAX_REAL_OPCODE def num_popped(opcode, oparg): """``_PyOpcode_num_popped``; ``-1`` for an unknown opcode.""" fn = _POPPED.get(opcode) - return -1 if fn is None else fn(oparg) + if fn is None: + return -1 + return fn if fn.__class__ is int else fn(oparg) def num_pushed(opcode, oparg): """``_PyOpcode_num_pushed``; ``-1`` for an unknown opcode.""" fn = _PUSHED.get(opcode) - return -1 if fn is None else fn(oparg) + if fn is None: + return -1 + return fn if fn.__class__ is int else fn(oparg) def get_stack_effects(opcode, oparg, jump): diff --git a/crates/weavepy-vm/src/stdlib/python/re_engine.py b/crates/weavepy-vm/src/stdlib/python/re_engine.py index 4e97d43d..60ad9e7c 100644 --- a/crates/weavepy-vm/src/stdlib/python/re_engine.py +++ b/crates/weavepy-vm/src/stdlib/python/re_engine.py @@ -2,11 +2,14 @@ # WeavePy: user-visible Pattern / Match objects for the re package. # # CPython implements `re.Pattern` and `re.Match` as C types inside the -# `_sre` extension. WeavePy instead keeps `_sre` as a pure-data -# backtracking core (compile + exec returning group spans) and builds -# the user-facing objects here, in Python. Doing so means callable -# `re.sub` replacements, `finditer`, `Scanner`, etc. all run on the -# normal interpreter without the engine ever re-entering the VM. +# `_sre` extension. WeavePy declares them here as Python classes with +# `__slots__`, and `_sre.install` (at the end of this module) replaces +# their hot methods with native bodies that work on the slot storage. +# The natives serve exact `str`/`bytes` subjects with plain arguments; +# everything else calls the Python methods below, which remain the +# reference implementation. Patterns, matches and scanners are always +# built by `_sre` (`make_pattern`, `exec_match`), never by calling the +# classes. # # Behaviour (group semantics, greedy/lazy scanning, empty-match # handling, split/sub/subn rules) follows CPython 3.13 exactly. @@ -37,8 +40,9 @@ def compile_pattern(pattern, flags, code, groups, groupindex, indexgroup): """Build a Pattern. Called by re._compiler.compile().""" - handle = _sre.compile(code, groups) - return Pattern(handle, pattern, flags, groups, groupindex, indexgroup) + # Issue 14260: a non-modifiable mapping, like CPython's C getter. + return _sre.make_pattern(Pattern, pattern, flags, code, groups, + _MappingProxy(groupindex), indexgroup) # Code-point length is O(n) for WeavePy's UTF-8 `str`, and regex-driven @@ -109,6 +113,12 @@ def _clamp_span(string, pos, endpos): class Pattern: __module__ = 're' + # The native methods read these by position (see `_sre.install`). + # `_code` is the compiled code's handle; `_indexgroup` maps a group + # number to its name or None. + __slots__ = ('_code', 'pattern', 'flags', 'groups', 'groupindex', + '_indexgroup', '__weakref__') + # PEP 585: `re.Pattern[str]` / `re.Pattern[bytes]` yield a # `types.GenericAlias` (CPython exposes this on the C `Pattern` type). __class_getitem__ = classmethod(_GenericAlias) @@ -118,15 +128,8 @@ class Pattern: def __init_subclass__(cls, /, **kwargs): raise TypeError("type 're.Pattern' is not an acceptable base type") - def __init__(self, handle, pattern, flags, groups, groupindex, indexgroup): - self._handle = handle - self.pattern = pattern - self.flags = flags - self.groups = groups - # Issue 14260: a non-modifiable mapping, like CPython's C getter. - self.groupindex = _MappingProxy(groupindex) - # tuple: group number (1-based) -> name or None - self._indexgroup = indexgroup + def __new__(cls, *args, **kwargs): + raise TypeError("cannot create 're.Pattern' instances") # CPython `_sre` pattern rich compare (bpo-35966): equal iff the # pattern string/bytes and effective flags agree; hash must follow. @@ -152,19 +155,22 @@ def __hash__(self): # -- internal -------------------------------------------------------- - def _exec(self, string, pos, endpos, mode, must_advance): + def _exec(self, string, pos, endpos, mode, must_advance, mpos=None): # CPython `_sre` rejects a str/bytes mismatch between the pattern and # the subject before matching (test_fnmatch.test_mix_bytes_str, plus - # the analogous re tests). `self.pattern` is the original str/bytes. + # the analogous re tests). `self.pattern` is the original str/bytes, + # or None for a `re.Scanner` pattern, which accepts either. if isinstance(self.pattern, str): if not isinstance(string, str): raise TypeError( "cannot use a string pattern on a bytes-like object") - elif isinstance(string, str): + elif self.pattern is not None and isinstance(string, str): raise TypeError( "cannot use a bytes pattern on a string-like object") - return _sre.exec(self._handle, string, pos, endpos, mode, - 1 if must_advance else 0) + # A Match reporting `mpos` (default `pos`) and `endpos`, or None. + return _sre.exec_match(self, string, pos, endpos, mode, + 1 if must_advance else 0, + pos if mpos is None else mpos) def _iter(self, string, pos, endpos, guard=None): # `guard` is a live buffer export pinning a mutable subject @@ -175,11 +181,12 @@ def _iter(self, string, pos, endpos, guard=None): must_advance = False opos, oendpos = pos, endpos while pos <= endpos: - r = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance) - if r is None: + m = self._exec(string, pos, endpos, _MODE_SEARCH, + must_advance, opos) + if m is None: break - start, end = r[0], r[1] - yield Match(self, string, opos, oendpos, r) + start, end = m._marks[0], m._marks[1] + yield m must_advance = start == end pos = end finally: @@ -190,24 +197,15 @@ def _iter(self, string, pos, endpos, guard=None): def match(self, string, pos=0, endpos=None): p, e = _clamp_span(string, pos, endpos) - r = self._exec(string, p, e, _MODE_MATCH, False) - if r is None: - return None - return Match(self, string, p, e, r) + return self._exec(string, p, e, _MODE_MATCH, False) def fullmatch(self, string, pos=0, endpos=None): p, e = _clamp_span(string, pos, endpos) - r = self._exec(string, p, e, _MODE_FULLMATCH, False) - if r is None: - return None - return Match(self, string, p, e, r) + return self._exec(string, p, e, _MODE_FULLMATCH, False) def search(self, string, pos=0, endpos=None): p, e = _clamp_span(string, pos, endpos) - r = self._exec(string, p, e, _MODE_SEARCH, False) - if r is None: - return None - return Match(self, string, p, e, r) + return self._exec(string, p, e, _MODE_SEARCH, False) def findall(self, string, pos=0, endpos=None): g = self.groups @@ -242,11 +240,19 @@ def subn(self, repl, string, count=0): return self._subx(repl, string, count) def _subx(self, repl, string, count): + # CPython substitutes nothing for a negative count. if count < 0: count = 0 + limit = 0 + else: + limit = None empty = _plain_slice(string, 0, 0) if callable(repl): filt = repl + elif _is_literal(repl): + # Like CPython, a replacement with no backslash is used as is, + # without parsing it as a template. + filt = lambda m, _l=repl: _l else: template = _compile_template(self, repl) if len(template) == 1 and not isinstance(template[0], int): @@ -262,15 +268,17 @@ def _subx(self, repl, string, count): endpos = _subject_len(string) must_advance = False while pos <= endpos: - if count and n >= count: + if (count and n >= count) or limit == 0: break - r = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance) - if r is None: + m = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance, 0) + if m is None: break - start, end = r[0], r[1] + start, end = m._marks[0], m._marks[1] out.append(_plain_slice(string, last, start)) - m = Match(self, string, 0, endpos, r) - out.append(filt(m)) + item = filt(m) + # CPython skips a None replacement. + if item is not None: + out.append(item) last = end n += 1 must_advance = start == end @@ -291,11 +299,10 @@ def split(self, string, maxsplit=0): while pos <= endpos: if maxsplit and n >= maxsplit: break - r = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance) - if r is None: + m = self._exec(string, pos, endpos, _MODE_SEARCH, must_advance, 0) + if m is None: break - start, end = r[0], r[1] - m = Match(self, string, 0, endpos, r) + start, end = m._marks[0], m._marks[1] out.append(_plain_slice(string, last, start)) for i in range(1, g + 1): out.append(m.group(i)) @@ -362,6 +369,11 @@ def _flags_repr(flags): class Match: __module__ = 're' + # The native methods read these by position (see `_sre.install`). + # `_marks` holds the match span, then each group's span, with -1 for a + # group that did not take part; `_lastindex` is -1 for none. + __slots__ = ('re', 'string', 'pos', 'endpos', '_marks', '_lastindex') + # PEP 585: `re.Match[str]` / `re.Match[bytes]` yield a `types.GenericAlias`. __class_getitem__ = classmethod(_GenericAlias) @@ -370,22 +382,13 @@ class Match: def __init_subclass__(cls, /, **kwargs): raise TypeError("type 're.Match' is not an acceptable base type") - def __init__(self, pattern, string, pos, endpos, r): - self.re = pattern - self.string = string - self.pos = pos - self.endpos = endpos - self._start = r[0] - self._end = r[1] - self._lastindex_raw = r[2] - self._marks = r[3] + def __new__(cls, *args, **kwargs): + raise TypeError("cannot create 're.Match' instances") # -- group span helpers --------------------------------------------- def _span_of(self, idx): - if idx == 0: - return (self._start, self._end) - i = (idx - 1) * 2 + i = idx * 2 return (self._marks[i], self._marks[i + 1]) def _index(self, group): @@ -439,14 +442,11 @@ def span(self, group=0): @property def regs(self): - spans = [(self._start, self._end)] - for i in range(1, self.re.groups + 1): - spans.append(self._span_of(i)) - return tuple(spans) + return tuple(self._span_of(i) for i in range(self.re.groups + 1)) @property def lastindex(self): - li = self._lastindex_raw + li = self._lastindex return None if li < 0 else li @property @@ -469,9 +469,10 @@ def __deepcopy__(self, memo): return self def __repr__(self): - text = _plain_slice(self.string, self._start, self._end) + start, end = self._marks[0], self._marks[1] + text = _plain_slice(self.string, start, end) return "" % ( - self._start, self._end, text) + start, end, text) # CPython's Pattern/Match are C static types whose `tp_name` carries the @@ -481,6 +482,23 @@ def __repr__(self): __weavepy_set_tp_name__(Match, "re.Match") +class SRE_Scanner: + """The scanner `finditer` and `Pattern.scanner` return for an exact + `str` or `bytes` subject. Built by `_sre` only; `match`, `search` and + the iterator protocol are native (see `_sre.install`).""" + + __module__ = '_sre' + __slots__ = ('pattern', '_string', '_start', '_pos', '_endpos', + '_must_advance') + + def __new__(cls, *args, **kwargs): + raise TypeError("cannot create '_sre.SRE_Scanner' instances") + + def __init_subclass__(cls, /, **kwargs): + raise TypeError( + "type '_sre.SRE_Scanner' is not an acceptable base type") + + class _Scanner: def __init__(self, pattern, string, pos, endpos): self.pattern = pattern @@ -499,14 +517,14 @@ def search(self): def _run(self, mode): if self._pos > self._endpos: return None - r = _sre.exec(self.pattern._handle, self._string, self._pos, - self._endpos, mode, 1 if self._must_advance else 0) - if r is None: - if mode == _MODE_MATCH: - return None + m = _sre.exec_match(self.pattern, self._string, self._pos, + self._endpos, mode, + 1 if self._must_advance else 0, self._opos) + if m is None: + # Exhausted, like CPython's scanner. + self._pos = self._endpos + 1 return None - start, end = r[0], r[1] - m = Match(self.pattern, self._string, self._opos, self._oendpos, r) + start, end = m._marks[0], m._marks[1] self._must_advance = start == end self._pos = end return m @@ -516,13 +534,24 @@ def _run(self, mode): # Replacement-template handling # --------------------------------------------------------------------------- +def _is_literal(repl): + """Whether `repl` is a str or bytes-like object with no backslash + (CPython's `pattern_subx` check).""" + if isinstance(repl, str): + return str.find(repl, '\\') < 0 + try: + return b'\\' not in memoryview(repl).tobytes() + except TypeError: + return False + + def _parse_template(pattern, repl): return _parser.parse_template(repl, pattern) def _compile_template(pattern, repl): try: - key = (pattern._handle, repl) + key = (pattern._code, repl) return _template_cache[key] except KeyError: pass @@ -553,3 +582,7 @@ def _expand_template(template, match): def clear_template_cache(): _template_cache.clear() + _sre.clear_templates(Pattern) + + +_sre.install(Pattern, Match, SRE_Scanner, _compile_template) diff --git a/crates/weavepy-vm/src/stdlib/python/re_init.py b/crates/weavepy-vm/src/stdlib/python/re_init.py index 62980985..bba3a862 100644 --- a/crates/weavepy-vm/src/stdlib/python/re_init.py +++ b/crates/weavepy-vm/src/stdlib/python/re_init.py @@ -45,6 +45,9 @@ from . import _engine import functools import _sre +# WeavePy: `_compile(pattern, flags)` answered natively from `_cache2` on a +# hit (see `_compile_front` below); a miss calls `_compile`. +from _sre import compile_cached as _compile_cached # public symbols @@ -83,17 +86,17 @@ class RegexFlag(enum.IntFlag, boundary=enum.KEEP): def match(pattern, string, flags=0): """Try to apply the pattern at the start of the string, returning a Match object, or None if no match was found.""" - return _compile(pattern, flags).match(string) + return _compile_cached(_compile_front, pattern, flags).match(string) def fullmatch(pattern, string, flags=0): """Try to apply the pattern to all of the string, returning a Match object, or None if no match was found.""" - return _compile(pattern, flags).fullmatch(string) + return _compile_cached(_compile_front, pattern, flags).fullmatch(string) def search(pattern, string, flags=0): """Scan through string looking for a match to the pattern, returning a Match object, or None if no match was found.""" - return _compile(pattern, flags).search(string) + return _compile_cached(_compile_front, pattern, flags).search(string) class _ZeroSentinel(int): pass @@ -128,7 +131,7 @@ def sub(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel if flags is _zero_sentinel: flags = 0 - return _compile(pattern, flags).sub(repl, string, count) + return _compile_cached(_compile_front, pattern, flags).sub(repl, string, count) sub.__text_signature__ = '(pattern, repl, string, count=0, flags=0)' def subn(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentinel): @@ -162,7 +165,7 @@ def subn(pattern, repl, string, *args, count=_zero_sentinel, flags=_zero_sentine if flags is _zero_sentinel: flags = 0 - return _compile(pattern, flags).subn(repl, string, count) + return _compile_cached(_compile_front, pattern, flags).subn(repl, string, count) subn.__text_signature__ = '(pattern, repl, string, count=0, flags=0)' def split(pattern, string, *args, maxsplit=_zero_sentinel, flags=_zero_sentinel): @@ -195,7 +198,7 @@ def split(pattern, string, *args, maxsplit=_zero_sentinel, flags=_zero_sentinel) if flags is _zero_sentinel: flags = 0 - return _compile(pattern, flags).split(string, maxsplit) + return _compile_cached(_compile_front, pattern, flags).split(string, maxsplit) split.__text_signature__ = '(pattern, string, maxsplit=0, flags=0)' def findall(pattern, string, flags=0): @@ -206,18 +209,18 @@ def findall(pattern, string, flags=0): has more than one group. Empty matches are included in the result.""" - return _compile(pattern, flags).findall(string) + return _compile_cached(_compile_front, pattern, flags).findall(string) def finditer(pattern, string, flags=0): """Return an iterator over all non-overlapping matches in the string. For each match, the iterator returns a Match object. Empty matches are included in the result.""" - return _compile(pattern, flags).finditer(string) + return _compile_cached(_compile_front, pattern, flags).finditer(string) def compile(pattern, flags=0): "Compile a regular expression pattern, returning a Pattern object." - return _compile(pattern, flags) + return _compile_cached(_compile_front, pattern, flags) def purge(): "Clear the regular expression caches" @@ -299,6 +302,10 @@ def _compile(pattern, flags): _cache2[key] = p return p +# What `_compile_cached` needs: the FIFO cache it reads, the full lookup it +# falls back to, and the flag type `_compile` normalizes. +_compile_front = (_cache2, _compile, RegexFlag) + # register myself for pickling import copyreg diff --git a/crates/weavepy-vm/src/stdlib/sre_mod.rs b/crates/weavepy-vm/src/stdlib/sre_mod.rs index 986fa982..b7d2d7c9 100644 --- a/crates/weavepy-vm/src/stdlib/sre_mod.rs +++ b/crates/weavepy-vm/src/stdlib/sre_mod.rs @@ -86,6 +86,11 @@ const OP_LITERAL_UNI_IGNORE: u32 = 40; const OP_NOT_LITERAL_UNI_IGNORE: u32 = 41; const OP_RANGE_UNI_IGNORE: u32 = 42; +// INFO block flags. +const SRE_INFO_PREFIX: u32 = 1; +const SRE_INFO_LITERAL: u32 = 2; +const SRE_INFO_CHARSET: u32 = 4; + // AT codes. const AT_BEGINNING: u32 = 0; const AT_BEGINNING_LINE: u32 = 1; @@ -134,9 +139,17 @@ const MAX_DEPTH: u32 = 200_000; // Compiled-pattern registry // --------------------------------------------------------------------------- -struct CompiledCode { - code: Vec, - groups: usize, +pub(crate) struct CompiledCode { + pub(crate) code: Vec, + pub(crate) groups: usize, + /// Per code position, the index in `tables` of the Latin-1 membership + /// table of the character set starting there ([`NO_TABLE`] for none). + table_of: Vec, + tables: Vec<[u64; 4]>, + /// Whether the pattern was a `str` (`Some(true)`) or a bytes-like + /// object (`Some(false)`); `None` for a pattern compiled from a parsed + /// tree (`re.Scanner`), which, like CPython's, accepts either subject. + pub(crate) is_str: Option, } /// Process-global compiled-pattern registry. @@ -148,13 +161,198 @@ struct CompiledCode { /// thread tries to `exec` it (e.g. `threading`/`logging` module-level /// regexes used from a `Thread.run`). Handles are indices that only /// ever grow, so they stay valid across threads; `CompiledCode` is -/// immutable after compilation, so sharing it is race-free. -fn registry() -> &'static parking_lot::Mutex>> { - static REGISTRY: std::sync::OnceLock>>> = +/// immutable after compilation, so sharing it is race-free. Entries are +/// never removed, so each one is leaked once and shared by reference. +fn registry() -> &'static parking_lot::Mutex> { + static REGISTRY: std::sync::OnceLock>> = std::sync::OnceLock::new(); REGISTRY.get_or_init(|| parking_lot::Mutex::new(Vec::new())) } +/// Lock-free handle lookup: chunks of entries, allocated on demand and +/// never freed, published with release stores (registration is +/// serialized by the registry lock). +const CHUNK_BITS: usize = 10; +const CHUNK: usize = 1 << CHUNK_BITS; +const CHUNKS: usize = 4096; + +type Chunk = [std::sync::atomic::AtomicPtr; CHUNK]; + +static LOOKUP: [std::sync::atomic::AtomicPtr; CHUNKS] = + [const { std::sync::atomic::AtomicPtr::new(std::ptr::null_mut()) }; CHUNKS]; + +fn publish(idx: usize, cc: &'static CompiledCode) { + use std::sync::atomic::{AtomicPtr, Ordering}; + let Some(slot) = LOOKUP.get(idx >> CHUNK_BITS) else { + return; + }; + let mut chunk = slot.load(Ordering::Acquire); + if chunk.is_null() { + let fresh: Box = Box::new([const { AtomicPtr::new(std::ptr::null_mut()) }; CHUNK]); + chunk = Box::into_raw(fresh); + slot.store(chunk, Ordering::Release); + } + // SAFETY: chunks are never freed. + let chunk = unsafe { &*chunk }; + chunk[idx & (CHUNK - 1)].store(std::ptr::from_ref(cc).cast_mut(), Ordering::Release); +} + +/// Register compiled code; the handle indexes the registry. +pub(crate) fn register_code(code: Vec, groups: usize, is_str: Option) -> i64 { + let (table_of, tables) = set_tables(&code); + let cc: &'static CompiledCode = Box::leak(Box::new(CompiledCode { + code, + groups, + table_of, + tables, + is_str, + })); + let mut reg = registry().lock(); + reg.push(cc); + let idx = reg.len() - 1; + publish(idx, cc); + idx as i64 +} + +const NO_TABLE: u32 = u32::MAX; + +/// Membership tables over the first 256 code points for the character +/// sets in `code`, so the matcher tests a Latin-1 character with one bit +/// lookup instead of walking the set. A table is built for every position +/// that follows an `IN`-family opcode word (and for the `INFO` block's +/// charset): the opcode value can also occur as data, but a table built +/// at a position that is not a set start is never consulted, since the +/// matcher only asks about real set starts. A position whose set does not +/// evaluate cleanly gets no table and keeps the full walk. +fn set_tables(code: &[u32]) -> (Vec, Vec<[u64; 4]>) { + let mut table_of = vec![NO_TABLE; code.len()]; + let mut tables = Vec::new(); + let mut starts: Vec = Vec::new(); + if code.first() == Some(&OP_INFO) && code.get(2).is_some_and(|f| f & SRE_INFO_CHARSET != 0) { + starts.push(5); + } + for (i, &op) in code.iter().enumerate() { + // ... , the skip landing past FAILURE. + if matches!(op, OP_IN | OP_IN_IGNORE | OP_IN_UNI_IGNORE) + && code + .get(i + 1) + .is_some_and(|&skip| skip >= 2 && code.get(i + skip as usize) == Some(&OP_FAILURE)) + { + starts.push(i + 2); + } + } + for set in starts { + if set >= code.len() || table_of[set] != NO_TABLE { + continue; + } + let mut t = [0u64; 4]; + let mut clean = true; + for ch in 0..256u32 { + match charset_checked(code, set, ch) { + Some(true) => t[(ch >> 6) as usize] |= 1 << (ch & 63), + Some(false) => {} + None => { + clean = false; + break; + } + } + } + if clean { + table_of[set] = tables.len() as u32; + tables.push(t); + } + } + (table_of, tables) +} + +/// `SRE(charset)` over possibly malformed code: `None` when the walk +/// leaves the code or meets an opcode a set cannot hold. +fn charset_checked(code: &[u32], mut set: usize, ch: u32) -> Option { + let at = |i: usize| code.get(i).copied(); + let mut ok = true; + // Real sets are short (the compiler folds long ones into bitmaps). + for _ in 0..256 { + let op = at(set)?; + set += 1; + match op { + OP_FAILURE => return Some(!ok), + OP_LITERAL => { + if ch == at(set)? { + return Some(ok); + } + set += 1; + } + OP_CATEGORY => { + if category(at(set)?, ch) { + return Some(ok); + } + set += 1; + } + OP_CHARSET => { + let word = at(set + (ch / 32) as usize)?; + at(set + 7)?; + if ch < 256 && word & (1u32 << (ch & 31)) != 0 { + return Some(ok); + } + set += 8; + } + OP_RANGE => { + if at(set)? <= ch && ch <= at(set + 1)? { + return Some(ok); + } + set += 2; + } + OP_RANGE_UNI_IGNORE => { + let (lo, hi) = (at(set)?, at(set + 1)?); + if lo <= ch && ch <= hi { + return Some(ok); + } + let uch = upper_unicode(ch); + if lo <= uch && uch <= hi { + return Some(ok); + } + set += 2; + } + OP_NEGATE => ok = !ok, + OP_BIGCHARSET => { + let count = at(set)? as usize; + set += 1; + let byte_index = (ch >> 8) as usize; + let word = at(set + byte_index / 4)?; + let block = ((word >> ((byte_index % 4) * 8)) & 0xff) as usize; + set += 64; + let bit = block * 256 + (ch as usize & 255); + if at(set + bit / 32)? & (1u32 << (bit & 31)) != 0 { + return Some(ok); + } + set += count * 8; + } + _ => return None, + } + } + None +} + +/// The compiled code behind `handle`. +#[inline] +pub(crate) fn compiled(handle: i64) -> Option<&'static CompiledCode> { + use std::sync::atomic::Ordering; + let idx = usize::try_from(handle).ok()?; + if let Some(slot) = LOOKUP.get(idx >> CHUNK_BITS) { + let chunk = slot.load(Ordering::Acquire); + if !chunk.is_null() { + // SAFETY: chunks are never freed. + let cc = unsafe { &*chunk }[idx & (CHUNK - 1)].load(Ordering::Acquire); + if !cc.is_null() { + // SAFETY: registered entries are leaked, never freed. + return Some(unsafe { &*cc }); + } + } + } + // Beyond the lookup table's reach. + registry().lock().get(idx).copied() +} + // --------------------------------------------------------------------------- // Case-folding helpers // --------------------------------------------------------------------------- @@ -259,7 +457,11 @@ fn loc_word(ch: u32) -> bool { } } +#[inline] fn uni_digit(ch: u32) -> bool { + if ch < 128 { + return ascii_digit(ch); + } match char::from_u32(ch) { // Py_UNICODE_ISDECIMAL — decimal digits: ASCII `0`-`9` plus the // Unicode Decimal_Number (Nd) category for non-ASCII scripts. @@ -293,16 +495,22 @@ fn nd_digit(c: char) -> bool { ) } +#[inline] fn uni_space(ch: u32) -> bool { + if ch < 128 { + // `char::is_whitespace` over ASCII. + return matches!(ch, 0x09..=0x0d | 0x20); + } match char::from_u32(ch) { Some(c) => c.is_whitespace(), None => false, } } +#[inline] fn uni_word(ch: u32) -> bool { - if ch == u32::from(b'_') { - return true; + if ch < 128 { + return ascii_word(ch); } match char::from_u32(ch) { Some(c) => c.is_alphanumeric(), @@ -317,6 +525,7 @@ fn uni_linebreak(ch: u32) -> bool { ) } +#[inline] fn category(chcode: u32, ch: u32) -> bool { match chcode { CAT_DIGIT => ascii_digit(ch), @@ -345,9 +554,49 @@ fn category(chcode: u32, ch: u32) -> bool { // The matcher // --------------------------------------------------------------------------- +/// A subject code unit: a byte (`bytes`, or an ASCII `str` matched over +/// its UTF-8 bytes in place) or a code point. +pub(crate) trait SreChar: Copy + Into {} +impl SreChar for u8 {} +impl SreChar for u32 {} + +#[inline(always)] +fn cu(c: C) -> u32 { + c.into() +} + +/// The length of `s`'s prefix whose units all satisfy `pred`. +#[inline(always)] +fn scan(s: &[C], pred: impl Fn(u32) -> bool) -> usize { + s.iter().position(|&c| !pred(cu(c))).unwrap_or(s.len()) +} + +/// What a tail's first consuming op requires of the character it starts +/// at, after any `MARK`s (which consume nothing): a `REPEAT_ONE` or +/// `MIN_REPEAT_ONE` skips the positions whose character cannot start the +/// tail instead of trying the tail there. +#[derive(Clone, Copy)] +enum TailFirst { + Any, + Literal(u32), + /// A literal compared after ASCII (`LITERAL_IGNORE`) or Unicode + /// (`LITERAL_UNI_IGNORE`) lowercasing. + LiteralIgnore(u32), + LiteralUniIgnore(u32), + /// A set (`IN`), by its code position, and its lowercasing forms. + Set(usize), + SetIgnore(usize), + SetUniIgnore(usize), + /// A `BRANCH` (by its code position) whose every alternative has a + /// requirement: any one of them. + Branch(usize), +} + +/// Saved capture state (CPython's `LASTMARK_SAVE`, plus `MARK_PUSH` of +/// the marks themselves while a `REPEAT` is active). #[derive(Clone)] struct MarkSnapshot { - marks: Vec, + marks: Option>, lastmark: isize, lastindex: isize, } @@ -360,14 +609,61 @@ struct RepeatCtx { prev: Option, } -struct Matcher<'a> { - s: &'a [u32], +/// The matcher's capture marks: inline for up to 16 groups. +// (The inline variant's size is the point: no allocation per match.) +#[allow(clippy::large_enum_variant)] +enum MarkBuf { + Inline { len: usize, buf: [isize; 32] }, + Heap(Vec), +} + +impl MarkBuf { + /// `n` marks, all unset (`-1`). + #[inline] + fn new(n: usize) -> Self { + if n <= 32 { + MarkBuf::Inline { + len: n, + buf: [-1; 32], + } + } else { + MarkBuf::Heap(vec![-1; n]) + } + } +} + +impl std::ops::Deref for MarkBuf { + type Target = [isize]; + #[inline] + fn deref(&self) -> &[isize] { + match self { + MarkBuf::Inline { len, buf } => &buf[..*len], + MarkBuf::Heap(v) => v, + } + } +} + +impl std::ops::DerefMut for MarkBuf { + #[inline] + fn deref_mut(&mut self) -> &mut [isize] { + match self { + MarkBuf::Inline { len, buf } => &mut buf[..*len], + MarkBuf::Heap(v) => v, + } + } +} + +struct Matcher<'a, C: SreChar> { + s: &'a [C], code: &'a [u32], + /// The compiled code's set tables (see [`set_tables`]). + table_of: &'a [u32], + tables: &'a [[u64; 4]], beginning: usize, start: usize, end: usize, ptr: usize, - marks: Vec, + marks: MarkBuf, lastmark: isize, lastindex: isize, must_advance: bool, @@ -482,16 +778,19 @@ fn code_range_has_mark(code: &[u32], start: usize, end: usize) -> bool { false } -impl<'a> Matcher<'a> { - fn new(s: &'a [u32], code: &'a [u32], groups: usize) -> Self { +impl<'a, C: SreChar> Matcher<'a, C> { + /// A matcher over `s`; `marks` holds `2 * groups` entries, all `-1`. + fn new(s: &'a [C], cc: &'a CompiledCode, marks: MarkBuf) -> Self { Matcher { s, - code, + code: &cc.code, + table_of: &cc.table_of, + tables: &cc.tables, beginning: 0, start: 0, end: s.len(), ptr: 0, - marks: vec![-1; groups * 2], + marks, lastmark: -1, lastindex: -1, must_advance: false, @@ -512,10 +811,16 @@ impl<'a> Matcher<'a> { self.lastindex = -1; } + /// Save the capture state before a choice point. Outside any `REPEAT` + /// only `lastmark`/`lastindex` are saved, as in CPython: a failed + /// alternative can only have set marks above the saved `lastmark` + /// (each group's marks appear once in the code, and nothing loops + /// back outside a repeat), and restoring `lastmark` invalidates + /// those. Inside a repeat the marks themselves are saved too. #[inline] fn snapshot(&self) -> MarkSnapshot { MarkSnapshot { - marks: self.marks.clone(), + marks: self.cur_repeat.is_some().then(|| self.marks.to_vec()), lastmark: self.lastmark, lastindex: self.lastindex, } @@ -523,7 +828,9 @@ impl<'a> Matcher<'a> { #[inline] fn restore(&mut self, snap: &MarkSnapshot) { - self.marks.clone_from(&snap.marks); + if let Some(marks) = &snap.marks { + self.marks.copy_from_slice(marks); + } self.lastmark = snap.lastmark; self.lastindex = snap.lastindex; } @@ -728,9 +1035,9 @@ impl<'a> Matcher<'a> { let s = self.s; match atcode { AT_BEGINNING | AT_BEGINNING_STRING => ptr == self.beginning, - AT_BEGINNING_LINE => ptr == self.beginning || is_linebreak(s[ptr - 1]), - AT_END => (self.end - ptr == 1 && is_linebreak(s[ptr])) || ptr == self.end, - AT_END_LINE => ptr == self.end || is_linebreak(s[ptr]), + AT_BEGINNING_LINE => ptr == self.beginning || is_linebreak(cu(s[ptr - 1])), + AT_END => (self.end - ptr == 1 && is_linebreak(cu(s[ptr]))) || ptr == self.end, + AT_END_LINE => ptr == self.end || is_linebreak(cu(s[ptr])), AT_END_STRING => ptr == self.end, AT_BOUNDARY => self.word_boundary(ptr, ascii_word), // 3.14 (gh-124130): `\B` matches the empty string again (the @@ -757,13 +1064,27 @@ impl<'a> Matcher<'a> { /// its exact complement (CPython `SRE_AT_NON_BOUNDARY`). #[inline] fn word_boundary_raw(&self, ptr: usize, is_word: fn(u32) -> bool) -> bool { - let thatp = ptr > self.beginning && is_word(self.s[ptr - 1]); - let thisp = ptr < self.end && is_word(self.s[ptr]); + let thatp = ptr > self.beginning && is_word(cu(self.s[ptr - 1])); + let thisp = ptr < self.end && is_word(cu(self.s[ptr])); thisp != thatp } /// `SRE(charset)` — is `ch` a member of the set starting at `set`? - fn charset(&self, mut set: usize, ch: u32) -> bool { + #[inline] + fn charset(&self, set: usize, ch: u32) -> bool { + if ch < 256 { + if let Some(&t) = self.table_of.get(set) { + if t != NO_TABLE { + let bits = self.tables[t as usize][(ch >> 6) as usize]; + return bits & (1 << (ch & 63)) != 0; + } + } + } + self.charset_walk(set, ch) + } + + /// [`Self::charset`] by walking the set. + fn charset_walk(&self, mut set: usize, ch: u32) -> bool { let code = self.code; let mut ok = true; loop { @@ -845,6 +1166,117 @@ impl<'a> Matcher<'a> { up != lo && self.charset(set, up) } + /// The first-character requirement of the tail at `tail` (see + /// [`TailFirst`]). + #[inline(always)] + fn tail_first(&self, tail: usize) -> TailFirst { + match Self::simple_first(self.code, tail) { + (TailFirst::Any, op) if self.code[op] == OP_BRANCH => self.branch_first(op), + (first, _) => first, + } + } + + /// [`Self::tail_first`] without the `BRANCH` case, with the position + /// of the op it looked at (past any `MARK`s). + #[inline(always)] + fn simple_first(code: &[u32], mut tail: usize) -> (TailFirst, usize) { + while code[tail] == OP_MARK { + tail += 2; + } + let first = match code[tail] { + // A single-character repeat of at least one item starts with + // that item: . + OP_REPEAT_ONE | OP_MIN_REPEAT_ONE | OP_POSSESSIVE_REPEAT_ONE if code[tail + 2] >= 1 => { + Self::item_first(code, tail + 4) + } + _ => Self::item_first(code, tail), + }; + (first, tail) + } + + /// [`Self::tail_first`] of the `BRANCH` at `op`: + /// ` ( )* <0>`. + #[inline(never)] + fn branch_first(&self, op: usize) -> TailFirst { + let code = self.code; + let mut alt = op + 1; + while code[alt] != 0 { + // (A nested branch counts as no requirement.) + if matches!(Self::simple_first(code, alt + 1).0, TailFirst::Any) { + return TailFirst::Any; + } + alt += code[alt] as usize; + } + TailFirst::Branch(op) + } + + /// Whether some alternative of the `BRANCH` at `op` can start at `pos`. + #[inline(never)] + fn branch_can_start(&self, op: usize, pos: usize) -> bool { + let code = self.code; + let mut alt = op + 1; + while code[alt] != 0 { + if self.simple_can_start(Self::simple_first(code, alt + 1).0, pos) { + return true; + } + alt += code[alt] as usize; + } + false + } + + /// Whether the alternative whose body starts at `body` can start at + /// `pos` (kept out of line: the `BRANCH` arm is in the matcher's hot + /// loop). + #[inline(never)] + fn alternative_can_start(&self, body: usize, pos: usize) -> bool { + self.can_start(self.tail_first(body), pos) + } + + /// The requirement of the single-character op at `op`. + #[inline(always)] + fn item_first(code: &[u32], op: usize) -> TailFirst { + match code[op] { + OP_LITERAL => TailFirst::Literal(code[op + 1]), + OP_LITERAL_IGNORE => TailFirst::LiteralIgnore(code[op + 1]), + OP_LITERAL_UNI_IGNORE => TailFirst::LiteralUniIgnore(code[op + 1]), + OP_IN => TailFirst::Set(op + 2), + OP_IN_IGNORE => TailFirst::SetIgnore(op + 2), + OP_IN_UNI_IGNORE => TailFirst::SetUniIgnore(op + 2), + _ => TailFirst::Any, + } + } + + /// Whether the character at `pos` can start a tail requiring `first`. + #[inline(always)] + fn can_start(&self, first: TailFirst, pos: usize) -> bool { + if let TailFirst::Branch(op) = first { + return self.branch_can_start(op, pos); + } + self.simple_can_start(first, pos) + } + + /// [`Self::can_start`] for a requirement other than a `BRANCH`. + #[inline(always)] + fn simple_can_start(&self, first: TailFirst, pos: usize) -> bool { + if matches!(first, TailFirst::Any) { + return true; + } + if pos >= self.end { + return false; + } + let ch = cu(self.s[pos]); + match first { + TailFirst::Any => true, + TailFirst::Literal(c) => ch == c, + TailFirst::LiteralIgnore(c) => lower_ascii(ch) == c, + TailFirst::LiteralUniIgnore(c) => lower_unicode(ch) == c, + TailFirst::Set(set) => self.charset(set, ch), + TailFirst::SetIgnore(set) => self.charset(set, lower_ascii(ch)), + TailFirst::SetUniIgnore(set) => self.charset(set, lower_unicode(ch)), + TailFirst::Branch(_) => true, + } + } + /// `SRE(count)` — count repeated single-character matches of the /// item at `pat`, starting at `self.ptr`, up to `maxcount`. fn count(&mut self, pat: usize, maxcount: u32) -> Result { @@ -858,15 +1290,37 @@ impl<'a> Matcher<'a> { let op = code[pat]; let counted = match op { OP_IN => { - let mut p = ptr; - while p < end && self.charset(pat + 2, s[p]) { - p += 1; + let set = pat + 2; + match (code[set], code.get(set + 2).copied()) { + // A lone category (`\w+`, `\d*`, `\s+`): one loop per + // class, with no set walk per character. + (OP_CATEGORY, Some(OP_FAILURE)) => { + let s = &s[ptr..end]; + match code[set + 1] { + CAT_UNI_WORD => scan(s, uni_word), + CAT_WORD => scan(s, ascii_word), + CAT_UNI_DIGIT => scan(s, uni_digit), + CAT_DIGIT => scan(s, ascii_digit), + CAT_UNI_SPACE => scan(s, uni_space), + CAT_SPACE => scan(s, ascii_space), + CAT_UNI_NOT_WORD => scan(s, |c| !uni_word(c)), + CAT_UNI_NOT_DIGIT => scan(s, |c| !uni_digit(c)), + CAT_UNI_NOT_SPACE => scan(s, |c| !uni_space(c)), + cat => scan(s, |c| category(cat, c)), + } + } + _ => { + let mut p = ptr; + while p < end && self.charset(set, cu(s[p])) { + p += 1; + } + p - ptr + } } - p - ptr } OP_ANY => { let mut p = ptr; - while p < end && !is_linebreak(s[p]) { + while p < end && !is_linebreak(cu(s[p])) { p += 1; } p - ptr @@ -875,7 +1329,7 @@ impl<'a> Matcher<'a> { OP_LITERAL => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && s[p] == chr { + while p < end && cu(s[p]) == chr { p += 1; } p - ptr @@ -883,7 +1337,7 @@ impl<'a> Matcher<'a> { OP_NOT_LITERAL => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && s[p] != chr { + while p < end && cu(s[p]) != chr { p += 1; } p - ptr @@ -891,7 +1345,7 @@ impl<'a> Matcher<'a> { OP_LITERAL_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && lower_ascii(s[p]) == chr { + while p < end && lower_ascii(cu(s[p])) == chr { p += 1; } p - ptr @@ -899,7 +1353,7 @@ impl<'a> Matcher<'a> { OP_LITERAL_UNI_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && lower_unicode(s[p]) == chr { + while p < end && lower_unicode(cu(s[p])) == chr { p += 1; } p - ptr @@ -907,7 +1361,7 @@ impl<'a> Matcher<'a> { OP_LITERAL_LOC_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && char_loc_ignore(chr, s[p]) { + while p < end && char_loc_ignore(chr, cu(s[p])) { p += 1; } p - ptr @@ -915,7 +1369,7 @@ impl<'a> Matcher<'a> { OP_NOT_LITERAL_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && lower_ascii(s[p]) != chr { + while p < end && lower_ascii(cu(s[p])) != chr { p += 1; } p - ptr @@ -923,7 +1377,7 @@ impl<'a> Matcher<'a> { OP_NOT_LITERAL_UNI_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && lower_unicode(s[p]) != chr { + while p < end && lower_unicode(cu(s[p])) != chr { p += 1; } p - ptr @@ -931,7 +1385,7 @@ impl<'a> Matcher<'a> { OP_NOT_LITERAL_LOC_IGNORE => { let chr = code[pat + 1]; let mut p = ptr; - while p < end && !char_loc_ignore(chr, s[p]) { + while p < end && !char_loc_ignore(chr, cu(s[p])) { p += 1; } p - ptr @@ -967,10 +1421,17 @@ impl<'a> Matcher<'a> { } // Segmented stack growth: linear-depth backtracking (one native // frame per repetition) must not overflow the OS stack before - // reaching MAX_DEPTH. - let r = stacker::maybe_grow(128 * 1024, 4 * 1024 * 1024, || { + // reaching MAX_DEPTH. Checking at every eighth level is enough: + // eight frames stay far inside the 128 KiB red zone. (The first + // check is at the eighth level, so a shallow match, the usual + // case, never asks for the stack pointer.) + let r = if self.depth.is_multiple_of(8) { + stacker::maybe_grow(128 * 1024, 4 * 1024 * 1024, || { + self.do_match_inner(pat, toplevel) + }) + } else { self.do_match_inner(pat, toplevel) - }); + }; self.depth -= 1; r } @@ -1011,14 +1472,14 @@ impl<'a> Matcher<'a> { pat += 1; } OP_LITERAL => { - if ptr >= end || self.s[ptr] != code[pat] { + if ptr >= end || cu(self.s[ptr]) != code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_NOT_LITERAL => { - if ptr >= end || self.s[ptr] == code[pat] { + if ptr >= end || cu(self.s[ptr]) == code[pat] { return Ok(false); } pat += 1; @@ -1041,14 +1502,14 @@ impl<'a> Matcher<'a> { pat += 1; } OP_CATEGORY => { - if ptr >= end || !category(code[pat], self.s[ptr]) { + if ptr >= end || !category(code[pat], cu(self.s[ptr])) { return Ok(false); } pat += 1; ptr += 1; } OP_ANY => { - if ptr >= end || is_linebreak(self.s[ptr]) { + if ptr >= end || is_linebreak(cu(self.s[ptr])) { return Ok(false); } ptr += 1; @@ -1060,70 +1521,70 @@ impl<'a> Matcher<'a> { ptr += 1; } OP_IN => { - if ptr >= end || !self.charset(pat + 1, self.s[ptr]) { + if ptr >= end || !self.charset(pat + 1, cu(self.s[ptr])) { return Ok(false); } pat += code[pat] as usize; ptr += 1; } OP_LITERAL_IGNORE => { - if ptr >= end || lower_ascii(self.s[ptr]) != code[pat] { + if ptr >= end || lower_ascii(cu(self.s[ptr])) != code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_LITERAL_UNI_IGNORE => { - if ptr >= end || lower_unicode(self.s[ptr]) != code[pat] { + if ptr >= end || lower_unicode(cu(self.s[ptr])) != code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_LITERAL_LOC_IGNORE => { - if ptr >= end || !char_loc_ignore(code[pat], self.s[ptr]) { + if ptr >= end || !char_loc_ignore(code[pat], cu(self.s[ptr])) { return Ok(false); } pat += 1; ptr += 1; } OP_NOT_LITERAL_IGNORE => { - if ptr >= end || lower_ascii(self.s[ptr]) == code[pat] { + if ptr >= end || lower_ascii(cu(self.s[ptr])) == code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_NOT_LITERAL_UNI_IGNORE => { - if ptr >= end || lower_unicode(self.s[ptr]) == code[pat] { + if ptr >= end || lower_unicode(cu(self.s[ptr])) == code[pat] { return Ok(false); } pat += 1; ptr += 1; } OP_NOT_LITERAL_LOC_IGNORE => { - if ptr >= end || char_loc_ignore(code[pat], self.s[ptr]) { + if ptr >= end || char_loc_ignore(code[pat], cu(self.s[ptr])) { return Ok(false); } pat += 1; ptr += 1; } OP_IN_IGNORE => { - if ptr >= end || !self.charset(pat + 1, lower_ascii(self.s[ptr])) { + if ptr >= end || !self.charset(pat + 1, lower_ascii(cu(self.s[ptr]))) { return Ok(false); } pat += code[pat] as usize; ptr += 1; } OP_IN_UNI_IGNORE => { - if ptr >= end || !self.charset(pat + 1, lower_unicode(self.s[ptr])) { + if ptr >= end || !self.charset(pat + 1, lower_unicode(cu(self.s[ptr]))) { return Ok(false); } pat += code[pat] as usize; ptr += 1; } OP_IN_LOC_IGNORE => { - if ptr >= end || !self.charset_loc_ignore(pat + 1, self.s[ptr]) { + if ptr >= end || !self.charset_loc_ignore(pat + 1, cu(self.s[ptr])) { return Ok(false); } pat += code[pat] as usize; @@ -1135,16 +1596,9 @@ impl<'a> Matcher<'a> { OP_BRANCH => { let save = self.snapshot(); while code[pat] != 0 { - // Fast skip when the branch can't possibly match. - if code[pat + 1] == OP_LITERAL - && (ptr >= end || self.s[ptr] != code[pat + 2]) - { - pat += code[pat] as usize; - continue; - } - if code[pat + 1] == OP_IN - && (ptr >= end || !self.charset(pat + 3, self.s[ptr])) - { + // Fast skip when the branch can't possibly match: + // its first consuming op rejects the character. + if !self.alternative_can_start(pat + 1, ptr) { pat += code[pat] as usize; continue; } @@ -1187,13 +1641,10 @@ impl<'a> Matcher<'a> { let save = self.snapshot(); let orig = ptr; let pmin_i = pmin as isize; - if code[tail] == OP_LITERAL { - let chr = code[tail + 1]; + let first = self.tail_first(tail); + if !matches!(first, TailFirst::Any) { loop { - while cnt >= pmin_i && { - let pos = orig + cnt as usize; - pos >= end || self.s[pos] != chr - } { + while cnt >= pmin_i && !self.can_start(first, orig + cnt as usize) { cnt -= 1; } if cnt < pmin_i { @@ -1252,15 +1703,18 @@ impl<'a> Matcher<'a> { return Ok(true); } let save = self.snapshot(); + let first = self.tail_first(tail); loop { if !(pmax == MAXREPEAT || (cnt as u32) <= pmax) { break; } self.ptr = ptr; - if self.do_match(tail, toplevel)? { - return Ok(true); + if self.can_start(first, ptr) { + if self.do_match(tail, toplevel)? { + return Ok(true); + } + self.restore(&save); } - self.restore(&save); self.ptr = ptr; let r = self.count(pat + 3, 1)?; if r == 0 { @@ -1567,8 +2021,8 @@ impl<'a> Matcher<'a> { if ptr >= end { return false; } - let a = self.s[ptr]; - let b = self.s[p]; + let a = cu(self.s[ptr]); + let b = cu(self.s[p]); let eq = match kind { GroupRefKind::Exact => a == b, GroupRefKind::Ascii => lower_ascii(a) == lower_ascii(b), @@ -1587,46 +2041,199 @@ impl<'a> Matcher<'a> { /// `SRE(search)` — scan for the leftmost match at or after /// `self.start`. Returns the start position of the match on success. + /// + /// Like CPython's, it uses the pattern's `INFO` block: a literal + /// prefix is found with its overlap table (and is the whole match + /// for a literal pattern), a leading character set skips the + /// positions that cannot start a match, and the minimum width stops + /// the scan early. fn search(&mut self) -> Result, RuntimeError> { - // Determine where the real pattern starts (after any INFO block) - // for the anchored-pattern fast reject. - let mut p = 0usize; - let mut min = 0usize; - if self.code[0] == OP_INFO { - min = self.code[3] as usize; - p = 1 + self.code[1] as usize; - } - let anchored = self.code.get(p) == Some(&OP_AT) - && matches!( - self.code.get(p + 1).copied(), - Some(AT_BEGINNING) | Some(AT_BEGINNING_STRING) - ); - + let code = self.code; let mut ptr = self.start; - let mut first = true; - loop { - if min != 0 && self.end.saturating_sub(ptr) < min { + let mut end = self.end; + if ptr > end { + return Ok(None); + } + let mut pat = 0usize; + let mut flags = 0u32; + let mut prefix_len = 0usize; + let mut prefix_skip = 0usize; + let mut charset = None; + if code[0] == OP_INFO { + // <1=skip> <2=flags> <3=min> <4=max> <5=prefix info> + flags = code[2]; + let min = code[3] as usize; + if min != 0 && end - ptr < min { return Ok(None); } - self.start = ptr; - self.ptr = ptr; - self.reset_capture(); - let matched = self.do_match(0, true)?; - if first { - self.must_advance = false; - first = false; + if min > 1 { + // Leave at least one character, so the literal search + // works. + end -= min - 1; + if end <= ptr { + end = ptr; + } } - if matched { - return Ok(Some(ptr)); + if flags & SRE_INFO_PREFIX != 0 { + // + prefix_len = code[5] as usize; + prefix_skip = code[6] as usize; + } else if flags & SRE_INFO_CHARSET != 0 { + charset = Some(5); } - if anchored { - return Ok(None); + pat = 1 + code[1] as usize; + } + + if prefix_len == 1 { + // The pattern starts with a literal character. + let c = code[7]; + let end = self.end; + self.must_advance = false; + while ptr < end { + while cu(self.s[ptr]) != c { + ptr += 1; + if ptr >= end { + return Ok(None); + } + } + self.start = ptr; + self.ptr = ptr + prefix_skip; + if flags & SRE_INFO_LITERAL != 0 { + return Ok(Some(ptr)); + } + if self.do_match(pat + 2 * prefix_skip, false)? { + return Ok(Some(ptr)); + } + ptr += 1; + self.reset_capture(); } - if ptr >= self.end { + return Ok(None); + } + + if prefix_len > 1 { + // The pattern starts with a known prefix: use the overlap + // table to skip forward as fast as possible. + let prefix = &code[7..7 + prefix_len]; + let overlap = &code[7 + prefix_len - 1..]; + let end = self.end; + if prefix_len > end - ptr { return Ok(None); } + while ptr < end { + let c = prefix[0]; + loop { + let ch = cu(self.s[ptr]); + ptr += 1; + if ch == c { + break; + } + if ptr >= end { + return Ok(None); + } + } + if ptr >= end { + return Ok(None); + } + let mut i = 1usize; + self.must_advance = false; + loop { + if cu(self.s[ptr]) == prefix[i] { + i += 1; + if i != prefix_len { + ptr += 1; + if ptr >= end { + return Ok(None); + } + continue; + } + // A potential match. + let start = ptr - (prefix_len - 1); + self.start = start; + // (`prefix_skip` may be the whole prefix.) + self.ptr = ptr + 1 + prefix_skip - prefix_len; + if flags & SRE_INFO_LITERAL != 0 { + return Ok(Some(start)); + } + if self.do_match(pat + 2 * prefix_skip, false)? { + return Ok(Some(start)); + } + // Close, but no match: try again. + ptr += 1; + if ptr >= end { + return Ok(None); + } + self.reset_capture(); + } + i = overlap[i] as usize; + if i == 0 { + break; + } + } + } + return Ok(None); + } + + if let Some(set) = charset { + // The pattern starts with a character from a known set. + let end = self.end; + self.must_advance = false; + loop { + while ptr < end && !self.charset(set, cu(self.s[ptr])) { + ptr += 1; + } + if ptr >= end { + return Ok(None); + } + self.start = ptr; + self.ptr = ptr; + if self.do_match(pat, false)? { + return Ok(Some(ptr)); + } + ptr += 1; + self.reset_capture(); + } + } + + // The general case. When the pattern's first consuming op + // requires a known character, the positions without one are + // skipped (a match attempt there would fail at that op). + let first = self.tail_first(pat); + if !matches!(first, TailFirst::Any) { + let start = ptr; + loop { + if self.can_start(first, ptr) { + self.start = ptr; + self.ptr = ptr; + if self.do_match(pat, ptr == start)? { + return Ok(Some(ptr)); + } + self.reset_capture(); + } + self.must_advance = false; + if ptr >= end { + return Ok(None); + } + ptr += 1; + } + } + self.start = ptr; + self.ptr = ptr; + let mut matched = self.do_match(pat, true)?; + self.must_advance = false; + if !matched + && code[pat] == OP_AT + && (code[pat + 1] == AT_BEGINNING || code[pat + 1] == AT_BEGINNING_STRING) + { + return Ok(None); + } + while !matched && ptr < end { ptr += 1; + self.reset_capture(); + self.start = ptr; + self.ptr = ptr; + matched = self.do_match(pat, false)?; } + Ok(matched.then_some(ptr)) } } @@ -1764,7 +2371,7 @@ fn subject_cache_key(obj: &Object) -> Option { /// Decode the subject into code points, reusing the thread-local cache when /// the same immutable subject is matched repeatedly. -fn decode_subject_cached(obj: &Object) -> Result>, RuntimeError> { +pub(crate) fn decode_subject_cached(obj: &Object) -> Result>, RuntimeError> { let Some(key) = subject_cache_key(obj) else { return Ok(Rc::new(subject_to_vec(obj)?)); }; @@ -1790,44 +2397,101 @@ fn sre_compile(args: &[Object]) -> Result { .ok_or_else(|| type_error("_sre.compile: code"))?, )?; let groups = arg_i64(args, 1, "groups")?.max(0) as usize; - let handle = { - let mut reg = registry().lock(); - reg.push(Rc::new(CompiledCode { code, groups })); - reg.len() - 1 + Ok(Object::Int(register_code(code, groups, None))) +} + +/// `_sre.compile_code(pattern, code, groups)`: [`sre_compile`] for a +/// pattern whose source (`str`, bytes-like, or `None`) fixes the subject +/// type it accepts. +pub(crate) fn compile_code( + pattern: &Object, + code: &Object, + groups: &Object, +) -> Result { + let code = codeseq_to_vec(code)?; + let groups = groups + .as_i64() + .ok_or_else(|| type_error("_sre: expected int for groups"))? + .max(0) as usize; + let is_str = match pattern { + Object::None => None, + Object::Str(_) | Object::WStr(_) => Some(true), + other => Some(matches!( + other.native_value(), + Some(Object::Str(_) | Object::WStr(_)) + )), }; - Ok(Object::Int(handle as i64)) + Ok(register_code(code, groups, is_str)) } -/// `_sre.exec(handle, string, pos, endpos, mode, must_advance)`. -/// -/// Returns `None` on no match, otherwise a tuple -/// `(start, end, lastindex, marks)` where `marks` is a tuple of -/// `2 * groups` code-point indices (`-1` for an unset group). -fn sre_exec(args: &[Object]) -> Result { - let handle = arg_i64(args, 0, "handle")? as usize; - let cc = registry().lock().get(handle).cloned(); - let cc = cc.ok_or_else(|| value_error("_sre.exec: invalid pattern handle"))?; - let subject = - decode_subject_cached(args.get(1).ok_or_else(|| type_error("_sre.exec: string"))?)?; - let slen = subject.len() as i64; - let pos = arg_i64(args, 2, "pos")?.clamp(0, slen) as usize; - let endpos = arg_i64(args, 3, "endpos")?.clamp(0, slen) as usize; - let mode = arg_i64(args, 4, "mode")?; - let must_advance = args - .get(5) - .map(|o| o.as_i64().unwrap_or(0) != 0) - .unwrap_or(false); +/// A successful match: its span, and the capture state its group spans +/// are read from. +pub(crate) struct Found<'m> { + pub(crate) start: usize, + pub(crate) end: usize, + /// The last matched group (`-1` for none). + pub(crate) lastindex: isize, + groups: usize, + marks: &'m [isize], + lastmark: isize, +} - if pos > endpos { - return Ok(Object::None); +impl Found<'_> { + /// The number of capturing groups. + #[inline] + pub(crate) fn groups(&self) -> usize { + self.groups + } + + /// Group `g`'s span (group 0 is the match), `(-1, -1)` for a group + /// that did not take part, normalized the way CPython's + /// `pattern_new_match` does. + #[inline] + pub(crate) fn span(&self, g: usize) -> (i64, i64) { + if g == 0 { + return (self.start as i64, self.end as i64); + } + let j = 2 * (g - 1); + if (j + 1) as isize <= self.lastmark && self.marks[j] >= 0 && self.marks[j + 1] >= 0 { + (self.marks[j] as i64, self.marks[j + 1] as i64) + } else { + (-1, -1) + } + } + + /// Entry `i` of the flat list of spans (`span(i / 2)`'s start or end). + #[inline] + pub(crate) fn mark(&self, i: usize) -> i64 { + let (a, b) = self.span(i / 2); + if i.is_multiple_of(2) { + a + } else { + b + } } +} - let mut m = Matcher::new(&subject[..], &cc.code, cc.groups); +/// One `SRE(match)` / `SRE(search)` over `subject[pos..endpos]`. +/// +/// `mode` 0 searches, 1 matches at `pos`, 2 must match all of +/// `[pos, endpos)`. On a match the result is `f` applied to it. +pub(crate) fn exec_found( + cc: &CompiledCode, + subject: &[C], + pos: usize, + endpos: usize, + mode: i64, + must_advance: bool, + f: impl FnOnce(&Found<'_>) -> R, +) -> Result, RuntimeError> { + if pos > endpos { + return Ok(None); + } + let mut m = Matcher::new(subject, cc, MarkBuf::new(cc.groups * 2)); m.end = endpos; m.start = pos; m.ptr = pos; m.must_advance = must_advance; - let (mstart, ok) = match mode { // 1 = match (anchored at pos) 1 => { @@ -1846,27 +2510,50 @@ fn sre_exec(args: &[Object]) -> Result { None => (0, false), }, }; - if !ok { - return Ok(Object::None); + return Ok(None); } + Ok(Some(f(&Found { + start: mstart, + end: m.ptr, + lastindex: m.lastindex, + groups: cc.groups, + marks: &m.marks, + lastmark: m.lastmark, + }))) +} - let mend = m.ptr; - let mut marks_out: Vec = Vec::with_capacity(cc.groups * 2); - for i in 0..cc.groups * 2 { - let v = if (i as isize) <= m.lastmark { - m.marks[i] - } else { - -1 - }; - marks_out.push(Object::Int(v as i64)); - } - Ok(Object::new_tuple_array([ - Object::Int(mstart as i64), - Object::Int(mend as i64), - Object::Int(m.lastindex as i64), - Object::new_tuple(marks_out), - ])) +/// `_sre.exec(handle, string, pos, endpos, mode, must_advance)`. +/// +/// Returns `None` on no match, otherwise a tuple +/// `(start, end, lastindex, marks)` where `marks` is a tuple of +/// `2 * groups` code-point indices (`-1` for an unset group). +fn sre_exec(args: &[Object]) -> Result { + let handle = arg_i64(args, 0, "handle")?; + let cc = compiled(handle).ok_or_else(|| value_error("_sre.exec: invalid pattern handle"))?; + let subject = + decode_subject_cached(args.get(1).ok_or_else(|| type_error("_sre.exec: string"))?)?; + let slen = subject.len() as i64; + let pos = arg_i64(args, 2, "pos")?.clamp(0, slen) as usize; + let endpos = arg_i64(args, 3, "endpos")?.clamp(0, slen) as usize; + let mode = arg_i64(args, 4, "mode")?; + let must_advance = args + .get(5) + .map(|o| o.as_i64().unwrap_or(0) != 0) + .unwrap_or(false); + let found = exec_found(cc, &subject[..], pos, endpos, mode, must_advance, |f| { + Object::new_tuple_array([ + Object::Int(f.start as i64), + Object::Int(f.end as i64), + Object::Int(f.lastindex as i64), + Object::new_tuple( + (2..2 + 2 * f.groups()) + .map(|i| Object::Int(f.mark(i))) + .collect(), + ), + ]) + })?; + Ok(found.unwrap_or(Object::None)) } fn sre_ascii_tolower(args: &[Object]) -> Result { @@ -1944,6 +2631,21 @@ pub fn build(_cache: &ModuleCache) -> Rc { b("compile", sre_compile), ); d.insert(DictKey(Object::from_static("exec")), b("exec", sre_exec)); + for (name, f) in [ + ( + "install", + super::sre_objects::install as fn(&[Object]) -> Result, + ), + ("make_pattern", super::sre_objects::make_pattern), + ("exec_match", super::sre_objects::exec_match), + ("clear_templates", super::sre_objects::clear_templates), + ] { + d.insert(DictKey(Object::from_static(name)), b(name, f)); + } + d.insert( + DictKey(Object::from_static("compile_cached")), + super::sre_objects::compile_cached_builtin(), + ); d.insert( DictKey(Object::from_static("ascii_tolower")), b("ascii_tolower", sre_ascii_tolower), diff --git a/crates/weavepy-vm/src/stdlib/sre_objects.rs b/crates/weavepy-vm/src/stdlib/sre_objects.rs new file mode 100644 index 00000000..bab10dd8 --- /dev/null +++ b/crates/weavepy-vm/src/stdlib/sre_objects.rs @@ -0,0 +1,1602 @@ +//! Native method bodies for `re.Pattern`, `re.Match` and the scanner. +//! +//! The classes are Python classes in the frozen `re._engine` module, with +//! `__slots__` storage, so their reprs, attributes, pickling and the rarer +//! paths stay in Python. `_sre.install` (called at the end of the module +//! body) saves the Python methods it replaces, puts native bodies in the +//! class dicts, and hangs this module's [`State`] on each class. Every +//! pattern, match and scanner is built natively over a shared slot layout +//! (see `SlotStorage::from_layout`), so a native reads all of an instance's +//! fields after one layout check. +//! +//! A native serves the common shapes: an exact `str` or `bytes` subject of +//! the pattern's type, plain `int` positions and counts, `int` or `str` +//! group names, and `str`/`bytes` replacement templates. Those fast halves +//! run no Python code, so they are registered as leaf entries the dispatch +//! loop calls inline. A callable replacement, or a template not parsed +//! yet, takes the native full body, which calls into the interpreter. +//! Anything else (other subject types, `__index__` positions, unexpected +//! keywords, and every error message the Python code owns) calls the +//! Python method the native replaced, so behavior is unchanged. +//! +//! The instances are never cycle-collector tracked, like `datetime`'s +//! natively built objects: a match holds its pattern and subject, a +//! pattern holds strings, ints and its name tables, and none of them can +//! reach back to a match. + +use std::collections::HashMap; + +use crate::error::{index_error, stop_iteration, type_error, RuntimeError}; +use crate::object::{BuiltinFn, DictData, DictKey, Object, StrKey}; +use crate::shared_value::{SharedSlice, SharedStr}; +use crate::sync::{Rc, RefCell, Weak}; +use crate::types::{PyInstance, SlotStorage, TypeObject}; + +use super::sre_mod::{self, CompiledCode, Found}; + +// Slot order of the natively built instances (the `__slots__` order the +// classes declare). +const PATTERN_SLOTS: [&str; 6] = [ + "_code", + "pattern", + "flags", + "groups", + "groupindex", + "_indexgroup", +]; +const P_CODE: usize = 0; +const P_GROUPINDEX: usize = 4; + +const MATCH_SLOTS: [&str; 6] = ["re", "string", "pos", "endpos", "_marks", "_lastindex"]; +const M_RE: usize = 0; +const M_STRING: usize = 1; +const M_MARKS: usize = 4; + +const SCANNER_SLOTS: [&str; 6] = [ + "pattern", + "_string", + "_start", + "_pos", + "_endpos", + "_must_advance", +]; +const S_PATTERN: usize = 0; +const S_STRING: usize = 1; +const S_START: usize = 2; +const S_POS: usize = 3; +const S_ENDPOS: usize = 4; +const S_MUST: usize = 5; + +/// A parsed replacement template: literals and group references. +enum Piece { + Lit(Object), + Group(usize), +} + +/// One interpreter's `re` classes (held weakly: the natives live in their +/// dicts), the instance layouts, and the Python methods the natives +/// replaced. +pub(crate) struct State { + match_cls: Weak, + scanner_cls: Weak, + pattern_layout: SharedSlice, + match_layout: SharedSlice, + scanner_layout: SharedSlice, + /// `"Pattern.search"` and so on: the replaced Python method. + orig: HashMap<&'static str, Object>, + /// `re._engine._compile_template(pattern, repl)`. + compile_template: Object, + /// Per pattern handle, the last replacement template it expanded. + templates: parking_lot::Mutex>)>>, +} + +#[inline] +fn state_of(cls: &TypeObject) -> Option<&State> { + cls.native_ext.get()?.downcast_ref::() +} + +type Ext = Rc; + +/// The state hanging on `inst`'s class, held without borrowing the class +/// cell (a callable replacement may run any Python code). +#[inline] +fn class_ext(inst: &PyInstance) -> Option { + inst.class.borrow().native_ext.get().cloned() +} + +/// Run `f` with the state on `inst`'s class, read in place. `f` must run +/// no Python code (nothing can then reassign `__class__` and free the +/// class while it runs). +#[inline] +fn with_state(inst: &PyInstance, f: impl FnOnce(&State) -> R) -> Option { + if crate::gil::free_threading_enabled() { + let ext = class_ext(inst)?; + return Some(f(ext.downcast_ref::()?)); + } + Some(f(state_of(inst.cls_raw())?)) +} + +fn with_interp( + f: impl FnOnce(&mut crate::Interpreter) -> Result, +) -> Result { + let ptr = crate::vm_singletons::current_interpreter_ptr() + .ok_or_else(|| type_error("re: no active interpreter"))?; + // SAFETY: published by the enclosing VM frame on this thread. + f(unsafe { &mut *ptr }) +} + +fn build(cls: Rc, layout: &SharedSlice, values: Vec) -> Object { + let mut i = PyInstance::new(cls); + i.slots = RefCell::new(SlotStorage::from_layout(layout.clone(), values)); + Object::Instance(Rc::new(i)) +} + +// --------------------------------------------------------------------- +// Subjects. + +/// Whether `o` is a subject the fast paths serve for `cc`: an exact `str` +/// or `bytes` of the type the pattern accepts. +#[inline] +fn fast_subject(o: &Object, cc: &CompiledCode) -> bool { + match o { + Object::Str(_) => cc.is_str != Some(false), + Object::Bytes(_) => cc.is_str != Some(true), + _ => false, + } +} + +fn empty_like(o: &Object) -> Object { + match o { + Object::Bytes(_) => Object::new_bytes(Vec::new()), + _ => Object::Str(SharedStr::from_ascii("")), + } +} + +/// `s[a:b]` in code points, or `None` when the span is out of range. +fn str_slice(s: &SharedStr, a: usize, b: usize) -> Option { + if b <= a { + return (a <= SharedStr::char_count(s)).then(|| Object::Str(SharedStr::from_ascii(""))); + } + if SharedStr::is_ascii(s) { + let bytes = s.as_bytes(); + if b > bytes.len() { + return None; + } + if b - a == 1 { + return Some(Object::from_char(char::from(bytes[a]))); + } + return Some(Object::Str(SharedStr::from_ascii(&s[a..b]))); + } + let (mut ba, mut bb) = (None, None); + for (n, (i, _)) in s.char_indices().enumerate() { + if n == a { + ba = Some(i); + } + if n == b { + bb = Some(i); + break; + } + } + let ba = ba?; + let bb = match bb { + Some(i) => i, + None if b == SharedStr::char_count(s) => s.len(), + None => return None, + }; + Some(Object::Str(SharedStr::with_count(&s[ba..bb], b - a))) +} + +/// The subject's `[a:b]` slice (CPython's `getslice`), for an exact `str` +/// or `bytes`; `None` for any other subject or an out-of-range span. +fn subject_slice(o: &Object, a: usize, b: usize) -> Option { + match o { + Object::Str(s) => str_slice(s, a, b), + Object::Bytes(bs) => { + if b > bs.len() || a > b { + return None; + } + if a == b { + return Some(Object::new_bytes(Vec::new())); + } + Some(Object::Bytes(SharedSlice::from(&bs[a..b]))) + } + _ => None, + } +} + +/// The code units a subject is matched over: the bytes of a `bytes` or +/// of an ASCII `str` (borrowed in place), or the code points of any other +/// subject. +enum Units<'a> { + Narrow(&'a [u8]), + Wide(Rc>), +} + +/// A subject ready for matching, with cheap slicing for `str`/`bytes`. +struct Subject<'a> { + obj: &'a Object, + units: Units<'a>, +} + +impl<'a> Subject<'a> { + fn new(obj: &'a Object) -> Result { + let units = match obj { + Object::Str(s) if SharedStr::is_ascii(s) => Units::Narrow(s.as_bytes()), + Object::Bytes(b) => Units::Narrow(&b[..]), + _ => Units::Wide(sre_mod::decode_subject_cached(obj)?), + }; + Ok(Self { obj, units }) + } + + fn len(&self) -> usize { + match &self.units { + Units::Narrow(b) => b.len(), + Units::Wide(w) => w.len(), + } + } + + /// One match attempt (see [`sre_mod::exec_found`]). + fn exec( + &self, + cc: &CompiledCode, + pos: usize, + endpos: usize, + mode: i64, + must_advance: bool, + f: impl FnOnce(&Found<'_>) -> R, + ) -> Result, RuntimeError> { + match &self.units { + Units::Narrow(b) => sre_mod::exec_found(cc, b, pos, endpos, mode, must_advance, f), + Units::Wide(w) => sre_mod::exec_found(cc, &w[..], pos, endpos, mode, must_advance, f), + } + } + + /// `[a:b]` as a new object (the subject itself for the whole range). + fn slice(&self, a: usize, b: usize) -> Object { + let (a, b) = (a.min(self.len()), b.min(self.len())); + if a == 0 && b == self.len() { + return self.obj.clone(); + } + match (self.obj, &self.units) { + (Object::Str(_), _) if b <= a => Object::Str(SharedStr::from_ascii("")), + (Object::Str(s), Units::Narrow(_)) => { + if b - a == 1 { + Object::from_char(char::from(s.as_bytes()[a])) + } else { + Object::Str(SharedStr::from_ascii(&s[a..b])) + } + } + (Object::Str(_), Units::Wide(w)) => { + let mut t = String::with_capacity(b - a); + push_cps(&mut t, &w[a..b]); + Object::Str(SharedStr::with_count(&t, b - a)) + } + (Object::Bytes(_), _) if b <= a => Object::new_bytes(Vec::new()), + (Object::Bytes(bs), _) => Object::Bytes(SharedSlice::from(&bs[a..b])), + _ => Object::None, + } + } + + /// Group `g`'s text from `marks`, or `default` when it did not match. + fn group(&self, found: &Found<'_>, g: usize, default: &Object) -> Object { + let (a, b) = found.span(g); + if a < 0 || b < 0 { + default.clone() + } else { + self.slice(a as usize, b as usize) + } + } +} + +fn push_cps(t: &mut String, cps: &[u32]) { + for &c in cps { + t.push(char::from_u32(c).unwrap_or('\u{FFFD}')); + } +} + +/// An output buffer for `sub`: text for a `str` subject, bytes for a +/// `bytes` one, or a list of parts to `join` through the interpreter once +/// a replacement of another type turns up. +enum Sink { + Str(String), + Bytes(Vec), + Parts(Vec), +} + +impl Sink { + fn new(subject: &Subject<'_>) -> Self { + match subject.obj { + Object::Bytes(_) => Sink::Bytes(Vec::new()), + _ => Sink::Str(String::new()), + } + } + + fn push_span(&mut self, subject: &Subject<'_>, a: usize, b: usize) { + if b <= a { + return; + } + match (self, subject.obj, &subject.units) { + (Sink::Str(t), Object::Str(s), Units::Narrow(_)) => t.push_str(&s[a..b]), + (Sink::Str(t), Object::Str(_), Units::Wide(w)) => push_cps(t, &w[a..b]), + (Sink::Bytes(v), Object::Bytes(bs), _) => v.extend_from_slice(&bs[a..b]), + (Sink::Parts(parts), _, _) => parts.push(subject.slice(a, b)), + _ => {} + } + } + + fn push_obj(&mut self, o: &Object) { + match (&mut *self, o) { + (Sink::Str(t), Object::Str(s)) => t.push_str(s), + (Sink::Bytes(v), Object::Bytes(b)) => v.extend_from_slice(b), + (Sink::Parts(parts), _) => parts.push(o.clone()), + _ => { + let done = std::mem::replace(self, Sink::Parts(Vec::new())).finish_plain(); + if let Sink::Parts(parts) = self { + parts.push(done); + parts.push(o.clone()); + } + } + } + } + + fn finish_plain(self) -> Object { + match self { + Sink::Str(t) => Object::Str(SharedStr::from(t.as_str())), + Sink::Bytes(v) => Object::new_bytes(v), + Sink::Parts(_) => Object::None, + } + } + + fn finish(self, empty: &Object) -> Result { + match self { + Sink::Parts(parts) => with_interp(|interp| { + let join = interp.load_attr_public(empty, "join")?; + interp.call_object(join, &[Object::new_list(parts)], &[]) + }), + plain => Ok(plain.finish_plain()), + } + } +} + +// --------------------------------------------------------------------- +// Receivers and arguments. + +/// The pattern's compiled code and handle. +fn pattern_code(st: &State, inst: &PyInstance) -> Option<(&'static CompiledCode, i64)> { + let slots = inst.slots.borrow(); + let v = slots.values_for_layout(&st.pattern_layout)?; + let Object::Int(h) = v[P_CODE] else { + return None; + }; + Some((sre_mod::compiled(h)?, h)) +} + +/// A position argument: a plain `int` (others take the Python path). +#[inline] +fn pos_arg(o: Option<&Object>, default: i64) -> Option { + match o { + None => Some(default), + Some(Object::Int(i)) => Some(*i), + Some(Object::Bool(b)) => Some(i64::from(*b)), + _ => None, + } +} + +#[inline] +fn clamp(pos: i64, endpos: i64, len: usize) -> (usize, usize) { + let n = len as i64; + (pos.clamp(0, n) as usize, endpos.clamp(0, n) as usize) +} + +/// Lay keyword arguments out positionally after `args` (receiver first) +/// following `names`, when they fill the next positions exactly. +fn merge_kwargs(args: &[Object], kw: &[(String, Object)], names: &[&str]) -> Option> { + let given = args.len().checked_sub(1)?; + if given + kw.len() > names.len() { + return None; + } + let mut out = args.to_vec(); + for name in &names[given..given + kw.len()] { + let (_, v) = kw.iter().find(|(k, _)| k == name)?; + out.push(v.clone()); + } + Some(out) +} + +/// The flat spans tuple a match keeps (`_marks`), built in one +/// fixed-size allocation for the common group counts. +fn marks_tuple(found: &Found<'_>) -> Object { + use crate::tuple_storage::TupleStorage; + let n = 2 * (found.groups() + 1); + macro_rules! sized { + ($($k:literal),*) => { + match n { + $($k => TupleStorage::from_array::<$k>(std::array::from_fn(|i| { + Object::Int(found.mark(i)) + })),)* + _ => TupleStorage::from_exact_iter((0..n).map(|i| Object::Int(found.mark(i)))), + } + }; + } + Object::Tuple(sized!(2, 4, 6, 8, 10, 12)) +} + +fn new_match( + st: &State, + pattern: &Object, + string: &Object, + pos: usize, + endpos: usize, + found: &Found<'_>, +) -> Result { + let cls = st + .match_cls + .upgrade() + .ok_or_else(|| type_error("re.Match is gone"))?; + let values = vec![ + pattern.clone(), + string.clone(), + Object::Int(pos as i64), + Object::Int(endpos as i64), + marks_tuple(found), + Object::Int(found.lastindex as i64), + ]; + Ok(build(cls, &st.match_layout, values)) +} + +// --------------------------------------------------------------------- +// Pattern methods. + +/// The receiver, subject and position arguments of `match`/`search`/ +/// `fullmatch`/`findall`/`finditer`/`scanner`, when the fast path serves +/// them. +struct ExecArgs<'a> { + recv: &'a Object, + string: &'a Object, + pos: i64, + endpos: i64, +} + +fn exec_args(args: &[Object]) -> Option> { + let [recv, string, rest @ ..] = args else { + return None; + }; + if rest.len() > 2 { + return None; + } + Some(ExecArgs { + recv, + string, + pos: pos_arg(rest.first(), 0)?, + endpos: pos_arg(rest.get(1), i64::MAX)?, + }) +} + +/// Run `f` with the pattern's state and code, when the receiver is a +/// natively built pattern and the subject a fast one. `f` runs no Python +/// code, so the state is read through the class in place. +fn with_pattern( + recv: &Object, + string: &Object, + f: impl FnOnce(&State, &'static CompiledCode, i64) -> R, +) -> Option { + let Object::Instance(inst) = recv else { + return None; + }; + with_state(inst, |st| { + let (cc, handle) = pattern_code(st, inst)?; + if !fast_subject(string, cc) { + return None; + } + Some(f(st, cc, handle)) + })? +} + +/// [`with_pattern`] for a body that may run Python code: the state is +/// held by its own reference, not through the class. +fn with_pattern_held( + recv: &Object, + string: &Object, + f: impl FnOnce(&State, &'static CompiledCode, i64) -> R, +) -> Option { + let Object::Instance(inst) = recv else { + return None; + }; + let ext = class_ext(inst)?; + let st = ext.downcast_ref::()?; + let (cc, handle) = pattern_code(st, inst)?; + if !fast_subject(string, cc) { + return None; + } + Some(f(st, cc, handle)) +} + +fn pattern_exec(args: &[Object], mode: i64) -> Option> { + let a = exec_args(args)?; + with_pattern(a.recv, a.string, |st, cc, _| { + let subject = Subject::new(a.string)?; + let (p, e) = clamp(a.pos, a.endpos, subject.len()); + subject + .exec(cc, p, e, mode, false, |f| { + new_match(st, a.recv, a.string, p, e, f) + })? + .unwrap_or(Ok(Object::None)) + }) +} + +fn pattern_match(args: &[Object]) -> Option> { + pattern_exec(args, 1) +} + +fn pattern_fullmatch(args: &[Object]) -> Option> { + pattern_exec(args, 2) +} + +fn pattern_search(args: &[Object]) -> Option> { + pattern_exec(args, 0) +} + +fn pattern_findall(args: &[Object]) -> Option> { + let a = exec_args(args)?; + with_pattern(a.recv, a.string, |_, cc, _| { + let subject = Subject::new(a.string)?; + let (mut p, e) = clamp(a.pos, a.endpos, subject.len()); + let empty = empty_like(a.string); + let mut out = Vec::with_capacity(8); + let mut must_advance = false; + while p <= e { + let Some((s, en)) = subject.exec(cc, p, e, 0, must_advance, |f| { + out.push(match cc.groups { + 0 => subject.slice(f.start, f.end), + 1 => subject.group(f, 1, &empty), + n => Object::new_tuple((1..=n).map(|g| subject.group(f, g, &empty)).collect()), + }); + (f.start, f.end) + })? + else { + break; + }; + must_advance = s == en; + p = en; + } + Ok(Object::new_list(out)) + }) +} + +/// `finditer` and `scanner`: a scanner over a fast subject. +fn pattern_scanner(args: &[Object]) -> Option> { + let a = exec_args(args)?; + with_pattern(a.recv, a.string, |st, _, _| { + let len = match a.string { + Object::Str(s) => SharedStr::char_count(s), + Object::Bytes(b) => b.len(), + _ => 0, + }; + let (p, e) = clamp(a.pos, a.endpos, len); + let cls = st + .scanner_cls + .upgrade() + .ok_or_else(|| type_error("re scanner type is gone"))?; + Ok(build( + cls, + &st.scanner_layout, + vec![ + a.recv.clone(), + a.string.clone(), + Object::Int(p as i64), + Object::Int(p as i64), + Object::Int(e as i64), + Object::Bool(false), + ], + )) + }) +} + +/// Subjects up to this many code units have their matches found when +/// `finditer` is called. +const EAGER_FINDITER: usize = 512; + +/// `finditer`: over a short subject, an iterator over the matches found +/// up front (a native tuple iterator, which the dispatch loop steps and +/// ends without a call or a `StopIteration`); over a longer one, a lazy +/// scanner. Only an immutable `str` or `bytes` subject is served, so +/// finding the matches early cannot observe a different subject. +fn pattern_finditer(args: &[Object]) -> Option> { + let a = exec_args(args)?; + let len = match a.string { + Object::Str(s) => SharedStr::char_count(s), + Object::Bytes(b) => b.len(), + _ => return None, + }; + if len > EAGER_FINDITER { + return pattern_scanner(args); + } + with_pattern(a.recv, a.string, |st, cc, _| { + let subject = Subject::new(a.string)?; + let (mut p, e) = clamp(a.pos, a.endpos, subject.len()); + let opos = p; + let mut out = Vec::with_capacity(8); + let mut must_advance = false; + while p <= e { + let Some((m, s, en)) = subject.exec(cc, p, e, 0, must_advance, |f| { + (new_match(st, a.recv, a.string, opos, e, f), f.start, f.end) + })? + else { + break; + }; + out.push(m?); + must_advance = s == en; + p = en; + } + let items = crate::tuple_storage::TupleStorage::from_vec(out); + Ok(Object::Iter(Rc::new(RefCell::new( + crate::object::PyIterator::Tuple { items, index: 0 }, + )))) + }) +} + +/// How `sub` produces each replacement. +enum Filter<'a> { + Literal(&'a Object), + Template(Rc>), + Callable(&'a Object), +} + +/// The cached template for `repl` (a `str` or `bytes` with a backslash). +fn cached_template(st: &State, handle: i64, repl: &Object) -> Option>> { + let cache = st.templates.lock(); + let (key, t) = cache.get(&handle)?; + let same = match (key, repl) { + (Object::Str(a), Object::Str(b)) => a == b, + (Object::Bytes(a), Object::Bytes(b)) => a[..] == b[..], + _ => false, + }; + same.then(|| t.clone()) +} + +/// Parse `repl` through `re._engine._compile_template` and cache it. +fn parse_template( + st: &State, + pattern: &Object, + handle: i64, + repl: &Object, +) -> Result>>, RuntimeError> { + let f = st.compile_template.clone(); + let parsed = with_interp(|i| i.call_object(f, &[pattern.clone(), repl.clone()], &[]))?; + let items: Vec = match &parsed { + Object::List(l) => l.borrow().clone(), + Object::Tuple(t) => t.to_vec(), + _ => return Ok(None), + }; + let mut pieces = Vec::with_capacity(items.len()); + for it in items { + match it { + Object::Int(g) if g >= 0 => pieces.push(Piece::Group(g as usize)), + Object::Str(ref s) if s.is_empty() => {} + Object::Bytes(ref b) if b.is_empty() => {} + o @ (Object::Str(_) | Object::Bytes(_)) => pieces.push(Piece::Lit(o)), + _ => return Ok(None), + } + } + let t = Rc::new(pieces); + let mut cache = st.templates.lock(); + if cache.len() >= 512 { + cache.clear(); + } + cache.insert(handle, (repl.clone(), t.clone())); + Ok(Some(t)) +} + +fn has_backslash(repl: &Object) -> bool { + match repl { + Object::Str(s) => s.as_bytes().contains(&b'\\'), + Object::Bytes(b) => b.contains(&b'\\'), + _ => true, + } +} + +/// The receiver, replacement, subject and count of `sub`/`subn`. +fn sub_args(args: &[Object]) -> Option<(&Object, &Object, &Object, i64)> { + let [recv, repl, string, rest @ ..] = args else { + return None; + }; + let count = match rest { + [] => 0, + [Object::Int(c)] => *c, + [Object::Bool(b)] => i64::from(*b), + _ => return None, + }; + Some((recv, repl, string, count)) +} + +/// Whether `repl` is a `str`/`bytes` replacement of the subject's type. +fn plain_repl(repl: &Object, string: &Object) -> bool { + matches!( + (repl, string), + (Object::Str(_), Object::Str(_)) | (Object::Bytes(_), Object::Bytes(_)) + ) +} + +#[allow(clippy::too_many_arguments)] +fn run_sub( + st: &State, + cc: &CompiledCode, + recv: &Object, + string: &Object, + filter: &Filter<'_>, + count: i64, + subn: bool, +) -> Result { + let subject = Subject::new(string)?; + let e = subject.len(); + let mut sink = Sink::new(&subject); + let (mut p, mut last, mut n) = (0usize, 0usize, 0i64); + let mut must_advance = false; + while p <= e && (count == 0 || n < count) { + // The literal and template replacements are written while the + // match is at hand; a callable gets a Match object, called after. + let Some((s, en, m)) = subject.exec(cc, p, e, 0, must_advance, |f| { + sink.push_span(&subject, last, f.start); + let m = match filter { + Filter::Literal(l) => { + sink.push_obj(l); + None + } + Filter::Template(t) => { + for piece in t.iter() { + match piece { + Piece::Lit(l) => sink.push_obj(l), + Piece::Group(g) => { + let (a, b) = f.span(*g); + if a >= 0 && b >= 0 { + sink.push_span(&subject, a as usize, b as usize); + } + } + } + } + None + } + Filter::Callable(_) => Some(new_match(st, recv, string, 0, e, f)), + }; + (f.start, f.end, m) + })? + else { + break; + }; + match (filter, m) { + (Filter::Callable(f), Some(m)) => { + let m = m?; + // A builtin call runs with the interpreter and its thread + // handles already published, so the callable is called + // directly (as `functools` does). + let item = with_interp(|i| { + let globals = i.builtins_dict(); + i.call(f, &[m], &[], &globals) + })?; + if !matches!(item, Object::None) { + sink.push_obj(&item); + } + } + _ => {} + } + last = en; + n += 1; + must_advance = s == en; + p = en; + } + sink.push_span(&subject, last, e); + let result = if n == 0 && !matches!(sink, Sink::Parts(_)) { + // No replacement: CPython hands back the subject itself. + string.clone() + } else { + sink.finish(&empty_like(string))? + }; + Ok(if subn { + Object::new_tuple_array([result, Object::Int(n)]) + } else { + result + }) +} + +fn sub_fast(args: &[Object], subn: bool) -> Option> { + let (recv, repl, string, count) = sub_args(args)?; + if count < 0 || !plain_repl(repl, string) { + return None; + } + with_pattern(recv, string, |st, cc, handle| { + let filter = if has_backslash(repl) { + Filter::Template(cached_template(st, handle, repl)?) + } else { + Filter::Literal(repl) + }; + Some(run_sub(st, cc, recv, string, &filter, count, subn)) + })? +} + +/// `sub`'s full body: a callable replacement, or a template to parse. +fn sub_full(args: &[Object], subn: bool) -> Option> { + let (recv, repl, string, count) = sub_args(args)?; + if count < 0 { + return None; + } + let callable = !plain_repl(repl, string); + if callable && !crate::builtins::object_is_callable(repl) { + return None; + } + with_pattern_held(recv, string, |st, cc, handle| { + let filter = if callable { + Filter::Callable(repl) + } else if has_backslash(repl) { + match cached_template(st, handle, repl) { + Some(t) => Filter::Template(t), + None => match parse_template(st, recv, handle, repl) { + Ok(Some(t)) => Filter::Template(t), + Ok(None) => return None, + Err(e) => return Some(Err(e)), + }, + } + } else { + Filter::Literal(repl) + }; + Some(run_sub(st, cc, recv, string, &filter, count, subn)) + })? +} + +fn pattern_sub(args: &[Object]) -> Option> { + sub_fast(args, false) +} + +fn pattern_subn(args: &[Object]) -> Option> { + sub_fast(args, true) +} + +fn pattern_sub_full(args: &[Object]) -> Option> { + sub_full(args, false) +} + +fn pattern_subn_full(args: &[Object]) -> Option> { + sub_full(args, true) +} + +fn pattern_split(args: &[Object]) -> Option> { + let [recv, string, rest @ ..] = args else { + return None; + }; + let maxsplit = match rest { + [] => 0, + [Object::Int(m)] => *m, + [Object::Bool(b)] => i64::from(*b), + _ => return None, + }; + with_pattern(recv, string, |_, cc, _| { + let subject = Subject::new(string)?; + let e = subject.len(); + let mut out = Vec::with_capacity(8); + let (mut p, mut last, mut n) = (0usize, 0usize, 0i64); + let mut must_advance = false; + while p <= e && (maxsplit == 0 || n < maxsplit) { + let Some((s, en)) = subject.exec(cc, p, e, 0, must_advance, |f| { + out.push(subject.slice(last, f.start)); + for g in 1..=cc.groups { + out.push(subject.group(f, g, &Object::None)); + } + (f.start, f.end) + })? + else { + break; + }; + n += 1; + must_advance = s == en; + p = en; + last = en; + } + out.push(subject.slice(last, e)); + Ok(Object::new_list(out)) + }) +} + +// --------------------------------------------------------------------- +// Match methods. + +/// Run `f` over a natively built match's slot values and its marks. +fn with_match( + recv: &Object, + f: impl FnOnce(&State, &[Object], &[Object]) -> Option, +) -> Option { + let Object::Instance(inst) = recv else { + return None; + }; + with_state(inst, |st| { + let slots = inst.slots.borrow(); + let v = slots.values_for_layout(&st.match_layout)?; + let Object::Tuple(marks) = &v[M_MARKS] else { + return None; + }; + if marks.len() < 2 || marks.len() % 2 != 0 { + return None; + } + f(st, v, marks) + })? +} + +enum GroupIdx { + Index(usize), + NoSuch, + Slow, +} + +/// CPython's `match_getindex` for an `int` or `str` group. +fn group_index(st: &State, v: &[Object], marks: &[Object], g: &Object) -> GroupIdx { + let groups = marks.len() / 2 - 1; + let i = match g { + Object::Int(i) => *i, + Object::Bool(b) => i64::from(*b), + Object::Str(name) => { + let Object::Instance(p) = &v[M_RE] else { + return GroupIdx::Slow; + }; + let slots = p.slots.borrow(); + let Some(pv) = slots.values_for_layout(&st.pattern_layout) else { + return GroupIdx::Slow; + }; + let Object::MappingProxy(d) = &pv[P_GROUPINDEX] else { + return GroupIdx::Slow; + }; + let found = match d.borrow().get(&StrKey(name)) { + Some(Object::Int(i)) => Some(*i), + Some(_) => return GroupIdx::Slow, + None => None, + }; + match found { + Some(i) => i, + None => return GroupIdx::NoSuch, + } + } + _ => return GroupIdx::Slow, + }; + if i < 0 || i as usize > groups { + GroupIdx::NoSuch + } else { + GroupIdx::Index(i as usize) + } +} + +/// Group `idx`'s span. +fn span_of(marks: &[Object], idx: usize) -> Option<(i64, i64)> { + match (&marks[2 * idx], &marks[2 * idx + 1]) { + (Object::Int(a), Object::Int(b)) => Some((*a, *b)), + _ => None, + } +} + +/// Group `idx`'s text, or `default` when it did not match. +fn group_text(v: &[Object], marks: &[Object], idx: usize, default: &Object) -> Option { + let (a, b) = span_of(marks, idx)?; + if a < 0 || b < 0 { + return Some(default.clone()); + } + subject_slice(&v[M_STRING], a as usize, b as usize) +} + +fn no_such_group() -> RuntimeError { + index_error("no such group") +} + +fn match_group(args: &[Object]) -> Option> { + let (recv, rest) = args.split_first()?; + with_match(recv, |st, v, marks| match rest { + [] => Some(Ok(group_text(v, marks, 0, &Object::None)?)), + [g] => match group_index(st, v, marks, g) { + GroupIdx::Index(i) => Some(Ok(group_text(v, marks, i, &Object::None)?)), + GroupIdx::NoSuch => Some(Err(no_such_group())), + GroupIdx::Slow => None, + }, + many => { + let mut out = Vec::with_capacity(many.len()); + for g in many { + match group_index(st, v, marks, g) { + GroupIdx::Index(i) => out.push(group_text(v, marks, i, &Object::None)?), + GroupIdx::NoSuch => return Some(Err(no_such_group())), + GroupIdx::Slow => return None, + } + } + Some(Ok(Object::new_tuple(out))) + } + }) +} + +fn match_getitem(args: &[Object]) -> Option> { + match args { + [_, _] => match_group(args), + _ => None, + } +} + +fn match_groups(args: &[Object]) -> Option> { + let (recv, rest) = args.split_first()?; + let default = match rest { + [] => &Object::None, + [d] => d, + _ => return None, + }; + with_match(recv, |_, v, marks| { + let n = marks.len() / 2 - 1; + let mut out = Vec::with_capacity(n); + for i in 1..=n { + out.push(group_text(v, marks, i, default)?); + } + Some(Ok(Object::new_tuple(out))) + }) +} + +fn match_groupdict(args: &[Object]) -> Option> { + let (recv, rest) = args.split_first()?; + let default = match rest { + [] => &Object::None, + [d] => d, + _ => return None, + }; + with_match(recv, |st, v, marks| { + let Object::Instance(p) = &v[M_RE] else { + return None; + }; + let slots = p.slots.borrow(); + let pv = slots.values_for_layout(&st.pattern_layout)?; + let Object::MappingProxy(d) = &pv[P_GROUPINDEX] else { + return None; + }; + let groups = marks.len() / 2 - 1; + let mut out = DictData::default(); + for (k, idx) in d.borrow().iter() { + let Object::Int(i) = idx else { + return None; + }; + if *i < 0 || *i as usize > groups { + return None; + } + out.insert(k.clone(), group_text(v, marks, *i as usize, default)?); + } + Some(Ok(Object::Dict(Rc::new(RefCell::new(out))))) + }) +} + +/// `start` (0), `end` (1) and `span` (2). +fn match_position(args: &[Object], which: u8) -> Option> { + let (recv, rest) = args.split_first()?; + with_match(recv, |st, v, marks| { + let idx = match rest { + [] => 0, + [g] => match group_index(st, v, marks, g) { + GroupIdx::Index(i) => i, + GroupIdx::NoSuch => return Some(Err(no_such_group())), + GroupIdx::Slow => return None, + }, + _ => return None, + }; + let (a, b) = span_of(marks, idx)?; + Some(Ok(match which { + 0 => Object::Int(a), + 1 => Object::Int(b), + _ => Object::new_tuple_array([Object::Int(a), Object::Int(b)]), + })) + }) +} + +fn match_start(args: &[Object]) -> Option> { + match_position(args, 0) +} + +fn match_end(args: &[Object]) -> Option> { + match_position(args, 1) +} + +fn match_span(args: &[Object]) -> Option> { + match_position(args, 2) +} + +// --------------------------------------------------------------------- +// The scanner (`finditer`, `Pattern.scanner`). + +/// One `match()` / `search()` step of a scanner; `iter` turns exhaustion +/// into `StopIteration` for `__next__`. +fn scanner_step(args: &[Object], mode: i64, iter: bool) -> Result { + let done = || { + if iter { + Err(stop_iteration()) + } else { + Ok(Object::None) + } + }; + let [Object::Instance(inst)] = args else { + return Err(type_error("SRE_Scanner method takes no arguments")); + }; + let bad = || type_error("descriptor requires a '_sre.SRE_Scanner' object"); + let ext = class_ext(inst).ok_or_else(bad)?; + let st = ext.downcast_ref::().ok_or_else(bad)?; + let (pattern, string, start, pos, endpos, must_advance) = { + let slots = inst.slots.borrow(); + let v = slots + .values_for_layout(&st.scanner_layout) + .ok_or_else(bad)?; + let int = |o: &Object| o.as_i64().unwrap_or(-1); + ( + v[S_PATTERN].clone(), + v[S_STRING].clone(), + int(&v[S_START]), + int(&v[S_POS]), + int(&v[S_ENDPOS]), + matches!(v[S_MUST], Object::Bool(true)), + ) + }; + if start < 0 || start > endpos { + return done(); + } + let Object::Instance(pinst) = &pattern else { + return Err(bad()); + }; + let (cc, _) = pattern_code(st, pinst).ok_or_else(bad)?; + if !fast_subject(&string, cc) { + return Err(bad()); + } + let subject = Subject::new(&string)?; + let endpos = (endpos as usize).min(subject.len()); + let start = start as usize; + let pos = pos.max(0) as usize; + let r = subject.exec(cc, start, endpos, mode, must_advance, |f| { + ( + new_match(st, &pattern, &string, pos, endpos, f), + f.start, + f.end, + ) + })?; + let mut slots = inst.slots.borrow_mut(); + let v = slots + .values_for_layout_mut(&st.scanner_layout) + .ok_or_else(bad)?; + let Some((m, s, e)) = r else { + v[S_START] = Object::Int(endpos as i64 + 1); + return done(); + }; + v[S_START] = Object::Int(e as i64); + v[S_MUST] = Object::Bool(s == e); + drop(slots); + m +} + +fn scanner_match(args: &[Object]) -> Result { + scanner_step(args, 1, false) +} + +fn scanner_search(args: &[Object]) -> Result { + scanner_step(args, 0, false) +} + +fn scanner_next(args: &[Object]) -> Result { + scanner_step(args, 0, true) +} + +fn scanner_iter(args: &[Object]) -> Result { + match args { + [it] => Ok(it.clone()), + _ => Err(type_error("__iter__ takes no arguments")), + } +} + +// --------------------------------------------------------------------- +// Module functions. + +type Fast = fn(&[Object]) -> Option>; + +/// One native method of `Pattern` (class 0) or `Match` (class 1). +struct Spec { + cls: usize, + name: &'static str, + key: &'static str, + fast: Fast, + /// The body that may call into the interpreter, tried after `fast`. + full: Option, + /// Parameter names after the receiver, for keyword arguments. + params: &'static [&'static str], +} + +const EXEC_PARAMS: &[&str] = &["string", "pos", "endpos"]; + +const SPECS: &[Spec] = &[ + Spec { + cls: 0, + name: "match", + key: "Pattern.match", + fast: pattern_match, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "fullmatch", + key: "Pattern.fullmatch", + fast: pattern_fullmatch, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "search", + key: "Pattern.search", + fast: pattern_search, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "findall", + key: "Pattern.findall", + fast: pattern_findall, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "finditer", + key: "Pattern.finditer", + fast: pattern_finditer, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "scanner", + key: "Pattern.scanner", + fast: pattern_scanner, + full: None, + params: EXEC_PARAMS, + }, + Spec { + cls: 0, + name: "sub", + key: "Pattern.sub", + fast: pattern_sub, + full: Some(pattern_sub_full as Fast), + params: &["repl", "string", "count"], + }, + Spec { + cls: 0, + name: "subn", + key: "Pattern.subn", + fast: pattern_subn, + full: Some(pattern_subn_full as Fast), + params: &["repl", "string", "count"], + }, + Spec { + cls: 0, + name: "split", + key: "Pattern.split", + fast: pattern_split, + full: None, + params: &["string", "maxsplit"], + }, + Spec { + cls: 1, + name: "group", + key: "Match.group", + fast: match_group, + full: None, + params: &[], + }, + Spec { + cls: 1, + name: "__getitem__", + key: "Match.__getitem__", + fast: match_getitem, + full: None, + params: &[], + }, + Spec { + cls: 1, + name: "groups", + key: "Match.groups", + fast: match_groups, + full: None, + params: &["default"], + }, + Spec { + cls: 1, + name: "groupdict", + key: "Match.groupdict", + fast: match_groupdict, + full: None, + params: &["default"], + }, + Spec { + cls: 1, + name: "start", + key: "Match.start", + fast: match_start, + full: None, + params: &["group"], + }, + Spec { + cls: 1, + name: "end", + key: "Match.end", + fast: match_end, + full: None, + params: &["group"], + }, + Spec { + cls: 1, + name: "span", + key: "Match.span", + fast: match_span, + full: None, + params: &["group"], + }, +]; + +fn dispatch( + st: &State, + spec: &Spec, + args: &[Object], + kw: &[(String, Object)], +) -> Result { + let merged; + let pos_args = if kw.is_empty() { + Some(args) + } else { + merged = merge_kwargs(args, kw, spec.params); + merged.as_deref() + }; + if let Some(a) = pos_args { + if let Some(r) = (spec.fast)(a) { + return r; + } + if let Some(r) = spec.full.and_then(|full| full(a)) { + return r; + } + } + let f = st + .orig + .get(spec.key) + .cloned() + .ok_or_else(|| type_error(format!("{} is not available", spec.key)))?; + with_interp(|i| i.call_object(f, args, kw)) +} + +fn interned(s: &str) -> SharedStr { + match crate::stdlib::sys::intern_name(s) { + Object::Str(s) => s, + _ => SharedStr::from(s), + } +} + +fn layout(names: &[&str]) -> SharedSlice { + names + .iter() + .map(|n| DictKey(Object::Str(interned(n)))) + .collect::>() + .into() +} + +fn native(name: &'static str, f: fn(&[Object]) -> Result) -> Rc { + Rc::new(BuiltinFn { + name, + binds_instance: true, + call: Box::new(f), + call_kw: None, + }) +} + +/// `_sre.install(Pattern, Match, SRE_Scanner, compile_template)`: put the +/// native bodies in the classes (see the module docs). +pub(crate) fn install(args: &[Object]) -> Result { + let [Object::Type(pattern), Object::Type(matchc), Object::Type(scanner), compile_template] = + args + else { + return Err(type_error( + "install() expects Pattern, Match, the scanner type and _compile_template", + )); + }; + if pattern.native_ext.get().is_some() { + return Ok(Object::None); + } + let classes = [pattern, matchc]; + let mut orig = HashMap::new(); + for spec in SPECS { + let v = classes[spec.cls] + .dict + .borrow() + .get(&StrKey(spec.name)) + .cloned() + .ok_or_else(|| type_error(format!("install(): no {}", spec.key)))?; + orig.insert(spec.key, v); + } + let state = Rc::new(State { + match_cls: Rc::downgrade(matchc), + scanner_cls: Rc::downgrade(scanner), + pattern_layout: layout(&PATTERN_SLOTS), + match_layout: layout(&MATCH_SLOTS), + scanner_layout: layout(&SCANNER_SLOTS), + orig, + compile_template: compile_template.clone(), + templates: parking_lot::Mutex::new(HashMap::new()), + }); + for cls in [pattern, matchc, scanner] { + let _ = cls + .native_ext + .set(crate::rc_unsize!(state.clone() => dyn std::any::Any + Send + Sync)); + } + for spec in SPECS { + let (st, st_kw) = (state.clone(), state.clone()); + // A method with no keyword parameters takes no keyword entry (the + // dispatch loop's native subscript requires that of + // `__getitem__`); the VM rejects keywords for it. + let call_kw: Option _ + Send + Sync>> = + if spec.params.is_empty() { + None + } else { + Some(Box::new(move |a: &[Object], kw: &[(String, Object)]| { + dispatch(&st_kw, spec, a, kw) + })) + }; + let b = Rc::new(BuiltinFn { + name: spec.name, + binds_instance: true, + call: Box::new(move |a: &[Object]| dispatch(&st, spec, a, &[])), + call_kw, + }); + crate::leaf_builtins::register_fast(&b, spec.fast); + classes[spec.cls].dict.borrow_mut().insert( + DictKey(Object::Str(interned(spec.name))), + Object::Builtin(b), + ); + } + for (name, f) in [ + ( + "match", + scanner_match as fn(&[Object]) -> Result, + ), + ("search", scanner_search), + ("__next__", scanner_next), + ("__iter__", scanner_iter), + ] { + let b = native(name, f); + crate::leaf_builtins::register(&b); + scanner + .dict + .borrow_mut() + .insert(DictKey(Object::Str(interned(name))), Object::Builtin(b)); + } + for cls in [pattern, matchc, scanner] { + cls.bump_attr_version(); + } + Ok(Object::None) +} + +/// `_sre.make_pattern(Pattern, pattern, flags, code, groups, groupindex, +/// indexgroup)`: compile `code` and build the pattern object. +pub(crate) fn make_pattern(args: &[Object]) -> Result { + let [Object::Type(cls), pattern, flags, code, groups, groupindex, indexgroup] = args else { + return Err(type_error("make_pattern() takes 7 arguments")); + }; + let st = state_of(cls).ok_or_else(|| type_error("make_pattern(): re is not installed"))?; + let handle = sre_mod::compile_code(pattern, code, groups)?; + Ok(build( + cls.clone(), + &st.pattern_layout, + vec![ + Object::Int(handle), + pattern.clone(), + flags.clone(), + groups.clone(), + groupindex.clone(), + indexgroup.clone(), + ], + )) +} + +/// `_sre.exec_match(pattern, string, pos, endpos, mode, must_advance, +/// mpos)`: one match attempt over any subject, as a `Match` reporting +/// `mpos` and `endpos`, or `None`. The Python paths' engine entry. +pub(crate) fn exec_match(args: &[Object]) -> Result { + let [pattern, string, pos, endpos, mode, must_advance, mpos] = args else { + return Err(type_error("exec_match() takes 7 arguments")); + }; + let Object::Instance(inst) = pattern else { + return Err(type_error("exec_match(): expected a pattern")); + }; + let ext = class_ext(inst).ok_or_else(|| type_error("exec_match(): expected a pattern"))?; + let st = ext + .downcast_ref::() + .ok_or_else(|| type_error("exec_match(): expected a pattern"))?; + let (cc, _) = + pattern_code(st, inst).ok_or_else(|| type_error("exec_match(): expected a pattern"))?; + let subject = Subject::new(string)?; + let int = |o: &Object| { + o.as_i64() + .ok_or_else(|| type_error("exec_match(): expected an int")) + }; + let (p, e) = clamp(int(pos)?, int(endpos)?, subject.len()); + let must = must_advance.as_i64().unwrap_or(0) != 0; + let mpos = int(mpos)?.max(0) as usize; + subject + .exec(cc, p, e, int(mode)?, must, |f| { + new_match(st, pattern, string, mpos, e, f) + })? + .unwrap_or(Ok(Object::None)) +} + +/// `_sre.clear_templates(Pattern)`: forget the parsed templates +/// (`re.purge()`). +pub(crate) fn clear_templates(args: &[Object]) -> Result { + if let [Object::Type(cls)] = args { + if let Some(st) = state_of(cls) { + st.templates.lock().clear(); + } + } + Ok(Object::None) +} + +// --------------------------------------------------------------------- +// The module-level cache (`re.search(pattern, ...)` and friends). + +/// The fast half of [`compile_cached`]: a hit in `re._cache2`. +fn compile_cached_fast(args: &[Object]) -> Option> { + let [Object::Tuple(front), pattern, flags] = args else { + return None; + }; + let [Object::Dict(cache), _, Object::Type(flag_type)] = &front[..] else { + return None; + }; + let bt = crate::builtin_types::builtin_types(); + let ty = match pattern { + Object::Str(_) => bt.str_.clone(), + Object::Bytes(_) => bt.bytes_.clone(), + _ => return None, + }; + // `_compile` keys on `flags.value` for a `RegexFlag`, and on the flags + // themselves otherwise (any other int subclass keeps the Python path). + let flags = match flags { + Object::Int(_) | Object::Bool(_) => flags.clone(), + Object::Instance(i) if Rc::ptr_eq(&i.cls(), flag_type) => match i.native.get() { + Some(v @ Object::Int(_)) => v.clone(), + _ => return None, + }, + _ => return None, + }; + let key = DictKey(Object::new_tuple_array([ + Object::Type(ty), + pattern.clone(), + flags, + ])); + let hit = cache.borrow().get(&key).cloned(); + hit.map(Ok) +} + +/// `_sre.compile_cached((cache, compile, RegexFlag), pattern, flags)`: +/// `re._compile(pattern, flags)`, answered from the `_cache2` dict when +/// the pattern is there (the hit `_compile` would return first), else by +/// calling `_compile`. +pub(crate) fn compile_cached(args: &[Object]) -> Result { + if let Some(r) = compile_cached_fast(args) { + return r; + } + let [Object::Tuple(front), pattern, flags] = args else { + return Err(type_error("compile_cached() takes 3 arguments")); + }; + let compile = front + .get(1) + .cloned() + .ok_or_else(|| type_error("compile_cached(): bad cache front"))?; + with_interp(|i| i.call_object(compile, &[pattern.clone(), flags.clone()], &[])) +} + +/// The `compile_cached` builtin, with its fast half registered as a leaf. +pub(crate) fn compile_cached_builtin() -> Object { + let b = Rc::new(BuiltinFn { + name: "compile_cached", + binds_instance: false, + call: Box::new(compile_cached), + call_kw: None, + }); + crate::leaf_builtins::register_fast(&b, compile_cached_fast); + Object::Builtin(b) +} diff --git a/crates/weavepy-vm/src/stdlib/sys.rs b/crates/weavepy-vm/src/stdlib/sys.rs index 0181d019..21f54627 100644 --- a/crates/weavepy-vm/src/stdlib/sys.rs +++ b/crates/weavepy-vm/src/stdlib/sys.rs @@ -304,7 +304,7 @@ pub fn build_with_state( // pinned its whole importlib graph (and everything reachable from // it) for the life of the process. let tracked = |o: Object| { - crate::gc_trace::track(o.clone()); + crate::gc_trace::track(&o); o }; d.insert( @@ -1417,8 +1417,10 @@ fn sys_setrecursionlimit(args: &[Object]) -> Result { // (pickle's `load_build` inserts `sys.intern(k)` keys — // test_pickle test_attribute_name_interning). thread_local! { - static INTERN_POOL: RefCell> = - RefCell::new(std::collections::HashSet::new()); + // (Fx-hashed: interning runs for every name of every code object a + // module loads, and SipHash showed in import profiles.) + static INTERN_POOL: RefCell> = + const { RefCell::new(std::collections::HashSet::with_hasher(crate::fasthash::FxBuildHasher)) }; } /// Canonicalize `name` through the interpreter's intern pool, seeding it @@ -1939,7 +1941,7 @@ pub(crate) fn sizeof_estimate(o: &Object) -> i64 { // wide struct, plus len+1 units of the kind width // (test_str.test_raiseMemError pins all four kinds). Object::Str(s) => { - let len = crate::builtins::str_char_len(s) as i64; + let len = crate::object::str_char_len(s) as i64; match s.chars().map(u32::from).max().unwrap_or(0) { 0..=0x7f => 40 + len + 1, 0x80..=0xff => 56 + (len + 1), @@ -2738,22 +2740,16 @@ fn stdlib_module_names_value() -> Object { } /// `sys.getrefcount(obj)` — best-effort, derived from the real -/// `Rc::strong_count` of the payload. Infrastructure references -/// (the cycle-GC registry's handle, weakref slots' strong clones) -/// are discounted so the number tracks *program-visible* bindings; -/// `+1` accounts for the argument reference, like CPython. The +/// `Rc::strong_count` of the payload. The collector and the weakref +/// registry hold no strong references; clones held only by the C-API +/// layer's caches are discounted so the number tracks *program-visible* +/// bindings. `+1` accounts for the argument reference, like CPython. The /// exact number is implementation-specific even in CPython. fn sys_getrefcount(args: &[Object]) -> Result { let Some(obj) = args.first() else { return Err(type_error("getrefcount() takes exactly 1 argument")); }; let strong = crate::gc_trace::strong_count_for(obj); - let id = crate::weakref_registry::id_of(obj); - let registry = usize::from(crate::gc_trace::is_tracked(id)); - let weak_clones = crate::weakref_registry::strong_clone_count(id); - // A dropped-but-registry-pinned memoryview (dead under CPython - // refcounting) must not count through its exporter edge. - let zombie_refs = crate::gc_trace::zombie_memoryview_refs_to(id); // Clones held only by the C-API layer's pin caches (a parked // argument-pinned identity box, a dead-except-pin scalar/tuple pin — // RFC 0076 WS1) are infrastructure too: on CPython the corresponding @@ -2768,9 +2764,6 @@ fn sys_getrefcount(args: &[Object]) -> Result { // unless CPython wouldn't have taken one at all (a borrowed load). let borrowed = borrowed_argument_discount(obj); let visible = strong - .saturating_sub(registry) - .saturating_sub(weak_clones) - .saturating_sub(zombie_refs) .saturating_sub(pinned) .saturating_sub(borrowed) .saturating_add(extra_c); diff --git a/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs index bcef325e..c7f2e974 100644 --- a/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs +++ b/crates/weavepy-vm/src/stdlib/testinternalcapi_mod.rs @@ -1259,6 +1259,21 @@ fn has_inline_values(args: &[Object]) -> Result { Ok(Object::Bool(inline)) } +/// `_testinternalcapi.instance_layout(obj)` (WeavePy-only): how an +/// instance stores its attributes, without changing it — `"split"` (the +/// class's shared names), `"dict"` (a published dictionary), or `"none"`. +fn instance_layout(args: &[Object]) -> Result { + let layout = match args.first() { + Some(Object::Instance(inst)) if inst.dict.published().is_some() => "dict", + Some(Object::Instance(inst)) => match inst.dict.split_cell().try_borrow() { + Ok(split) if !split.is_empty() => "split", + _ => "none", + }, + _ => "none", + }; + Ok(Object::from_static(layout)) +} + /// `_testinternalcapi.run_in_subinterp_with_config(code, config)` — /// CPython's helper spins up a `Py_NewInterpreterFromConfig` interpreter, /// runs `code` with `PyRun_SimpleString`, and tears it down, returning @@ -2139,6 +2154,15 @@ pub fn build(_cache: &ModuleCache) -> Rc { call_kw: None, })), ); + d.insert( + DictKey(Object::from_static("instance_layout")), + Object::Builtin(Rc::new(BuiltinFn { + name: "instance_layout", + binds_instance: false, + call: Box::new(instance_layout), + call_kw: None, + })), + ); d.insert( DictKey(Object::from_static("has_inline_values")), Object::Builtin(Rc::new(BuiltinFn { diff --git a/crates/weavepy-vm/src/stdlib/thread_real.rs b/crates/weavepy-vm/src/stdlib/thread_real.rs index 06eebcda..0e528ee0 100644 --- a/crates/weavepy-vm/src/stdlib/thread_real.rs +++ b/crates/weavepy-vm/src/stdlib/thread_real.rs @@ -1434,16 +1434,19 @@ fn spawn_python_worker( let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { worker_interp.run_thread_local_death_cleanup(synth_id as i64); })); - // Run any `__del__`/weakref-callback finalizers this worker - // deferred onto its *thread-local* pending queue (the prompt-reap - // cascade and the `PyInstance` `Drop` safety net both enqueue - // there). A blocked pool handler thread — `_handle_results` + // Release the target and its arguments, then run the + // `__del__`s and weakref callbacks this worker queued on its + // *thread-local* pending queue — the target function's death + // included (test_thread.test__count waits on its weakref + // callback). A blocked pool handler thread — `_handle_results` // parked in `outqueue.get()` — never reaches an eval-loop tick // to drain its own queue, so without this flush an object whose - // last reference died on that thread (e.g. a `test_release_task_refs` - // result copy) would be freed silently at thread teardown with its - // `__del__` skipped, leaking it. Drain while the GIL and the - // worker interpreter are both still live; isolate panics. + // last reference died on that thread would be freed with its + // `__del__` skipped. Drain while the GIL and the worker + // interpreter are both still live; isolate panics. + drop(worker_func); + drop(positional); + drop(kwargs_pairs); let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { worker_interp.run_pending_finalizers(); })); @@ -1455,17 +1458,13 @@ fn spawn_python_worker( let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe( super::greenlet_native::on_thread_teardown, )); - // Release this worker's Python references *while the GIL is - // still held*. The interpreter snapshot pins a large shared - // object graph; dropping it after the GIL is gone lets its - // `Arc` decrements (and the prompt-reap bookkeeping they - // trigger) race a peer thread's in-flight mark phase — which + // Release this worker's interpreter *while the GIL is still + // held*. Its snapshot pins a large shared object graph, and + // dropping it after the GIL is gone lets its `Arc` decrements + // race a peer thread's in-flight mark phase — which // intermittently classified the peer's *live* suspended // generators as garbage and closed them mid-`for` loop // (test_threading.test_foreign_thread's `wait_threads_exit`). - drop(worker_func); - drop(positional); - drop(kwargs_pairs); drop(worker_interp); crate::vm_singletons::clear_thread_python_tls(); // Drop the guard before marking finished so the parent's @@ -1730,13 +1729,6 @@ fn make_thread_handle_object(state: Arc, ident: Object) -> Ob return Err(runtime_error("thread not started")); } if join_state.done.load(Ordering::Acquire) { - // Already-finished fast path: still sweep — the worker's teardown - // (`del self._target, self._args, self._kwargs` in `Thread.run`) - // dropped containers that stay pinned by their cycle-GC handles - // until a sweep, and callers expect join() to have restored - // CPython's refcount timing regardless of who won the race - // (test_threading.test_no_refcycle_through_target). - crate::gc_trace::reap_dead_acyclic(); return Ok(Object::None); } // Joining a thread from itself would deadlock; CPython's handle @@ -1786,18 +1778,6 @@ fn make_thread_handle_object(state: Arc, ident: Object) -> Ob } }, } - // RFC 0039 (WS4): the just-joined worker ran its target's - // teardown — e.g. `threading.Thread.run`'s - // `del self._target, self._args, self._kwargs` — on its own - // thread, dropping the last *program* reference to any argument - // cycle. Objects this (joining) thread allocated are still pinned - // by its cycle-GC handle until a collection, so a `del` right - // after the join would still see the dead container as a live - // referrer. Refcounting would have freed it instantly in CPython; - // sweep this thread's dead acyclic garbage now so the caller's - // subsequent `del` finalizes the cycle without an explicit - // `gc.collect()` (test_threading.test_no_refcycle_through_target). - crate::gc_trace::reap_dead_acyclic(); Ok(Object::None) }; diff --git a/crates/weavepy-vm/src/stdlib/warnings_mod.rs b/crates/weavepy-vm/src/stdlib/warnings_mod.rs index 7f92cd7d..6e9739b3 100644 --- a/crates/weavepy-vm/src/stdlib/warnings_mod.rs +++ b/crates/weavepy-vm/src/stdlib/warnings_mod.rs @@ -614,16 +614,8 @@ fn call_show_warning( )?; let shown = ip.call_object_with_globals(&show_fn, &[msg.clone()], &[], &globals); // CPython decrefs the transient `WarningMessage` the moment - // `_showwarnmsg` returns; when the hook did not retain it (the stock - // stderr writer), the message — and, through `source=`, the very - // object whose finalizer emitted the warning — dies right here. A - // plain Rust drop would leave the tracked message pinned by its own - // GC handle until the next cyclic collection, keeping e.g. an - // unclosed `SpooledTemporaryFile`'s buffered fd alive across tests - // (test_tempfile.test_warnings_on_cleanup). The refcount guard - // inside leaves a *recorded* message (a `catch_warnings(record=True)` - // log holds it) untouched. - ip.maybe_prompt_reap_replaced(msg); + // `_showwarnmsg` returns: unless the hook kept it, it dies here. + drop(msg); shown?; Ok(()) } diff --git a/crates/weavepy-vm/src/stdlib/weakref_real.rs b/crates/weavepy-vm/src/stdlib/weakref_real.rs index 3eeda5a7..f7f5ef48 100644 --- a/crates/weavepy-vm/src/stdlib/weakref_real.rs +++ b/crates/weavepy-vm/src/stdlib/weakref_real.rs @@ -1104,9 +1104,9 @@ fn make_ref_object_with_class( class_override: Option>, ) -> Object { let target_id = id_of(&target); - // The registry's strong clone keeps the referent alive until the - // collector's prompt reap clears it, so a deferred instance is - // tracked from here on. + // A weakly referenced instance may die in a cycle, and only a + // collection can find that, so a deferred instance is tracked from + // here on. if let Object::Instance(inst) = &target { inst.ensure_gc_tracked(); } @@ -1124,27 +1124,6 @@ fn make_ref_object_with_class( let slot = Arc::new(slot); register(slot.clone()); - // RFC 0040 (GC arc): a weakref *with a callback* (`weakref.ref(obj, cb)`, - // `weakref.finalize`, `multiprocessing.util.Finalize`) must fire that - // callback the instant the referent's last strong reference drops, not at - // the next cyclic collection. Enroll the (tracked) referent in the cycle - // GC's prompt-finalization index so a refcount-death between bytecodes - // fires it — matching CPython's `tp_dealloc` weakref clear. - // - // Callback-*less* refs must clear promptly too (leak tests observe - // `wr()` going `None` right after the owning scope exits — test_ssl's - // SSLContext checks), but NOT via this index: enrolling every weakref - // target would leave `has_any_finalizable()` permanently true from the - // interpreter-boot weakrefs (`abc`/`typing` registries), turning the - // eval loop's drop safe point into a full index scan on every - // reference-dropping opcode (a 3-4x interpreter-wide slowdown, RFC 0054 - // WS5 re-measure). They are served by the opcode-level prompt-reap - // cascade (`reap_dead_subgraph` clears weakrefs) plus the suspect - // re-probe for deaths inside Rust transients. - if callback.is_some() { - crate::gc_trace::note_weakref_finalizable(target_id); - } - // The getter owns the slot. Registry entries and the wrapper's // back-pointer are weak; callback ownership stays in traced storage. let slot_for_get = slot.clone(); @@ -1236,7 +1215,7 @@ fn make_ref_object_with_class( // is its own callback) and bound methods are tracked as well. let callback_can_cycle = !matches!(&callback, None | Some(Object::Builtin(_))); if callback_can_cycle { - crate::gc_trace::track(wrapper.clone()); + crate::gc_trace::track(&wrapper); } wrapper } @@ -1536,7 +1515,7 @@ mod tests { TypeObject::new_user("WeakrefTarget", vec![], DictData::default()).unwrap(), ); let r = c_new_ref(target.clone(), None).unwrap(); - crate::gc_trace::sweep_weakref_only_targets(); + crate::weakref_registry::sweep_dead_targets(); let live = ref_type_call(std::slice::from_ref(&r)).unwrap(); assert!(live.is_same(&target)); let id = id_of(&target); diff --git a/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs b/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs index 6575d610..f3129168 100644 --- a/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs +++ b/crates/weavepy-vm/src/stdlib/weave_frame_mod.rs @@ -208,6 +208,7 @@ fn frame_new(args: &[Object]) -> Result { on_stack: Cell::new(0), extra_locals: RefCell::new(None), cleared: Cell::new(false), + lazy_back: RefCell::new(None), }); Ok(Object::Frame(frame)) } diff --git a/crates/weavepy-vm/src/stdlib/win_console.rs b/crates/weavepy-vm/src/stdlib/win_console.rs index 33426e7b..eb1d0d9a 100644 --- a/crates/weavepy-vm/src/stdlib/win_console.rs +++ b/crates/weavepy-vm/src/stdlib/win_console.rs @@ -767,6 +767,6 @@ fn wcio_new(args: &[Object]) -> Result { } }; let inst = Object::Instance(Rc::new(PyInstance::new(cls))); - crate::gc_trace::track(inst.clone()); + crate::gc_trace::track(&inst); Ok(inst) } diff --git a/crates/weavepy-vm/src/sync.rs b/crates/weavepy-vm/src/sync.rs index 1d8b2ef0..e9a6a243 100644 --- a/crates/weavepy-vm/src/sync.rs +++ b/crates/weavepy-vm/src/sync.rs @@ -342,10 +342,16 @@ fn cells_shared() -> bool { static CELLS_UNGUARDED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); #[inline] -fn cells_unguarded() -> bool { +pub(crate) fn cells_unguarded() -> bool { CELLS_UNGUARDED.load(Ordering::Relaxed) } +/// The flag [`GilCell::peek`] tests, for native code that peeks (a byte: +/// nonzero once cells are shared between threads). +pub(crate) fn cells_unguarded_flag() -> *const bool { + CELLS_UNGUARDED.as_ptr() +} + /// Whether reference counts must be updated atomically: set whenever the /// cell bias is revoked, and also before spawning a thread that may touch /// objects before it registers (see [`crate::rc`]). Never cleared. @@ -658,6 +664,11 @@ impl fmt::Display for BorrowMutError { impl std::error::Error for BorrowMutError {} impl GilCell { + /// Where the borrow counter (an `i32`: negative while mutably + /// borrowed) and the payload sit, for native code that peeks. + pub(crate) const BORROW_OFFSET: usize = std::mem::offset_of!(Self, borrow); + pub(crate) const DATA_OFFSET: usize = std::mem::offset_of!(Self, data); + /// Build a cell holding `value`. `const`-callable so the /// codebase's existing `thread_local!{ static FOO: RefCell<…> = /// const { RefCell::new(None) }; }` patterns keep working. diff --git a/crates/weavepy-vm/src/tier2.rs b/crates/weavepy-vm/src/tier2.rs index ae5f98b0..b9960a7f 100644 --- a/crates/weavepy-vm/src/tier2.rs +++ b/crates/weavepy-vm/src/tier2.rs @@ -88,6 +88,11 @@ struct AttrGuard { /// The class value can't acquire descriptor hooks independently of /// the receiver's class version. Only native scalar updates use this. stable_descriptor: bool, + /// The index the name holds in its class's shared names, when the + /// site's index is that one (`u32::MAX` otherwise): compiled code + /// reads and writes a split-layout instance's field there in line + /// (see [`obj_layout`]). + split_idx: u32, /// The value lane the site was compiled with. lane: JitType, /// The class's `attr_version` at compile time. Tokens are globally unique @@ -110,9 +115,12 @@ struct AttrGuard { /// instance-dict shadowing, and `__code__` rebinding introduced after /// compilation stay exact. struct MethodEntry { - func: Rc, - /// The function's `__code__` at compile time (rebindable). - code: Rc, + /// What the class resolves the name to. + callee: MethodCallee, + /// The class forbids an instance `__dict__` (its `__slots__` cover + /// every base), so no instance attribute can shadow the method: the + /// guard skips that probe. + no_dict: bool, /// The method name (for the shadow check and the span rebuild). name: String, /// `hash(name)`, so the per-call shadow probe hashes nothing. @@ -126,11 +134,80 @@ struct MethodEntry { /// A globally unique resolution token. The live receiver owns the class /// checked at access; this cached method must not keep that class alive. ver: u64, + /// What compiled code needs to run this method's callback-free field + /// update (`self.n += k; return self.n`) in line, armed by + /// `wpjit_call_method` once the helper has run it (see [`arm_update`]). + update: InlineUpdate, +} + +/// [`MethodEntry::update`]: compiled code reads these in place (see +/// [`obj_layout`]) and calls the helper unless the receiver's class still +/// has the entry's version, its values are split over the class's names +/// and are too few to shadow the method's name, the function still wears +/// the entry's code, and no observer or exotic key could tell. +#[repr(C)] +struct InlineUpdate { + /// The split index of the field the update adds to (`u32::MAX` while + /// unarmed). + idx: Cell, + /// A receiver holding more split values than this may shadow the + /// method's name. + shadow: Cell, + /// `1` when the increment is the call's argument; `0` when it's + /// [`Self::inc`]. + from_arg: Cell, + /// The literal increment. + inc: Cell, + /// Where the function keeps its code pointer, and the entry's code + /// pointer it must equal. + code_at: Cell, + code: Cell, +} + +impl InlineUpdate { + fn unarmed() -> Self { + Self { + idx: Cell::new(u32::MAX), + shadow: Cell::new(0), + from_arg: Cell::new(0), + inc: Cell::new(0), + code_at: Cell::new(0), + code: Cell::new(0), + } + } +} + +/// The callee a burned-in method site resolved to. +enum MethodCallee { + /// A plain Python function and its `__code__` at compile time + /// (rebindable, so the guard compares it per call). + Py { + func: Rc, + code: Rc, + }, + /// A native accelerator's method: a builtin that binds its instance, + /// registered as a leaf and opted into this lane + /// (`leaf_builtins::register_jit_method`), such as `deque.append`. + /// Its body runs no Python code, so the helper calls it directly + /// with no interpreter frame; `fast` is its pure fast half when the + /// full body may reach Python (an argument's `__index__`). + Native { + builtin: Rc, + fast: Option, + /// The builtin's direct operation + /// (`collections_native::fast_op`), `0` for none. + op: u8, + }, } /// One slot per method token (parallel to `cf.method_sites`). type MethodTable = Vec; +/// The widest positional arity a native method site admits (receiver +/// included). The builtin validates its own arity, so this only bounds +/// the marshal buffer. +const NATIVE_METHOD_MAX_ARGS: u32 = 8; + /// One burned-in math intrinsic. The ordinary global snapshot guards the /// module's identity; this guard reads its current attribute by a checked /// dictionary index. Neither a value replacement nor a moved key is hidden. @@ -207,17 +284,33 @@ struct GuardSnapshot { /// value epoch)` of the last full validation that held; all-zero /// until one has. last_ok: std::cell::Cell<(usize, u64, usize, u64, u64)>, + /// Callees whose defaults the compiled code may have burned in (a + /// direct leaf call that leaves parameters out): they must keep them. + defaults: Vec>, + /// Per callee-table entry, the class version at which a burned-in + /// constructor's full probe last held (`u64::MAX` for none): the + /// construction plan is a function of it, so an unchanged class only + /// needs its `__init__` code and metaclass rechecked. + ctor_vers: Vec>, } impl GuardSnapshot { - fn new(entries: Vec<(String, Object)>) -> Self { + fn new(entries: Vec<(String, Object)>, defaults: Vec>, callees: usize) -> Self { Self { entries, last_ok: std::cell::Cell::new((0, 0, 0, 0, 0)), + defaults, + ctor_vers: (0..callees).map(|_| Cell::new(u64::MAX)).collect(), } } } +/// Whether `f.__defaults__` no longer is the tuple `f` was defined with. +#[inline] +fn defaults_overridden(f: &PyFunction) -> bool { + f.defaults_maybe_overridden() && f.slot("__defaults__").is_some() +} + impl std::ops::Deref for GuardSnapshot { type Target = [(String, Object)]; fn deref(&self) -> &[(String, Object)] { @@ -360,6 +453,15 @@ struct CacheEntry { /// (it would observe the same frame state); other entries still /// do, with their own live values. probe_misses: Vec, + /// A compiled frame took a cold exit (see `CompiledFrame::cold_exits`): + /// the code after it was unsupported *with the values live when it was + /// compiled* (typically a later loop's receiver, still unbound when an + /// earlier loop triggered the compile). The next OSR request at a loop + /// header the frame can't enter recompiles from there, with that + /// loop's live values, at most [`COLD_RECOMPILE_BUDGET`] times. + recompile_at_osr: bool, + /// Recompiles taken for [`Self::recompile_at_osr`]. + cold_recompiles: u8, /// Native side exits taken by this code's compiled frame. Healthy /// compiled code exits by *returning* (deopt is exceptional — a /// type-lane surprise or invalidated guard), so a frame that keeps @@ -785,6 +887,12 @@ fn compile_allowed(counter: u32, threshold: u32) -> bool { || counter >= threshold.saturating_mul(16) } +/// [`compile_allowed`] for the frame compiler (see `frame_jit`), which +/// keeps tier 2's start-up and import budgets. +pub(crate) fn frame_compile_allowed(counter: u32, threshold: u32) -> bool { + compile_allowed(counter, threshold) +} + /// True when no thread's JIT can be enabled (see [`JIT_PROCESS_GATE`]). #[inline] pub(crate) fn jit_off_for_process() -> bool { @@ -856,6 +964,7 @@ impl JitState { weavepy_jit::register_const_str_helper(wpjit_const_str); weavepy_jit::register_tuple_read_helpers(wpjit_const_tuple, wpjit_tuple_len); weavepy_jit::register_unbox_int_helper(wpjit_unbox_int); + weavepy_jit::register_is_obj_helper(wpjit_is_obj); weavepy_jit::register_dict_iter_helper(wpjit_dict_iter_new); // RFC 0073 WS3 — the string write lanes. weavepy_jit::register_str_write_helpers( @@ -868,6 +977,11 @@ impl JitState { weavepy_jit::register_self_call_helpers(wpjit_self_enter, wpjit_self_exit, wpjit_self_slow); // RFC 0069 WS1 — the guarded method-call lane. weavepy_jit::register_call_method_helper(wpjit_call_method); + weavepy_jit::register_call_native_method_helper(wpjit_call_native_method); + // Native container subscripts (`deque.__getitem__`) and native + // method-site load guards. + weavepy_jit::register_obj_getitem_helper(wpjit_obj_getitem); + weavepy_jit::register_guard_method_helper(wpjit_guard_method); // RFC 0073 WS3 — the native `str`-method lane. weavepy_jit::register_str_method_helper(wpjit_str_method); // RFC 0069 WS2 — the libm sin/cos intrinsics and the Python- @@ -926,6 +1040,8 @@ impl JitState { native_entries: 0, generic_dyn_calls: 0, probe_misses: Vec::new(), + recompile_at_osr: false, + cold_recompiles: 0, native: None, method_native: None, direct: None, @@ -936,7 +1052,22 @@ impl JitState { // compile even when an individual activation is short. entry.defer_osr = false; } + // A cold exit handed the activation to the interpreter, which + // now asks to enter a loop the frame can't: compile again from + // here (see `CacheEntry::recompile_at_osr`). + let recompile = entry.recompile_at_osr + && entry_pc != 0 + && matches!(&entry.tier, Tier::Compiled(a) + if !a.cf.osr_entries.iter().any(|e| e.pc == entry_pc)); + if recompile { + entry.recompile_at_osr = false; + entry.cold_recompiles = entry.cold_recompiles.saturating_add(1); + entry.osr_failures = 0; + entry.tier = Tier::Cold; + entry.counter = entry.counter.max(self.threshold); + } match &entry.tier { + Tier::Cold if recompile => {} Tier::Compiled(a) => { let out = CompiledEntry { cf: a.cf.clone(), @@ -1137,6 +1268,7 @@ impl JitState { arg_count: e.arg_count, min_args: e.min_args, ret: e.ret, + native: matches!(e.callee, MethodCallee::Native { .. }), }); } let e = method(slot, path, name)?; @@ -1147,6 +1279,7 @@ impl JitState { arg_count: e.arg_count, min_args: e.min_args, ret: e.ret, + native: matches!(e.callee, MethodCallee::Native { .. }), }; methods.push(e); Some(res) @@ -1221,17 +1354,41 @@ impl JitState { // A callee already compiled as a guard-free scalar leaf is // entered directly by native code (its identity is guarded // with the callee table like any burned-in callee). + // A site that leaves trailing parameters out binds the + // function's scalar defaults there (guarded through + // `GuardSnapshot::defaults`). let mut direct = |token: u32| { let callees = callees.borrow(); - let (Object::Function(_), fcode) = callees.get(token as usize)? else { + let (Object::Function(f), fcode) = callees.get(token as usize)? else { return None; }; let k = Rc::as_ptr(fcode).cast::(); - match &cache_ref.get(&k)?.tier { - Tier::Compiled(a) => a.cf.direct_leaf(fcode.arg_count), - _ => None, + let leaf = match &cache_ref.get(&k)?.tier { + Tier::Compiled(a) => a.cf.direct_leaf(fcode.arg_count)?, + _ => return None, + }; + if f.defaults.is_empty() || defaults_overridden(f) { + return Some(leaf); } + let n = leaf.params().len(); + let first = n.saturating_sub(f.defaults.len()); + let defaults = (0..n) + .map(|i| { + let d = f.defaults.get((i + f.defaults.len()).checked_sub(n)?)?; + let lane = leaf.params()[i]; + // `bool` is not `int` here (the lanes are exact). + let exact = matches!( + (lane, d), + (JitType::Int, Object::Int(_)) + | (JitType::Float, Object::Float(_)) + | (JitType::Bool, Object::Bool(_)) + ); + (i >= first && exact).then(|| pack(d, lane)).flatten() + }) + .collect(); + Some(leaf.with_defaults(defaults)) }; + ensure_obj_layout(); let r = engine.compile_frame_direct(code, &mut classify, &mut jit_probes, &mut direct); if let Some(t0) = t0 { eprintln!("jit compile-time {:?} {:?}", code.name, t0.elapsed()); @@ -1327,10 +1484,23 @@ impl JitState { } let scalar_update = scalar_field_update_plan(code, &cf, &attr_guards).map(Box::new); + // Every callee whose defaults a direct leaf call could + // have bound (see `direct` above). + let burned_defaults = callees + .iter() + .filter_map(|(f, _)| match f { + Object::Function(f) + if !f.defaults.is_empty() && !defaults_overridden(f) => + { + Some(f.clone()) + } + _ => None, + }) + .collect(); let artifacts = StdRc::new(Artifacts { cf: StdRc::new(cf), scalar_update, - snap: StdRc::new(GuardSnapshot::new(snap)), + snap: StdRc::new(GuardSnapshot::new(snap, burned_defaults, callees.len())), callees: StdRc::new(callees), obj_globals: StdRc::new(obj_globals), attr_guards: StdRc::new(attr_guards), @@ -1476,7 +1646,10 @@ impl JitState { }; let table: NativeTable = methods .iter() - .map(|m| self.resolve_native_func(&m.func, &m.code, true)) + .map(|m| match &m.callee { + MethodCallee::Py { func, code } => self.resolve_native_func(func, code, true), + MethodCallee::Native { .. } => None, + }) .collect(); let tbl = StdRc::new(table); if let Some(entry) = self.cache.get_mut(&key) { @@ -1609,6 +1782,8 @@ impl JitState { native_entries: 0, generic_dyn_calls: 0, probe_misses: Vec::new(), + recompile_at_osr: false, + cold_recompiles: 0, native: None, method_native: None, direct: None, @@ -2165,6 +2340,9 @@ fn method_ret_info( /// slot bits tagged [`SlotTag::ListPin`] / [`SlotTag::ObjPin`] index /// this table, which keeps the object alive and reachable for the /// access helpers and the deopt rebuild. +// `repr(C, u8)`: compiled code reads an object pin's discriminant and +// value in place (see `obj_layout`). +#[repr(C, u8)] enum Pin { /// A pinned list plus the element lane the compile assumed. List(Rc>>, JitType), @@ -2185,6 +2363,47 @@ impl Pin { /// One activation's pinned objects (RFC 0061/0065 WS5). type PinTable = Vec; +/// Whether a pin holds an object that something else keeps alive too (or +/// one that owns nothing): releasing it would free nothing and finalize +/// nothing, so holding it costs only the table's slot. +fn pin_is_shared(p: &Pin) -> bool { + match p { + Pin::List(l, _) => Rc::strong_count(l) > 1, + Pin::Obj(o) => match o { + Object::None | Object::Int(_) | Object::Float(_) | Object::Bool(_) => true, + Object::Instance(i) => Rc::strong_count(i) > 1, + Object::List(l) => Rc::strong_count(l) > 1, + _ => false, + }, + } +} + +/// At the soft pin limit, decide whether the activation may keep going: +/// the limit bounds how many objects a long activation keeps alive past +/// their last other reference (delaying their release and finalizers), so +/// pins whose objects are still referenced elsewhere (a loop over a list's +/// elements, results stored as they're made) don't count. When few of the +/// temporaries are last references, the limit rises by another soft +/// limit's worth (up to half the hard cap); otherwise the poll leaves. +/// Each pin is counted once, when the limit is first reached past it. +fn relax_pin_limit(ctx: &mut CallCtx) -> bool { + let from = ctx.pins_counted.max(ctx.entry_pin_count); + if let Some(fresh) = ctx.pins.get(from..) { + ctx.last_ref_pins += fresh.iter().filter(|p| !pin_is_shared(p)).count(); + } + ctx.pins_counted = ctx.pins.len(); + let temporaries = ctx.pins.len().saturating_sub(ctx.entry_pin_count); + // Half the hard cap stays free: a poll interval that pins heavily + // must still reach the next poll before the helpers refuse to pin. + if ctx.last_ref_pins < RUNTIME_PIN_SOFT_LIMIT / 2 + && ctx.pins.len() + RUNTIME_PIN_SOFT_LIMIT <= RUNTIME_PIN_CAP / 2 + { + ctx.pin_limit = temporaries + RUNTIME_PIN_SOFT_LIMIT; + return true; + } + false +} + /// RFC 0070 WS1 — hard cap on an activation's pin table. Object-lane /// attribute loads append pins for changing results (a list traversal /// appends one per node), so an unbounded loop needs a bound: at the cap the @@ -2198,62 +2417,20 @@ const RUNTIME_PIN_CAP: usize = 1 << 16; const RUNTIME_PIN_SOFT_LIMIT: usize = 1 << 12; /// Retire every pin after the activation's result or complete interpreter -/// state has been reconstructed. Entry pins can also become dead temporaries -/// when native code replaces a local or detaches an object from its owner. -/// Give them the same prompt-reap treatment as runtime pins and interpreter -/// locals. A suspended native activation must keep its entire table instead. -/// Returns `true` when a cascade could have run Python, requiring the caller -/// to revalidate its compiled resolutions. -fn drain_activation_pins(interp: &mut super::Interpreter, pins: &mut PinTable) -> bool { - let mut dirty = false; - for p in pins.drain(..) { - // Fresh lists are tracked just like interpreter-built lists. - // Dropping only the native pin leaves the collector's strong - // handle retaining a dead temporary until a later collection. - let o = match p { - Pin::List(list, _) => Object::List(list), - Pin::Obj(o) => o, - }; - // The common pin, a receiver or argument others still hold, - // releases plainly (the grades below would conclude the same). - if crate::gc_trace::drop_survives_plainly(&o) { - continue; - } - if super::Interpreter::local_needs_prompt_reap(&o) - && super::Interpreter::looks_reapable_temporary(&o) - { - dirty = true; - interp.maybe_prompt_reap_replaced(o); - } else { - crate::gc_trace::note_dropped(&o); - } - } - dirty -} - -/// Retire an activation's pins after its complete -/// interpreter frame has been rebuilt. Leaves can drop immediately, but -/// a dying container or finalizable object must wait for the interpreter -/// to install that frame and charge its recursion tick. The existing -/// pending-drop queue runs its cascade at the next bytecode safe point. -/// No Python runs here, and all live values already have rebuilt roots. +/// state has been reconstructed. A pin that held an object's last +/// reference frees it; a dying object only queues work for the next safe +/// point, so no Python runs here. Returns whether Python could have run +/// (it can't), for the callers that revalidate their compiled resolutions. +/// A suspended native activation must keep its entire table instead. +fn drain_activation_pins(_interp: &mut super::Interpreter, pins: &mut PinTable) -> bool { + pins.clear(); + false +} + +/// Retire an activation's pins after its complete interpreter frame has +/// been rebuilt (see [`drain_activation_pins`]). fn defer_activation_pins(pins: &mut PinTable) { - for pin in pins.drain(..) { - let obj = match pin { - Pin::List(list, _) => Object::List(list), - Pin::Obj(obj) => obj, - }; - if crate::gc_trace::drop_survives_plainly(&obj) { - continue; - } - if super::Interpreter::local_needs_prompt_reap(&obj) - && super::Interpreter::looks_reapable_temporary(&obj) - { - crate::vm_singletons::queue_parked_drop(&obj); - } else { - crate::gc_trace::note_dropped(&obj); - } - } + pins.clear(); } /// Reconstruct an [`Object`] from a `(bits, tag)` slot. `Boxed` never @@ -2387,8 +2564,14 @@ fn probe_list_lane(frame: &super::Frame, slot: u32) -> Option { if items.is_empty() { return Some(JitType::Unknown); } + list_elem_lane(&items) +} + +/// The uniform element lane of a non-empty list's `items` (see +/// [`probe_list_lane`]), `None` for a mixed or unsupported list. +fn list_elem_lane(items: &[Object]) -> Option { let mut lane: Option = None; - for it in items.iter() { + for it in items { let t = match it { Object::Int(_) => JitType::Int, Object::Float(_) => JitType::Float, @@ -2541,13 +2724,8 @@ fn attr_chain_step(obj: &Object, name: &str) -> Option { }; let v = match storage { AttrStorage::Slot(_) => inst.slot_get(name)?, - AttrStorage::Indexed(key_idx) => inst - .dict - .get()? - .borrow() - .get_index(key_idx as usize)? - .1 - .clone(), + // Either layout, without materializing a split one. + AttrStorage::Indexed(key_idx) => inst.attr_index_map(key_idx as usize, |_, v| v.clone())?, AttrStorage::NewKey => return None, }; matches!(v, Object::Instance(_)).then_some(v) @@ -2661,6 +2839,7 @@ fn attr_site_guard( lane: site.lane, ver: cls.attr_version.get(), storage: AttrStorage::Indexed(*field_idx), + split_idx: split_index(&cls, AttrStorage::Indexed(*field_idx), &site.name), last_result_pin: Cell::new(usize::MAX), }); } @@ -2679,7 +2858,7 @@ fn attr_site_guard( else { return None; }; - let Object::Instance(_) = &recv else { + let Object::Instance(inst) = &recv else { return None; }; return Some(AttrGuard { @@ -2689,6 +2868,7 @@ fn attr_site_guard( lane: site.lane, ver, storage: AttrStorage::Indexed(field_idx), + split_idx: split_index(&inst.cls(), AttrStorage::Indexed(field_idx), &site.name), last_result_pin: Cell::new(usize::MAX), }); } @@ -2704,9 +2884,16 @@ fn attr_site_guard( return None; } let stable_descriptor = site.store - && matches!(storage, AttrStorage::Indexed(_)) && scalar_field_update_shape(&frame.code) - && scalar_update_class_value_stable(&recv, &site.name, ver); + && match storage { + AttrStorage::Indexed(_) => scalar_update_class_value_stable(&recv, &site.name, ver), + AttrStorage::Slot(_) => scalar_update_slot_stable(&recv, &site.name, ver), + AttrStorage::NewKey => false, + }; + let split_idx = match &recv { + Object::Instance(inst) => split_index(&inst.cls(), storage, &site.name), + _ => u32::MAX, + }; Some(AttrGuard { name: shared_name(), name_hash: crate::object::py_str_hash(&site.name), @@ -2714,10 +2901,303 @@ fn attr_site_guard( lane: site.lane, ver, storage, + split_idx, last_result_pin: Cell::new(usize::MAX), }) } +/// [`AttrGuard::split_idx`] for an indexed site of `cls`: the index, when +/// the class's shared names hold `name` there (they never move). +/// +/// A new-key site's is the name's position among its class's shared names +/// (compiled code appends the field there, in line, while a fresh instance +/// sets its fields in the class's order). +fn split_index(cls: &TypeObject, storage: AttrStorage, name: &str) -> u32 { + let i = match storage { + AttrStorage::Indexed(i) => i, + AttrStorage::NewKey if cls.native_kind.get() == 0 => { + let Some(keys) = cls.shared_keys.get() else { + return u32::MAX; + }; + match u32::try_from(keys.names_before(name, crate::object::py_str_hash(name))) { + Ok(i) => i, + Err(_) => return u32::MAX, + } + } + _ => return u32::MAX, + }; + match cls.shared_keys.get().and_then(|keys| keys.get(i as usize)) { + Some(DictKey(Object::Str(s))) if &**s == name => i, + _ => u32::MAX, + } +} + +/// Measure where pinned instances keep what compiled code's in-line field +/// reads and writes touch (see [`weavepy_jit::ObjLayout`]), check every +/// offset against `obj` (a live instance), and publish the layout. Once: +/// later calls return straight away. +#[inline] +pub(crate) fn ensure_obj_layout() { + static DONE: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + if DONE.load(std::sync::atomic::Ordering::Relaxed) { + return; + } + DONE.store(true, std::sync::atomic::Ordering::Relaxed); + publish_obj_layout(); +} + +#[cold] +#[inline(never)] +fn publish_obj_layout() { + // A private instance of `object` to measure against (never seen by + // Python code; nothing tracks it). + let cls = crate::builtin_types::builtin_types().object_.clone(); + let inst = Rc::new(crate::types::PyInstance::new(cls)); + let obj = Object::Instance(inst.clone()); + if let Some(layout) = obj_layout(&obj, &inst) { + let _ = PUBLISHED_LAYOUT.set(layout); + weavepy_jit::set_obj_layout(layout); + } +} + +static PUBLISHED_LAYOUT: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// The object layout [`ensure_obj_layout`] published, for the VM's other +/// native code (the leaf plans'). +pub(crate) fn published_obj_layout() -> Option<&'static weavepy_jit::ObjLayout> { + PUBLISHED_LAYOUT.get() +} + +/// The layout [`ensure_obj_layout`] publishes, `None` when any offset +/// doesn't read back what the safe accessors say about `obj`. +fn obj_layout(obj: &Object, inst: &Rc) -> Option { + use crate::inst_dict::{InstDict, SplitValues}; + use crate::sync::GilCell; + use crate::types::PyInstance; + if crate::gil::free_threading_enabled() + || std::env::var_os("WEAVEPY_JIT_NO_INLINE_ATTRS").is_some() + { + return None; + } + let word = |p: *const u8, off: usize| -> usize { + // SAFETY: every read below is inside a live object the caller + // holds, at an offset measured from its own type. + unsafe { p.add(off).cast::().read_unaligned() } + }; + let tag = |v: &Object| -> u8 { + // SAFETY: `Object` is `repr(u8)`: its first byte is the tag. + unsafe { *std::ptr::from_ref(v).cast::() } + }; + let obj_p = std::ptr::from_ref(obj).cast::(); + // The payload word is the `Arc` allocation; the instance follows its + // two counts. + let arc = word(obj_p, 8); + let data = Rc::as_ptr(inst) as usize; + let arc_data = data.checked_sub(arc)?; + let i32_of = |v: usize| i32::try_from(v).ok(); + let inst_class = + arc_data + std::mem::offset_of!(PyInstance, class) + GilCell::>::DATA_OFFSET; + let dict = arc_data + std::mem::offset_of!(PyInstance, dict); + let inst_dict_lazy = dict + InstDict::LAZY_OFFSET; + let inst_split_borrow = dict + InstDict::SPLIT_OFFSET + GilCell::::BORROW_OFFSET; + let inst_split_block = dict + + InstDict::SPLIT_OFFSET + + GilCell::::DATA_OFFSET + + SplitValues::BLOCK_OFFSET; + let arc_p = arc as *const u8; + // The class pointer is its `Arc` allocation too. + let cls = inst.cls(); + let cls_arc = word(arc_p, inst_class); + if cls_arc + arc_data != Rc::as_ptr(&cls) as usize { + return None; + } + // A class value's payload is the same allocation pointer. + let cls_obj = Object::Type(cls.clone()); + if word(std::ptr::from_ref(&cls_obj).cast::(), 8) != cls_arc { + return None; + } + // The native body's word: a `Cell` alone in its wrapper, + // checked on a sample. + let inst_c_body = arc_data + std::mem::offset_of!(PyInstance, c_body); + { + let sample = crate::types::CBody::default(); + sample.set(0x5eed_c0de); + if std::mem::size_of::() != 8 + || word(std::ptr::from_ref(&sample).cast::(), 0) != 0x5eed_c0de + || word(arc as *const u8, inst_c_body) != inst.c_body.get() + { + return None; + } + } + let type_attr_version = arc_data + std::mem::offset_of!(TypeObject, attr_version); + let type_shared_keys = arc_data + + std::mem::offset_of!(TypeObject, shared_keys) + + crate::sync::LazyArc::::POINTER_OFFSET; + let cls_p = cls_arc as *const u8; + // SAFETY: as `word`, a `u64` field. + let ver = unsafe { cls_p.add(type_attr_version).cast::().read_unaligned() }; + let keys = cls + .shared_keys + .get() + .map_or(0, |k| std::ptr::from_ref(k) as usize); + let published = inst.dict.published().is_some(); + if ver != cls.attr_version.get() + || word(cls_p, type_shared_keys) != keys + || (word(arc_p, inst_dict_lazy) != 0) != published + { + return None; + } + // SAFETY: as `word`, the `i32` borrow counter. + let borrow = unsafe { arc_p.add(inst_split_borrow).cast::().read_unaligned() }; + if borrow != 0 { + return None; + } + // A pin: `repr(C, u8)`. + let sample = Pin::Obj(obj.clone()); + let sample_p = std::ptr::from_ref(&sample).cast::(); + let pin_obj = match &sample { + Pin::Obj(o) => std::ptr::from_ref(o) as usize - sample_p as usize, + Pin::List(..) => return None, + }; + // SAFETY: the discriminant byte of a `repr(C, u8)` enum. + let pin_obj_tag = unsafe { *sample_p }; + if word(sample_p, pin_obj + 8) != arc { + return None; + } + drop(sample); + // A vector's buffer pointer, length and capacity, measured on one + // whose three words all differ. + let mut v: Vec = Vec::with_capacity(4); + v.push(Pin::Obj(Object::None)); + let vp = std::ptr::from_ref(&v).cast::(); + let find = |want: usize| (0..3).map(|k| k * 8).find(|&off| word(vp, off) == want); + let (vec_ptr, vec_len, vec_cap) = (find(v.as_ptr() as usize)?, find(1)?, find(v.capacity())?); + drop(v); + // A list pin: the list's `Arc` allocation and its element lane. + let list = Rc::new(GilRefCell::new(vec![Object::Int(1)])); + let list_arc = { + // SAFETY: `Rc` is one pointer to its allocation. + unsafe { std::mem::transmute_copy::>>, usize>(&list) } + }; + let sample = Pin::List(list.clone(), JitType::Float); + let sample_p = std::ptr::from_ref(&sample).cast::(); + // SAFETY: the discriminant byte of a `repr(C, u8)` enum. + let pin_list_tag = unsafe { *sample_p }; + let (pin_list, pin_list_elem) = match &sample { + Pin::List(l, e) => ( + std::ptr::from_ref(l) as usize - sample_p as usize, + std::ptr::from_ref(e) as usize - sample_p as usize, + ), + Pin::Obj(_) => return None, + }; + // SAFETY: the element lane's byte. + let lane_byte = unsafe { *sample_p.add(pin_list_elem) }; + if word(sample_p, pin_list) != list_arc + || lane_byte != JitType::Float as u8 + || pin_list_tag == pin_obj_tag + || std::mem::size_of::() != 1 + { + return None; + } + drop(sample); + let list_data = list_arc_data(&list, list_arc)?; + let list_borrow = list_data + GilCell::>::BORROW_OFFSET; + let list_vec = list_data + GilCell::>::DATA_OFFSET; + let list_p = list_arc as *const u8; + // SAFETY: as `word`, the `i32` borrow counter. + let list_free = unsafe { list_p.add(list_borrow).cast::().read_unaligned() } == 0; + let items = list.borrow(); + if !list_free + || word(list_p, list_vec + vec_ptr) != items.as_ptr() as usize + || word(list_p, list_vec + vec_len) != items.len() + || word(list_p, list_vec + vec_cap) != items.capacity() + { + return None; + } + drop(items); + let guards: StdRc> = StdRc::new(Vec::with_capacity(2)); + let rc_box = { + // SAFETY: `std::rc::Rc` is one pointer to its allocation. + unsafe { std::mem::transmute_copy::>, usize>(&guards) } + }; + let guards_vec = std::ptr::from_ref::>(&guards) as usize - rc_box; + let guards_buf = guards_vec + vec_ptr; + if word(rc_box as *const u8, guards_buf) != guards.as_ptr() as usize { + return None; + } + // The method entries hang off the same shape of shared vector. + let methods: StdRc = StdRc::new(Vec::with_capacity(3)); + let methods_box = { + // SAFETY: as for the guards above. + unsafe { std::mem::transmute_copy::, usize>(&methods) } + }; + let methods_vec = std::ptr::from_ref::(&methods) as usize - methods_box; + if word(methods_box as *const u8, methods_vec + vec_ptr) != methods.as_ptr() as usize + || word(methods_box as *const u8, methods_vec + vec_len) != 0 + { + return None; + } + let update = std::mem::offset_of!(MethodEntry, update); + let ctx_pins = std::mem::offset_of!(CallCtx, pins); + Some(weavepy_jit::ObjLayout { + ctx_pins_ptr: i32_of(ctx_pins + vec_ptr)?, + ctx_pins_len: i32_of(ctx_pins + vec_len)?, + pin_size: i32_of(std::mem::size_of::())?, + pin_tag: 0, + pin_obj_tag, + pin_obj: i32_of(pin_obj)?, + ctx_guards: i32_of(std::mem::offset_of!(CallCtx, attr_guards))?, + guards_buf: i32_of(guards_buf)?, + guard_size: i32_of(std::mem::size_of::())?, + guard_ver: i32_of(std::mem::offset_of!(AttrGuard, ver))?, + guard_split_idx: i32_of(std::mem::offset_of!(AttrGuard, split_idx))?, + tag_instance: tag(obj), + tag_int: tag(&Object::Int(0)), + tag_float: tag(&Object::Float(0.0)), + tag_bool: tag(&Object::Bool(false)), + tag_none: tag(&Object::None), + tag_type: tag(&cls_obj), + inst_class: i32_of(inst_class)?, + inst_dict_lazy: i32_of(inst_dict_lazy)?, + inst_split_borrow: i32_of(inst_split_borrow)?, + inst_split_block: i32_of(inst_split_block)?, + type_attr_version: i32_of(type_attr_version)?, + type_shared_keys: i32_of(type_shared_keys)?, + split_keys: i32_of(SplitValues::KEYS_OFFSET)?, + split_len: i32_of(SplitValues::LEN_OFFSET)?, + split_cap: i32_of(SplitValues::CAP_OFFSET)?, + inst_c_body: i32_of(inst_c_body)?, + split_values: i32_of(SplitValues::VALUES_OFFSET)?, + cells_unguarded: crate::sync::cells_unguarded_flag() as usize, + pin_list_tag, + pin_list: i32_of(pin_list)?, + pin_list_elem: i32_of(pin_list_elem)?, + list_borrow: i32_of(list_borrow)?, + list_ptr: i32_of(list_vec + vec_ptr)?, + list_len: i32_of(list_vec + vec_len)?, + list_cap: i32_of(list_vec + vec_cap)?, + ctx_methods: i32_of(std::mem::offset_of!(CallCtx, methods))?, + methods_buf: i32_of(methods_vec + vec_ptr)?, + methods_len: i32_of(methods_vec + vec_len)?, + method_size: i32_of(std::mem::size_of::())?, + method_ver: i32_of(std::mem::offset_of!(MethodEntry, ver))?, + method_upd_idx: i32_of(update + std::mem::offset_of!(InlineUpdate, idx))?, + method_upd_shadow: i32_of(update + std::mem::offset_of!(InlineUpdate, shadow))?, + method_upd_from_arg: i32_of(update + std::mem::offset_of!(InlineUpdate, from_arg))?, + method_upd_inc: i32_of(update + std::mem::offset_of!(InlineUpdate, inc))?, + method_upd_code_at: i32_of(update + std::mem::offset_of!(InlineUpdate, code_at))?, + method_upd_code: i32_of(update + std::mem::offset_of!(InlineUpdate, code))?, + observers: crate::trace::observer_count_flag() as usize, + dict_watchers: crate::capi_watchers::dicts_active_flag() as usize, + exotic_keys: crate::object::exotic_str_keys_flag() as usize, + }) +} + +/// Where a list's cell sits in its `Arc` allocation at `arc`. +fn list_arc_data(list: &Rc>>, arc: usize) -> Option { + (Rc::as_ptr(list) as usize).checked_sub(arc) +} + /// RFC 0069 WS1 — the compile-time method probe: resolve `name` on the /// class of the instance currently in local `slot`, when the shape is /// eligible for a burned-in method call: @@ -2749,15 +3229,45 @@ fn probe_method_entry( if crate::specialize::type_has_attr_override(&cls) { return None; } - if inst - .dict - .get() - .is_some_and(|dict| dict.borrow().get(&StrKey(name)).is_some()) - { + // Read without materializing a split layout: a probe must not change + // how the instance stores its attributes. + if inst.attr_get_str(name).is_some() { return None; } - let Some(Object::Function(f)) = cls.lookup(name) else { - return None; + let f = match cls.lookup(name) { + // The analyzer probes `__getitem__` for native subscripts only + // (`TOp::ObjGetItem`); a Python-level one must not take a token + // that no site would use. + Some(Object::Function(_)) if name == "__getitem__" => return None, + Some(Object::Function(f)) => f, + // A native accelerator's leaf method (`deque.append`): any + // positional arity (the body validates it and raises exactly), + // and an object-lane result (`None` rides the nullable `-1`, so a + // procedure's result costs no pin). + Some(Object::Builtin(b)) if b.binds_instance => { + let op = crate::leaf_builtins::jit_method_op(&b)?; + let fast = match interp.leaf_call_kind(&b)? { + super::LeafKind::Opaque => None, + super::LeafKind::Fast(fast) => Some(fast), + _ => return None, + }; + return Some(MethodEntry { + callee: MethodCallee::Native { + builtin: b, + fast, + op, + }, + no_dict: cls.forbids_dict, + name: name.to_owned(), + name_hash: crate::object::py_str_hash(name), + arg_count: NATIVE_METHOD_MAX_ARGS, + min_args: 1, + ret: MethodRet::Scalar(JitType::Obj), + ver: cls.attr_version.get(), + update: InlineUpdate::unarmed(), + }); + } + _ => return None, }; let fcode = f.code.borrow().clone(); if !py_callee_ok(&fcode) || fcode.arg_count == 0 { @@ -2781,14 +3291,18 @@ fn probe_method_entry( let ver = cls.attr_version.get(); let arg_count = fcode.arg_count; Some(MethodEntry { - func: f, - code: fcode, + callee: MethodCallee::Py { + func: f, + code: fcode, + }, + no_dict: cls.forbids_dict, name: name.to_owned(), name_hash: crate::object::py_str_hash(name), arg_count, min_args, ret, ver, + update: InlineUpdate::unarmed(), }) } @@ -2903,13 +3417,7 @@ pub(crate) fn gc_sweep() { let dead: Vec<*const CodeObject> = st .cache .iter() - .filter(|(key, e)| { - // Python weakref slots temporarily hold strong clones; - // the collector discounts those from reachability too. - let weak_clones = - crate::weakref_registry::strong_clone_count(**key as usize as u64); - Rc::strong_count(&e.code) == 1 + weak_clones - }) + .filter(|(_, e)| Rc::strong_count(&e.code) == 1) .map(|(k, _)| *k) .collect(); for k in dead { @@ -2992,6 +3500,8 @@ pub(crate) fn warm_compile(interp: &mut super::Interpreter, frame: &mut super::F native_entries: 0, generic_dyn_calls: 0, probe_misses: Vec::new(), + recompile_at_osr: false, + cold_recompiles: 0, native: None, method_native: None, direct: None, @@ -3107,9 +3617,23 @@ fn resolve_plain_dicts( /// ops (`CallDyn`, `DynAttrGet`/`Set`, iterator capture) apply and /// every other access helper deopts on the lane surprise. The identity /// guard makes the grade stable for the compilation's whole life. +/// +/// A non-empty list with a uniform element lane rides its pinned-list +/// lane, as a list local does (see [`probe_list_lane`]): the identity +/// guard keeps the list object, and every element access re-validates +/// the element lane, so later changes to the list's contents deopt. fn grade_obj_global(obj: &Object) -> JitType { match obj { Object::Str(_) => JitType::Str, + Object::List(l) => { + let items = l.borrow(); + if items.is_empty() { + return JitType::Obj; + } + list_elem_lane(&items) + .and_then(JitType::list_of) + .unwrap_or(JitType::Obj) + } _ => JitType::Obj, } } @@ -3191,6 +3715,14 @@ struct CallCtx { pins: PinTable, /// Entry roots do not count toward the temporary-pin soft limit. entry_pin_count: usize, + /// How many temporary pins this activation may hold before its next + /// poll leaves (see [`relax_pin_limit`]): the soft limit, raised while + /// most of them aren't the last reference to their object. + pin_limit: usize, + /// The temporary pins [`relax_pin_limit`] has counted so far, and how + /// many of those held their object's last reference when it did. + pins_counted: usize, + last_ref_pins: usize, /// Set only when a helper is committed to a reconstruction exit. /// Generated code exits before another native operation can run. pin_pressure_exit: bool, @@ -3266,11 +3798,79 @@ struct CallCtx { /// a local) re-enters without re-resolving or rebuilding the handle. /// Taken out for the call's duration and put back after. dyn_callee: Option<(usize, usize, bool, NativeCallee)>, + /// Pin indices recently handed out for instances and strings, by + /// object address (see [`pin_reusing`]): a loop over the same few + /// objects reuses their pins instead of growing the table. Advisory; + /// every hit is checked against the pin it names. + pin_memo: [u32; PIN_MEMO], +} + +/// The size of [`CallCtx::pin_memo`]. +const PIN_MEMO: usize = 16; + +/// The identity [`pin_reusing`] keys an object by (`0` for objects whose +/// pins aren't reused). +#[inline] +fn pin_identity(v: &Object) -> usize { + match v { + Object::Instance(i) => Rc::as_ptr(i) as usize, + Object::Str(s) => SharedStr::as_ptr(s).cast::() as usize, + _ => 0, + } +} + +/// The [`CallCtx::pin_memo`] entry for an object identity. +#[inline] +fn pin_memo_slot(id: usize) -> usize { + (id >> 4 ^ id >> 9) % PIN_MEMO +} + +/// The pin that already holds this very object, when `memo` remembers +/// one. A pin only owns its object, so two sites that pin the same object +/// can share one. +#[inline] +fn pin_memo_hit(v: &Object, pins: &PinTable, memo: &[u32; PIN_MEMO]) -> Option { + let id = pin_identity(v); + if id == 0 { + return None; + } + let hint = memo[pin_memo_slot(id)] as usize; + match pins.get(hint) { + Some(Pin::Obj(p)) if pin_identity(p) == id => Some(hint as u64), + _ => None, + } +} + +/// Remember that pin `ix` will hold `p`'s object. +#[inline] +fn pin_memo_note(p: &Pin, ix: usize, memo: &mut [u32; PIN_MEMO]) { + if let Pin::Obj(v) = p { + let id = pin_identity(v); + if id != 0 { + memo[pin_memo_slot(id)] = ix as u32; + } + } +} + +/// Pin `v` in `pins`, reusing the pin that already holds this very object +/// (see [`pin_memo_hit`]); `None` at the pin cap. +#[inline] +fn pin_reusing(v: &Object, pins: &mut PinTable, memo: &mut [u32; PIN_MEMO]) -> Option { + if let Some(bits) = pin_memo_hit(v, pins, memo) { + return Some(bits); + } + if pins.len() >= RUNTIME_PIN_CAP { + return None; + } + let p = Pin::Obj(v.clone()); + pin_memo_note(&p, pins.len(), memo); + pins.push(p); + Some((pins.len() - 1) as u64) } impl CallCtx { fn temporary_pin_limit_reached(&self) -> bool { - self.pins.len().saturating_sub(self.entry_pin_count) >= RUNTIME_PIN_SOFT_LIMIT + self.pins.len().saturating_sub(self.entry_pin_count) >= self.pin_limit } /// RFC 0073 WS1 — re-resolve this activation's native-callee @@ -3295,6 +3895,8 @@ impl CallCtx { /// code-rebindable; a swap invalidates arity/lane assumptions), and /// each burned-in math intrinsic's `name.attr` still resolves to the /// snapshotted function (RFC 0069 WS2 — module dicts are mutable). +/// In line up to the namespaces' stamps, which settle the common case. +#[inline] fn guards_hold( interp: &super::Interpreter, globals: &Rc>, @@ -3315,17 +3917,48 @@ fn guards_hold( // A rebinding in place leaves the stamps alone (see `STORE_GLOBAL`). crate::object::global_value_epoch(), ); - if guard_snapshot.last_ok.get() != key || interp.globals_missing_any.get() { - for (name, expected) in guard_snapshot.entries.iter() { - let ok = resolve_plain_dicts(interp, globals, builtins, name) - .is_some_and(|cur| cur.is_same(expected)); - if !ok { - return false; - } + if (guard_snapshot.last_ok.get() != key || interp.globals_missing_any.get()) + && !global_guards_hold(interp, globals, builtins, guard_snapshot, key) + { + return false; + } + (callees.is_empty() || callee_guards_hold(interp, callees, &guard_snapshot.ctor_vers)) + && math.iter().all(MathGuard::holds) + && !guard_snapshot + .defaults + .iter() + .any(|f| defaults_overridden(f)) +} + +/// [`guards_hold`]'s globals, resolved name by name (and remembered as +/// holding at `key` when they do). +#[inline(never)] +fn global_guards_hold( + interp: &super::Interpreter, + globals: &Rc>, + builtins: &Rc>, + guard_snapshot: &GuardSnapshot, + key: (usize, u64, usize, u64, u64), +) -> bool { + for (name, expected) in guard_snapshot.entries.iter() { + let ok = resolve_plain_dicts(interp, globals, builtins, name) + .is_some_and(|cur| cur.is_same(expected)); + if !ok { + return false; } - guard_snapshot.last_ok.set(key); } - for (f, code_snap) in callees { + guard_snapshot.last_ok.set(key); + true +} + +/// [`guards_hold`]'s burned-in callees. +#[inline(never)] +fn callee_guards_hold( + interp: &super::Interpreter, + callees: &CalleeTable, + ctor_vers: &[Cell], +) -> bool { + for (i, (f, code_snap)) in callees.iter().enumerate() { match f { Object::Function(pf) => { // SAFETY: GIL-serialized raw read of the code cell (see @@ -3345,6 +3978,12 @@ fn guards_hold( // is memoised on `attr_version`, so an unchanged class // revalidates with a version compare. Object::Type(t) => { + let ver = t.attr_version.get(); + let memo = ctor_vers.get(i); + if memo.is_some_and(|m| m.get() == ver) && ctor_still_inits_with(t, ver, code_snap) + { + continue; + } let ok = matches!( probe_class_ctor_shape(interp, t), Some((init_code, ..)) if Rc::ptr_eq(&init_code, code_snap) @@ -3352,18 +3991,40 @@ fn guards_hold( if !ok { return false; } + if let Some(m) = memo { + m.set(ver); + } } _ => return false, } } - for guard in math { - if !guard.holds() { - return false; - } - } true } +/// Whether class `t`, unchanged since its constructor's probe held at +/// version `ver`, still constructs through type's own metaclass with the +/// plain `__init__` wearing `code`: the rest of the probe is a function of +/// the version (the memoised plan says so for the plan itself). +fn ctor_still_inits_with(t: &TypeObject, ver: u64, code: &Rc) -> bool { + if !t.metaclass_is_type() { + return false; + } + let Ok(plan) = t.instance_plan.try_borrow() else { + return false; + }; + match plan.as_ref() { + Some((v, plan)) if *v == ver => match plan.init_fn.as_ref() { + // SAFETY: GIL-serialized raw read of the code cell; only the + // pointer is compared. + Some(Object::Function(f)) => { + std::ptr::eq(unsafe { Rc::as_ptr(&*f.code.as_ptr()) }, Rc::as_ptr(code)) + } + _ => false, + }, + _ => false, + } +} + /// RFC 0067 WS1 — pooled exchange buffers for nested native entries /// (and the top-level `enter_compiled`), so a call-recursive program /// doesn't `malloc` five vectors per call. @@ -3441,6 +4102,39 @@ fn put_u32(v: Vec) { }); } +thread_local! { + /// The pin table of the last framed activation to finish, drained and + /// kept for the next one: a long loop that leaves at the pin limit and + /// re-enters would otherwise regrow its table from empty each time. + static SPARE_PINS: Cell = const { Cell::new(Vec::new()) }; +} + +/// The most pins [`SPARE_PINS`] keeps capacity for: as many as an +/// activation reaches before a pressure exit (see [`relax_pin_limit`]), +/// with room for its entry pins (under a megabyte). +const SPARE_PINS_CAP: usize = RUNTIME_PIN_CAP / 2 + 2 * RUNTIME_PIN_SOFT_LIMIT; + +/// An empty pin table, with the spare's capacity when there is one. +fn take_pins() -> PinTable { + SPARE_PINS.with(Cell::take) +} + +/// Keep `pins`' allocation for the next activation (see [`SPARE_PINS`]). +fn put_pins(mut pins: PinTable) { + if pins.capacity() == 0 || pins.capacity() > SPARE_PINS_CAP { + return; + } + pins.clear(); + SPARE_PINS.with(|spare| { + let old = spare.take(); + spare.set(if old.capacity() >= pins.capacity() { + old + } else { + pins + }); + }); +} + #[cfg(test)] mod scratch_pool_tests { use super::*; @@ -3598,7 +4292,7 @@ unsafe extern "C" fn wpjit_poll(frame: *mut JitFrame) -> i64 { ) { return 1; } - if ctx.temporary_pin_limit_reached() { + if ctx.temporary_pin_limit_reached() && !relax_pin_limit(ctx) { // The generated poll immediately spills live locals and the // boundary stack, then exits. No pin is freed while native // registers can still refer to its index. @@ -3628,14 +4322,12 @@ fn lane_tag(t: JitType) -> u32 { /// RFC 0071 WS1 — pack a call result for an `ObjPin` return lane: the /// `None` singleton is the nullable lane's `-1`; an instance pins into -/// the caller's table (capped). Anything else can't ride the lane. -fn obj_ret_bits(v: &Object, pins: &mut PinTable) -> Option { +/// the caller's table (capped), reusing the pin that already holds it +/// (a method returning `self`). Anything else can't ride the lane. +fn obj_ret_bits(v: &Object, pins: &mut PinTable, memo: &mut [u32; PIN_MEMO]) -> Option { match v { Object::None => Some(u64::MAX), - Object::Instance(_) if pins.len() < RUNTIME_PIN_CAP => { - pins.push(Pin::Obj(v.clone())); - Some((pins.len() - 1) as u64) - } + Object::Instance(_) => pin_reusing(v, pins, memo), _ => None, } } @@ -3768,6 +4460,23 @@ fn scalar_update_class_value_stable(receiver: &Object, name: &str, ver: u64) -> && cls.attr_version.get() == ver } +/// [`scalar_update_class_value_stable`] for a `__slots__` member: the +/// class value is the member's own slot descriptor, which has no hooks a +/// class mutation wouldn't version. +fn scalar_update_slot_stable(receiver: &Object, name: &str, ver: u64) -> bool { + if crate::object::exotic_str_keys_possible() { + return false; + } + let Object::Instance(inst) = receiver else { + return false; + }; + let cls = inst.cls(); + cls.native_kind.get() == 0 + && super::Interpreter::default_getattribute(&cls) + && matches!(cls.lookup(name), Some(Object::SlotDescriptor(sd)) if sd.name == name) + && cls.attr_version.get() == ver +} + fn scalar_field_update_plan( code: &CodeObject, cf: &CompiledFrame, @@ -3969,6 +4678,8 @@ unsafe fn native_scalar_field_update( let (_, slot) = slots.get_index_mut(index as usize)?; // An exact integer owns no destructor or GC edge. *slot = Object::Int(value); + #[cfg(test)] + crate::SCALAR_FIELD_UPDATE_NATIVE_CALLS.with(|hits| hits.set(hits.get() + 1)); return Some(value); } // SAFETY: no callback, allocation, or Python execution can overlap this @@ -3989,6 +4700,56 @@ unsafe fn native_scalar_field_update( Some(value) } +/// Arm `entry`'s in-line update (see [`InlineUpdate`]) after the helper +/// ran it on `receiver`: the method guard and [`native_scalar_field_update`] +/// have just held, so what compiled code still checks per call (the +/// receiver's class version and split names, the shadow bound, the code +/// identity, the observer and key gates) is all that can change. A split +/// field and the plain increments only; anything else stays unarmed. +fn arm_update(entry: &MethodEntry, nc: &NativeCallee, receiver: &Object, argc: u32) { + let Some(plan) = nc.scalar_update.as_deref() else { + return; + }; + // (A native method's body is no field update.) + let MethodCallee::Py { func, code } = &entry.callee else { + return; + }; + let Some(guard) = nc.attr_guards.get(plan.store_token) else { + return; + }; + let Object::Instance(inst) = receiver else { + return; + }; + let cls = inst.cls_raw(); + let Some(keys) = cls.shared_keys.get() else { + return; + }; + let (from_arg, inc) = match (plan.increment, argc) { + (Some(inc), 0) => (0, inc), + (None, 1) => (1, 0), + _ => return, + }; + if guard.split_idx == u32::MAX + || guard.ver != entry.ver + || cls.attr_version.get() != entry.ver + || !matches!(guard.storage, AttrStorage::Indexed(i) if i == guard.split_idx) + { + return; + } + let shadow = keys.names_before(&entry.name, entry.name_hash); + let u = &entry.update; + u.shadow.set(u32::try_from(shadow).unwrap_or(0)); + u.from_arg.set(from_arg); + u.inc.set(inc); + u.code_at.set(func.code.as_ptr() as usize); + // SAFETY: `Rc` is one pointer, compared and never dereferenced. + u.code + .set(unsafe { std::mem::transmute_copy::, usize>(code) }); + u.idx.set(guard.split_idx); + #[cfg(test)] + crate::SCALAR_FIELD_UPDATE_ARMED.with(|armed| armed.set(armed.get() + 1)); +} + /// Bind the slots a keyword call skipped (tagged [`SlotTag::Default`] /// under [`weavepy_jit::CALL_GAPS`]) to the callee's current /// `__defaults__` when those are scalars, so the call is a positional @@ -4155,35 +4916,8 @@ unsafe fn try_native_call( if supplied > n_params { return None; } - let first_default = n_params - nc.func.defaults.len().min(n_params); - if supplied < n_params { - if nc.func.defaults_maybe_overridden() && nc.func.slot("__defaults__").is_some() { - return None; - } - if supplied < first_default { - // Unbindable — the interpreter path raises the faithful - // TypeError. - return None; - } - // Every spliced default must fit the callee's compiled - // parameter lane (checked before any buffer is taken). List - // lanes stay interpreted: a mutable default whose *identity* - // matters must go through the generic binder. - for k in supplied..n_params { - // Retained defaults can outnumber parameters after __code__ - // replacement. Match the parameter against the trailing suffix. - let d = &nc.func.defaults[nc.func.defaults.len() - (n_params - k)]; - match nc.cf.local_types.get(k).copied().flatten() { - None | Some(JitType::Obj) => {} - Some(ty @ (JitType::Int | JitType::Bool | JitType::Float)) => { - pack(d, ty)?; - } - Some(JitType::Str) if matches!(d, Object::Str(_)) => {} - Some(JitType::Bytes) if matches!(d, Object::Bytes(_)) => {} - Some(JitType::Dict) if matches!(d, Object::Dict(_)) => {} - Some(_) => return None, - } - } + if supplied < n_params && !native_defaults_fit(nc, supplied, n_params) { + return None; } // The receiver slot must be the object-pin lane the eligibility // check admitted (defensive — `native_method_callable` verified @@ -4249,36 +4983,15 @@ unsafe fn try_native_call( && nc.cf.n_locals as usize <= SCALAR_LEAF_SLOTS && (nc.cf.max_stack as usize) < SCALAR_LEAF_SLOTS { - // Grow before entering the separate function that owns the fixed - // arrays, preserving the ordinary native call's stack discipline. - native_stat(|s| s.scalar_leaf_calls.set(s.scalar_leaf_calls.get() + 1)); - // SAFETY: the checks above establish the helper's size, scalar + // SAFETY: the checks above establish the leaf's size, scalar // lane, default-binding, code-lifetime, and recursion invariants. - let enter = || unsafe { enter_scalar_leaf(&nc.cf, &nc.func, &nc.code, jf, argc_usize) }; - let result = if crate::stdlib::greenlet_native::on_greenlet_stack() { - enter() - } else { - stacker::maybe_grow(512 * 1024, 8 * 1024 * 1024, enter) - }; - return match result { - Some((bits, tag)) if tag == expect_tag => { - jf.ret_bits = bits; - jf.ret_tag = tag; - Some(CallStatus::Ok as i64) - } - Some((bits, tag)) => { - ctx.parked = Some(unpack(bits, tag)); - Some(CallStatus::Boxed as i64) - } - None => { - // Only pure numeric work has run. The ordinary call path - // can restart to compute a bignum or raise with a complete - // interpreter frame. This return releases the recursion - // guard before that path charges its own tick. - native_stat(|s| s.deopts.set(s.deopts.get() + 1)); - None - } - }; + let status = unsafe { scalar_leaf_call(jf, ctx, nc, argc_usize, expect_tag) }; + // Only pure numeric work has run when the leaf declines: the + // ordinary call path can restart it to compute a bignum or raise + // with a complete interpreter frame. This return releases the + // recursion guard before that path charges its own tick. + drop(recursion_guard); + return status; } // A framed native call of a *tiny* callee (a getter, a comparison) @@ -4290,9 +5003,11 @@ unsafe fn try_native_call( // callee whatever its shape. A tiny *method* that stores — the // attribute fixtures' `tick` — is not, and its loop keeps its // native lane. - if crate::code_is_pure_leaf_pub(&nc.code) - || (recv.is_none() && nc.code.instructions.len() <= TINY_CALLEE_OPS) - { + let pure_leaf = match nc.code.jit_hint.pure_leaf() { + Some(known) => known, + None => crate::code_is_pure_leaf_pub(&nc.code), + }; + if pure_leaf || (recv.is_none() && nc.code.instructions.len() <= TINY_CALLEE_OPS) { ctx.dyn_py_calls = ctx.dyn_py_calls.saturating_add(1); } // The framed native call below is itself expensive (buffers, pins, a @@ -4380,11 +5095,46 @@ unsafe fn try_native_call( let (locals_buf, rest) = u64s.split_at_mut(n_locals); let (spill, call_args) = rest.split_at_mut(cap); let (tags, call_tags) = u32s.split_at_mut(cap); - // The cached child's (drained) pin table keeps its capacity. - let mut pins: PinTable = match ctx.child.as_deref_mut() { - Some(c) => std::mem::take(&mut c.pins), - None => Vec::new(), + let callee_key = Rc::as_ptr(&nc.code).cast::(); + // The callee's context: this activation's cached child when it was + // built for this very callee (same code, artifact tables and + // namespaces — the child holds those handles, so pointer identity + // is sound), else a fresh one. Reuse resets only the per-activation + // state; the handles and resolved tables carry over (stale tables + // re-resolve on demand, as for any long-lived activation). + let reusable = ctx.child.as_deref().is_some_and(|c| { + c.code_ptr == callee_key + && StdRc::ptr_eq(&c.callees, &nc.callees) + && StdRc::ptr_eq(&c.guard_snapshot, &nc.snap) + && Rc::ptr_eq(&c.globals, &nc.func.globals) + && Rc::ptr_eq(&c.builtins, &nc.func.builtins) + }); + let mut child: Box = if reusable { + let mut c = ctx.child.take().expect("checked above"); + // (A previous call left `parked`/`raised` empty.) + if !c.const_pins.is_empty() { + c.const_pins.clear(); + } + c.pin_pressure_exit = false; + c.pin_limit = RUNTIME_PIN_SOFT_LIMIT; + c.pins_counted = 0; + c.last_ref_pins = 0; + if !c.obj_global_pins.is_empty() { + c.obj_global_pins.clear(); + } + c.cf = StdRc::as_ptr(&nc.cf); + c.dirty = false; + c.interp_calls = 0; + c.dyn_py_calls = 0; + c.native_calls = 0; + c.polls = 0; + c + } else { + fresh_child(ctx, nc, callee_key) }; + // The callee's pins: the receiver, then the arguments' objects (the + // reused child's drained table keeps its capacity). + let pins = &mut child.pins; if !pins.is_empty() { pins.clear(); } @@ -4446,103 +5196,7 @@ unsafe fn try_native_call( _ => 0, }; } - let entry_pin_count = pins.len(); - let callee_key = Rc::as_ptr(&nc.code).cast::(); - // The callee's context: this activation's cached child when it was - // built for this very callee (same code, artifact tables and - // namespaces — the child holds those handles, so pointer identity - // is sound), else a fresh one. Reuse resets only the per-activation - // state; the handles and resolved tables carry over (stale tables - // re-resolve on demand, as for any long-lived activation). - let reusable = ctx.child.as_deref().is_some_and(|c| { - c.code_ptr == callee_key - && StdRc::ptr_eq(&c.callees, &nc.callees) - && StdRc::ptr_eq(&c.guard_snapshot, &nc.snap) - && Rc::ptr_eq(&c.globals, &nc.func.globals) - && Rc::ptr_eq(&c.builtins, &nc.func.builtins) - }); - let mut child: Box = if reusable { - let mut c = ctx.child.take().expect("checked above"); - // (A previous call left `parked`/`raised` empty.) - if !c.const_pins.is_empty() { - c.const_pins.clear(); - } - c.pins = pins; - c.entry_pin_count = entry_pin_count; - c.pin_pressure_exit = false; - if !c.obj_global_pins.is_empty() { - c.obj_global_pins.clear(); - } - c.cf = StdRc::as_ptr(&nc.cf); - c.dirty = false; - c.interp_calls = 0; - c.dyn_py_calls = 0; - c.native_calls = 0; - c.polls = 0; - c - } else { - // Self-recursion reuses this activation's own tables. An immutable - // compilation with no call tokens needs no resolution, even when - // another function compiles and advances the cache generation. - // Nonempty tables retain the generation-checked cache lookup. - let (native, method_native) = if callee_key == ctx.code_ptr { - (ctx.native.clone(), ctx.method_native.clone()) - } else { - ( - if nc.callees.is_empty() { - None - } else { - resolved_native_table(callee_key) - }, - if nc.methods.is_empty() { - None - } else { - resolved_method_native_table(callee_key) - }, - ) - }; - let table_gen = if callee_key == ctx.code_ptr { - ctx.table_gen - } else { - current_compile_gen() - }; - Box::new(CallCtx { - interp: ctx.interp, - callees: nc.callees.clone(), - cf: StdRc::as_ptr(&nc.cf), - guard_snapshot: nc.snap.clone(), - globals: nc.func.globals.clone(), - builtins: nc.func.builtins.clone(), - // Native callees are cell-free by `py_callee_ok`. - cells: crate::object::empty_cells(), - parked: None, - raised: None, - const_pins: Vec::new(), - pins, - entry_pin_count, - pin_pressure_exit: false, - obj_globals: nc.obj_globals.clone(), - obj_global_pins: Vec::new(), - attr_guards: nc.attr_guards.clone(), - methods: nc.methods.clone(), - math: nc.math.clone(), - dirty: false, - interp_calls: 0, - dyn_py_calls: 0, - native_calls: 0, - polls: 0, - child: None, - depth_cell: ctx.depth_cell, - code_ptr: callee_key, - native, - method_native, - table_gen, - // The native call lanes push no interpreter frame for the - // callee — keep it observable to callee-side stack walkers. - frameless_code: Some(nc.code.clone()), - dyn_callee: None, - }) - }; + child.entry_pin_count = child.pins.len(); child.interp = ctx.interp; let nctx: &mut CallCtx = &mut child; let mut njf = JitFrame { @@ -4578,6 +5232,259 @@ unsafe fn try_native_call( }) }; + // The common returns, from a callee that ran no Python (nothing needs + // revalidating): a scalar in the expected lane, or `None` or an + // instance on the object lane where the site takes `None` (the + // procedure shape) or an object. + if status == JitStatus::Returned && !child.dirty { + let expect = SlotTag::from_raw(expect_tag); + let ret = if njf.ret_tag == expect_tag + && matches!( + expect, + SlotTag::Int | SlotTag::Float | SlotTag::Bool | SlotTag::None + ) { + Some(njf.ret_bits) + } else if njf.ret_tag == SlotTag::ObjPin as u32 && njf.ret_bits == u64::MAX { + // `None`. + match expect { + SlotTag::None => Some(0), + SlotTag::ObjPin => Some(u64::MAX), + _ => None, + } + } else if njf.ret_tag == SlotTag::ObjPin as u32 && expect == SlotTag::ObjPin { + // An instance pins into this activation's table (a method + // returning `self` finds its existing pin). + match child.pins.get(njf.ret_bits as usize) { + Some(Pin::Obj(v @ Object::Instance(_))) => { + pin_reusing(v, &mut ctx.pins, &mut ctx.pin_memo) + } + _ => None, + } + } else { + None + }; + if let Some(bits) = ret { + note_callee_exit(&nc.art, &nc.code, &child); + if !inline_bufs { + put_u64(u64_buf); + put_u32(u32_buf); + } + child.pins.clear(); + ctx.child = Some(child); + jf.ret_bits = bits; + jf.ret_tag = expect_tag; + return Some(CallStatus::Ok as i64); + } + } + // SAFETY: as above; the buffers are the ones the callee ran on. + let status = unsafe { + finish_native_call( + jf, + ctx, + interp, + nc, + child, + NativeExit { + status, + njf: &njf, + locals: locals_buf, + spill, + tags, + expect_tag, + }, + recursion_guard, + ) + }; + if !inline_bufs { + put_u64(u64_buf); + put_u32(u32_buf); + } + Some(status) +} + +/// [`try_native_call`]'s under-arity check: every parameter from +/// `supplied` on must bind a default, and each must fit its compiled lane +/// (checked before any buffer is taken). +#[cold] +#[inline(never)] +fn native_defaults_fit(nc: &NativeCallee, supplied: usize, n_params: usize) -> bool { + if nc.func.defaults_maybe_overridden() && nc.func.slot("__defaults__").is_some() { + return false; + } + let first_default = n_params - nc.func.defaults.len().min(n_params); + if supplied < first_default { + // Unbindable — the interpreter path raises the faithful + // TypeError. + return false; + } + // List lanes stay interpreted: a mutable default whose *identity* + // matters must go through the generic binder. + (supplied..n_params).all(|k| { + // Retained defaults can outnumber parameters after __code__ + // replacement. Match the parameter against the trailing suffix. + let d = &nc.func.defaults[nc.func.defaults.len() - (n_params - k)]; + match nc.cf.local_types.get(k).copied().flatten() { + None | Some(JitType::Obj) => true, + Some(ty @ (JitType::Int | JitType::Bool | JitType::Float)) => pack(d, ty).is_some(), + Some(JitType::Str) => matches!(d, Object::Str(_)), + Some(JitType::Bytes) => matches!(d, Object::Bytes(_)), + Some(JitType::Dict) => matches!(d, Object::Dict(_)), + Some(_) => false, + } + }) +} + +/// [`try_native_call`] of a certified scalar leaf, on bounded stack +/// storage; `None` when the leaf declines (only pure numeric work ran). +/// +/// # Safety +/// +/// As [`enter_scalar_leaf`]; the caller holds the recursion tick. +#[inline(never)] +unsafe fn scalar_leaf_call( + jf: &mut JitFrame, + ctx: &mut CallCtx, + nc: &NativeCallee, + argc: usize, + expect_tag: u32, +) -> Option { + // Grow before entering the separate function that owns the fixed + // arrays, preserving the ordinary native call's stack discipline. + native_stat(|s| s.scalar_leaf_calls.set(s.scalar_leaf_calls.get() + 1)); + // SAFETY: the caller's contract. + let enter = || unsafe { enter_scalar_leaf(&nc.cf, &nc.func, &nc.code, jf, argc) }; + let result = if crate::stdlib::greenlet_native::on_greenlet_stack() { + enter() + } else { + stacker::maybe_grow(512 * 1024, 8 * 1024 * 1024, enter) + }; + match result { + Some((bits, tag)) if tag == expect_tag => { + jf.ret_bits = bits; + jf.ret_tag = tag; + Some(CallStatus::Ok as i64) + } + Some((bits, tag)) => { + ctx.parked = Some(unpack(bits, tag)); + Some(CallStatus::Boxed as i64) + } + None => { + native_stat(|s| s.deopts.set(s.deopts.get() + 1)); + None + } + } +} + +/// A new context for a native callee this activation hasn't cached (see +/// [`try_native_call`]): the callee's handles and resolved tables, and +/// empty per-activation state. +#[cold] +#[inline(never)] +fn fresh_child(ctx: &CallCtx, nc: &NativeCallee, callee_key: *const CodeObject) -> Box { + // Self-recursion reuses this activation's own tables. An immutable + // compilation with no call tokens needs no resolution, even when + // another function compiles and advances the cache generation. + // Nonempty tables retain the generation-checked cache lookup. + let (native, method_native) = if callee_key == ctx.code_ptr { + (ctx.native.clone(), ctx.method_native.clone()) + } else { + ( + if nc.callees.is_empty() { + None + } else { + resolved_native_table(callee_key) + }, + if nc.methods.is_empty() { + None + } else { + resolved_method_native_table(callee_key) + }, + ) + }; + let table_gen = if callee_key == ctx.code_ptr { + ctx.table_gen + } else { + current_compile_gen() + }; + Box::new(CallCtx { + interp: ctx.interp, + callees: nc.callees.clone(), + cf: StdRc::as_ptr(&nc.cf), + guard_snapshot: nc.snap.clone(), + globals: nc.func.globals.clone(), + builtins: nc.func.builtins.clone(), + // Native callees are cell-free by `py_callee_ok`. + cells: crate::object::empty_cells(), + parked: None, + raised: None, + const_pins: Vec::new(), + pins: Vec::new(), + entry_pin_count: 0, + pin_pressure_exit: false, + pin_limit: RUNTIME_PIN_SOFT_LIMIT, + pins_counted: 0, + last_ref_pins: 0, + obj_globals: nc.obj_globals.clone(), + obj_global_pins: Vec::new(), + attr_guards: nc.attr_guards.clone(), + methods: nc.methods.clone(), + math: nc.math.clone(), + dirty: false, + interp_calls: 0, + dyn_py_calls: 0, + native_calls: 0, + polls: 0, + child: None, + depth_cell: ctx.depth_cell, + code_ptr: callee_key, + native, + method_native, + table_gen, + // The native call lanes push no interpreter frame for the + // callee — keep it observable to callee-side stack walkers. + frameless_code: Some(nc.code.clone()), + dyn_callee: None, + pin_memo: [u32::MAX; PIN_MEMO], + }) +} + +/// How a native callee's activation ended, and the buffers it ran on. +struct NativeExit<'a> { + status: JitStatus, + njf: &'a JitFrame, + locals: &'a [u64], + spill: &'a [u64], + tags: &'a [u32], + expect_tag: u32, +} + +/// [`try_native_call`] after the callee left native code any other way +/// than a clean scalar return: materialize a deopt or a raise, translate +/// an object result into this activation's lanes, revalidate this +/// activation's guards when Python ran, and put the callee's context back. +/// +/// # Safety +/// +/// Same contract as [`try_native_call`]; `child` is the context the callee +/// ran with and `exit` its frame and buffers. +#[inline(never)] +unsafe fn finish_native_call( + jf: &mut JitFrame, + ctx: &mut CallCtx, + interp: &mut super::Interpreter, + nc: &NativeCallee, + mut child: Box, + exit: NativeExit<'_>, + recursion_guard: crate::recursion::Guard, +) -> i64 { + let NativeExit { + status, + njf, + expect_tag, + .. + } = exit; + let nctx: &mut CallCtx = &mut child; + /// How the nested call concluded, before result-lane translation. enum Done { Scalar(u64, u32), @@ -4634,17 +5541,22 @@ unsafe fn try_native_call( } else { None }; - match finish_deopted_callee(interp, nc, nctx, locals_buf, spill, tags, &njf, pending) { + match finish_deopted_callee( + interp, + nc, + nctx, + exit.locals, + exit.spill, + exit.tags, + njf, + pending, + ) { Ok(v) => Done::Obj(v), Err(e) => Done::Raised(e), } } }; note_callee_exit(&nc.art, &nc.code, nctx); - if !inline_bufs { - put_u64(u64_buf); - put_u32(u32_buf); - } // Reap the callee's pins (after every // pin-based rebuild above); a reap cascade runs Python, so it // dirties the call like any interpreter-path work. @@ -4659,7 +5571,7 @@ unsafe fn try_native_call( ctx.child = Some(child); ctx.dirty |= child_dirty; - Some(match done { + match done { Done::Raised(e) => { ctx.raised = Some(e); CallStatus::Raised as i64 @@ -4717,12 +5629,12 @@ unsafe fn try_native_call( // The procedure shape: nothing to write back. jf.ret_bits = 0; jf.ret_tag = expect_tag; - return Some(CallStatus::Ok as i64); + return CallStatus::Ok as i64; } // RFC 0071 WS1 — an object-lane return pins into the // *caller's* table (`None` rides as the nullable `-1`). if guards_ok && expect_tag == SlotTag::ObjPin as u32 { - return Some(match obj_ret_bits(&v, &mut ctx.pins) { + return match obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo) { Some(bits) => { jf.ret_bits = bits; jf.ret_tag = expect_tag; @@ -4732,7 +5644,7 @@ unsafe fn try_native_call( ctx.parked = Some(v); CallStatus::Boxed as i64 } - }); + }; } let expect = match SlotTag::from_raw(expect_tag) { SlotTag::Int => JitType::Int, @@ -4756,7 +5668,7 @@ unsafe fn try_native_call( } } } - }) + } } /// RFC 0071 WS2 — the native class-construction fast path: allocate @@ -4800,11 +5712,30 @@ unsafe fn try_native_ctor( // Threshold collection ran finalizers — arbitrary Python. ctx.dirty = true; } + // Room for every field the class's instances set, so the compiled + // `__init__` appends even its first one in line. + if let (Object::Instance(i), Some(keys)) = (&inst, cls.shared_keys.get()) { + let n = keys.len(); + if n > 0 && cls.native_kind.get() == 0 && i.dict.published().is_none() { + // SAFETY: the instance is fresh and unshared; nothing else + // borrows its values. + if let Some(split) = unsafe { i.dict.split_cell().peek_mut() } { + split.reserve_for(|| cls.shared_keys.share(), n); + } + } + } // `__init__` is a procedure: its `None` rides the procedure lane. // A `try_native_call` rejection discards the fresh (empty, never // `__init__`-ed) instance and re-allocates on the interpreter path. - let status = - unsafe { try_native_call(jf, ctx, interp, nc, argc, SlotTag::None as u32, Some(&inst)) }?; + // Python never saw it, so it dies without its finalizer. + let Some(status) = + (unsafe { try_native_call(jf, ctx, interp, nc, argc, SlotTag::None as u32, Some(&inst)) }) + else { + if let Object::Instance(i) = &inst { + i.finalize_ran.set(true); + } + return None; + }; if status == CallStatus::Raised as i64 { // A raising `__init__` discards the instance (CPython's // `type_call` propagates before returning it). @@ -4814,7 +5745,7 @@ unsafe fn try_native_ctor( // `__init__` completed and returned `None` with guards intact; // the call site's value is the fresh instance. if expect_tag == SlotTag::ObjPin as u32 { - if let Some(bits) = obj_ret_bits(&inst, &mut ctx.pins) { + if let Some(bits) = obj_ret_bits(&inst, &mut ctx.pins, &mut ctx.pin_memo) { jf.ret_bits = bits; jf.ret_tag = expect_tag; return Some(CallStatus::Ok as i64); @@ -4999,10 +5930,13 @@ fn call_with_activation_shell( lasti: std::sync::atomic::AtomicU32::new(jf.deopt_pc), has_materialized: std::sync::atomic::AtomicBool::new(false), materialized: GilRefCell::new(None), + tb_refs: std::sync::atomic::AtomicU32::new(0), }); interp.frame_stack.borrow_mut().push(shell); let out = f(interp); - interp.frame_stack.borrow_mut().pop(); + // The general pop: a traceback entry or a callee frame's `f_back` may + // hold the shell lazily, and gets its frame object on the way out. + interp.pop_frame_shell(); out } @@ -5153,7 +6087,7 @@ unsafe extern "C" fn wpjit_call_py( // RFC 0071 WS1 — an object-lane result pins into this // activation's table. if expect_tag == SlotTag::ObjPin as u32 { - if let Some(bits) = obj_ret_bits(&v, &mut ctx.pins) { + if let Some(bits) = obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo) { jf.ret_bits = bits; jf.ret_tag = expect_tag; return CallStatus::Ok as i64; @@ -5257,7 +6191,7 @@ fn deliver_call_result(jf: &mut JitFrame, ctx: &mut CallCtx, v: Object, expect_t // RFC 0071 WS1 — an object-lane result pins into this // activation's table. SlotTag::ObjPin => { - if let Some(bits) = obj_ret_bits(&v, &mut ctx.pins) { + if let Some(bits) = obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo) { jf.ret_bits = bits; jf.ret_tag = expect_tag; return CallStatus::Ok as i64; @@ -5533,13 +6467,20 @@ unsafe extern "C" fn wpjit_self_slow( /// interpreter calls is a thin native driver around them. Each such call pays pin /// traffic, an activation shell and a generic call that the interpreter's /// inline call path avoids, so the callee retires to tier-1. -#[inline] +#[inline(always)] fn note_callee_exit(art: &Artifacts, code: &Rc, child: &CallCtx) { let entries = art.callee_entries.get().saturating_add(1); art.callee_entries.set(entries); - if child.dyn_py_calls == 0 { - return; + if child.dyn_py_calls != 0 { + note_callee_roundtrips(art, code, child, entries); } +} + +/// [`note_callee_exit`] for an activation that made interpreter +/// round-trips, out of line. +#[cold] +#[inline(never)] +fn note_callee_roundtrips(art: &Artifacts, code: &Rc, child: &CallCtx, entries: u32) { let trips = art .callee_roundtrips .get() @@ -5601,6 +6542,12 @@ fn retire_native_driver(ctx: &CallCtx) { unsafe { (*ctx.code_ptr).jit_hint.mark_not_jitable() }; } +/// Recompiles one code object may take after cold exits (see +/// `CacheEntry::recompile_at_osr`): one per later loop is the common +/// shape (a function running several loops over containers it builds in +/// turn), and the budget bounds compile cost for anything else. +const COLD_RECOMPILE_BUDGET: u8 = 3; + /// Generic native-callee calls one activation may make between two /// loop polls before its code retires (see [`charge_native_roundtrip`]). const NATIVE_CALL_RETIRE_BUDGET: u32 = 4096; @@ -5692,6 +6639,9 @@ unsafe extern "C" fn wpjit_call_method( // SAFETY: the `&mut Interpreter` that entered native code is dormant // while the helper runs; this is the only live path to it. let interp = unsafe { &mut *ctx.interp }; + // A native site's deferred-guard miss (see `wpjit_call_native_method`) + // takes the surprise-receiver lane below. + let token = token & !(weavepy_jit::METHOD_GUARD_AT_CALL | weavepy_jit::METHOD_NATIVE); native_stat(|s| s.method_calls.set(s.method_calls.get() + 1)); let guard_miss = || { native_stat(|s| s.method_guard_misses.set(s.method_guard_misses.get() + 1)); @@ -5719,9 +6669,13 @@ unsafe extern "C" fn wpjit_call_method( // code (see `GilCell::peek`). attr_class_ok(inst, entry.ver) && unsafe { inst.attr_peek_has(probe.s, probe.hash) } == Some(false) - && match unsafe { entry.func.code.peek() } { - Some(code) => Rc::ptr_eq(code, &entry.code), - None => Rc::ptr_eq(&entry.func.code.borrow(), &entry.code), + && match &entry.callee { + MethodCallee::Py { func, code: burned } => match unsafe { func.code.peek() } { + Some(code) => Rc::ptr_eq(code, burned), + None => Rc::ptr_eq(&func.code.borrow(), burned), + }, + // Only a native site's deferred-guard miss gets here. + MethodCallee::Native { .. } => false, } } _ => false, @@ -5753,6 +6707,9 @@ unsafe extern "C" fn wpjit_call_method( }); return finish_interp_call(jf, ctx, interp, res, expect_tag); } + let MethodCallee::Py { func, code } = &entry.callee else { + unreachable!("native methods returned above"); + }; // A pure-leaf method (a getter, a predicate) evaluates frameless. // SAFETY: `argc` marshaled entries are live (the function contract), @@ -5762,8 +6719,8 @@ unsafe extern "C" fn wpjit_call_method( jf, ctx, interp, - &entry.func, - &entry.code, + func, + code, Some(&raw const recv), argc, expect_tag, @@ -5782,15 +6739,17 @@ unsafe extern "C" fn wpjit_call_method( .and_then(Option::as_ref) { if nc.scalar_update.is_some() - && entry.code.arg_count == argc + 1 - && Rc::ptr_eq(&nc.func, &entry.func) - && Rc::ptr_eq(&nc.code, &entry.code) + && code.arg_count == argc + 1 + && Rc::ptr_eq(&nc.func, func) + && Rc::ptr_eq(&nc.code, code) { // SAFETY: the method guard above pinned the binding; the update // checks its receiver, argument lane, and observers itself. if let Some(value) = unsafe { native_scalar_field_update(jf, ctx, nc, &recv, argc as usize) } { + // Compiled code runs the next ones in line. + arm_update(entry, nc, &recv, argc); if expect_tag == SlotTag::Int as u32 { jf.ret_bits = value as u64; jf.ret_tag = SlotTag::Int as u32; @@ -5818,7 +6777,7 @@ unsafe extern "C" fn wpjit_call_method( .and_then(|t| t.get(token as usize)) .and_then(Option::as_ref) { - if Rc::ptr_eq(&nc.func, &entry.func) && Rc::ptr_eq(&nc.code, &entry.code) { + if Rc::ptr_eq(&nc.func, func) && Rc::ptr_eq(&nc.code, code) { // SAFETY: `jf`/`ctx` are this activation's live buffers // (see the function contract) and `nc` came from this // thread's tier cache via the activation's resolved table. @@ -5839,34 +6798,474 @@ unsafe extern "C" fn wpjit_call_method( if tried_refresh || !ctx.refresh_tables() { break; } - tried_refresh = true; - method_native = ctx.method_native.clone(); + tried_refresh = true; + method_native = ctx.method_native.clone(); + } + + // Interpreter path: arbitrary Python runs on behalf of this + // activation, so burned-in resolutions must be revalidated after + // the call (RFC 0067 WS1's dirtiness discipline). + NATIVE_CALL_STATS.with(|s| { + s.method_call_fallbacks + .set(s.method_call_fallbacks.get() + 1); + }); + ctx.dirty = true; + let callee = Object::Function(func.clone()); + let mut args: Vec = Vec::with_capacity(argc as usize + 1); + args.push(recv.clone()); + for j in 0..argc as usize { + // SAFETY: native code wrote `argc` entries, and the buffers are + // `max_call_args` wide. + let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; + // RFC 0071 WS1 — pin-tagged arguments resolve against this + // activation's pin table. + args.push(unpack_pins(bits, tag, &ctx.pins)); + } + + let res = call_with_activation_shell(interp, ctx, jf, |i| { + i.call(&callee, &args, &[], &ctx.globals) + }); + finish_interp_call(jf, ctx, interp, res, expect_tag) +} + +/// Deliver a native method's or subscript's completed result `v` to +/// the compiled caller on its `expect_tag` lane: `None` for the +/// procedure lane, an unboxed scalar, or an object-lane pin (`None` +/// rides the nullable `-1`, so a procedure's result costs no pin). A +/// value the lane can't carry, or pin pressure, parks `v` (`Boxed`: the +/// caller resumes after the operation, which never runs again). +#[inline(always)] +fn deliver_native_result(jf: &mut JitFrame, ctx: &mut CallCtx, v: Object, expect_tag: u32) -> i64 { + // The scalar arms own nothing, so they skip `v`'s drop glue. + match (SlotTag::from_raw(expect_tag), &v) { + (SlotTag::None, Object::None) => { + std::mem::forget(v); + return CallStatus::Ok as i64; + } + (SlotTag::Int, &Object::Int(value)) => { + std::mem::forget(v); + jf.ret_bits = value as u64; + jf.ret_tag = expect_tag; + return CallStatus::Ok as i64; + } + (SlotTag::ObjPin, Object::None) => { + std::mem::forget(v); + jf.ret_bits = u64::MAX; + jf.ret_tag = expect_tag; + return CallStatus::Ok as i64; + } + (SlotTag::ObjPin, _) => { + if ctx.temporary_pin_limit_reached() { + ctx.pin_pressure_exit = true; + } else if ctx.pins.len() < RUNTIME_PIN_CAP { + jf.ret_bits = ctx.pins.len() as u64; + jf.ret_tag = expect_tag; + ctx.pins.push(Pin::Obj(v)); + return CallStatus::Ok as i64; + } + } + _ => {} + } + ctx.parked = Some(v); + CallStatus::Boxed as i64 +} + +/// A bitwise view of one marshaled argument: scalars rebuild by value +/// and an object pin aliases its pinned object without taking a +/// reference (the pin owns it for the whole activation). The view must +/// never be dropped. `None` for a lane it can't alias (a list pin). +#[inline(always)] +fn arg_view(bits: u64, tag: u32, pins: &PinTable) -> Option { + Some(match SlotTag::from_raw(tag) { + SlotTag::Int => Object::Int(bits as i64), + SlotTag::Float => Object::Float(f64::from_bits(bits)), + SlotTag::Bool => Object::Bool(bits != 0), + SlotTag::ObjPin if bits == u64::MAX => Object::None, + SlotTag::ObjPin => match pins.get(bits as usize)? { + // SAFETY: a bitwise alias, never dropped (see above). + Pin::Obj(o) => unsafe { std::ptr::read(o) }, + Pin::List(..) => return None, + }, + _ => return None, + }) +} + +/// A native leaf body's direct call on bitwise views of the receiver pin +/// and the marshaled arguments (see [`arg_view`]): no references are +/// taken or released around the call, which runs no Python code. `None` +/// (nothing ran) when an argument has no view, the arity exceeds the +/// inline buffer, or the fast half declines. +/// +/// # Safety +/// +/// `argc` marshal entries are initialized (the helper contract). +#[inline(always)] +unsafe fn direct_native_call( + jf: &JitFrame, + ctx: &CallCtx, + builtin: &crate::object::BuiltinFn, + fast: Option, + recv_pin: i64, + argc: u32, +) -> Option> { + const N: usize = 4; + let n = argc as usize + 1; + if n > N { + return None; + } + let mut views = [const { std::mem::MaybeUninit::::uninit() }; N]; + match ctx.pins.get(usize::try_from(recv_pin).ok()?)? { + // SAFETY: a bitwise alias, never dropped (the pin owns it). + Pin::Obj(o) => views[0].write(unsafe { std::ptr::read(o) }), + Pin::List(..) => return None, + }; + for j in 0..argc as usize { + // SAFETY: native code wrote `argc` entries. + let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; + views[j + 1].write(arg_view(bits, tag, &ctx.pins)?); + } + // SAFETY: the first `n` entries were written; they are views, so the + // `MaybeUninit` array never drops them. + let args = unsafe { std::slice::from_raw_parts(views.as_ptr().cast::(), n) }; + match fast { + Some(fast) => fast(args), + None => Some(match builtin.call_kw.as_ref() { + Some(ckw) => ckw(args, &[]), + None => (builtin.call)(args), + }), + } +} + +/// A native method's direct operation (`collections_native::fast_op`) on +/// the receiver pin and at most one marshaled argument, viewed in place +/// (see [`arg_view`]). `None` (nothing ran) sends the call to the full +/// body. +/// +/// # Safety +/// +/// `argc` marshal entries are initialized (the helper contract). +#[inline(always)] +unsafe fn direct_native_op( + jf: &JitFrame, + ctx: &CallCtx, + op: u8, + recv_pin: i64, + argc: u32, +) -> Option { + let Pin::Obj(recv) = ctx.pins.get(usize::try_from(recv_pin).ok()?)? else { + return None; + }; + match argc { + 0 => crate::stdlib::collections_native::fast_op(op, recv, None), + 1 => { + // SAFETY: native code wrote one entry. + let (bits, tag) = unsafe { (*jf.call_args, *jf.call_tags) }; + let arg = std::mem::ManuallyDrop::new(arg_view(bits, tag, &ctx.pins)?); + crate::stdlib::collections_native::fast_op(op, recv, Some(&arg)) + } + _ => None, + } +} + +/// A burned-in native leaf method (see [`MethodCallee::Native`]), +/// called directly on the receiver bound at its load and the marshaled +/// arguments. No Python runs on the direct path, so the activation stays +/// clean and no round trip is charged. +/// +/// A fast half that declines (an argument whose conversion could run +/// Python), or an active profiler or tracer that must see the call, +/// needs the interpreter, and Python must see this activation's locals, +/// which live lane-packed in the native frame. While the receiver still +/// resolves the name to this builtin, the call rejects, and the +/// interpreter performs it (and the attribute load, which binds the same +/// builtin) on a materialized frame. A class rebound by the argument +/// evaluation since the load calls the builtin bound there through the +/// interpreter's profiled call path instead. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_method`]; `entry` lives in the +/// activation's method table. +unsafe fn call_native_method( + jf: &mut JitFrame, + ctx: &mut CallCtx, + interp: &mut super::Interpreter, + entry: &MethodEntry, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + let MethodCallee::Native { op, .. } = &entry.callee else { + return CallStatus::Reject as i64; + }; + if *op != 0 && !crate::trace::any_observers_active() { + // SAFETY: per the function contract. + if let Some(v) = unsafe { direct_native_op(jf, ctx, *op, recv_pin, argc) } { + return deliver_native_result(jf, ctx, v, expect_tag); + } + } + // SAFETY: per the function contract. + unsafe { call_native_body(jf, ctx, interp, entry, recv_pin, argc, expect_tag) } +} + +/// [`call_native_method`] past its direct operation: the builtin's body +/// (or fast half) on views of the operands, else the interpreter. +/// +/// # Safety +/// +/// Same contract as [`call_native_method`]. +#[cold] +#[inline(never)] +unsafe fn call_native_body( + jf: &mut JitFrame, + ctx: &mut CallCtx, + interp: &mut super::Interpreter, + entry: &MethodEntry, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + let MethodCallee::Native { builtin, fast, .. } = &entry.callee else { + return CallStatus::Reject as i64; + }; + if !crate::trace::any_observers_active() { + // SAFETY: per the function contract. + match unsafe { direct_native_call(jf, ctx, builtin, *fast, recv_pin, argc) } { + Some(Ok(v)) => return deliver_native_result(jf, ctx, v, expect_tag), + Some(Err(err)) => { + ctx.raised = Some(err); + return CallStatus::Raised as i64; + } + None => {} + } + } + let Some(recv) = ctx.pins.get(recv_pin as usize).map(Pin::to_object) else { + return CallStatus::Reject as i64; + }; + if matches!(&recv, Object::Instance(inst) if method_guard_ok(entry, inst)) { + return CallStatus::Reject as i64; + } + let mut args: Vec = Vec::with_capacity(argc as usize); + for j in 0..argc as usize { + // SAFETY: native code wrote `argc` entries, and the buffers are + // `max_call_args` wide. + let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; + args.push(unpack_pins(bits, tag, &ctx.pins)); + } + ctx.dirty = true; + let bound = Object::BoundMethod(Rc::new(crate::object::BoundMethod::new( + recv, + Object::Builtin(builtin.clone()), + ))); + let res = call_with_activation_shell(interp, ctx, jf, |i| { + i.call_c_profiled(&bound, &args, &[], &ctx.globals) + }); + finish_interp_call(jf, ctx, interp, res, expect_tag) +} + +/// The `wpjit_call_native_method` helper: [`wpjit_call_method`] for a +/// site that resolved a native leaf method (`METHOD_NATIVE` in the +/// token), kept apart so the common call pays none of the Python path's +/// setup. The method was guarded and bound at its load +/// (`TOp::GuardMethod`), or, when nothing ran in between, is guarded +/// here (`METHOD_GUARD_AT_CALL`): call the builtin bound there. A +/// deferred guard's miss is exactly the load the interpreter would +/// perform now, so it takes the generic helper's surprise-receiver lane. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_method`]. +unsafe extern "C" fn wpjit_call_native_method( + frame: *mut JitFrame, + token: u32, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &mut *jf.ctx.cast::() }; + let guard_at_call = token & weavepy_jit::METHOD_GUARD_AT_CALL != 0; + let index = + (token & !(weavepy_jit::METHOD_GUARD_AT_CALL | weavepy_jit::METHOD_NATIVE)) as usize; + // The common case in one pass: the entry's direct operation on the + // receiver pin (read once) and at most one argument view. + if let Some( + entry @ MethodEntry { + callee: MethodCallee::Native { op, .. }, + .. + }, + ) = ctx.methods.get(index) + { + if let Some(Pin::Obj(recv)) = ctx.pins.get(recv_pin as usize) { + let bound = !guard_at_call + || matches!(recv, Object::Instance(inst) if method_guard_ok(entry, inst)); + if bound && *op != 0 && argc <= 1 && !crate::trace::any_observers_active() { + let arg = if argc == 1 { + // SAFETY: native code wrote one entry. + let (bits, tag) = unsafe { (*jf.call_args, *jf.call_tags) }; + arg_view(bits, tag, &ctx.pins).map(std::mem::ManuallyDrop::new) + } else { + None + }; + if argc == 0 || arg.is_some() { + let arg = arg.as_deref(); + if let Some(v) = crate::stdlib::collections_native::fast_op(*op, recv, arg) { + return deliver_native_result(jf, ctx, v, expect_tag); + } + } + } + } + } + // SAFETY: per the function contract. + unsafe { call_native_method_slow(frame, token, recv_pin, argc, expect_tag) } +} + +/// [`wpjit_call_native_method`] past its direct operation (see there). +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_method`]. +#[cold] +#[inline(never)] +unsafe fn call_native_method_slow( + frame: *mut JitFrame, + token: u32, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &mut *jf.ctx.cast::() }; + let guard_at_call = token & weavepy_jit::METHOD_GUARD_AT_CALL != 0; + let index = + (token & !(weavepy_jit::METHOD_GUARD_AT_CALL | weavepy_jit::METHOD_NATIVE)) as usize; + if let Some( + entry @ MethodEntry { + callee: MethodCallee::Native { .. }, + .. + }, + ) = ctx.methods.get(index) + { + let bound = !guard_at_call + || matches!( + ctx.pins.get(recv_pin as usize), + Some(Pin::Obj(Object::Instance(inst))) if method_guard_ok(entry, inst) + ); + if bound { + let entry: *const MethodEntry = entry; + // SAFETY: the `&mut Interpreter` that entered native code is + // dormant while the helper runs; the activation's method table + // (and so the entry) lives until the activation ends. + return unsafe { + let interp = &mut *ctx.interp; + call_native_method(jf, ctx, interp, &*entry, recv_pin, argc, expect_tag) + }; + } + } + // SAFETY: per the function contract. + unsafe { wpjit_call_method(frame, token, recv_pin, argc, expect_tag) } +} + +/// The `wpjit_guard_method` helper (see `TOp::GuardMethod`): `0` when the +/// receiver pin is an instance of the class version the native method +/// site `token` burned, with no instance attribute shadowing the name; +/// `1` sends the load to the interpreter (nothing has run). +/// +/// # Safety +/// +/// Same live-activation contract as [`wpjit_call_py`]. +unsafe extern "C" fn wpjit_guard_method(frame: *mut JitFrame, pin: i64, token: i64) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &*jf.ctx.cast::() }; + let Some(entry) = usize::try_from(token).ok().and_then(|t| ctx.methods.get(t)) else { + return 1; + }; + let Some(Pin::Obj(Object::Instance(inst))) = + usize::try_from(pin).ok().and_then(|p| ctx.pins.get(p)) + else { + return 1; + }; + i64::from(!method_guard_ok(entry, inst)) +} + +/// A burned method entry still describes `inst`: the class version it +/// was resolved against (which pins the MRO hit), and no instance +/// attribute shadowing the name. +#[inline(always)] +fn method_guard_ok(entry: &MethodEntry, inst: &crate::types::PyInstance) -> bool { + attr_class_ok(inst, entry.ver) + && (entry.no_dict + // SAFETY: a read between two native ops; nothing here runs + // code (see `GilCell::peek`). + || unsafe { inst.attr_peek_has(&entry.name, entry.name_hash) } == Some(false)) +} + +/// The `wpjit_obj_getitem` helper: `container[index]` on an object-lane +/// container whose class resolved `__getitem__` to a native leaf method +/// (`deque.__getitem__`) at compile time. Shares the method helper's +/// shape: `token` names the burned resolution and the index rides +/// `call_args[0]`. Unlike a method call, a guard miss (another class, a +/// non-instance) or a fast half that declines rejects, so the +/// interpreter re-executes the `BINARY_SUBSCR` itself: nothing has run. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_method`]; `argc` is 1. +unsafe extern "C" fn wpjit_obj_getitem( + frame: *mut JitFrame, + token: u32, + recv_pin: i64, + argc: u32, + expect_tag: u32, +) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &mut *jf.ctx.cast::() }; + if argc != 1 || crate::trace::any_observers_active() { + return CallStatus::Reject as i64; + } + let Some(MethodEntry { + callee: MethodCallee::Native { builtin, fast, op }, + ver, + .. + }) = ctx.methods.get(token as usize) + else { + return CallStatus::Reject as i64; + }; + // Subscription looks the dunder up on the type: no instance + // shadowing check. + let recv = match ctx.pins.get(recv_pin as usize) { + Some(Pin::Obj(recv @ Object::Instance(inst))) if attr_class_ok(inst, *ver) => recv, + _ => return CallStatus::Reject as i64, + }; + if *op != 0 { + // SAFETY: native code wrote one entry (the index). + let (bits, tag) = unsafe { (*jf.call_args, *jf.call_tags) }; + if let Some(index) = arg_view(bits, tag, &ctx.pins).map(std::mem::ManuallyDrop::new) { + if let Some(v) = crate::stdlib::collections_native::fast_op(*op, recv, Some(&index)) { + return deliver_native_result(jf, ctx, v, expect_tag); + } + } } - - // Interpreter path: arbitrary Python runs on behalf of this - // activation, so burned-in resolutions must be revalidated after - // the call (RFC 0067 WS1's dirtiness discipline). - NATIVE_CALL_STATS.with(|s| { - s.method_call_fallbacks - .set(s.method_call_fallbacks.get() + 1); - }); - ctx.dirty = true; - let callee = Object::Function(entry.func.clone()); - let mut args: Vec = Vec::with_capacity(argc as usize + 1); - args.push(recv.clone()); - for j in 0..argc as usize { - // SAFETY: native code wrote `argc` entries, and the buffers are - // `max_call_args` wide. - let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; - // RFC 0071 WS1 — pin-tagged arguments resolve against this - // activation's pin table. - args.push(unpack_pins(bits, tag, &ctx.pins)); + let (builtin, fast) = (Rc::as_ptr(builtin), *fast); + // SAFETY: the activation's method table keeps the builtin alive; one + // marshaled argument (the index). + match unsafe { direct_native_call(jf, ctx, &*builtin, fast, recv_pin, 1) } { + Some(Ok(v)) => deliver_native_result(jf, ctx, v, expect_tag), + Some(Err(err)) => { + ctx.raised = Some(err); + CallStatus::Raised as i64 + } + // A declined fast half ran nothing: the interpreter re-executes. + None => CallStatus::Reject as i64, } - - let res = call_with_activation_shell(interp, ctx, jf, |i| { - i.call(&callee, &args, &[], &ctx.globals) - }); - finish_interp_call(jf, ctx, interp, res, expect_tag) } /// RFC 0073 WS3 — the per-method resolved `str` builtin bodies, @@ -6129,7 +7528,10 @@ unsafe extern "C" fn wpjit_list_get(frame: *mut JitFrame, pin: i64, idx: i64) -> // RFC 0073 WS3 — exact-`str` elements pin on the object // lane (indexing into a `split` result). (v @ (Object::Instance(_) | Object::Str(_) | Object::Tuple(_)), JitType::Obj) => { - Err(Pin::Obj(v.clone())) + match pin_memo_hit(v, &ctx.pins, &ctx.pin_memo) { + Some(bits) => Ok(bits), + None => Err(Pin::Obj(v.clone())), + } } // The nested lane: a list element pins on the float-list // lane (its reads re-validate each float). @@ -6149,6 +7551,7 @@ unsafe extern "C" fn wpjit_list_get(frame: *mut JitFrame, pin: i64, idx: i64) -> return 1; } jf.ret_bits = ctx.pins.len() as u64; + pin_memo_note(&p, ctx.pins.len(), &mut ctx.pin_memo); ctx.pins.push(p); 0 } @@ -6416,6 +7819,42 @@ unsafe extern "C" fn wpjit_str_eq(frame: *mut JitFrame, a: i64, b: i64) -> i64 { i64::from(sa == sb) } +/// The `wpjit_is_obj` helper ([`weavepy_jit::TOp::IsObj`]): whether two +/// object-lane values (pin indices, `-1` for `None`) are the same object; +/// `2` deopts on a pin miss. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_py`]. +unsafe extern "C" fn wpjit_is_obj(frame: *mut JitFrame, a: i64, b: i64) -> i64 { + // SAFETY: see wpjit_call_py — same live-buffer contract. + let jf = unsafe { &mut *frame }; + #[allow(clippy::cast_ptr_alignment)] + let ctx = unsafe { &*jf.ctx.cast::() }; + let none = Object::None; + let at = |p: i64| -> Option> { + if p == -1 { + return Some(Ok(&none)); + } + match ctx.pins.get(usize::try_from(p).ok()?)? { + Pin::Obj(o) => Some(Ok(o)), + Pin::List(l, _) => Some(Err(Rc::as_ptr(l).cast())), + } + }; + let (Some(x), Some(y)) = (at(a), at(b)) else { + return 2; + }; + let same = match (x, y) { + (Ok(x), Ok(y)) => x.is_same(y), + (Err(x), Err(y)) => x == y, + (Ok(Object::List(l)), Err(y)) | (Err(y), Ok(Object::List(l))) => { + Rc::as_ptr(l).cast::<()>() == y + } + _ => false, + }; + i64::from(same) +} + /// The `wpjit_str_len` helper (RFC 0071 WS6): `len` of a pinned `str` /// — the *character* count, matching `str.__len__`. Negative return /// deopts (pin miss). @@ -6643,25 +8082,11 @@ unsafe extern "C" fn wpjit_dict_set( let Some((d, key)) = dict_pin_and_key(ctx, pin, key_bits, key_tag) else { return 1; }; - // Python-free pre-probe: displaced-value discipline + deferral - // detection (a deferral means the insert below could run Python). - let old = match dict_probe_native(&d, &key) { - Ok(f) => f, - Err(()) => return 1, - }; - if let Some(old) = &old { - if !matches!( - old, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(old) - && super::Interpreter::looks_reapable_temporary(old) - { - return 1; - } + // Python-free pre-probe: a deferral means the insert below could run + // Python. + if dict_probe_native(&d, &key).is_err() { + return 1; } - // The displaced value (probed non-reapable above) drops by plain - // refcount, exactly like the no-op arm of the interpreter's - // prompt-reap check. match crate::builtins::dict_insert(&d, key, v) { Ok(_) => 0, Err(_) => 1, @@ -6700,21 +8125,9 @@ unsafe extern "C" fn wpjit_dict_del( Ok(Some(v)) => v, _ => return 1, }; - if !matches!( - old, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(&old) - && super::Interpreter::looks_reapable_temporary(&old) - { - return 1; - } drop(old); match crate::builtins::dict_remove(&d, &key) { - Ok(Some((k, v))) => { - crate::vm_singletons::queue_container_removed(&k); - crate::vm_singletons::queue_container_removed(&v); - 0 - } + Ok(Some(_)) => 0, _ => 1, } } @@ -6829,10 +8242,25 @@ unsafe extern "C" fn wpjit_tuple_len(frame: *mut JitFrame, pin: i64) -> i64 { let jf = unsafe { &mut *frame }; #[allow(clippy::cast_ptr_alignment)] let ctx = unsafe { &mut *jf.ctx.cast::() }; - let Some(Pin::Obj(Object::Tuple(items))) = ctx.pins.get(pin as usize) else { - return -1; - }; - items.len() as i64 + match ctx.pins.get(pin as usize) { + Some(Pin::Obj(Object::Tuple(items))) => items.len() as i64, + // An instance whose class's `__len__` is a registered native leaf + // (`deque.__len__`): its body runs no Python and has no effects, + // so anything but a plain length deopts for the interpreter to + // call it again. + Some(Pin::Obj(obj @ Object::Instance(_))) if !crate::gil::free_threading_enabled() => { + // SAFETY: the `&mut Interpreter` is dormant while the helper runs. + let interp = unsafe { &*ctx.interp }; + match interp + .leaf_instance_dunder(obj, "__len__") + .map(|b| (b.call)(std::slice::from_ref(obj))) + { + Some(Ok(Object::Int(n))) if n >= 0 => n, + _ => -1, + } + } + _ => -1, + } } #[cfg(test)] @@ -6963,8 +8391,12 @@ unsafe extern "C" fn wpjit_list_next(frame: *mut JitFrame, pin: i64, idx: i64) - (Object::None, JitType::Obj) => Ok(u64::MAX), // RFC 0073 WS3 — exact-`str` elements pin on the object // lane (a `split` result's `ForList` consumer). + // A loop over the same few objects reuses their pins. (v @ (Object::Instance(_) | Object::Str(_) | Object::Tuple(_)), JitType::Obj) => { - Err(Pin::Obj(v.clone())) + match pin_memo_hit(v, &ctx.pins, &ctx.pin_memo) { + Some(bits) => Ok(bits), + None => Err(Pin::Obj(v.clone())), + } } // The nested lane: a list element pins on the float-list // lane (see `wpjit_list_get`). @@ -6974,19 +8406,20 @@ unsafe extern "C" fn wpjit_list_next(frame: *mut JitFrame, pin: i64, idx: i64) - _ => return 2, } }; - match outcome { - Ok(bits) => { - jf.ret_bits = bits; - 0 - } - Err(p) => { - if ctx.pins.len() >= RUNTIME_PIN_CAP { - return 2; - } - jf.ret_bits = ctx.pins.len() as u64; + let pinned = match outcome { + Ok(bits) => Some(bits), + Err(p) => (ctx.pins.len() < RUNTIME_PIN_CAP).then(|| { + pin_memo_note(&p, ctx.pins.len(), &mut ctx.pin_memo); ctx.pins.push(p); + (ctx.pins.len() - 1) as u64 + }), + }; + match pinned { + Some(bits) => { + jf.ret_bits = bits; 0 } + None => 2, } } @@ -7012,6 +8445,22 @@ unsafe extern "C" fn wpjit_get_iter(frame: *mut JitFrame, pin: i64) -> i64 { } } +/// One step of a list, tuple or range iterator, without the generic +/// dispatch: the next item, or `None` once exhausted (the iterator then +/// detached as [`PyIterator::next_value`] leaves it). `None` outside for +/// any other iterator, or one already borrowed. +#[inline] +fn builtin_seq_step(cell: &Rc>) -> Option> { + use crate::object::PyIterator; + let mut it = cell.try_borrow_mut().ok()?; + match &*it { + PyIterator::List { .. } | PyIterator::Tuple { .. } | PyIterator::Range { .. } => { + Some(it.next_value()) + } + _ => None, + } +} + /// The `wpjit_iter_next` helper (RFC 0071 WS4): one step of a /// [`weavepy_jit::TTerm`]`::ForIter` loop over a pinned identity /// iterable. **Runs Python code** for a generator source (the resume @@ -7034,31 +8483,103 @@ unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i // SAFETY: the `&mut Interpreter` that entered native code is // dormant while the helper runs. let interp = unsafe { &mut *ctx.interp }; - // The loop's poll point (the header's countdown poll covers the - // native back edge; this covers the Python the step may run): - // pending interpreter work and active observers route the loop - // through the interpreter. - crate::gil::yield_checkpoint(); - if crate::hot_gates::load() != 0 || crate::trace::any_observers_active() { - return 2; + // A native deque iterator's step runs no Python: no checkpoint (the + // loop header's countdown poll covers the back edge), and the element + // is read in place through the pin. + if let Some(Pin::Obj(it @ Object::Instance(inst))) = ctx.pins.get(pin as usize) { + if !crate::gil::free_threading_enabled() + && crate::hot_gates::load() == 0 + && !crate::trace::any_observers_active() + && super::Interpreter::default_getattribute(inst.cls_raw()) + && !crate::object::exotic_str_keys_possible() + { + if let Some((_, op)) = interp.core_leaf_next_op(inst) { + if op != 0 { + if let Some(v) = crate::stdlib::collections_native::fast_op(op, it, None) { + if let Some(bits) = + pack_iter_elem(&v, elem_tag, &mut ctx.pins, &mut ctx.pin_memo) + { + jf.ret_bits = bits; + return 0; + } + // Consumed but outside the lane: resume at the store. + jf.ret_bits = if matches!(v, Object::None) { + u64::MAX + } else { + ctx.pins.push(Pin::Obj(v)); + (ctx.pins.len() - 1) as u64 + }; + return 3; + } + } + } + } } - let it = match ctx.pins.get(pin as usize) { - Some(Pin::Obj(o @ (Object::Generator(_) | Object::Iter(_)))) => o.clone(), - _ => return 2, + // A list, tuple or range iterator steps in place: no Python runs, so + // the header's countdown poll covers the step, and nothing needs the + // iterator cloned. + let fast = match ctx.pins.get(pin as usize) { + Some(Pin::Obj(Object::Iter(cell))) => builtin_seq_step(cell), + _ => None, }; - // A builtin-iterator step is pure native code; everything else - // (generator resume) runs arbitrary Python on behalf of this - // activation. - let runs_python = !matches!(it, Object::Iter(_)); - if runs_python { - ctx.dirty = true; - // A generator resume from native code rebuilds a whole interpreter - // activation, several times what the interpreter's own inline - // resume costs: charged like an interpreter call, so a loop that - // drives a generator retires at the next poll (see `wpjit_poll`). - ctx.dyn_py_calls = ctx.dyn_py_calls.saturating_add(1); - } - match interp.iter_next(&it, &ctx.globals) { + let (step, runs_python) = match fast { + Some(step) => (Ok(step), false), + None => { + // The loop's poll point (the header's countdown poll covers the + // native back edge; this covers the Python the step may run): + // pending interpreter work and active observers route the loop + // through the interpreter. + crate::gil::yield_checkpoint(); + if crate::hot_gates::load() != 0 || crate::trace::any_observers_active() { + return 2; + } + let it = match ctx.pins.get(pin as usize) { + Some(Pin::Obj(o @ (Object::Generator(_) | Object::Iter(_)))) => o.clone(), + // An instance iterator whose `__next__` is a registered native + // leaf (a `deque` iterator): stepped below without Python. + Some(Pin::Obj(o @ Object::Instance(_))) + if !crate::gil::free_threading_enabled() => + { + o.clone() + } + _ => return 2, + }; + // A builtin-iterator step is pure native code; everything else + // (generator resume) runs arbitrary Python on behalf of this + // activation. + let runs_python = matches!(it, Object::Generator(_)); + if runs_python { + ctx.dirty = true; + // A generator resume from native code rebuilds a whole interpreter + // activation, several times what the interpreter's own inline + // resume costs: charged like an interpreter call, so a loop that + // drives a generator retires at the next poll (see `wpjit_poll`). + ctx.dyn_py_calls = ctx.dyn_py_calls.saturating_add(1); + } + let step = if let Object::Instance(inst) = &it { + // The core loop's cached `__next__` resolution (by class version). + // Any other `__next__` belongs to the interpreter (nothing was + // consumed). + if !super::Interpreter::default_getattribute(inst.cls_raw()) + || crate::object::exotic_str_keys_possible() + { + return 2; + } + match interp.leaf_next_step(&it, inst) { + None => return 2, + Some(Ok(v)) => Ok(Some(v)), + Some(Err(RuntimeError::PyException(e))) if e.type_name() == "StopIteration" => { + Ok(None) + } + Some(Err(e)) => Err(e), + } + } else { + interp.iter_next(&it, &ctx.globals) + }; + (step, runs_python) + } + }; + match step { Err(err) => { ctx.raised = Some(err); 4 @@ -7095,7 +8616,7 @@ unsafe extern "C" fn wpjit_iter_next(frame: *mut JitFrame, pin: i64, elem_tag: i SlotTag::Int => pack(&v, JitType::Int), SlotTag::Float => pack(&v, JitType::Float), SlotTag::Bool => pack(&v, JitType::Bool), - SlotTag::ObjPin => obj_ret_bits(&v, &mut ctx.pins), + SlotTag::ObjPin => obj_ret_bits(&v, &mut ctx.pins, &mut ctx.pin_memo), _ => None, } }; @@ -7214,7 +8735,7 @@ unsafe extern "C" fn wpjit_build_list( // (`gc.is_tracked([])` is True and any list can close a cycle by // later mutation); a natively built list is no different, and a // comprehension accumulator in particular always escapes. - crate::gc_trace::track(Object::List(list.clone())); + crate::gc_trace::track(&Object::List(list.clone())); let idx = ctx.pins.len() as i64; ctx.pins.push(Pin::List(list, elem)); idx @@ -7407,7 +8928,7 @@ unsafe extern "C" fn wpjit_build_map(frame: *mut JitFrame, n: i64) -> i64 { let obj = Object::Dict(Rc::new(GilRefCell::new(d))); // The interpreter tracks every dict it builds; a natively built // dict is no different. - crate::gc_trace::track(obj.clone()); + crate::gc_trace::track(&obj); let idx = ctx.pins.len() as i64; ctx.pins.push(Pin::Obj(obj)); idx @@ -7451,7 +8972,7 @@ unsafe extern "C" fn wpjit_list_repeat(frame: *mut JitFrame, pin: i64, count: i6 drop(items); let fresh = Rc::new(crate::sync::RefCell::new(out)); // See `wpjit_build_list`: every built list is GC-tracked. - crate::gc_trace::track(Object::List(fresh.clone())); + crate::gc_trace::track(&Object::List(fresh.clone())); let idx = ctx.pins.len() as i64; ctx.pins.push(Pin::List(fresh, elem)); idx @@ -7483,7 +9004,13 @@ unsafe extern "C" fn wpjit_list_from_range(frame: *mut JitFrame, start: i64, sto let items: Vec = (start..start + len).map(Object::Int).collect(); let list = Rc::new(crate::sync::RefCell::new(items)); let obj = Object::List(list.clone()); - crate::gc_trace::track(obj.clone()); + // A short list of integers can't close a cycle (a long one is + // registered without the scan as well). + if len <= 32 { + crate::gc_trace::track_inert(&obj); + } else { + crate::gc_trace::track(&obj); + } if crate::stdlib::tracemalloc_real::is_tracking() { crate::stdlib::tracemalloc_real::track_new_object(&obj); } @@ -7539,7 +9066,7 @@ unsafe extern "C" fn wpjit_list_slice( drop(items); let fresh = Rc::new(crate::sync::RefCell::new(out)); // See `wpjit_build_list`: every built list is GC-tracked. - crate::gc_trace::track(Object::List(fresh.clone())); + crate::gc_trace::track(&Object::List(fresh.clone())); let idx = ctx.pins.len() as i64; ctx.pins.push(Pin::List(fresh, elem)); idx @@ -7747,21 +9274,38 @@ unsafe extern "C" fn wpjit_attr_get(frame: *mut JitFrame, pin: i64, site: i64) - ctx.pins.get(pin as usize), ctx.attr_guards.get(site as usize), ) { - if let (AttrStorage::Indexed(key_idx), JitType::Int | JitType::Float | JitType::Bool) = - (g.storage, g.lane) - { - if attr_class_ok(inst, g.ver) { - // SAFETY: a read between two native ops; nothing here runs - // code (see `GilCell::peek`). - if let Some((k, v)) = unsafe { inst.attr_peek_index(key_idx as usize) } { - if key_is(k, &g.name) { - if let Some(bits) = pack(v, g.lane) { - jf.ret_bits = bits; - return 0; + match (g.storage, g.lane) { + (AttrStorage::Indexed(key_idx), JitType::Int | JitType::Float | JitType::Bool) => { + if attr_class_ok(inst, g.ver) { + // SAFETY: a read between two native ops; nothing here + // runs code (see `GilCell::peek`). + if let Some((k, v)) = unsafe { inst.attr_peek_index(key_idx as usize) } { + if key_is(k, &g.name) { + if let Some(bits) = pack(v, g.lane) { + jf.ret_bits = bits; + return 0; + } } } } } + // A `__slots__` field at its usual position. + (AttrStorage::Slot(key_idx), JitType::Int | JitType::Float | JitType::Bool) => { + if attr_class_ok(inst, g.ver) { + // SAFETY: as above. + if let Some((k, v)) = + unsafe { inst.slots.peek() }.and_then(|s| s.get_index(key_idx as usize)) + { + if key_is(k, &g.name) { + if let Some(bits) = pack(v, g.lane) { + jf.ret_bits = bits; + return 0; + } + } + } + } + } + _ => {} } } // Scoped so the receiver borrow of `ctx.pins` ends before an @@ -8450,14 +9994,6 @@ unsafe extern "C" fn wpjit_attr_set(frame: *mut JitFrame, pin: i64, site: i64) - // and the reap; otherwise the overwrite drops it by plain // refcount, exactly like the no-op arm of // `maybe_prompt_reap_replaced`. - if !matches!( - dst, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(dst) - && super::Interpreter::looks_reapable_temporary(dst) - { - return 1; - } let old = std::mem::replace(dst, v); drop(old); 0 @@ -8476,18 +10012,8 @@ unsafe extern "C" fn wpjit_attr_set(frame: *mut JitFrame, pin: i64, site: i64) - // The split layout (see `Interpreter::core_store_new_attr`). let v = if inst.dict.published().is_none() { // SAFETY: a read between two native ops. - let Some(split) = (unsafe { inst.dict.split_cell().peek() }) else { + if unsafe { inst.dict.split_cell().peek() }.is_none() { return 1; - }; - if let Some(dst) = split.get(&g.name) { - if !matches!( - dst, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(dst) - && super::Interpreter::looks_reapable_temporary(dst) - { - return 1; - } } match inst.split_store(&g.name, v) { Ok(old) => { @@ -8521,14 +10047,6 @@ unsafe extern "C" fn wpjit_attr_set(frame: *mut JitFrame, pin: i64, site: i64) - RawEntryMut::Occupied(mut entry) => { let dst = entry.get_mut(); // The displaced-value discipline of the indexed arm. - if !matches!( - dst, - Object::Int(_) | Object::Float(_) | Object::Bool(_) | Object::None - ) && super::Interpreter::local_needs_prompt_reap(dst) - && super::Interpreter::looks_reapable_temporary(dst) - { - return 1; - } Some(std::mem::replace(dst, v)) } RawEntryMut::Vacant(entry) => { @@ -8566,6 +10084,9 @@ unsafe extern "C" fn wpjit_global_obj(frame: *mut JitFrame, token: i64) -> i64 { let jf = unsafe { &mut *frame }; #[allow(clippy::cast_ptr_alignment)] let ctx = unsafe { &mut *jf.ctx.cast::() }; + // A list-lane global's element lane rides the high word (see + // `weavepy_jit::global_obj_list_code`). + let elem = weavepy_jit::global_obj_list_elem(token); let token = token as u32; if let Some(&(_, pin)) = ctx.obj_global_pins.iter().find(|&&(t, _)| t == token) { return pin as i64; @@ -8580,7 +10101,11 @@ unsafe extern "C" fn wpjit_global_obj(frame: *mut JitFrame, token: i64) -> i64 { return -2; } let pin = ctx.pins.len() as u64; - ctx.pins.push(Pin::Obj(obj)); + match (elem, obj) { + (Some(elem), Object::List(l)) => ctx.pins.push(Pin::List(l, elem)), + (Some(_), _) => return -2, + (None, obj) => ctx.pins.push(Pin::Obj(obj)), + } ctx.obj_global_pins.push((token, pin)); pin as i64 } @@ -8989,6 +10514,14 @@ unsafe fn call_dyn_impl( return status; } } + // A keyword call of a warm pure leaf evaluates frameless on its + // marshaled values (no Python runs, so nothing needs revalidating). + if kwc > 0 { + // SAFETY: per the function contract — same live buffers. + if let Some(v) = unsafe { dyn_kw_pure_leaf(jf, ctx, interp, &callee, argc, kwc, names) } { + return dyn_leaf_result(jf, ctx, v, int_result); + } + } // A keyword call of a plain function whose site carries the // interpreter's `CallPyKwNames` permutation binds through it, as the // `CALL_KW` handler's hit does: no name strings, no generic binder. @@ -9118,6 +10651,86 @@ unsafe fn dyn_call_result( } } +/// A dynamic keyword call of a warm pure-leaf function (or a bound method +/// over one), evaluated frameless on the marshaled operands through the +/// call site's `CallPyKwNames` permutation (see +/// [`super::Interpreter::kw_pure_leaf_call`]): its result, or `None` +/// having done nothing observable. +/// +/// # Safety +/// +/// Same contract as [`wpjit_call_py`]; the call site's pc is in +/// `jf.deopt_pc` (stored before every call helper runs). +unsafe fn dyn_kw_pure_leaf( + jf: &JitFrame, + ctx: &CallCtx, + interp: &super::Interpreter, + callee: &Object, + argc: u32, + kwc: u32, + names: u32, +) -> Option { + const MAX: usize = 9; + // SAFETY: per the function contract, the activation keeps its code + // object alive. + let code = unsafe { &*ctx.code_ptr }; + let Some(Object::Tuple(name_items)) = crate::code_const_objects(code).get(names as usize) + else { + return None; + }; + let (f, recv) = match callee { + Object::Function(f) => (f, None), + Object::BoundMethod(bm) if !bm.redispatch_descriptor => match &bm.function { + Object::Function(f) => (f, Some(&bm.receiver)), + _ => return None, + }, + _ => return None, + }; + let n = (argc + kwc) as usize; + let eff_argc = argc as usize + usize::from(recv.is_some()); + if n + usize::from(recv.is_some()) > MAX || name_items.len() != kwc as usize { + return None; + } + let mut vals: [Object; MAX] = std::array::from_fn(|_| Object::None); + let offset = usize::from(recv.is_some()); + if let Some(r) = recv { + vals[0] = r.clone(); + } + for j in 0..n { + // SAFETY: native code wrote `argc + kwc` entries, and the buffers + // are `max_call_args` wide. + let (bits, tag) = unsafe { (*jf.call_args.add(j), *jf.call_tags.add(j)) }; + vals[offset + j] = unpack_pins(bits, tag, &ctx.pins); + } + interp.kw_pure_leaf_call( + code, + jf.deopt_pc as usize, + f, + name_items, + &vals[..offset + n], + eff_argc, + ctx.depth_cell, + ) +} + +/// Deliver a pure leaf's result at a dynamic call site: it ran no Python, +/// so the activation's guards still hold. +fn dyn_leaf_result(jf: &mut JitFrame, ctx: &mut CallCtx, v: Object, int_result: bool) -> i64 { + if int_result { + if let Object::Int(value) = &v { + jf.ret_bits = *value as u64; + jf.ret_tag = SlotTag::Int as u32; + return CallStatus::Ok as i64; + } + } else if let Some(bits) = pin_any(v.clone(), &mut ctx.pins) { + jf.ret_bits = bits; + jf.ret_tag = SlotTag::ObjPin as u32; + return CallStatus::Ok as i64; + } + ctx.parked = Some(v); + CallStatus::Boxed as i64 +} + /// Bind a dynamic keyword call's marshaled operands for a plain function /// (or a bound method over one) through the call site's cached /// `CallPyKwNames` permutation, re-verified against the live function @@ -9437,18 +11050,21 @@ unsafe extern "C" fn wpjit_truth(frame: *mut JitFrame, pin: i64, _reserved: i64) jf.ret_bits = 0; return 0; } + // A registered native `__bool__`/`__len__` (or neither): the + // interpreter's cached answer, which runs no Python (read through the + // pin: the table owns the object throughout). + if let Some(Pin::Obj(obj @ Object::Instance(_))) = ctx.pins.get(pin as usize) { + if !crate::gil::free_threading_enabled() { + if let Some(b) = interp.leaf_instance_truth(obj) { + jf.ret_bits = u64::from(b); + return 0; + } + } + } let v = match ctx.pins.get(pin as usize) { Some(p) => p.to_object(), None => return 3, }; - // A registered native `__bool__`/`__len__` (or neither): the - // interpreter's cached answer, which runs no Python. - if matches!(v, Object::Instance(_)) && !crate::gil::free_threading_enabled() { - if let Some(b) = interp.leaf_instance_truth(&v) { - jf.ret_bits = u64::from(b); - return 0; - } - } let pure = match &v { Object::Foreign(_) | Object::MappingProxyObj(_) => false, Object::Instance(_) => { @@ -9464,6 +11080,12 @@ unsafe extern "C" fn wpjit_truth(frame: *mut JitFrame, pin: i64, _reserved: i64) jf.ret_bits = u64::from(v.is_truthy()); return 0; } + // A Python `__bool__`/`__len__` may inspect its caller's frame, whose + // locals live lane-packed in the native activation: the interpreter + // re-executes the test with a materialized frame. + if matches!(v, Object::Instance(_)) { + return 3; + } ctx.dirty = true; match interp.obj_truthy(&v, &ctx.globals) { Err(err) => { @@ -9628,13 +11250,29 @@ unsafe extern "C" fn wpjit_iter_new(frame: *mut JitFrame, pin: i64) -> i64 { Some(Pin::List(l, _)) => Object::List(l.clone()), None => return -1, }; + // An instance whose class's `__iter__` is a registered native leaf + // (`deque.__iter__`) builds its iterator without Python and without + // effects: a failure deopts for the interpreter to raise it. + if let Object::Instance(_) = &recv { + if crate::gil::free_threading_enabled() { + return -1; + } + let Some(b) = interp.leaf_instance_dunder(&recv, "__iter__") else { + return -1; + }; + return match (b.call)(std::slice::from_ref(&recv)) { + Ok(it) => { + let idx = ctx.pins.len() as i64; + ctx.pins.push(Pin::Obj(it)); + idx + } + Err(_) => -1, + }; + } // `make_iter` dispatches Python for exactly these receiver // shapes; a fresh compile would double `__iter__`'s side effects // on a post-hoc deopt, so they never enter the helper. - if matches!( - recv, - Object::Instance(_) | Object::Type(_) | Object::MappingProxyObj(_) - ) { + if matches!(recv, Object::Type(_) | Object::MappingProxyObj(_)) { return -1; } match interp.make_iter(&recv, &ctx.globals) { @@ -9650,7 +11288,12 @@ unsafe extern "C" fn wpjit_iter_new(frame: *mut JitFrame, pin: i64) -> i64 { /// Pack one yielded element into a compiled loop-variable lane /// (RFC 0074 WS3 — the [`weavepy_jit::ITER_ELEM_STR`]-aware sibling /// of `wpjit_iter_next`'s packing). `None` = outside the lane. -fn pack_iter_elem(v: &Object, tag: i64, pins: &mut PinTable) -> Option { +fn pack_iter_elem( + v: &Object, + tag: i64, + pins: &mut PinTable, + memo: &mut [u32; PIN_MEMO], +) -> Option { if tag == weavepy_jit::ITER_ELEM_STR { return match v { Object::Str(_) if pins.len() < RUNTIME_PIN_CAP => { @@ -9676,7 +11319,7 @@ fn pack_iter_elem(v: &Object, tag: i64, pins: &mut PinTable) -> Option { | Object::Str(_) | Object::WStr(_) | Object::Bytes(_) => pin_any(v.clone(), pins), - _ => obj_ret_bits(v, pins), + _ => obj_ret_bits(v, pins, memo), }, _ => None, } @@ -9840,6 +11483,42 @@ unsafe extern "C" fn wpjit_iter_next_pair( let ctx = unsafe { &mut *jf.ctx.cast::() }; // SAFETY: the `&mut Interpreter` is dormant while the helper runs. let interp = unsafe { &mut *ctx.interp }; + // A `dict.items()` step yields the key and value straight into the + // two lanes, without the tuple; no Python runs, so the header's poll + // covers it. + let items = match ctx.pins.get(pin as usize) { + Some(Pin::Obj(Object::Iter(cell))) => cell + .try_borrow_mut() + .ok() + .and_then(|mut it| it.next_item_pair()), + _ => None, + }; + if let Some(step) = items { + return match step { + Err(err) => { + ctx.raised = Some(err); + 4 + } + Ok(None) => 1, + Ok(Some((k, v))) => { + let b1 = pack_iter_elem(&k, tag1, &mut ctx.pins, &mut ctx.pin_memo); + let b2 = + b1.and_then(|_| pack_iter_elem(&v, tag2, &mut ctx.pins, &mut ctx.pin_memo)); + if let (Some(b1), Some(b2)) = (b1, b2) { + jf.ret_bits = b1; + // SAFETY: the marshal buffer is at least one slot + // wide (`max_call_args.max(1)`). + unsafe { *jf.call_args = b2 }; + return 0; + } + // Not in the lanes: the pair surrenders as the tuple the + // erased `UNPACK_SEQUENCE` consumes. + ctx.pins.push(Pin::Obj(Object::new_tuple_array([k, v]))); + jf.ret_bits = (ctx.pins.len() - 1) as u64; + 3 + } + }; + } // The loop's poll point — see `wpjit_iter_next`. crate::gil::yield_checkpoint(); if crate::hot_gates::load() != 0 || crate::trace::any_observers_active() { @@ -9926,9 +11605,11 @@ unsafe extern "C" fn wpjit_iter_next_pair( // store-pc deopt and unpack generically). if let Object::Tuple(items) = &v { if items.len() == 2 { - let packed1 = pack_iter_elem(&items[0], tag1, &mut ctx.pins); - let packed2 = - packed1.and_then(|_| pack_iter_elem(&items[1], tag2, &mut ctx.pins)); + let packed1 = + pack_iter_elem(&items[0], tag1, &mut ctx.pins, &mut ctx.pin_memo); + let packed2 = packed1.and_then(|_| { + pack_iter_elem(&items[1], tag2, &mut ctx.pins, &mut ctx.pin_memo) + }); if let (Some(b1), Some(b2)) = (packed1, packed2) { jf.ret_bits = b1; // SAFETY: the marshal buffer is at least @@ -10234,6 +11915,9 @@ pub(crate) fn try_call_native_direct( pins, entry_pin_count, pin_pressure_exit: false, + pin_limit: RUNTIME_PIN_SOFT_LIMIT, + pins_counted: 0, + last_ref_pins: 0, obj_globals: entry.art.obj_globals.clone(), obj_global_pins: Vec::new(), attr_guards: entry.art.attr_guards.clone(), @@ -10254,6 +11938,7 @@ pub(crate) fn try_call_native_direct( // keep the activation observable to callee-side stack walkers. frameless_code: Some(code.clone()), dyn_callee: None, + pin_memo: [u32::MAX; PIN_MEMO], }; let mut jf = JitFrame { locals: locals_buf.as_mut_ptr(), @@ -10934,7 +12619,7 @@ fn enter_compiled( // slot carries an index into `pins`, and the table (not the slot) // keeps the object alive and reachable for the access helpers and // the deopt rebuild. - let mut pins: PinTable = Vec::new(); + let mut pins: PinTable = take_pins(); { let locals = frame.locals.borrow(); for (slot, dst) in locals_buf.iter_mut().enumerate() { @@ -11042,6 +12727,9 @@ fn enter_compiled( pins, entry_pin_count, pin_pressure_exit: false, + pin_limit: RUNTIME_PIN_SOFT_LIMIT, + pins_counted: 0, + last_ref_pins: 0, obj_globals: entry.obj_globals.clone(), obj_global_pins: Vec::new(), attr_guards: entry.attr_guards.clone(), @@ -11062,6 +12750,7 @@ fn enter_compiled( // spine already. frameless_code: None, dyn_callee: None, + pin_memo: [u32::MAX; PIN_MEMO], }; let mut jf = JitFrame { locals: locals_buf.as_mut_ptr(), @@ -11147,6 +12836,7 @@ fn enter_compiled( // ones dying with the activation (after every pin-based rebuild // above, so nothing is unpacked from a drained table). drain_activation_pins(interp, &mut ctx.pins); + put_pins(std::mem::take(&mut ctx.pins)); put_u64(locals_buf); put_u64(spill); put_u32(tags); @@ -11170,6 +12860,12 @@ fn note_native_exit( st.stats.native_entries += 1; if cold_exit { st.stats.cold_exits += 1; + let key = Rc::as_ptr(&frame.code).cast::(); + if let Some(ce) = st.cache.get_mut(&key) { + if ce.cold_recompiles < COLD_RECOMPILE_BUDGET { + ce.recompile_at_osr = true; + } + } return; } if pin_pressure_exit { @@ -11607,11 +13303,6 @@ impl NativeActivation { visit(&o); } } - - /// The pinned objects, cloned (for [`super::frame_reapables`]). - pub(crate) fn pinned_objects(&self) -> impl Iterator + '_ { - self.pins.iter().map(Pin::to_object) - } } /// Decide whether a `Yielded` exit can park (RFC 0073 WS4) and, if @@ -11931,6 +13622,9 @@ fn resume_parked(interp: &mut super::Interpreter, frame: &mut super::Frame) -> J pins, entry_pin_count: act.entry_pin_count, pin_pressure_exit: false, + pin_limit: RUNTIME_PIN_SOFT_LIMIT, + pins_counted: 0, + last_ref_pins: 0, obj_globals: entry.obj_globals.clone(), obj_global_pins: std::mem::take(&mut act.obj_global_pins), attr_guards: entry.attr_guards.clone(), @@ -11951,6 +13645,7 @@ fn resume_parked(interp: &mut super::Interpreter, frame: &mut super::Frame) -> J // is on the spine already. frameless_code: None, dyn_callee: None, + pin_memo: [u32::MAX; PIN_MEMO], }; let mut jf = JitFrame { locals: act.locals_buf.as_mut_ptr(), diff --git a/crates/weavepy-vm/src/trace.rs b/crates/weavepy-vm/src/trace.rs index 13a06750..17d15409 100644 --- a/crates/weavepy-vm/src/trace.rs +++ b/crates/weavepy-vm/src/trace.rs @@ -445,6 +445,13 @@ pub fn any_observers_active() -> bool { any_observers_active_slow() } +/// The observer count [`any_observers_active`] tests first, for native +/// code that tests it in line (a `usize`: nonzero while any observer may +/// be active). +pub(crate) fn observer_count_flag() -> *const usize { + OBSERVER_COUNT.as_ptr() +} + #[cold] #[inline(never)] fn any_observers_active_slow() -> bool { diff --git a/crates/weavepy-vm/src/types.rs b/crates/weavepy-vm/src/types.rs index c0543ad7..711d78b6 100644 --- a/crates/weavepy-vm/src/types.rs +++ b/crates/weavepy-vm/src/types.rs @@ -155,15 +155,18 @@ fn next_type_version() -> u64 { /// Never publish zero or reuse a token, including at counter exhaustion. fn allocate_type_version(counter: &std::sync::atomic::AtomicU64) -> Option { use std::sync::atomic::Ordering::Relaxed; - counter - .fetch_update(Relaxed, Relaxed, |value| { - if value == 0 { - None - } else { - value.checked_add(1) - } - }) - .ok() + let mut value = counter.load(Relaxed); + loop { + let next = if value == 0 { + None + } else { + value.checked_add(1) + }?; + match counter.compare_exchange_weak(value, next, Relaxed, Relaxed) { + Ok(previous) => return Some(previous), + Err(current) => value = current, + } + } } #[cfg(test)] @@ -447,6 +450,10 @@ pub enum LeafAttrKind { ValueInstance(crate::sync::Weak), /// A class stored on the class, likewise held weakly. ValueType(crate::sync::Weak), + /// A dict stored on the class (`Enum._member_map_`), likewise. + ValueDict(crate::sync::Weak>), + /// A list stored on the class (`Enum._member_names_`), likewise. + ValueList(crate::sync::Weak>>), /// A `property` on the MRO (a data descriptor: it wins over the /// instance dict); its getter is read at access time. Property(crate::sync::Weak), @@ -741,6 +748,13 @@ pub struct TypeObject { pub exc_families: Cell, } +/// A dying TypeObject clears the weak references watching it. +impl Drop for TypeObject { + fn drop(&mut self) { + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); + } +} + /// Per-class resolution of the `type.__call__` protocol, cached on the /// class ([`TypeObject::instance_plan`]) and invalidated by /// [`TypeObject::attr_version`]. Everything here is a pure function of @@ -1100,7 +1114,7 @@ impl TypeObject { // free it — and weakrefs to it (or to methods in its dict) // would never clear. Built-ins are immortal; skip them. if !ty.flags.is_builtin { - crate::gc_trace::track(Object::Type(ty.clone())); + crate::gc_trace::track(&Object::Type(ty.clone())); } Ok(ty) } @@ -1537,12 +1551,20 @@ impl TypeObject { /// conservative `true` on any borrow conflict: a spurious resurrection /// is harmless because `Vm::invoke_finalizer` simply no-ops when the /// instance turns out to have no `__del__`. + #[inline] pub fn instances_need_finalize(&self) -> bool { match self.has_del.get() { - 1 => return false, - 2 => return true, - _ => {} + 1 => false, + 2 => true, + _ => self.instances_need_finalize_slow(), } + } + + /// [`Self::instances_need_finalize`] with no cached verdict: the MRO + /// walk, which caches one. + #[cold] + #[inline(never)] + fn instances_need_finalize_slow(&self) -> bool { let Ok(mro) = self.mro.try_borrow() else { return true; }; @@ -1989,8 +2011,16 @@ enum SlotData { layout: SharedSlice, values: Box<[Object]>, }, + /// A natively built `datetime` value's fields, packed into words (see + /// [`crate::stdlib::datetime_native::PackedSlots`]). Readers that + /// need `&Object`s get a lazily built copy; any write first converts + /// to [`Self::Fixed`]. + Packed(crate::stdlib::datetime_native::PackedSlots), } +#[cfg(target_pointer_width = "64")] +const _: () = assert!(std::mem::size_of::() == 32); + /// Is `key` the slot name `name`? (Interned names usually share the /// probe's storage, settled without reading either length.) #[inline(always)] @@ -2030,7 +2060,7 @@ fn slot_name_eq(stored: &str, name: &str) -> bool { /// The key for a newly populated slot `name`. The slots every raise /// populates (`args`, `__traceback__`, the chaining links) share one /// interned key each instead of allocating a string per exception. -fn slot_key(name: &str) -> DictKey { +pub(crate) fn slot_key(name: &str) -> DictKey { const COMMON: [&str; 6] = [ "args", "__traceback__", @@ -2094,6 +2124,7 @@ impl SlotStorage { SlotData::Fixed { layout, values } if SharedSlice::ptr_eq(layout, expected) => { Some(values) } + SlotData::Packed(p) if SharedSlice::ptr_eq(p.layout(), expected) => Some(p.values()), _ => None, } } @@ -2102,6 +2133,7 @@ impl SlotStorage { &mut self, expected: &SharedSlice, ) -> Option<&mut [Object]> { + self.unpack(); match &mut self.data { SlotData::Fixed { layout, values } if SharedSlice::ptr_eq(layout, expected) => { Some(values) @@ -2110,9 +2142,58 @@ impl SlotStorage { } } + /// Storage holding a natively packed value (see [`SlotData::Packed`]). + pub(crate) fn from_packed(packed: crate::stdlib::datetime_native::PackedSlots) -> Self { + Self { + data: SlotData::Packed(packed), + } + } + + /// The packed value, while the storage still holds one. + #[inline] + pub(crate) fn as_packed(&self) -> Option<&crate::stdlib::datetime_native::PackedSlots> { + match &self.data { + SlotData::Packed(p) => Some(p), + _ => None, + } + } + + /// [`Self::as_packed`], mutably. + #[inline] + pub(crate) fn as_packed_mut( + &mut self, + ) -> Option<&mut crate::stdlib::datetime_native::PackedSlots> { + match &mut self.data { + SlotData::Packed(p) => Some(p), + _ => None, + } + } + + /// Turn a [`SlotData::Packed`] storage into [`SlotData::Fixed`] + /// (before a slot is written). + #[inline] + fn unpack(&mut self) { + if let SlotData::Packed(_) = &self.data { + self.unpack_slow(); + } + } + + #[cold] + #[inline(never)] + fn unpack_slow(&mut self) { + if let SlotData::Packed(p) = std::mem::take(self).data { + let layout = p.layout().clone(); + self.data = SlotData::Fixed { + layout, + values: p.into_values(), + }; + } + } + /// Turn a [`SlotData::Fixed`] storage into the per-key form (before /// a key is added or removed). fn unfix(&mut self) { + self.unpack(); if let SlotData::Fixed { layout, values } = &mut self.data { let values = std::mem::take(values); let entries: Vec<(DictKey, Object)> = @@ -2139,6 +2220,24 @@ impl SlotStorage { pub fn from_layout(layout: SharedSlice, values: Vec) -> Self { Self::from_entries(layout.iter().cloned().zip(values).collect()) } + + /// Storage holding a packed value's slots (stored per key here). + pub(crate) fn from_packed(packed: crate::stdlib::datetime_native::PackedSlots) -> Self { + let layout = packed.layout().clone(); + Self::from_layout(layout, packed.into_values().into_vec()) + } + + #[inline] + pub(crate) fn as_packed(&self) -> Option<&crate::stdlib::datetime_native::PackedSlots> { + None + } + + #[inline] + pub(crate) fn as_packed_mut( + &mut self, + ) -> Option<&mut crate::stdlib::datetime_native::PackedSlots> { + None + } } #[cfg(not(target_pointer_width = "64"))] @@ -2186,6 +2285,13 @@ impl SlotStorage { |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)), ) .map(|index| index as u32), + SlotData::Packed(p) => p + .layout() + .iter() + .position( + |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)), + ) + .map(|index| index as u32), } } @@ -2198,12 +2304,14 @@ impl SlotStorage { SlotData::Small(entries) => entries.get(index).map(|(key, value)| (key, value)), SlotData::Many(table) => table.get_index(index), SlotData::Fixed { layout, values } => layout.get(index).zip(values.get(index)), + SlotData::Packed(p) => p.layout().get(index).zip(p.values().get(index)), _ => None, } } #[inline] pub fn get_index_mut(&mut self, index: usize) -> Option<(&DictKey, &mut Object)> { + self.unpack(); match &mut self.data { SlotData::Single { key, value } if index == 0 => key.as_ref().map(|key| (key, value)), SlotData::Small(entries) => entries.get_mut(index).map(|(key, value)| (&*key, value)), @@ -2229,6 +2337,13 @@ impl SlotStorage { |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)), ) .and_then(|i| values.get(i)), + SlotData::Packed(p) => p + .layout() + .iter() + .position( + |key| matches!(&key.0, Object::Str(stored) if slot_name_eq(stored.as_ref(), name)), + ) + .and_then(|i| p.values().get(i)), _ => None, } } @@ -2301,16 +2416,87 @@ impl SlotStorage { } Some(std::array::from_fn(|i| &values[i])) } + SlotData::Packed(p) => { + let (keys, values) = (p.layout().get(..N)?, p.values().get(..N)?); + if !keys + .iter() + .zip(names) + .all(|(key, name)| key_named(key, name)) + { + return None; + } + Some(std::array::from_fn(|i| &values[i])) + } _ => None, } } + /// Whether the first `N` keys of a small store are the given + /// interned names themselves (pointer identity, as attribute stores + /// intern them). + #[inline(always)] + fn leads_with_interned( + &self, + names: &[&crate::shared_value::SharedStr; N], + ) -> bool { + let SlotData::Small(entries) = &self.data else { + return false; + }; + if entries.len() < N { + return false; + } + let mut same = true; + for (i, name) in names.iter().enumerate() { + same &= matches!(&entries[i].0 .0, Object::Str(s) + if crate::shared_value::SharedStr::ptr_eq(s, name)); + } + same + } + + /// [`Self::leading`] for its common shape, inlined: a small store + /// whose first `N` keys are the given interned names themselves. Any + /// other shape takes the full comparison. + #[inline(always)] + pub fn leading_interned( + &self, + names: [&crate::shared_value::SharedStr; N], + ) -> Option<[&Object; N]> { + if !self.leads_with_interned(&names) { + return self.leading(names); + } + let SlotData::Small(entries) = &self.data else { + return None; + }; + let base = entries.as_ptr(); + // SAFETY: the store holds at least `N` entries (checked above). + Some(std::array::from_fn(|i| unsafe { &(*base.add(i)).1 })) + } + + /// [`Self::leading_interned`], mutably. + #[inline(always)] + pub fn leading_interned_mut( + &mut self, + names: [&crate::shared_value::SharedStr; N], + ) -> Option<[&mut Object; N]> { + if !self.leads_with_interned(&names) { + return self.leading_mut(names); + } + let SlotData::Small(entries) = &mut self.data else { + return None; + }; + let base = entries.as_mut_ptr(); + // SAFETY: the store holds at least `N` entries (checked above), + // and each index yields a distinct entry. + Some(std::array::from_fn(|i| unsafe { &mut (*base.add(i)).1 })) + } + /// [`Self::leading`], mutably. #[inline] pub fn leading_mut( &mut self, names: [&crate::shared_value::SharedStr; N], ) -> Option<[&mut Object; N]> { + self.unpack(); let values: &mut [Object] = match &mut self.data { SlotData::Small(entries) => { let entries = entries.get_mut(..N)?; @@ -2343,6 +2529,7 @@ impl SlotStorage { /// Mutable access to a populated slot's value, by name. pub fn get_mut(&mut self, name: &str) -> Option<&mut Object> { + self.unpack(); match &mut self.data { SlotData::Single { key: Some(DictKey(Object::Str(stored))), @@ -2382,6 +2569,7 @@ impl SlotStorage { value: Object, make_key: impl FnOnce() -> DictKey, ) -> Option { + self.unpack(); if let SlotData::Fixed { .. } = &self.data { if let Some(slot) = self.get_mut(name) { return Some(std::mem::replace(slot, value)); @@ -2468,7 +2656,8 @@ impl SlotStorage { } SlotData::Small(entries) => (None, Some(entries), None, None), SlotData::Many(table) => (None, None, Some(table), None), - SlotData::Fixed { layout, values } => (None, None, None, Some((layout, values))), + SlotData::Fixed { layout, values } => (None, None, None, Some((layout, &values[..]))), + SlotData::Packed(p) => (None, None, None, Some((p.layout(), p.values()))), }; single .into_iter() @@ -2605,13 +2794,7 @@ pub struct PyInstance { /// `_PyGC_FINALIZED` bit. Set by `Vm::invoke_finalizer` the moment the /// finalizer is dispatched; read by [`PyInstance`]'s `Drop` to decide /// whether the dying instance still needs its `__del__` resurrected onto - /// the pending-finalizer queue. Without this, an acyclic finalizable - /// instance whose last `Arc` is dropped on a code path that *didn't* - /// route through the prompt-reap cascade (e.g. a cross-thread handoff - /// where a transient clone briefly inflated the refcount so the reap - /// bailed) is freed silently, skipping `__del__` (RFC 0040: - /// `test_multiprocessing_*` `test_release_task_refs` leaked one - /// `CountedObject` per race). + /// the pending-finalizer queue. pub finalize_ran: Cell, /// Cycle-collector tracking is still deferred: the collector has /// never seen this instance, and it holds only atomic values (see @@ -2755,6 +2938,13 @@ impl PyInstance { /// The caller established that `obj` is its only reference, the /// instance is still deferred and owns no C body. pub fn try_recycle(mut inst: Rc) { + // SAFETY: the caller holds the only reference. + if unsafe { &*inst.class.as_ptr() }.native_kind.get() != 0 { + if let Err(inst) = crate::stdlib::datetime_native::recycle(inst) { + drop(Rc::into_arc(inst)); + } + return; + } let Some(m) = Rc::get_mut(&mut inst) else { return; }; @@ -2939,19 +3129,8 @@ impl PyInstance { } impl Drop for PyInstance { - /// Last-resort finalizer safety net, mirroring - /// [`crate::object::PyGenerator`]'s `Drop`. WeavePy normally runs an - /// instance's `__del__` through the prompt-reap cascade the instant its - /// last program reference is dropped (matching CPython's refcount - /// timing). But that cascade is driven from specific eval-loop sites and - /// is gated on a refcount-dead test; when the final `Arc` is released - /// somewhere else — most often a cross-thread object handoff where a - /// transient clone on another thread briefly inflated the strong count so - /// the reap conservatively bailed — the instance would otherwise be freed - /// by a plain `Arc` drop with its `__del__` silently skipped (the cycle - /// collector never revisits acyclic objects). Catch that here: resurrect - /// a shallow copy that shares the dying instance's `__dict__`/slots/native - /// value onto the VM's pending-finalizer queue so `__del__` still runs. + /// An instance dies when its last reference goes, as in CPython, and + /// the weak references watching it clear. fn drop(&mut self) { // A deferred-tracking record names this instance; the dict may // outlive it (`d = obj.__dict__`), so retire the record first. @@ -2968,12 +3147,6 @@ impl Drop for PyInstance { } } } - if crate::hot_gates::env_flags::reap_trace() { - let name = self.cls().name.clone(); - if name.contains("Block") || name.contains("DataFrame") { - eprintln!("[INST-DROP] {name} body={:#x}", self.c_body.get()); - } - } // RFC 0045 (wave 3): release the faithful C inline body this // instance owns, if it ever crossed into a C extension that reads // its fields at fixed `tp_basicsize` offsets. Runs before the @@ -2988,44 +3161,9 @@ impl Drop for PyInstance { free(body); } } - // Already finalized (cascade/GC/this net's resurrected copy): the - // common case for finalizable instances and the *only* path for the - // overwhelmingly common finalizer-free instance — a single `Cell` - // read keeps the hot drop path cheap. - if self.finalize_ran.get() { - return; - } - // No `__del__` anywhere in the MRO ⇒ nothing to do. Cached on the - // type, so this is one more `Cell` read after the first instance. - if !self.cls().instances_need_finalize() { - return; - } - // CPython runs `tp_finalize` once: claim it so the resurrected copy - // (and any re-drop after the finalizer completes) can't loop. - self.finalize_ran.set(true); - // Can't run Python from `Drop`; resurrect onto the pending queue. The - // copy shares `dict`/`slots`/`native` (cloning the `Arc`s/contents), - // so `__del__` observes the same attributes; `try_*` tolerates TLS - // teardown and re-entrant borrows by dropping the request. - let resurrected = Object::Instance(Rc::new(PyInstance { - class: RefCell::new(self.cls()), - dict: self.dict.clone(), - native: match self.native.get() { - Some(v) => crate::sync::OnceBox::from(v.clone()), - None => crate::sync::OnceBox::new(), - }, - inline_values: Cell::new(self.inline_values.get()), - slots: RefCell::new(self.slots.borrow().clone()), - hash_cache: crate::sync::CachedHash::new(self.hash_cache.get()), - // This copy still owes its queued invocation. Dispatch claims - // its flag atomically; a failed enqueue suppresses its Drop. - finalize_ran: Cell::new(false), - // The resurrected copy is a distinct object that owns no C - // body (the dying `self` already freed its own above). - deferred: crate::sync::Cell::new(false), - c_body: CBody::default(), - })); - crate::vm_singletons::try_push_pending_finalizer(resurrected); + // A finalizer, if the class has one, already ran: `rc::Rc`'s `Drop` + // queued this instance for it at its last release. + crate::weakref_registry::on_death(std::ptr::from_ref(self) as usize as u64); } } diff --git a/crates/weavepy-vm/src/unicode_case.rs b/crates/weavepy-vm/src/unicode_case.rs index bc01b413..f0b03296 100644 --- a/crates/weavepy-vm/src/unicode_case.rs +++ b/crates/weavepy-vm/src/unicode_case.rs @@ -6,6 +6,7 @@ //! (multi-code-point) SpecialCasing expansions. use crate::stdlib::ucd; +use std::mem::MaybeUninit; const CAPITAL_SIGMA: u32 = 0x3A3; const FINAL_SIGMA: u32 = 0x3C2; @@ -61,7 +62,7 @@ fn lower_at(cps: &[char], i: usize) -> ucd::CaseMap { pub fn lower(s: &str) -> String { if s.is_ascii() { - return s.to_ascii_lowercase(); + return ascii_case_map(s, AsciiCase::Lower); } let cps: Vec = s.chars().collect(); let mut out = String::with_capacity(s.len()); @@ -73,7 +74,7 @@ pub fn lower(s: &str) -> String { pub fn upper(s: &str) -> String { if s.is_ascii() { - return s.to_ascii_uppercase(); + return ascii_case_map(s, AsciiCase::Upper); } let mut out = String::with_capacity(s.len()); for c in s.chars() { @@ -85,7 +86,7 @@ pub fn upper(s: &str) -> String { /// `str.casefold()` — context-free full fold (no Final_Sigma). pub fn casefold(s: &str) -> String { if s.is_ascii() { - return s.to_ascii_lowercase(); + return ascii_case_map(s, AsciiCase::Lower); } let mut out = String::with_capacity(s.len()); for c in s.chars() { @@ -96,16 +97,7 @@ pub fn casefold(s: &str) -> String { pub fn title(s: &str) -> String { if s.is_ascii() { - let mut previous_is_cased = false; - return ascii_case_map(s, |b| { - let mapped = if previous_is_cased { - b.to_ascii_lowercase() - } else { - b.to_ascii_uppercase() - }; - previous_is_cased = b.is_ascii_alphabetic(); - mapped - }); + return ascii_case_map(s, AsciiCase::Title); } let cps: Vec = s.chars().collect(); let mut out = String::with_capacity(s.len()); @@ -125,11 +117,7 @@ pub fn title(s: &str) -> String { /// lowered with the sigma rule. pub fn capitalize(s: &str) -> String { if s.is_ascii() { - let mut out = s.to_ascii_lowercase(); - if let Some(first) = out.get_mut(..1) { - first.make_ascii_uppercase(); - } - return out; + return ascii_case_map(s, AsciiCase::Capitalize); } let cps: Vec = s.chars().collect(); let mut out = String::with_capacity(s.len()); @@ -144,13 +132,7 @@ pub fn capitalize(s: &str) -> String { pub fn swapcase(s: &str) -> String { if s.is_ascii() { - return ascii_case_map(s, |b| { - if b.is_ascii_lowercase() { - b.to_ascii_uppercase() - } else { - b.to_ascii_lowercase() - } - }); + return ascii_case_map(s, AsciiCase::Swap); } let cps: Vec = s.chars().collect(); let mut out = String::with_capacity(s.len()); @@ -168,10 +150,87 @@ pub fn swapcase(s: &str) -> String { out } -/// ASCII case mappings preserve both length and UTF-8 validity. Keep this -/// safe even if a future caller passes a non-ASCII byte through the mapper. -fn ascii_case_map(s: &str, map: impl FnMut(u8) -> u8) -> String { - String::from_utf8(s.bytes().map(map).collect()).expect("ASCII case mapping") +/// The case operations' ASCII forms, which [`ascii_case_into`] applies +/// to a whole buffer at once. (`casefold` of ASCII is `Lower`.) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum AsciiCase { + Upper, + Lower, + Title, + Capitalize, + Swap, +} + +/// Write the `case` mapping of ASCII `src` into `dst` (of equal length), +/// initializing all of it. +/// +/// Each output byte depends only on its input byte and, for `Title`, the +/// one before it (a letter follows a cased letter exactly when the byte +/// before it is alphabetic), so every loop below is branch-free and +/// vectorizes, where the per-character case-table lookups of the general +/// path don't. Each mapping flips the case bit (`0x20`) of selected +/// letters only, so ASCII stays ASCII; a non-ASCII byte is never a letter +/// here and passes through unchanged. +pub fn ascii_case_into(case: AsciiCase, src: &[u8], dst: &mut [MaybeUninit]) { + #[inline(always)] + fn is_lower(b: u8) -> bool { + b.wrapping_sub(b'a') < 26 + } + #[inline(always)] + fn is_upper(b: u8) -> bool { + b.wrapping_sub(b'A') < 26 + } + #[inline(always)] + fn is_alpha(b: u8) -> bool { + (b | 0x20).wrapping_sub(b'a') < 26 + } + #[inline(always)] + fn flip(b: u8, when: bool) -> MaybeUninit { + MaybeUninit::new(b ^ (u8::from(when) << 5)) + } + assert_eq!(src.len(), dst.len()); + let (Some(&first), Some(head)) = (src.first(), dst.first_mut()) else { + return; + }; + match case { + AsciiCase::Upper => { + for (d, &c) in dst.iter_mut().zip(src) { + *d = flip(c, is_lower(c)); + } + } + AsciiCase::Lower => { + for (d, &c) in dst.iter_mut().zip(src) { + *d = flip(c, is_upper(c)); + } + } + AsciiCase::Swap => { + for (d, &c) in dst.iter_mut().zip(src) { + *d = flip(c, is_alpha(c)); + } + } + AsciiCase::Capitalize => { + *head = flip(first, is_lower(first)); + for (d, &c) in dst[1..].iter_mut().zip(&src[1..]) { + *d = flip(c, is_upper(c)); + } + } + AsciiCase::Title => { + *head = flip(first, is_lower(first)); + for ((d, &c), &prev) in dst[1..].iter_mut().zip(&src[1..]).zip(src) { + let cased = is_alpha(prev); + *d = flip(c, (cased && is_upper(c)) || (!cased && is_lower(c))); + } + } + } +} + +/// [`ascii_case_into`] into a new `String`. +fn ascii_case_map(s: &str, case: AsciiCase) -> String { + let mut out = Vec::with_capacity(s.len()); + ascii_case_into(case, s.as_bytes(), &mut out.spare_capacity_mut()[..s.len()]); + // SAFETY: `ascii_case_into` initialized all `s.len()` bytes. + unsafe { out.set_len(s.len()) }; + String::from_utf8(out).expect("ASCII case mapping") } /// `Py_UNICODE_ISSPACE` — differs from Rust's `char::is_whitespace` (e.g. diff --git a/crates/weavepy-vm/src/vm_singletons.rs b/crates/weavepy-vm/src/vm_singletons.rs index 4ef06da4..a6f984b2 100644 --- a/crates/weavepy-vm/src/vm_singletons.rs +++ b/crates/weavepy-vm/src/vm_singletons.rs @@ -51,12 +51,15 @@ thread_local! { const { RefCell::new(Vec::new()) }; } -/// Process-wide count of parked `__del__` requests across all threads' -/// [`PENDING_FINALIZERS`] queues (RFC 0058 WS2). The eval loop probes -/// for pending finalizers *every instruction*; a macOS thread-local +/// Process-wide count of parked `__del__` requests and weakref callbacks +/// across all threads' [`PENDING_FINALIZERS`] and +/// [`PENDING_WEAKREF_CALLBACKS`] queues (RFC 0058 WS2). The eval loop +/// probes for pending work *every instruction*; a macOS thread-local /// access plus a `RefCell` borrow there is measurably expensive, so -/// this relaxed atomic is the fast gate and the thread-local queue -/// stays the precise, per-thread source of truth. +/// this relaxed atomic is the fast gate and the thread-local queues +/// stay the precise, per-thread source of truth. A drain re-raises the +/// shared hot gate while it's nonzero, so one thread's drain never +/// strands work another thread queued. static PENDING_FINALIZER_COUNT: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); @@ -68,6 +71,7 @@ pub fn push_pending_finalizer(obj: Object) { }); PENDING_FINALIZER_COUNT.fetch_add(1, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::PENDING_FINALIZERS); + crate::gc_trace::mark_maybe_dead(); } /// Like [`push_pending_finalizer`], but callable from `Drop` impls: @@ -88,6 +92,7 @@ pub fn try_push_pending_finalizer(obj: Object) { if pushed { PENDING_FINALIZER_COUNT.fetch_add(1, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::PENDING_FINALIZERS); + crate::gc_trace::mark_maybe_dead(); } else if let Some(obj) = pending { // No safe point can run this request during TLS teardown or a // reentrant queue borrow. Suppress its Drop fallback before releasing @@ -102,6 +107,69 @@ pub fn try_push_pending_finalizer(obj: Object) { } } +/// The last reference to `arc`'s object, a `PyInstance` or a `PyGenerator` +/// (see `rc::Rc`'s `Drop`), is going. One that owes a finalizer (an +/// instance whose class has `__del__`, a suspended generator, a coroutine +/// never awaited) is queued, still alive at its own address, for the next +/// safe point; anything else is released. +pub(crate) fn finalize_on_last_release(arc: std::sync::Arc) { + use crate::object::{CoroutineKind, GeneratorState, PyGenerator}; + use crate::types::PyInstance; + use std::any::TypeId; + use std::sync::Arc; + let t = TypeId::of::(); + if t == TypeId::of::() { + // SAFETY: `T` is `PyInstance` (the cast drops no metadata). + let inst = unsafe { Arc::from_raw(Arc::into_raw(arc).cast::()) }; + // A natively served `datetime` value: back to its pool (see + // `stdlib::datetime_native::recycle`). With no weak reference the + // last owner is the only one who can reach the class. + let inst = if Arc::weak_count(&inst) == 0 + // SAFETY: as above. + && unsafe { &*inst.class.as_ptr() }.native_kind.get() != 0 + { + match crate::stdlib::datetime_native::recycle(Rc::from_arc(inst)) { + Ok(()) => return, + Err(inst) => Rc::into_arc(inst), + } + } else { + inst + }; + let owes = !inst.finalize_ran.get() + && inst + .class + .try_borrow() + .is_ok_and(|cls| cls.instances_need_finalize()); + if owes { + try_push_pending_finalizer(Object::Instance(Rc::from_arc(inst))); + } else { + drop(inst); + } + } else if t == TypeId::of::() { + // SAFETY: as above, for `PyGenerator`. + let g = unsafe { Arc::from_raw(Arc::into_raw(arc).cast::()) }; + let owes = !g.finalize_ran.get() + && g.state.try_borrow().is_ok_and(|state| match &*state { + GeneratorState::Suspended(_) => true, + GeneratorState::Created(_) => g.kind == CoroutineKind::Coroutine, + GeneratorState::Finished | GeneratorState::Running => false, + }); + if owes { + let kind = g.kind; + let g = Rc::from_arc(g); + try_push_pending_finalizer(match kind { + CoroutineKind::Generator => Object::Generator(g), + CoroutineKind::Coroutine => Object::Coroutine(g), + CoroutineKind::AsyncGenerator => Object::AsyncGenerator(g), + }); + } else { + drop(g); + } + } else { + drop(arc); + } +} + /// Drain the pending-finalizer queue. The eval loop calls this /// at every eval-breaker tick that has the GC flag set. pub fn drain_pending_finalizers() -> Vec { @@ -136,11 +204,18 @@ pub fn has_pending_finalizers() -> bool { /// Queue a weakref callback `(callback, weakref_obj)` for invocation at /// the next safe point. Teardown-safe (callable from sweep paths). pub fn push_pending_weakref_callback(callback: Object, weakref_obj: Object) { - let _ = PENDING_WEAKREF_CALLBACKS.try_with(|cell| { - if let Ok(mut queue) = cell.try_borrow_mut() { - queue.push((callback, weakref_obj)); - } - }); + let pushed = PENDING_WEAKREF_CALLBACKS + .try_with(|cell| { + cell.try_borrow_mut() + .map(|mut queue| queue.push((callback, weakref_obj))) + .is_ok() + }) + .unwrap_or(false); + if pushed { + PENDING_FINALIZER_COUNT.fetch_add(1, std::sync::atomic::Ordering::Release); + crate::hot_gates::set(crate::hot_gates::PENDING_FINALIZERS); + crate::gc_trace::mark_maybe_dead(); + } } /// Is anything parked in this thread's weakref-callback queue? Cheap @@ -155,7 +230,11 @@ pub fn has_pending_weakref_callbacks() -> bool { /// Drain the pending weakref-callback queue. pub fn drain_pending_weakref_callbacks() -> Vec<(Object, Object)> { - PENDING_WEAKREF_CALLBACKS.with(|cell| std::mem::take(&mut *cell.borrow_mut())) + let taken = PENDING_WEAKREF_CALLBACKS.with(|cell| std::mem::take(&mut *cell.borrow_mut())); + if !taken.is_empty() { + PENDING_FINALIZER_COUNT.fetch_sub(taken.len(), std::sync::atomic::Ordering::Release); + } + taken } /// Build a singleton instance of the given built-in registry type. @@ -399,7 +478,13 @@ pub fn clear_thread_python_tls() { PENDING_FINALIZER_COUNT.fetch_sub(n, std::sync::atomic::Ordering::Release); } } - let _ = PENDING_WEAKREF_CALLBACKS.try_with(|cell| cell.borrow_mut().clear()); + let dropped_callbacks = + PENDING_WEAKREF_CALLBACKS.try_with(|cell| std::mem::take(&mut *cell.borrow_mut()).len()); + if let Ok(n) = dropped_callbacks { + if n > 0 { + PENDING_FINALIZER_COUNT.fetch_sub(n, std::sync::atomic::Ordering::Release); + } + } let _ = CURRENT_THREAD_HANDLES.try_with(|cell| cell.borrow_mut().clear()); let dropped_cext = PENDING_CEXT_DROPS.try_with(|cell| std::mem::take(&mut *cell.borrow_mut()).len()); @@ -638,6 +723,7 @@ pub fn push_pending_resource_warning(message: String) { PENDING_RESOURCE_WARNINGS.with(|cell| cell.borrow_mut().push((message, None))); PENDING_RW_FLAG.store(true, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::RESOURCE_WARNINGS); + crate::gc_trace::mark_maybe_dead(); } /// As [`push_pending_resource_warning`], carrying the dying object's @@ -651,6 +737,7 @@ pub fn push_pending_resource_warning_with_source(message: String, source_key: us PENDING_RESOURCE_WARNINGS.with(|cell| cell.borrow_mut().push((message, Some(source_key)))); PENDING_RW_FLAG.store(true, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::RESOURCE_WARNINGS); + crate::gc_trace::mark_maybe_dead(); } /// Cheap probe for the eval-loop safe point: are any deferred resource @@ -805,65 +892,6 @@ thread_local! { static PENDING_CEXT_FLAG: std::cell::Cell = const { std::cell::Cell::new(false) }; } -thread_local! { - /// Object ids currently being torn down by the prompt reaper's cascade - /// on this thread (RFC 0047, wave 5). While an object is mid-cascade, - /// the teardown itself crosses it back into C — `traverse_object` runs - /// the GC bridge's `tp_traverse`, which mints a transient self box and - /// releases it — and that release would *re-queue* the object through - /// [`queue_cext_dropped`]. The queued clone then makes the cascade's - /// own deadness re-check see the object as externally alive, aborting - /// the teardown, and the drained clone starts the cascade over: a - /// livelock that pinned `repr(DataFrame)` at 100% CPU indefinitely. - /// A queue request for an id in this set is the cascade observing - /// itself and is dropped; requests for *other* objects (a child body - /// whose last C pin fell during the teardown) still queue normally. - static CASCADING_IDS: RefCell> = - RefCell::new(crate::fasthash::FxHashSet::default()); -} - -/// RAII marker for one object's trip through the prompt reaper's cascade; -/// see [`CASCADING_IDS`]. Created by [`enter_cascade`]. -#[derive(Debug)] -pub struct CascadeGuard { - id: u64, - owner: bool, -} - -impl Drop for CascadeGuard { - fn drop(&mut self) { - if self.owner { - let _ = CASCADING_IDS.try_with(|c| { - if let Ok(mut set) = c.try_borrow_mut() { - set.remove(&self.id); - } - }); - } - } -} - -/// Mark `id` as mid-cascade for the guard's lifetime. Nesting-safe: a -/// guard for an id already in the set is a no-op on drop (the outer -/// guard owns the entry). -pub fn enter_cascade(id: u64) -> CascadeGuard { - let owner = CASCADING_IDS - .try_with(|c| { - c.try_borrow_mut() - .map(|mut set| set.insert(id)) - .unwrap_or(false) - }) - .unwrap_or(false); - CascadeGuard { id, owner } -} - -/// Is `id` currently being torn down by the prompt reaper's cascade on -/// this thread? -fn in_cascade(id: u64) -> bool { - CASCADING_IDS - .try_with(|c| c.try_borrow().map(|set| set.contains(&id)).unwrap_or(true)) - .unwrap_or(false) -} - /// Park an object dropped by C extension code for a prompt-reap pass at /// the next eval-loop safe point. Only object kinds that can carry (or /// anchor) finalizers/weakrefs/tracked children are queued; scalars and @@ -887,23 +915,6 @@ pub fn queue_cext_dropped(obj: &Object) { queue_parked_drop(obj); } -/// Park a value evicted from a native container by a *mutating* method or -/// opcode — `dict.clear`/`__delitem__`/replacing `__setitem__`, `del d[k]`, -/// `list.remove`/`clear`/slice assignment, `set.discard`, … — for a -/// prompt-reap pass at the next eval-loop safe point. These mutators run as -/// plain builtin fns without interpreter access, so the reference they drop -/// can't go through `prompt_reap_dropped` inline; without the park, an -/// object whose *last* reference lived in the container stayed pinned by -/// its weakref registry entry / GC handle until the next cyclic collection. -/// CPython frees it on the spot (pandas' `_item_cache.clear()` relies on -/// the evicted Series — and its CoW `Block` — dying immediately: a stale -/// block kept `refs.has_reference()` true, misfiring the chained-assignment -/// `FutureWarning` in `Series.__setitem__`). Same kind filter as -/// [`queue_cext_dropped`]: scalars and other leaves drop inline as before. -pub fn queue_container_removed(obj: &Object) { - queue_cext_dropped(obj); -} - /// As [`queue_cext_dropped`] but with **no kind filter**. Native side exits /// also use this queue to retire temporary pins before the interpreter /// continuation runs, with callbacks deferred until its frame is installed. @@ -933,12 +944,6 @@ pub fn queue_parked_drop(obj: &Object) { if crate::gc_trace::collector_active() { return; } - // The prompt reaper's own teardown of this object (see - // [`CASCADING_IDS`]): the transient C crossings the cascade performs - // must not re-queue the object it is in the middle of freeing. - if in_cascade(crate::weakref_registry::id_of(obj)) { - return; - } let pushed = PENDING_CEXT_DROPS .try_with(|cell| { if let Ok(mut queue) = cell.try_borrow_mut() { @@ -950,33 +955,6 @@ pub fn queue_parked_drop(obj: &Object) { }) .unwrap_or(false); if pushed { - if crate::hot_gates::env_flags::reap_trace() { - eprintln!( - "[CEXT-DROP] queued {} id={:#x}", - obj.type_name_owned(), - crate::weakref_registry::id_of(obj) - ); - if std::env::var_os("WEAVEPY_REAP_BT").is_some() { - thread_local! { - static N: std::cell::Cell = const { std::cell::Cell::new(0) }; - } - let n = N.with(|c| { - let v = c.get() + 1; - c.set(v); - v - }); - let every: usize = std::env::var("WEAVEPY_REAP_BT_EVERY") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(5000); - if n.is_multiple_of(every) { - eprintln!( - "[CEXT-DROP-BT]\n{}", - std::backtrace::Backtrace::force_capture() - ); - } - } - } let _ = PENDING_CEXT_FLAG.try_with(|c| c.set(true)); PENDING_CEXT_COUNT.fetch_add(1, std::sync::atomic::Ordering::Release); crate::hot_gates::set(crate::hot_gates::PENDING_CEXT); diff --git a/crates/weavepy-vm/src/weak_object.rs b/crates/weavepy-vm/src/weak_object.rs new file mode 100644 index 00000000..65cefc96 --- /dev/null +++ b/crates/weavepy-vm/src/weak_object.rs @@ -0,0 +1,136 @@ +//! A non-owning reference to any heap-backed [`Object`]. +//! +//! The cycle collector and the weak-reference registry hold their targets +//! through this type, so neither keeps an object alive: an object dies by +//! reference count the moment its last strong owner goes, as in CPython, +//! and the collector only has to find and break cycles. +//! +//! The weak handle keeps the allocation (not the payload) reserved, so an +//! object's address can't be reused while a handle to it exists. That's +//! what lets the collector and the registry key their tables by address. + +use crate::object::Object; +use crate::shared_value::ThinWeak; +use crate::sync::{Rc, Weak}; + +macro_rules! weak_object { + ($($variant:ident),* $(,)?) => { + /// A weak handle on a heap-backed [`Object`]; see the module docs. + #[allow(missing_debug_implementations)] + pub enum WeakObject { + Tuple(ThinWeak), + $($variant(weak_object!(@payload $variant)),)* + } + + impl WeakObject { + /// A weak handle on `obj`, or `None` for a value with no heap + /// allocation of its own (scalars, strings, bytes). + pub fn new(obj: &Object) -> Option { + Some(match obj { + Object::Tuple(t) => Self::Tuple(crate::shared_value::ThinArc::downgrade(t)), + $(Object::$variant(rc) => Self::$variant(Rc::downgrade(rc)),)* + _ => return None, + }) + } + + /// The object, while it is alive. + pub fn upgrade(&self) -> Option { + Some(match self { + Self::Tuple(w) => Object::Tuple(w.upgrade()?), + $(Self::$variant(w) => Object::$variant(w.upgrade()?),)* + }) + } + + /// The number of strong owners left (0 once the object died). + pub fn strong_count(&self) -> usize { + match self { + Self::Tuple(w) => w.strong_count(), + $(Self::$variant(w) => w.strong_count(),)* + } + } + + /// Whether the object has died. + #[inline] + pub fn is_dead(&self) -> bool { + self.strong_count() == 0 + } + } + }; + (@payload Long) => { Weak }; + (@payload Complex) => { Weak }; + (@payload List) => { Weak>> }; + (@payload Dict) => { Weak> }; + (@payload Range) => { Weak }; + (@payload Function) => { Weak }; + (@payload Builtin) => { Weak }; + (@payload BoundMethod) => { Weak }; + (@payload Code) => { Weak }; + (@payload Cell) => { Weak> }; + (@payload Iter) => { Weak> }; + (@payload Slice) => { Weak }; + (@payload Type) => { Weak }; + (@payload Instance) => { Weak }; + (@payload Module) => { Weak }; + (@payload Generator) => { Weak }; + (@payload Coroutine) => { Weak }; + (@payload AsyncGenerator) => { Weak }; + (@payload AsyncGenAwait) => { Weak }; + (@payload ByteArray) => { Weak>> }; + (@payload Set) => { Weak> }; + (@payload FrozenSet) => { Weak }; + (@payload File) => { Weak }; + (@payload Property) => { Weak }; + (@payload StaticMethod) => { Weak }; + (@payload ClassMethod) => { Weak }; + (@payload SlotDescriptor) => { Weak }; + (@payload Frame) => { Weak }; + (@payload Traceback) => { Weak }; + (@payload MemoryView) => { Weak }; + (@payload MappingProxy) => { Weak> }; + (@payload MappingProxyObj) => { Weak }; + (@payload DictView) => { Weak }; + (@payload SimpleNamespace) => { Weak> }; + (@payload LazyIter) => { Weak }; + (@payload Capsule) => { Weak }; + (@payload Foreign) => { Weak }; +} + +weak_object!( + Long, + Complex, + List, + Dict, + Range, + Function, + Builtin, + BoundMethod, + Code, + Cell, + Iter, + Slice, + Type, + Instance, + Module, + Generator, + Coroutine, + AsyncGenerator, + AsyncGenAwait, + ByteArray, + Set, + FrozenSet, + File, + Property, + StaticMethod, + ClassMethod, + SlotDescriptor, + Frame, + Traceback, + MemoryView, + MappingProxy, + MappingProxyObj, + DictView, + SimpleNamespace, + LazyIter, + Capsule, + Foreign, +); diff --git a/crates/weavepy-vm/src/weakref_registry.rs b/crates/weavepy-vm/src/weakref_registry.rs index 71ed558c..0dfc486c 100644 --- a/crates/weavepy-vm/src/weakref_registry.rs +++ b/crates/weavepy-vm/src/weakref_registry.rs @@ -93,9 +93,11 @@ pub struct WeakRefSlot { /// Identity of the referent. Used to look up the registry /// list when the referent dies. pub target_id: ObjectId, - /// `Some(strong_clone_of_target)` while the referent is - /// alive. Set to `None` by `notify_clear`. - pub target: RefCell>, + /// The referent, held weakly, while the slot is live. Set to `None` + /// by `notify_clear`. A weak reference never keeps its referent + /// alive: the referent dies by reference count, and its `Drop` (or the + /// collector's sweep, for kinds without one) clears the slot. + pub target: RefCell>, /// Whether a callback was supplied at creation. The callback /// *object* itself lives in the user-visible wrapper's traced slots /// or dictionary (under `__callback__`), never here: the wrapper is @@ -166,11 +168,45 @@ pub mod kind { pub const CALLABLE_PROXY: u8 = 2; } +/// A slot's referent. Every Python-visible referent has a heap allocation +/// and is held weakly; a value without one (only the registry's own unit +/// tests make such slots) is simply held. +#[allow(missing_debug_implementations)] +pub enum WeakTarget { + Weak(crate::weak_object::WeakObject), + Strong(Object), +} + +impl WeakTarget { + pub fn new(target: &Object) -> Self { + match crate::weak_object::WeakObject::new(target) { + Some(w) => Self::Weak(w), + None => Self::Strong(target.clone()), + } + } + + /// The referent, while it is alive. + pub fn upgrade(&self) -> Option { + match self { + Self::Weak(w) => w.upgrade(), + Self::Strong(o) => Some(o.clone()), + } + } + + /// Whether the referent has died. + pub fn is_dead(&self) -> bool { + match self { + Self::Weak(w) => w.is_dead(), + Self::Strong(_) => false, + } + } +} + impl WeakRefSlot { pub fn new(target_id: ObjectId, target: Object, has_callback: bool, kind: u8) -> Self { Self { target_id, - target: RefCell::new(Some(target)), + target: RefCell::new(Some(WeakTarget::new(&target))), has_callback, fixed_wrapper: false, identity_hash: target_id as i64, @@ -188,7 +224,17 @@ impl WeakRefSlot { if self.is_dead() { return None; } - self.target.borrow().clone() + self.target.borrow().as_ref()?.upgrade() + } + + /// Whether the referent died without this slot being cleared yet (a + /// kind with no death hook; see `sweep_dead_targets`). + pub fn target_died(&self) -> bool { + !self.is_dead() + && self + .target + .try_borrow() + .is_ok_and(|t| t.as_ref().is_some_and(WeakTarget::is_dead)) } /// Clear the slot. Returns the callback (if any) so the caller can @@ -354,17 +400,33 @@ impl WeakRefRegistry { /// list of `(weakref_object, callback_object)` pairs the /// caller should invoke. Removes the entry for `id`. pub fn notify_clear(&self, id: ObjectId) -> Vec<(Arc, Option)> { + let entries = Self::take_entries(&mut self.inner.borrow_mut(), id); + Self::clear_entries(entries) + } + + /// [`Self::notify_clear`] from a `Drop`, which may run while the + /// registry is borrowed: `None` then, leaving the slots for the + /// collector's sweep. + fn try_notify_clear(&self, id: ObjectId) -> Option, Option)>> { let entries = { - let mut g = self.inner.borrow_mut(); - // Hot path: this runs for every object the prompt reaper - // frees, and almost none of them was ever weakly referenced. - // Skip the map removal on the miss-filter's say-so, exactly - // as `count` does. - if g.slots.is_empty() || !g.filter.may_contain(id) { - return Vec::new(); - } - g.slots.remove(&id).unwrap_or_default() + let Ok(mut g) = self.inner.try_borrow_mut() else { + return None; + }; + Self::take_entries(&mut g, id) }; + Some(Self::clear_entries(entries)) + } + + fn take_entries(g: &mut RegistryInner, id: ObjectId) -> Vec> { + // Almost no object was ever weakly referenced: skip the map + // removal on the miss-filter's say-so, exactly as `count` does. + if g.slots.is_empty() || !g.filter.may_contain(id) { + return Vec::new(); + } + g.slots.remove(&id).unwrap_or_default() + } + + fn clear_entries(entries: Vec>) -> Vec<(Arc, Option)> { let mut out = Vec::with_capacity(entries.len()); // CPython fires a referent's weakref callbacks newest-first (its // weakref list inserts at the head), so a chain of @@ -394,28 +456,23 @@ impl WeakRefRegistry { .unwrap_or(0) } - /// How many *strong clones* of the referent the registry is - /// currently holding for `id`. Every live, un-cleared slot keeps - /// one `Object` clone of its target alive (see - /// [`WeakRefSlot::target`]). The cycle collector subtracts this - /// from an object's outer refcount so a weakref does **not** keep - /// its referent reachable — otherwise `weakref.ref(obj)()` would - /// stay live forever and `WeakKeyDictionary`/`WeakValueDictionary` - /// would never self-clean after `del obj; gc.collect()`. - pub fn strong_clone_count(&self, id: ObjectId) -> usize { - let g = self.inner.borrow(); - if g.slots.is_empty() || !g.filter.may_contain(id) { - return 0; - } + /// The ids whose referent died without clearing its slots: the kinds + /// with no death hook (`set`, `bytearray`, `SimpleNamespace`, code + /// objects), or a death that found the registry busy. Their slots are cleared by + /// [`sweep_dead_targets`]. + fn dead_target_ids(&self) -> Vec { + let Ok(g) = self.inner.try_borrow() else { + return Vec::new(); + }; g.slots - .get(&id) - .map(|v| { + .iter() + .filter(|(_, v)| { v.iter() .filter_map(Weak::upgrade) - .filter(|s| !s.is_dead() && s.target.borrow().is_some()) - .count() + .any(|slot| slot.target_died()) }) - .unwrap_or(0) + .map(|(id, _)| *id) + .collect() } /// Snapshot the live weakrefs targeting `id` as @@ -486,7 +543,7 @@ impl WeakRefRegistry { .filter_map(|(id, v)| { v.iter() .filter_map(Weak::upgrade) - .find_map(|s| s.target.borrow().clone()) + .find_map(|s| s.target.borrow().as_ref()?.upgrade()) .map(|t| (*id, t)) }) .collect() @@ -509,8 +566,8 @@ static REGISTRY: std::sync::LazyLock = std::sync::LazyLock::new(WeakRefRegistry::new); /// RFC 0065 (WS4): insert-only miss-filter over every id a weakref was -/// ever registered for. The prompt-reap drop paths ask "is anything -/// watching this object?" for *every* dropped container; the +/// ever registered for. Every death hook asks "is anything watching +/// this object?"; the /// overwhelming "no" answer is two relaxed loads here instead of the /// registry cell's borrow protocol. Stale bits (dead referents) are /// false positives that take the precise path — never a correctness @@ -575,14 +632,34 @@ pub fn count_for(id: ObjectId) -> usize { with_registry(|r| r.count(id)) } -/// Convenience: count of registry-held strong clones of `id` in the -/// current thread. Used by the cycle collector's refcount accounting. -pub fn strong_clone_count(id: ObjectId) -> usize { - // RFC 0065 (WS4): see `count_for`. - if !WEAKREF_FILTER.may_contain(id) { - return 0; +/// The death hook every weakly referenceable object's `Drop` calls with +/// its own id: clear the slots watching it and queue their callbacks for +/// the next safe point, as CPython's `tp_dealloc` does. One relaxed load +/// when no weakref was ever registered for `id`. +#[inline] +pub fn on_death(id: ObjectId) { + if WEAKREF_FILTER.may_contain(id) { + on_death_slow(id); + } +} + +#[cold] +fn on_death_slow(id: ObjectId) { + // A death during a registry borrow leaves the slots for + // `sweep_dead_targets`. + if let Some(cleared) = REGISTRY.try_notify_clear(id) { + queue_callbacks(cleared); + } +} + +/// Clear the slots of referents that died without their death hook +/// clearing them, and queue the callbacks. Run by every collection. +pub fn sweep_dead_targets() -> usize { + let ids = REGISTRY.dead_target_ids(); + for &id in &ids { + queue_callbacks(notify_clear(id)); } - with_registry(|r| r.strong_clone_count(id)) + ids.len() } /// Convenience: collect every live weakref for `id` in the @@ -705,11 +782,9 @@ mod tests { reg.register(last.clone()); drop(dead); assert_eq!(reg.count(64), 2); - assert_eq!(reg.strong_clone_count(64), 2); first.clear(); - // A still-owned, cleared slot is a live wrapper, not a target owner. + // A still-owned, cleared slot is a live wrapper. assert_eq!(reg.count(64), 2); - assert_eq!(reg.strong_clone_count(64), 1); reg.shrink(); let watchers = reg.collect_strong(64); assert_eq!(watchers.len(), 2); @@ -721,7 +796,6 @@ mod tests { assert!(Arc::ptr_eq(&cleared[1].0, &first)); assert!(last.upgrade().is_none()); assert_eq!(reg.count(64), 0); - assert_eq!(reg.strong_clone_count(64), 0); assert!(reg.notify_clear(64).is_empty()); } @@ -765,9 +839,7 @@ mod tests { let replacement = Arc::new(WeakRefSlot::new(16, Object::Int(7), false, kind::REF)); reg.register(replacement.clone()); assert_eq!(reg.count(16), 1); - assert_eq!(reg.strong_clone_count(16), 1); assert_eq!(reg.count(17), 0); - assert_eq!(reg.strong_clone_count(17), 0); assert!(reg.collect_strong(17).is_empty()); assert!(reg.notify_clear(17).is_empty()); } diff --git a/docs/PERFORMANCE-CPYTHON-PARITY.md b/docs/PERFORMANCE-CPYTHON-PARITY.md index 3c922af1..cf41bcf5 100644 --- a/docs/PERFORMANCE-CPYTHON-PARITY.md +++ b/docs/PERFORMANCE-CPYTHON-PARITY.md @@ -10,17 +10,13 @@ the reports it links to. ## Method -Timings compare a profile-guided release build of the branch -(`tools/pgo_build.py --skip-regrtest`, JIT on) with CPython 3.14.7 on the -macOS x86-64 development host (Intel Core i9-9980HK). Both executables are -native x86-64. - -Wall-time ratios come from paired, interleaved runs. Each cycle runs every -fixture once under each interpreter at the harness work sizes, alternating -which goes first; one warmup cycle is discarded and five are measured. A -fixture's ratio is the median of its per-cycle WeavePy/CPython ratios, using -each fixture's own timer (`WEAVEPY_BENCH_NS`), which excludes startup and -imports. Values below 1.00 mean WeavePy is faster. +Timings come from the CI bench gate (`weavepy-bench gate`), run on the +macOS x86-64 development host (Intel Core i9-9980HK). It compares the +branch's release build with a release build of `main` (the merge base) and +with CPython 3.14.8, interleaving the three interpreters' runs on the same +machine. Each fixture's time is the median of its samples, measured by the +fixture's own timer (`WEAVEPY_BENCH_NS`), which excludes startup and +imports. Values below 1.00 mean the branch is faster. Instruction counts come from `/usr/bin/time -l` ("instructions retired") on the host, or from callgrind in a Linux container when a per-function or @@ -32,57 +28,76 @@ checkpoint confirms them. ## Benchmark fixtures -Geometric mean over the 23 timed fixtures: **0.592**. - -| Fixture | Work | WeavePy | CPython | Ratio | -| --- | ---: | ---: | ---: | ---: | -| `deltablue` | 50 | 213.8 ms | 67.8 ms | 3.15 | -| `deque_ops` | 200,000 | 97.9 ms | 64.1 ms | 1.53 | -| `pickle_bench` | 40 | 11.6 ms | 8.5 ms | 1.34 | -| `generators` | 300,000 | 59.3 ms | 49.6 ms | 1.19 | -| `datetime_ops` | 60,000 | 50.7 ms | 52.7 ms | 0.98 | -| `str_methods` | 15,000 | 57.0 ms | 59.9 ms | 0.95 | -| `float_math` | 100,000 | 78.0 ms | 81.6 ms | 0.94 | -| `dict_ops` | 100,000 | 55.9 ms | 61.7 ms | 0.90 | -| `call_overhead` | 150,000 | 69.8 ms | 82.2 ms | 0.84 | -| `fannkuch` | 100,000 | 17.1 ms | 20.9 ms | 0.83 | -| `richards` | 50,000 | 16.1 ms | 20.3 ms | 0.80 | -| `nbody` | 20,000 | 33.0 ms | 42.9 ms | 0.77 | -| `attr_access` | 200,000 | 35.4 ms | 47.1 ms | 0.77 | -| `list_ops` | 10,000 | 33.8 ms | 45.1 ms | 0.75 | -| `json_bench` | 150 | 57.4 ms | 81.1 ms | 0.71 | -| `pidigits` | 500,000 | 1,421.0 ms | 2,167.8 ms | 0.67 | -| `pyaes` | 400 | 15.8 ms | 28.9 ms | 0.54 | -| `jitkernels` | 2,000 | 24.3 ms | 49.0 ms | 0.48 | -| `fib` | 27 | 8.0 ms | 21.6 ms | 0.37 | -| `spectral_norm` | 100 | 12.1 ms | 57.7 ms | 0.21 | -| `nested_loops` | 120 | 6.2 ms | 69.3 ms | 0.09 | -| `jitloop` | 1,000 | 7.5 ms | 92.0 ms | 0.08 | -| `sumvm` | 2,000,000 | 3.8 ms | 72.5 ms | 0.05 | - -Times are the medians of each interpreter's samples; ratios are the medians -of the paired ratios, so they needn't equal the quotient of the two times. +Geometric mean over the 24 fixtures: **0.50** of CPython's time and +**0.75** of `main`'s. + +| Fixture | Work | Branch | `main` | CPython | vs `main` | vs CPython | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `deltablue` | 50 | 244.3 ms | 292.5 ms | 80.5 ms | 0.83 | 3.03 | +| `generators` | 300,000 | 64.0 ms | 76.8 ms | 58.0 ms | 0.83 | 1.10 | +| `dict_ops` | 100,000 | 68.1 ms | 82.2 ms | 64.2 ms | 0.83 | 1.06 | +| `pickle_bench` | 40 | 9.7 ms | 15.9 ms | 10.2 ms | 0.61 | 0.95 | +| `float_math` | 100,000 | 86.8 ms | 112.8 ms | 92.5 ms | 0.77 | 0.94 | +| `deque_ops` | 200,000 | 65.7 ms | 134.5 ms | 74.7 ms | 0.49 | 0.88 | +| `json_bench` | 150 | 76.8 ms | 93.3 ms | 96.8 ms | 0.82 | 0.79 | +| `list_ops` | 10,000 | 36.4 ms | 38.7 ms | 46.0 ms | 0.94 | 0.79 | +| `startup` | 1 | 19.4 ms | 20.6 ms | 24.5 ms | 0.94 | 0.79 | +| `fannkuch` | 100,000 | 16.8 ms | 24.4 ms | 22.4 ms | 0.69 | 0.75 | +| `nbody` | 20,000 | 37.9 ms | 50.1 ms | 50.9 ms | 0.76 | 0.74 | +| `str_methods` | 15,000 | 48.7 ms | 83.1 ms | 70.5 ms | 0.59 | 0.69 | +| `call_overhead` | 150,000 | 67.3 ms | 109.3 ms | 97.5 ms | 0.62 | 0.69 | +| `attr_access` | 200,000 | 37.8 ms | 64.2 ms | 57.0 ms | 0.59 | 0.66 | +| `pyaes` | 400 | 19.5 ms | 19.1 ms | 30.4 ms | 1.02 | 0.64 | +| `pidigits` | 500,000 | 1,580 ms | 1,680 ms | 2,570 ms | 0.94 | 0.61 | +| `richards` | 50,000 | 11.7 ms | 20.3 ms | 21.5 ms | 0.58 | 0.55 | +| `datetime_ops` | 60,000 | 30.7 ms | 84.2 ms | 60.8 ms | 0.36 | 0.51 | +| `jitkernels` | 2,000 | 21.6 ms | 34.9 ms | 58.8 ms | 0.62 | 0.37 | +| `fib` | 27 | 8.9 ms | 9.3 ms | 24.6 ms | 0.96 | 0.36 | +| `spectral_norm` | 100 | 12.8 ms | 13.3 ms | 63.9 ms | 0.97 | 0.20 | +| `nested_loops` | 120 | 7.3 ms | 7.4 ms | 86.2 ms | 0.99 | 0.08 | +| `jitloop` | 1,000 | 8.7 ms | 8.9 ms | 111.7 ms | 0.99 | 0.08 | +| `sumvm` | 2,000,000 | 4.3 ms | 4.4 ms | 85.8 ms | 0.99 | 0.05 | + +The largest gains over `main` came from these changes: + +- **A baseline frame compiler** (`frame_jit`) runs loops that tier 2 + declines, such as loops of method calls and container operations, in + native code. It compiles a loop only when most of its instructions stay + native, since every instruction that leaves for the core loop costs a + round trip. +- **Native containers and values.** `collections.deque`, `datetime` values, + `pickle`, compiled regular expressions and their matches, and `map` and + `filter` over native containers run natively instead of through their + pure-Python implementations. +- **Cheaper calls and exceptions.** Small functions that native code calls + (`key=lambda t: t[1]`) run without an activation, tracebacks are built + only when something reads them, and the core loop catches exceptions + without leaving. +- **Class bodies and metaclass reads.** Class creation and metaclass + attribute reads no longer pass through the general call path. ## Startup and imports -Whole-process instructions retired, warm stdlib cache: - -| Command | WeavePy | CPython | -| --- | ---: | ---: | -| `-c pass` | 91M | 127M | -| `import json` | 205M | 153M | -| `import dataclasses` | 349M | 196M | -| `import logging` | 513M | 258M | -| `import asyncio` | 689M | 365M | -| `import unittest` | 496M | 270M | +Whole-process instructions retired and peak memory, warm stdlib cache: -Startup is cheaper than CPython's. Importing large parts of the standard -library still costs about twice as much. Three changes on this branch cut -those costs by 20% to 60%: +| Command | WeavePy | CPython | WeavePy RSS | CPython RSS | +| --- | ---: | ---: | ---: | ---: | +| `-c pass` | 77M | 103M | 11.7 MiB | 10.5 MiB | +| `import json` | 177M | 141M | 24.3 MiB | 12.6 MiB | +| `import dataclasses` | 282M | 183M | 32.9 MiB | 13.9 MiB | +| `import logging` | 421M | 241M | 38.8 MiB | 16.0 MiB | +| `import asyncio` | 577M | 350M | 48.0 MiB | 19.1 MiB | +| `import unittest` | 407M | 253M | 40.4 MiB | 16.0 MiB | + +Startup is cheaper than CPython's, and imports retire 14% to 19% fewer +instructions than when this report was first written. Importing large parts +of the standard library still costs 1.3 to 1.8 times as much as in CPython. +These changes did most of the work: - Compiled stdlib modules are cached in a WeavePy-native code format - (`weavepy_compiler::native_code`), which loads without unmarshalling CPython - bytecode and transcoding it back into WeavePy instructions. + (`weavepy_compiler::native_code`, now version 3), which loads without + unmarshalling CPython bytecode and transcoding it back into WeavePy + instructions. Column tables stay encoded until something reads them. - Slicing a tuple, `bytes`, `bytearray`, or a string with surrogates no longer copies the whole source sequence. `re.compile` with `IGNORECASE` slices a 64K-entry charset map 256 times, which had made `import logging` cost 1.19B @@ -90,29 +105,39 @@ those costs by 20% to 60%: - The collector's per-drop sweep of suspected-dead objects skips entries that have used up their probe budget, so it no longer walks up to 256 entries at every drop during imports. +- The opcode shim builds its stack-effect tables on first use, and the frame + compiler stays within tier 2's start-up and import budget. -Peak memory after these imports is about twice CPython's (for example, -23 MiB against 12 MiB for `import json`). An allocation-site profile -attributes most of the difference to compiled code objects: their per-code -structure, per-instruction line and column tables, and the materialized -constants and names built on first use. +Peak memory is where WeavePy trails most: two to two and a half times +CPython's after a large import, and up to twice CPython's on the fixtures. An +allocation-site profile attributes most of the difference to compiled code +objects: their per-code structure, per-instruction line tables, and the +materialized constants and names built on first use. ## What still trails CPython -The four slower fixtures share a cause: operations that CPython's specialized -bytecodes do in 10 to 30 instructions take WeavePy two to four times as many. -Per operation, with the JIT off: +`deltablue` (3.03) and `generators` (1.10) are still slower than CPython, +and `dict_ops` is at parity (between 0.92 and 1.06 across runs). They +spend their time in operations that CPython's specialized bytecodes do in +a few dozen instructions. Per operation, with +the default settings, measured in a loop that tier 2 compiles: | Operation | WeavePy | CPython | | --- | ---: | ---: | -| Instance attribute read (`o.x`) | 160 | 64 | -| Class attribute read through an instance (`o.K`) | 240 | 60 | -| Global read | 150 | 50 | -| `isinstance(o, C)` | 980 | 240 | -| Call of a small non-leaf function | 1,700 | 440 | +| Instance attribute read (`o.x`) | 64 | 37 | +| Class attribute read through an instance (`o.K`) | 357 | 36 | +| Global read | under 5 | 20 | +| `isinstance(o, C)` | 1,635 | 271 | +| Call of a small non-leaf function | 1,389 | 763 | + +Global reads and the arithmetic around them compile to native code. The +rest leave compiled code for a general helper: a class attribute read +through an instance resolves through the class on every read, and +`isinstance` and calls to Python functions go through the generic call +path. Teaching tier 2 and the frame compiler those three shapes is the +next step for `deltablue`. -The costs come from the interpreter's structure rather than from any single -slow path: +The interpreter's structure adds to these costs where nothing compiles: - **Loop state.** The core loop keeps about 17 values live across its arms, which contain calls; x86-64 has six callee-saved registers, so most of the @@ -131,11 +156,19 @@ branches predict poorly. ## Validation -Every change on the branch passes the VM unit tests (420), the bundled -regression suite (248), the semantics comparison scripts, and the CPython -regression suites most exposed to it. The changes described above were -checked with, among others, `test_bytes`, `test_tuple`, `test_slice`, -`test_str`, `test_re`, `test_marshal`, `test_import`, `test_code`, -`test_compile`, `test_logging`, `test_gc`, `test_weakref`, `test_io`, -`test_ssl`, `test_asyncio`, `test_scope`, `test_global`, `test_module`, -`test_builtin`, `test_sys_settrace`, and `test_monitoring`. +The branch passes the workspace's unit, integration, and doc tests, the +bundled regression suite (272 tests), its free-threaded lane, the tools' +tests, and the bench gate against `main` with no fixture regressing. + +The curated CPython 3.14 gate (616 suites, among them `test_re`, +`test_pickle`, `test_deque`, `test_datetime`, `test_generators`, +`test_exceptions`, `test_threading`, `test_signal`, `test_asyncio`, and +`test_sys_settrace`) passes except where `main` fails the same way on the +development host: `test_capi`, `test_genexps`, `test_tempfile`, and +`test_utf8_mode` fail there too, and the `multiprocessing` spawn and +forkserver suites time out on both. + +New regression tests cover the frame compiler, frameless subscripts, +native `map` and `filter`, packed `datetime` values, deque lanes in +compiled code, unraisable `OSError` reports, and interrupts that arrive +while code only calls and catches. diff --git a/tests/regrtest/test_datetime_packed.py b/tests/regrtest/test_datetime_packed.py new file mode 100644 index 00000000..9b165f6b --- /dev/null +++ b/tests/regrtest/test_datetime_packed.py @@ -0,0 +1,283 @@ +"""Natively built datetime values (packed storage) behave like Python-built ones.""" +import copy +import gc +import pickle +from datetime import date, datetime, time, timedelta, timezone, tzinfo + + +class SubDate(date): + pass + + +class SubDateTime(datetime): + pass + + +class SubDelta(timedelta): + pass + + +utc = timezone.utc +east = timezone(timedelta(hours=5, minutes=30)) +odd = timezone(timedelta(hours=-3, seconds=7, microseconds=11)) + +# Slot reads from Python methods see the packed fields. +d = datetime(2024, 2, 29, 23, 59, 58, 999999, tzinfo=east) +# The pure-Python classes' private slots (CPython's C types have none). +PURE = hasattr(d, "_year") +if PURE: + assert (d._year, d._month, d._day, d._hour, d._minute) == (2024, 2, 29, 23, 59) + assert (d._second, d._microsecond, d._tzinfo, d._fold) == (58, 999999, east, 0) +assert repr(d) == ("datetime.datetime(2024, 2, 29, 23, 59, 58, 999999, " + "tzinfo=datetime.timezone(datetime.timedelta(seconds=19800)))") +assert str(d) == "2024-02-29 23:59:58.999999+05:30" +assert d.utcoffset() == timedelta(hours=5, minutes=30) +assert d.timetuple()[:6] == (2024, 2, 29, 23, 59, 58) +assert d.astimezone(utc) == d and d.astimezone(utc).hour == 18 +t = timedelta(days=-3, seconds=5, microseconds=7) +assert (t.days, t.seconds, t.microseconds) == (-3, 5, 7) +if PURE: + assert (t._days, t._seconds, t._microseconds) == (-3, 5, 7) +assert repr(t) == "datetime.timedelta(days=-3, seconds=5, microseconds=7)" +assert t.total_seconds() == -259194.999993 + +# Pickling and copying round-trip, through every protocol. +for value in [d, t, date(1, 1, 1), datetime(9999, 12, 31, 23, 59, 59, 999999), + datetime(2000, 1, 1, fold=1), datetime(2000, 1, 1, tzinfo=odd)]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + back = pickle.loads(pickle.dumps(value, proto)) + assert back == value and type(back) is type(value) + assert copy.copy(value) == value and copy.deepcopy(value) == value +assert pickle.loads(pickle.dumps(datetime(2000, 1, 1, fold=1), 4)).fold == 1 + +# A written slot keeps the other fields. +w = datetime(2021, 3, 4, 5, 6, 7) +if PURE: + w._hashcode = 12345 + assert w._hashcode == 12345 and w == datetime(2021, 3, 4, 5, 6, 7) +assert (w.year, w.month, w.day, w.hour, w.minute, w.second) == (2021, 3, 4, 5, 6, 7) + +# Native hashes equal the Python implementation's (which subclasses run). +for a, b in [(date(2024, 2, 29), SubDate(2024, 2, 29)), + (datetime(2024, 2, 29, 1, 2, 3, 456789), + SubDateTime(2024, 2, 29, 1, 2, 3, 456789)), + (datetime(2024, 2, 29, 1, 2, 3, tzinfo=odd), + SubDateTime(2024, 2, 29, 1, 2, 3, tzinfo=odd)), + (datetime(1, 1, 1, tzinfo=east), SubDateTime(1, 1, 1, tzinfo=east)), + (datetime(2000, 1, 1, fold=1), SubDateTime(2000, 1, 1, fold=1)), + (timedelta(1, 2, 3), SubDelta(1, 2, 3)), + (timedelta(-999999999), SubDelta(-999999999))]: + assert a == b and hash(a) == hash(b), (a, b) +assert hash(datetime(2020, 1, 1, 12, tzinfo=utc)) == hash( + datetime(2020, 1, 1, 17, 30, tzinfo=east)) + +# The collector sees the tzinfo (CPython's C type is not a GC type). +assert not PURE or east in gc.get_referents(d) + +# Arithmetic within and across days, both directions. +base = datetime(2024, 1, 15, 8, 30, tzinfo=utc) +step = timedelta(minutes=37, seconds=11) +cur = base +for i in range(200): + cur = cur + step +assert cur == datetime(2024, 1, 20, 12, 26, 40, tzinfo=utc) +assert cur - base == step * 200 and base - cur == -(step * 200) +assert (cur - step * 200) == base +assert datetime(2024, 1, 1) - timedelta(microseconds=1) == datetime(2023, 12, 31, 23, 59, 59, 999999) +assert datetime(2000, 3, 1, fold=1) + timedelta(0) == datetime(2000, 3, 1) +assert (datetime(2000, 3, 1, fold=1) + timedelta(0)).fold == 0 +assert datetime(2024, 1, 1, 12, tzinfo=utc) - datetime(2024, 1, 1, 12, tzinfo=east) == timedelta(hours=5, minutes=30) +for big in [timedelta(days=4000000), timedelta(days=-800000)]: + try: + datetime(2000, 1, 1) + big + except OverflowError: + pass + else: + raise AssertionError("out-of-range datetime accepted") +try: + timedelta(days=999999999) + timedelta(days=1) +except OverflowError: + pass +else: + raise AssertionError("out-of-range timedelta accepted") + +# Comparisons: fixed offsets, naive and aware, mixed kinds. +assert datetime(2024, 1, 1, 12, tzinfo=utc) == datetime(2024, 1, 1, 17, 30, tzinfo=east) +assert datetime(2024, 1, 1, 12, tzinfo=utc) < datetime(2024, 1, 1, 18, tzinfo=east) +assert datetime(2024, 1, 1) != datetime(2024, 1, 1, tzinfo=utc) +try: + datetime(2024, 1, 1) < datetime(2024, 1, 1, tzinfo=utc) +except TypeError: + pass +else: + raise AssertionError("naive/aware ordering accepted") +assert datetime(2000, 1, 1, fold=1) == datetime(2000, 1, 1) +assert timedelta(-1) < timedelta(0) < timedelta(microseconds=1) < timedelta(seconds=1) +assert date(2024, 1, 1) != datetime(2024, 1, 1) + +# Constructors: keywords, defaults, fallbacks and errors. +assert timedelta(weeks=1, days=-1, hours=1, minutes=2, seconds=3, + milliseconds=4, microseconds=5) == timedelta(6, 3723, 4005) +assert timedelta(hours=1.5) == timedelta(seconds=5400) +assert timedelta(True) == timedelta(1) +for bad in [dict(days=10**9), dict(days=2**70)]: + try: + timedelta(**bad) + except OverflowError: + pass + else: + raise AssertionError("out-of-range timedelta accepted") +assert date(year=2020, day=2, month=1) == date(2020, 1, 2) +assert datetime(2020, 5, 3, hour=4, tzinfo=utc) == datetime(2020, 5, 3, 4, 0, 0, 0, utc) +assert datetime(2020, 1, 1, fold=1).fold == 1 +for args, kwargs, exc in [((2020, 2, 30), {}, ValueError), ((2020, 1, 1, 24), {}, ValueError), + ((2020, 1, 1), {"fold": 2}, ValueError), + ((2020, 1, 1), {"year": 2020}, TypeError), + ((2020, 1, 1), {"bogus": 1}, TypeError), + ((2020, 1, 1, 0, 0, 0, 0, None, 0), {}, TypeError)]: + try: + datetime(*args, **kwargs) + except exc: + pass + else: + raise AssertionError((args, kwargs)) +assert type(SubDateTime(2020, 1, 1)) is SubDateTime +assert type(SubDelta(hours=1)) is SubDelta + +# replace(), positional and keyword. +x = date(2024, 2, 29) +assert x.replace(day=1) == date(2024, 2, 1) +assert x.replace(2023, 3) == date(2023, 3, 29) +assert x.replace(year=2023, month=3, day=31) == date(2023, 3, 31) +try: + x.replace(year=2023) +except ValueError: + pass +else: + raise AssertionError("2023-02-29 accepted") +assert type(SubDate(2024, 1, 1).replace(day=2)) is SubDate + +# fromisoformat: bound to the class it is read from. +m = datetime.fromisoformat +assert m.__self__ is datetime and m == datetime.fromisoformat +assert datetime.fromisoformat("2024-01-15T08:30:00+00:00") == base +assert datetime.fromisoformat("2024-01-15T08:30:00+00:00").tzinfo is utc +assert datetime.fromisoformat("2024-02-29T23:59:58.999999+05:30") == d +assert type(SubDateTime.fromisoformat("2024-01-15T08:30:00")) is SubDateTime +assert type(base.fromisoformat("2024-01-15")) is datetime + +# Formatting. +assert d.isoformat() == "2024-02-29T23:59:58.999999+05:30" +assert d.isoformat(" ", "minutes") == "2024-02-29 23:59+05:30" +assert d.isoformat("é", "milliseconds") == "2024-02-29é23:59:58.999+05:30" +assert datetime(2020, 1, 1, tzinfo=odd).isoformat() == "2020-01-01T00:00:00-02:59:52.999989" +assert datetime(5, 1, 2, 3, 4, 5).isoformat() == "0005-01-02T03:04:05" +assert d.strftime("%Y-%m-%d %H:%M:%S.%f %z|%:z %j %y %I%%") == \ + "2024-02-29 23:59:58.999999 +0530|+05:30 060 24 11%" +assert date(2024, 2, 29).isoformat() == "2024-02-29" +assert date(2024, 2, 29).strftime("%F é %T") == "2024-02-29 é 00:00:00" + +# A tzinfo held only by a native result is released (and finalized) with it. +deaths = [] + + +class Zone(tzinfo): + def utcoffset(self, dt): + return timedelta(hours=1) + + def dst(self, dt): + return timedelta(0) + + def __del__(self): + deaths.append(1) + + +v = datetime(2020, 1, 1, tzinfo=Zone()) + timedelta(hours=1) +assert v.utcoffset() == timedelta(hours=1) +del v +gc.collect() +assert deaths == [1], deaths + +# Values outlive their pooled neighbours. +keep = [datetime(2000, 1, 1) + timedelta(days=i) for i in range(300)] +for i in range(1000): + datetime(2000, 1, 1) + timedelta(seconds=i) +assert keep[-1] == datetime(2000, 10, 26) and len(set(keep)) == 300 + + +# Hot loops (compiled to native code) over datetime operators and fields, +# and operator sites that see datetimes and other operands in turn. +def walk(n): + cur = base = datetime(2024, 1, 1, tzinfo=utc) + step = timedelta(minutes=37, seconds=11) + day = timedelta(days=1) + hours = days = resets = 0 + for i in range(n): + cur = cur + step + if cur - base > day * 30: + base = base + day + resets += 1 + if cur.hour == 12: + hours += 1 + days += cur.weekday() + return cur, base, hours, days, resets + + +assert walk(20000) == (datetime(2025, 5, 31, 10, 26, 40, tzinfo=utc), + datetime(2025, 5, 2, tzinfo=utc), 835, 59850, 487) + + +def add(a, b, n): + out = None + for i in range(n): + out = a + b + return out + + +def less(a, b, n): + out = None + for i in range(n): + out = a < b + return out + + +def hour(obj, n): + out = None + for i in range(n): + out = obj.hour + return out + + +class Clock: + def __init__(self): + self.hour = 7 + + +for _ in range(2): + assert add(1, 2, 3000) == 3 + assert add(base, step, 3000) == base + step + assert add(2**62, 2**62, 3000) == 2**63 + assert add("a", "b", 3000) == "ab" + assert add(1.5, 2, 3000) == 3.5 + assert add(step, step, 3000) == timedelta(minutes=74, seconds=22) + assert less(1, 2, 3000) is True + assert less(step, timedelta(0), 3000) is False + assert less(2.5, 1.0, 3000) is False + assert less(base, base + step, 3000) is True + try: + less(datetime(2024, 1, 1), base, 3000) + except TypeError: + pass + else: + raise AssertionError("naive/aware ordering accepted") + try: + add(datetime(9999, 12, 31), timedelta(days=1), 3000) + except OverflowError: + pass + else: + raise AssertionError("out-of-range datetime accepted") + assert hour(base, 3000) == 8 + assert hour(Clock(), 3000) == 7 + assert hour(datetime(2020, 1, 1, 23), 3000) == 23 + +print("datetime packed values: ok") diff --git a/tests/regrtest/test_finalizer_constructor_fallback.py b/tests/regrtest/test_finalizer_constructor_fallback.py new file mode 100644 index 00000000..d8b5cc50 --- /dev/null +++ b/tests/regrtest/test_finalizer_constructor_fallback.py @@ -0,0 +1,38 @@ +# A constructor fast path that falls back to the generic call must not +# finalize the instance it allocated and then abandoned: Python never saw +# it, so its `__del__` would run on an object `__init__` never touched. + +log = [] +missing = [] + + +class Finalized: + def __init__(self, value): + self.value = value + + def __del__(self): + try: + log.append(self.value) + except AttributeError: + missing.append(len(log)) + + +class Holder: + pass + + +def rebind(n): + holder = Holder() + holder.item = 0 + for i in range(n): + holder.item = Finalized(i) + + +for unused in range(3): + log.clear() + rebind(3000) + assert missing == [], f"{len(missing)} unborn instances finalized" + # The last instance is still bound until `holder` dies with the frame. + assert log == list(range(3000)), (len(log), log[:5]) + +print("constructor fallback finalization: ok") diff --git a/tests/regrtest/test_frame_jit.py b/tests/regrtest/test_frame_jit.py new file mode 100644 index 00000000..e434f22d --- /dev/null +++ b/tests/regrtest/test_frame_jit.py @@ -0,0 +1,191 @@ +"""The core loop's native code keeps the interpreter's semantics. + +Each function runs long enough to be compiled, then meets the shapes the +native code hands back to the core loop: overflows, NaNs, unbound +locals, heap values where it expects scalars, identity of every kind of +value, finalizers released by stores, and attribute caches that go +stale. +""" + +import gc +import unittest + +N = 2000 + + +def ints(n): + t = 0 + for i in range(n): + t += i * 3 - (i >> 2) + (i & 7) ^ (i | 1) + if i % 5 == 0: + t -= i // 3 + t = t % 1000003 + return t + + +def floats(n): + x = 0.5 + for i in range(n): + x = x * 1.0001 + i / 7.0 - (x / 3.0) + if x > 1e6: + x = x - 1e6 + return round(x, 6) + + +def grows(n): + t = 1 + for i in range(n): + t = t * 3 + i + return t + + +class Box: + def __init__(self, v): + self.v = v + self.next = None + + +class FrameJitTests(unittest.TestCase): + def test_arithmetic(self): + self.assertEqual(ints(N), 363158) + self.assertEqual(floats(N), floats(N)) + + def test_overflow_promotes(self): + self.assertEqual(grows(N), grows(N)) + self.assertGreater(grows(200), 2**64) + + def test_nan_and_zero_division(self): + def f(n, d): + t = 0.0 + for i in range(n): + t = t + float("inf") - float("inf") if i == n - 1 else t + i / d + return t + + r = f(N, 2.0) + self.assertNotEqual(r, r) + with self.assertRaises(ZeroDivisionError): + f(N, 0.0) + + def test_unbound_local(self): + def f(n, flag): + for i in range(n): + x = i + if flag and i == n - 1: + del x + return x + + self.assertEqual(f(N, False), N - 1) + with self.assertRaises(UnboundLocalError): + f(N, True) + + def test_identity(self): + a = object() + s = "x" * 3 + big = 2**70 + + def f(n, x, y): + c = 0 + for _ in range(n): + if x is y: + c += 1 + if x is not None: + c += 2 + return c + + self.assertEqual(f(N, a, a), 3 * N) + self.assertEqual(f(N, a, object()), 2 * N) + self.assertEqual(f(N, s, s), 3 * N) + self.assertEqual(f(N, 5, 5), 3 * N) + self.assertEqual(f(N, 5, 5.0), 2 * N) + self.assertEqual(f(N, 1.5, 1.5), 3 * N) + self.assertEqual(f(N, 0.0, -0.0), 2 * N) + self.assertEqual(f(N, True, True), 3 * N) + self.assertEqual(f(N, True, 1), 2 * N) + self.assertEqual(f(N, None, None), N) + self.assertEqual(f(N, big, big), 3 * N) + + def test_finalizer_on_store(self): + log = [] + + class D: + def __del__(self): + log.append(1) + + def f(n): + x = None + for i in range(n): + x = D() + x = i + return x + + self.assertEqual(f(N), N - 1) + gc.collect() + self.assertEqual(len(log), N) + + def test_fields(self): + def total(b, n): + t = 0 + for _ in range(n): + t += b.v + return t + + b = Box(3) + self.assertEqual(total(b, N), 3 * N) + b.v = 2.5 + self.assertEqual(total(b, 4), 10.0) + b.v = "s" + with self.assertRaises(TypeError): + total(b, 1) + + def test_field_chains(self): + def walk(head, n): + t = 0 + for _ in range(n): + node = head + while node is not None: + t += node.v + node = node.next + return t + + head = Box(1) + head.next = Box(2) + head.next.next = Box(3) + self.assertEqual(walk(head, N), 6 * N) + Box.v = property(lambda self: 10) + try: + self.assertEqual(walk(head, 2), 60) + finally: + del Box.v + + def test_copy_and_swap(self): + def f(n): + a, b = 1, 2 + t = 0 + for i in range(n): + a, b = b, a + i + a, b = b % 997, a % 991 + t += a + return t, a, b + + self.assertEqual(f(N), f(N)) + + def test_truth(self): + def f(n, x): + c = 0 + for _ in range(n): + if x: + c += 1 + if not x: + c += 2 + return c + + self.assertEqual(f(N, 0), 2 * N) + self.assertEqual(f(N, 3), N) + self.assertEqual(f(N, 0.0), 2 * N) + self.assertEqual(f(N, None), 2 * N) + self.assertEqual(f(N, [1]), N) + self.assertEqual(f(N, ""), 2 * N) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_function_creation.py b/tests/regrtest/test_function_creation.py new file mode 100644 index 00000000..31933ed2 --- /dev/null +++ b/tests/regrtest/test_function_creation.py @@ -0,0 +1,172 @@ +"""Functions made in a hot loop keep their attributes and call shapes. + +A `def` or `lambda` that runs many times builds each function from a +template of its code's name slots, attaches its defaults and closure as it +is built, and is called through the call site's shape for its code. Every +function made here must still behave as CPython's do. +""" + +import types +import unittest + + +def make_adder(n, scale=2): + def add(x, y=10): + return (x + y + n) * scale + + return add + + +def make_lambda(k): + return lambda x, z=k: x * z + + +class FunctionCreationTests(unittest.TestCase): + def test_closures_and_defaults(self): + total = 0 + for i in range(5000): + total += make_adder(i)(1) + self.assertEqual(total, sum((1 + 10 + i) * 2 for i in range(5000))) + f = make_adder(3, scale=5) + self.assertEqual(f(1, 2), 30) + self.assertEqual(f.__defaults__, (10,)) + self.assertEqual(f.__closure__[0].cell_contents, 3) + self.assertEqual([make_lambda(k)(3) for k in range(5)], [0, 3, 6, 9, 12]) + g = make_lambda(4) + self.assertEqual(g(2, 5), 10) + self.assertEqual(g.__defaults__, (4,)) + + def test_names_and_identity(self): + fs = [make_adder(i) for i in range(100)] + self.assertEqual({f.__name__ for f in fs}, {"add"}) + self.assertEqual(fs[0].__qualname__, "make_adder..add") + self.assertEqual(fs[0].__module__, __name__) + self.assertIs(fs[0].__name__, fs[1].__name__) + fs[0].__name__ = "renamed" + fs[0].__qualname__ = "q" + self.assertEqual(fs[0].__name__, "renamed") + self.assertEqual(fs[1].__name__, "add") + self.assertEqual(fs[1].__qualname__, "make_adder..add") + self.assertEqual(fs[2].__dict__, {}) + fs[2].tag = 1 + self.assertEqual(fs[2].__dict__, {"tag": 1}) + self.assertEqual(fs[3].__dict__, {}) + + def test_module_name_follows_globals(self): + src = "def f():\n return __name__\n" + code = compile(src, "", "exec") + first, second = {"__name__": "first"}, {"__name__": "second"} + for _ in range(100): + exec(code, first) + exec(code, second) + self.assertEqual(first["f"].__module__, "first") + self.assertEqual(second["f"].__module__, "second") + self.assertEqual(second["f"](), "second") + anon = {} + exec(code, anon) + self.assertIsNone(anon["f"].__module__) + clone = types.FunctionType(first["f"].__code__, {"__name__": "third"}) + self.assertEqual(clone.__module__, "third") + + def test_keyword_defaults_and_annotations(self): + def build(i): + def h(a: int, *, b=i) -> int: + return a + b + + return h + + hs = [build(i) for i in range(2000)] + self.assertEqual(hs[7](1), 8) + self.assertEqual(hs[7].__kwdefaults__, {"b": 7}) + self.assertEqual(hs[7].__annotations__, {"a": int, "return": int}) + + def test_name_slots_read_and_written_later(self): + import functools + import pickle + + fs = [make_adder(i) for i in range(50)] + + @functools.wraps(fs[0]) + def wrapper(*args): + return fs[0](*args) + + self.assertEqual(wrapper.__name__, "add") + self.assertEqual(wrapper.__qualname__, "make_adder..add") + self.assertEqual(wrapper.__module__, __name__) + self.assertIs(wrapper.__wrapped__, fs[0]) + fs[1].__module__ = "elsewhere" + self.assertEqual(fs[1].__module__, "elsewhere") + self.assertEqual(fs[2].__module__, __name__) + fs[3].__doc__ = "doc" + self.assertEqual((fs[3].__doc__, fs[4].__doc__), ("doc", None)) + self.assertIs(pickle.loads(pickle.dumps(make_adder)), make_adder) + g = [make_lambda(k) for k in range(3)] + self.assertEqual({h.__name__ for h in g}, {""}) + self.assertEqual(g[0].__qualname__, "make_lambda..") + + def test_free_variable_reads(self): + def counter(): + n = 0 + + def get(): + return n + + def bump(): + nonlocal n + n += 1 + + return get, bump + + get, bump = counter() + for _ in range(5000): + self.assertEqual(get(), 0) + bump() + self.assertEqual(get(), 1) + + def late(): + def read(): + return value + + out = [read] + try: + read() + except NameError as e: + out.append(type(e)) + value = 7 + out.append(read()) + return out + + for _ in range(3000): + r = late() + self.assertEqual(r[1:], [NameError, 7]) + + def scaled(k): + return lambda x: x * k + len(box) + + box = [1, 2] + fs = [scaled(i) for i in range(4000)] + self.assertEqual(sum(f(1) for f in fs), sum(range(4000)) + 2 * 4000) + box.append(3) + self.assertEqual(fs[5](2), 13) + + def test_fresh_functions_share_call_shapes(self): + def caller(n): + t = 0 + for i in range(n): + t += (lambda x, y=2: x * y)(i) + return t + + self.assertEqual(caller(5000), 2 * sum(range(5000))) + + def defaults_change(n): + out = [] + for i in range(n): + f = lambda x, y=i: x + y + out.append(f(1)) + return out + + self.assertEqual(defaults_change(300)[-3:], [298, 299, 300]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_inline_calls.py b/tests/regrtest/test_inline_calls.py new file mode 100644 index 00000000..0fc1254f --- /dev/null +++ b/tests/regrtest/test_inline_calls.py @@ -0,0 +1,338 @@ +"""Python calls the core loop switches to in place. + +A call of a function that its call site has seen before runs in a pooled +activation (with its own cells when it has cell variables, and its +`*args` tuple built by the site), and its return hands the result +straight back; `with` loads its `__enter__` and `__exit__` from the site, +and a method call site remembers the leaf methods of several classes. +Each loop here runs long enough for the sites to settle; then the callee, +its arguments or its frame change, and every call must still behave as +the interpreter's general call does. +""" + +import sys +import unittest + +WARM = 3000 + + +def add(a, b): + return [a + b][0] + + +def ident(x): + return [x][0] + + +class Box: + def __init__(self, v): + self.v = v + + def get(self): + return [self.v][0] + + def put(self, v): + self.v = [v][0] + return self + + +def loop_add(n): + t = 0 + for i in range(n): + t += add(i, 1) + return t + + +def loop_methods(n): + b = Box(0) + t = 0 + for i in range(n): + t += b.put(i).get() + return t + + +def depth(n): + if n == 0: + return [sys._getframe(0).f_code.co_name, sys._getframe(1).f_code.co_name] + return depth(n - 1) + + +class InlineCallTests(unittest.TestCase): + def test_results(self): + self.assertEqual(loop_add(WARM), sum(range(WARM)) + WARM) + self.assertEqual(loop_methods(WARM), sum(range(WARM))) + + def test_rebinding_the_callee(self): + global add + saved = add + try: + self.assertEqual(loop_add(WARM), sum(range(WARM)) + WARM) + add = lambda a, b: a - b + self.assertEqual(loop_add(10), sum(range(10)) - 10) + + def add(a, b=100): + return a + b + + self.assertEqual(loop_add(10), sum(range(10)) + 10) + finally: + add = saved + + def test_code_replaced(self): + def f(x): + return [x][0] + + def g(x): + return [x * 2][0] + + def run(n): + t = 0 + for i in range(n): + t += f(i) + return t + + self.assertEqual(run(WARM), sum(range(WARM))) + f.__code__ = g.__code__ + self.assertEqual(run(10), 2 * sum(range(10))) + + def test_frames_and_errors(self): + for _ in range(WARM): + ident(1) + self.assertEqual(depth(5), ["depth", "depth"]) + + def boom(x): + return [x][1] + + def caller(n): + for i in range(n): + boom(i) + + try: + caller(5) + except IndexError as e: + tb = e.__traceback__ + else: + self.fail("no IndexError") + names = [] + while tb: + names.append(tb.tb_frame.f_code.co_name) + tb = tb.tb_next + self.assertEqual(names[-2:], ["caller", "boom"]) + + def test_recursion_limit(self): + def down(n): + return [down(n + 1)][0] + + with self.assertRaises(RecursionError): + down(0) + self.assertEqual(loop_add(10), sum(range(10)) + 10) + + def test_cell_variables(self): + def adder(n): + def add(x): + return x + n + return add + + def counter(start): + count = start + + def bump(): + nonlocal count + count += 1 + return count + bump() + return [bump, count] + + def shadow(a, b): + def get(): + return a, b + a = a * 2 + return get + + def peek(n): + def inner(): + return n + return sorted(sys._getframe().f_locals) + + def fails(n): + def inner(): + return n + return [inner][n] + + made = [adder(i) for i in range(WARM)] + self.assertEqual([f(1) for f in made[:3]], [1, 2, 3]) + self.assertEqual(sum(f(0) for f in made), sum(range(WARM))) + for i in range(WARM): + bump, seen = counter(i) + self.assertEqual(seen, i + 1) + self.assertEqual(bump(), i + 2) + self.assertEqual(shadow(i, 1)(), (2 * i, 1)) + self.assertEqual(peek(3), ["inner", "n"]) + for i in range(WARM): + self.assertEqual(fails(0)(), 0) + with self.assertRaises(IndexError): + fails(1) + + def test_cell_variable_finalizers(self): + seen = [] + + class Note: + def __del__(self): + seen.append(1) + + def hold(i): + n = Note() + + def keep(): + return n + return i + + def escape(i): + n = Note() + + def keep(): + return n + return keep + + for i in range(WARM): + hold(i) + self.assertEqual(len(seen), WARM) + kept = [escape(i) for i in range(100)] + self.assertEqual(len(seen), WARM) + del kept + self.assertEqual(len(seen), WARM + 100) + + def test_with_statements(self): + log = [] + + class CM: + def __enter__(self): + return self + + def __exit__(self, *exc): + log.append(exc[0]) + return False + + def use(cm): + with cm as c: + return c + + cm = CM() + for _ in range(WARM): + self.assertIs(use(cm), cm) + self.assertEqual(log, [None] * WARM) + log.clear() + # Special lookup reads the class, never the instance. + cm.__enter__ = lambda: "instance" + self.assertIs(use(cm), cm) + # A rebound class attribute takes effect at once. + CM.__enter__ = lambda self: "rebound" + self.assertEqual(use(cm), "rebound") + CM.__enter__ = staticmethod(lambda: "static") + self.assertEqual(use(cm), "static") + + class Suppress(CM): + def __exit__(self, kind, value, tb): + log.append(kind) + return True + + def boom(cm): + with cm: + raise KeyError(1) + return "suppressed" + + for _ in range(WARM): + self.assertEqual(boom(Suppress()), "suppressed") + self.assertEqual(log[-1], KeyError) + with self.assertRaises(KeyError): + boom(CM()) + + def test_star_args(self): + def va(*args): + return args + + def some(a, b=2, *rest): + return a, b, rest + + def run(n): + t = 0 + for i in range(n): + t += len(va(i, i, i)) + len(va()) + some(i)[1] + len(some(i, 1, 2, 3)[2]) + return t + + self.assertEqual(run(WARM), WARM * 7) + self.assertEqual(va(1, 2), (1, 2)) + self.assertEqual(some(1, 5, 6), (1, 5, (6,))) + self.assertIs(type(va()), tuple) + + def test_polymorphic_leaf_sites(self): + class Base: + FORWARD = 1 + + def __init__(self, k): + self.k = k + self.direction = 1 + + def output(self): + return self.k if self.direction == Base.FORWARD else -self.k + + class A(Base): + pass + + class B(Base): + pass + + class C: + def __init__(self, k): + self.k = k + + def output(self): + return self.k * 10 + + class D(C): + pass + + objs = [A(1), B(2), C(3), D(4)] + + def total(n): + t = 0 + for i in range(n): + for o in objs: + t += o.output() + return t + + self.assertEqual(total(WARM), WARM * (1 + 2 + 30 + 40)) + # A class's method changes: only its instances see the new one. + B.output = lambda self: 100 + self.assertEqual(total(10), 10 * (1 + 100 + 30 + 40)) + # A function's code changes under the same class version. + def other(self): + return 7 + C.output.__code__ = other.__code__ + self.assertEqual(total(10), 10 * (1 + 100 + 7 + 7)) + # An instance attribute shadows its class's method. + objs[0].output = lambda: 1000 + self.assertEqual(total(10), 10 * (1000 + 100 + 7 + 7)) + del objs[0].output + objs[1].direction = 0 + self.assertEqual(total(10), 10 * (1 + 100 + 7 + 7)) + del B.output + self.assertEqual(total(10), 10 * (1 - 2 + 7 + 7)) + + def test_finalizers_run_on_return(self): + seen = [] + + class Note: + def __del__(self): + seen.append(1) + + def make(i): + n = Note() + return [i][0] + + for i in range(WARM): + make(i) + self.assertEqual(len(seen), WARM) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_interrupt_while_catching.py b/tests/regrtest/test_interrupt_while_catching.py new file mode 100644 index 00000000..1ee4ede2 --- /dev/null +++ b/tests/regrtest/test_interrupt_while_catching.py @@ -0,0 +1,50 @@ +"""An interrupt reaches code that only calls and catches exceptions. + +Recursion retried from its own `RecursionError` handler never returns +normally and has no loop, so the interpreter must still give up the GIL +and look for signals when it catches an exception (gh-102056's shape, +from `test_threading`). +""" + +import subprocess +import sys +import unittest + +SCRIPT = r""" +import time +import threading +import _thread + +def f(): + try: + f() + except RecursionError: + f() + +def h(): + time.sleep(0.5) + _thread.interrupt_main() + +t = threading.Thread(target=h) +t.start() +try: + f() +except KeyboardInterrupt: + print("interrupted") +t.join() +""" + + +class InterruptWhileCatching(unittest.TestCase): + def test_interrupt_main_lands(self): + proc = subprocess.run( + [sys.executable, "-c", SCRIPT], + capture_output=True, + text=True, + timeout=45, + ) + self.assertIn("interrupted", proc.stdout, proc.stderr[-2000:]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_ctor_guards.py b/tests/regrtest/test_jit_ctor_guards.py new file mode 100644 index 00000000..f60c2400 --- /dev/null +++ b/tests/regrtest/test_jit_ctor_guards.py @@ -0,0 +1,133 @@ +"""Compiled code that constructs a class keeps checking how it constructs. + +A loop builds instances of a global class whose construction tier 2 burns +in, and an interpreted hook changes the class or its `__init__` partway +through: the constructions after the change must see it. +""" + +import unittest + + +class Plain: + def __init__(self, v): + self.v = v + + +class Rebound: + def __init__(self, v): + self.v = v + + +class Swapped: + def __init__(self, v): + self.v = v + + +class GainsNew: + def __init__(self, v): + self.v = v + + +def build_plain(n, hook): + t = 0 + for i in range(n): + t += Plain(i).v + hook(i) + return t + + +def build_rebound(n, hook): + t = 0 + for i in range(n): + t += Rebound(i).v + hook(i) + return t + + +def build_swapped(n, hook): + t = 0 + for i in range(n): + t += Swapped(i).v + hook(i) + return t + + +def build_gains_new(n, hook): + t = 0 + for i in range(n): + t += GainsNew(i).v + hook(i) + return t + + +def noop(i): + pass + + +def triangle(n): + # The sum of 0, 1, ..., n - 1. + return n * (n - 1) // 2 + + +class CtorGuardTests(unittest.TestCase): + N = 3000 + + def test_plain(self): + self.assertEqual(build_plain(self.N, noop), triangle(self.N)) + self.assertEqual(build_plain(self.N, noop), triangle(self.N)) + + def test_init_rebound_midway(self): + def init(self, v): + self.v = -1 + + half = self.N // 2 + + def hook(i): + if i == half: + Rebound.__init__ = init + + self.assertEqual(build_rebound(self.N, noop), triangle(self.N)) + expected = triangle(half + 1) - (self.N - half - 1) + saved = Rebound.__init__ + try: + self.assertEqual(build_rebound(self.N, hook), expected) + finally: + Rebound.__init__ = saved + + def test_init_code_swapped_midway(self): + def other(self, v): + self.v = 2 * v + + half = self.N // 2 + + def hook(i): + if i == half: + Swapped.__init__.__code__ = other.__code__ + + self.assertEqual(build_swapped(self.N, noop), triangle(self.N)) + expected = triangle(half + 1) + 2 * (triangle(self.N) - triangle(half + 1)) + saved = Swapped.__init__.__code__ + try: + self.assertEqual(build_swapped(self.N, hook), expected) + finally: + Swapped.__init__.__code__ = saved + + def test_new_added_midway(self): + half = self.N // 2 + + def new(cls, v): + o = object.__new__(cls) + o.extra = True + return o + + def hook(i): + if i == half: + GainsNew.__new__ = new + + self.assertEqual(build_gains_new(self.N, noop), triangle(self.N)) + self.assertEqual(build_gains_new(self.N, hook), triangle(self.N)) + self.assertTrue(GainsNew(1).extra) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_deque_lanes.py b/tests/regrtest/test_jit_deque_lanes.py new file mode 100644 index 00000000..293c3a94 --- /dev/null +++ b/tests/regrtest/test_jit_deque_lanes.py @@ -0,0 +1,257 @@ +"""Compiled loops over deques keep exact results, errors, and fallbacks. + +Hot loops call `deque` methods, subscripts, `len`, truth tests, and +iteration natively from compiled code. Each check below runs long enough +to compile and then changes the shape (a subclass override, a non-integer +element, a mutation during iteration) so the compiled code must fall back +without changing what Python observes. +""" + +from collections import deque +import gc +import pickle +import weakref + +N = 3000 + + +def fifo(n): + total = 0 + q = deque() + for i in range(n): + q.append(i) + q.append(i + 1) + total += q.popleft() + if i & 7 == 0: + total += q[0] + q[-1] + while q: + total += q.popleft() + return total + + +def fifo_list(n): + total = 0 + q = [] + for i in range(n): + q.append(i) + q.append(i + 1) + total += q.pop(0) + if i & 7 == 0: + total += q[0] + q[-1] + while q: + total += q.pop(0) + return total + + +assert fifo(N) == fifo_list(N) + + +def ring(n): + total = 0 + r = deque(maxlen=64) + for i in range(n): + r.append(i) + if i & 15 == 15: + r.rotate(3) + total += r[5] + return total + sum(r), list(r) + + +def ring_list(n): + total = 0 + r = [] + for i in range(n): + r.append(i) + if len(r) > 64: + del r[0] + if i & 15 == 15: + r[:] = r[-3:] + r[:-3] + total += r[5] + return total + sum(r), r + + +assert ring(N) == ring_list(N) + + +def stack(n): + total = 0 + s = deque() + for i in range(n): + s.appendleft(i) + if i & 3 == 3: + total += s.pop() + s.popleft() + total += len(s) + for x in s: + total += x + return total + + +def stack_list(n): + total = 0 + s = [] + for i in range(n): + s.insert(0, i) + if i & 3 == 3: + total += s.pop() + s.pop(0) + total += len(s) + for x in s: + total += x + return total + + +assert stack(N) == stack_list(N) + + +# Elements that are not integers: the integer speculation misses and the +# exact generic operation runs instead. +def concat(q, n): + out = None + for _ in range(n): + out = q[0] + q[-1] + return out + + +assert concat(deque([1, 2]), N) == 3 +assert concat(deque(["a", "b"]), N) == "ab" +assert concat(deque([1.5, 2]), 3) == 3.5 + + +def pops(q, n): + total = 0 + for i in range(n): + q.append(i) + total += q.pop() + return total + + +assert pops(deque(), N) == sum(range(N)) +# A non-integer element below the integers is never popped. +q = deque(["x"]) +assert pops(q, N) == sum(range(N)) +assert list(q) == ["x"] + + +# Errors raised by a native method inside a compiled loop. +def drain(q, n): + caught = 0 + for _ in range(n): + try: + q.popleft() + except IndexError as e: + assert str(e) == "pop from an empty deque", e + caught += 1 + return caught + + +assert drain(deque(range(10)), N) == N - 10 + + +def peek(q, i, n): + total = 0 + for _ in range(n): + total += q[i] + return total + + +assert peek(deque([5, 6, 7]), -1, N) == 7 * N +try: + peek(deque([5]), 3, 2) +except IndexError as e: + assert str(e) == "deque index out of range", e +else: + raise AssertionError("IndexError expected") + + +# Subclasses that override the operations keep their overrides. +class Doubling(deque): + def __getitem__(self, i): + return 2 * deque.__getitem__(self, i) + + def __len__(self): + return 100 + + def __iter__(self): + yield from (2 * x for x in deque.__iter__(self)) + + +class Scaling(deque): + def append(self, x): + deque.append(self, x * 10) + + +d = Doubling([1, 2, 3]) +assert peek(d, 0, N) == 2 * N + + +def lengths(q, n): + total = 0 + for _ in range(n): + total += len(q) + return total + + +assert lengths(deque([1, 2]), N) == 2 * N +assert lengths(d, 3) == 300 + + +def appends(q, n): + for i in range(n): + q.append(i) + return q + + +assert list(appends(deque(), 5)) == [0, 1, 2, 3, 4] +assert list(appends(Scaling(), 3)) == [0, 10, 20] + + +def iterate(q): + total = 0 + for x in q: + total += x + return total + + +assert iterate(deque(range(N))) == sum(range(N)) +assert iterate(Doubling([1, 2])) == 6 + +# Mutation during compiled iteration raises exactly. +def mutate(q): + seen = 0 + for x in q: + seen += 1 + if seen == 5: + q.append(x) + return seen + + +try: + mutate(deque(range(N))) +except RuntimeError as e: + assert str(e) == "deque mutated during iteration", e +else: + raise AssertionError("RuntimeError expected") + + +# Native iterators keep the iterator classes' surface. +q = deque([1, 2, 3]) +it = iter(q) +assert type(it).__name__ == "_deque_iterator", type(it) +assert next(it) == 1 +assert list(pickle.loads(pickle.dumps(it))) == [2, 3] +assert list(it) == [2, 3] +rit = reversed(q) +assert type(rit).__name__ == "_deque_reverse_iterator", type(rit) +assert list(rit) == [3, 2, 1] +assert it.__length_hint__() == 0 +assert iter(q).__length_hint__() == 3 + + +# A deque holding its own iterator is collectable. +q = deque() +q.append(iter(q)) +ref = weakref.ref(q) +del q +gc.collect() +assert ref() is None + +print("Compiled deque lanes: ok") diff --git a/tests/regrtest/test_jit_global_lists.py b/tests/regrtest/test_jit_global_lists.py new file mode 100644 index 00000000..bae7c48e --- /dev/null +++ b/tests/regrtest/test_jit_global_lists.py @@ -0,0 +1,127 @@ +"""Compiled code reads a module-global list on its list lane. + +A global holding a uniform list (all ints, all floats, all instances) +compiles like a list local: loops, comprehensions and reads index it +natively. Each loop here runs long enough to compile; then the list's +contents or the global itself change, and every result must still match +what the interpreter computes. +""" + +import unittest + +WARM = 3000 + +INTS = list(range(50)) +FLOATS = [i * 0.5 for i in range(20)] + + +class Box: + def __init__(self, v): + self.v = v + + +BOXES = [Box(i) for i in range(10)] + + +def doubled(): + return [x * 2 for x in INTS] + + +def total(): + t = 0 + for x in INTS: + t += x + return t + + +def ftotal(): + t = 0.0 + for x in FLOATS: + t += x + return t + + +def btotal(): + t = 0 + for b in BOXES: + t += b.v + return t + + +def nth(i): + return INTS[i] + len(INTS) + + +def run(fn, n, *args): + r = None + for _ in range(n): + r = fn(*args) + return r + + +class GlobalListTests(unittest.TestCase): + def test_comprehension_then_mixed_elements(self): + global INTS + saved = INTS + try: + self.assertEqual(run(doubled, WARM), [x * 2 for x in range(50)]) + INTS.append("a") + self.assertEqual(doubled()[-1], "aa") + INTS = [1.5, 2] + self.assertEqual(doubled(), [3.0, 4]) + INTS = [] + self.assertEqual(doubled(), []) + finally: + INTS = saved + if INTS and INTS[-1] == "a": + INTS.pop() + + def test_loop_sees_mutations(self): + global INTS + saved = list(INTS) + try: + self.assertEqual(run(total, WARM), sum(range(50))) + INTS[0] = 1000 + self.assertEqual(total(), sum(range(50)) + 1000) + INTS[1] = 2**70 + self.assertEqual(total(), sum(range(2, 50)) + 1000 + 2**70) + del INTS[10:] + self.assertEqual(total(), 1000 + 2**70 + sum(range(2, 10))) + finally: + INTS[:] = saved + + def test_float_and_instance_lists(self): + global FLOATS, BOXES + self.assertEqual(run(ftotal, WARM), sum(i * 0.5 for i in range(20))) + FLOATS[3] = 7 + self.assertEqual(ftotal(), sum(i * 0.5 for i in range(20)) - 1.5 + 7) + self.assertEqual(run(btotal, WARM), sum(range(10))) + saved = BOXES + BOXES = BOXES + [None] + try: + with self.assertRaises(AttributeError): + btotal() + finally: + BOXES = saved + self.assertEqual(btotal(), sum(range(10))) + + def test_index_and_len(self): + global INTS + self.assertEqual(run(nth, WARM, 3), 3 + 50) + INTS.append(99) + try: + self.assertEqual(nth(-1), 99 + 51) + with self.assertRaises(IndexError): + nth(100) + finally: + INTS.pop() + saved = INTS + INTS = list(range(5)) + try: + self.assertEqual(nth(4), 4 + 5) + finally: + INTS = saved + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_inline_fields.py b/tests/regrtest/test_jit_inline_fields.py new file mode 100644 index 00000000..903293b1 --- /dev/null +++ b/tests/regrtest/test_jit_inline_fields.py @@ -0,0 +1,129 @@ +"""Compiled code reads and writes instance fields in line. + +Each loop runs long enough for the tier-2 compiler to take it, then the +receiver or its class changes shape under the compiled code: every change +must reach the generic path with the interpreter's result. +""" + +import unittest + + +class Point: + def __init__(self, x, y): + self.x = x + self.y = y + + +class Other: + def __init__(self): + self.y = 100 + self.x = 200 + + +def read_sum(p, n): + t = 0 + for _ in range(n): + t += p.x + return t + + +def bump(p, n): + for _ in range(n): + p.y = p.y + 1.5 + return p.y + + +class InlineFieldTests(unittest.TestCase): + N = 3000 + + def test_plain_reads_and_writes(self): + p = Point(2, 0.5) + self.assertEqual(read_sum(p, self.N), 2 * self.N) + self.assertEqual(bump(p, self.N), 0.5 + 1.5 * self.N) + + def test_class_gains_a_property(self): + class C: + def __init__(self): + self.x = 1 + + def loop(o, n): + t = 0 + for _ in range(n): + t += o.x + return t + + c = C() + self.assertEqual(loop(c, self.N), self.N) + C.x = property(lambda self: 7) + self.assertEqual(loop(c, self.N), 7 * self.N) + + def test_class_swap(self): + class A: + def __init__(self): + self.x = 1 + self.y = 2 + + class B: + def __init__(self): + self.y = 3 + self.x = 4 + + def loop(o, n): + t = 0 + for _ in range(n): + t += o.x + return t + + a = A() + self.assertEqual(loop(a, self.N), self.N) + b = B() + self.assertEqual(loop(b, self.N), 4 * self.N) + a.__class__ = B + self.assertEqual(loop(a, self.N), self.N) + + def test_materialized_dict(self): + p = Point(3, 1.0) + self.assertEqual(read_sum(p, self.N), 3 * self.N) + d = vars(p) + d["x"] = 5 + self.assertEqual(read_sum(p, self.N), 5 * self.N) + del d["y"] + d["y"] = 1.0 + self.assertEqual(bump(p, 10), 16.0) + + def test_deleted_and_readded_field(self): + p = Point(1, 0.0) + self.assertEqual(read_sum(p, self.N), self.N) + del p.x + with self.assertRaises(AttributeError): + read_sum(p, 1) + p.x = 9 + self.assertEqual(read_sum(p, self.N), 9 * self.N) + + def test_non_scalar_store_and_read(self): + p = Point(1, 0.0) + self.assertEqual(bump(p, self.N), 1.5 * self.N) + p.y = [1] + with self.assertRaises(TypeError): + bump(p, 1) + p.y = 2.0 + self.assertEqual(bump(p, 2), 5.0) + + def test_field_order_differs_between_instances(self): + p = Point(6, 0.0) + o = Other() + self.assertEqual(read_sum(p, self.N), 6 * self.N) + self.assertEqual(read_sum(o, self.N), 200 * self.N) + self.assertEqual(read_sum(p, self.N), 6 * self.N) + + def test_type_change_of_field(self): + p = Point(1, 0.0) + self.assertEqual(read_sum(p, self.N), self.N) + p.x = 2.5 + self.assertEqual(read_sum(p, 4), 10.0) + p.x = True + self.assertEqual(read_sum(p, 4), 4) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_inline_lists.py b/tests/regrtest/test_jit_inline_lists.py new file mode 100644 index 00000000..c46f72e9 --- /dev/null +++ b/tests/regrtest/test_jit_inline_lists.py @@ -0,0 +1,126 @@ +"""Compiled code reads, writes, appends to and iterates scalar lists in line. + +Each loop runs long enough for the tier-2 compiler to take it; the lists +then change shape under the compiled code, and every change must reach +the generic path with the interpreter's result. +""" + +import unittest + + +def total(xs, n): + t = 0 + for i in range(n): + t += xs[i % len(xs)] + xs[-1] + return t + + +def scale(xs, n, k): + for i in range(n): + j = i % len(xs) + xs[j] = xs[j] * k + return xs + + +def fill(n): + out = [] + for i in range(n): + out.append(i * 2) + return out + + +def fill_floats(n): + out = [] + for i in range(n): + out.append(i * 0.5) + return out + + +def walk(xs): + t = 0 + for x in xs: + t += x + return t + + +class InlineListTests(unittest.TestCase): + N = 3000 + + def test_reads(self): + xs = list(range(10)) + self.assertEqual(total(xs, self.N), sum(xs[i % 10] + 9 for i in range(self.N))) + + def test_out_of_range(self): + xs = [1, 2, 3] + + def get(xs, n, idx): + t = 0 + for _ in range(n): + t += xs[idx] + return t + + self.assertEqual(get(xs, self.N, -3), self.N) + with self.assertRaises(IndexError): + get(xs, 1, 3) + with self.assertRaises(IndexError): + get(xs, 1, -4) + + def test_element_type_changes(self): + xs = list(range(10)) + self.assertEqual(total(xs, self.N), total(list(range(10)), self.N)) + xs[3] = 2.5 + self.assertEqual(total(xs, 10), sum(xs) + 9 * 10) + xs[3] = "a" + with self.assertRaises(TypeError): + total(xs, 10) + + def test_writes(self): + xs = [1.0] * 8 + scale(xs, self.N, 1.0001) + ref = [1.0] * 8 + for i in range(self.N): + ref[i % 8] = ref[i % 8] * 1.0001 + self.assertEqual(xs, ref) + + def test_write_over_heap_element(self): + xs = [1.0] * 4 + scale(xs, self.N, 1.0) + marker = [1.0] + xs[1] = marker + with self.assertRaises(TypeError): + scale(xs, 4, 2.0) + + def test_appends_grow(self): + self.assertEqual(fill(self.N), [i * 2 for i in range(self.N)]) + self.assertEqual(fill_floats(self.N), [i * 0.5 for i in range(self.N)]) + self.assertEqual(fill(5), [0, 2, 4, 6, 8]) + + def test_iteration(self): + xs = list(range(self.N)) + self.assertEqual(walk(xs), sum(xs)) + self.assertEqual(walk([]), 0) + ys = [1, 2, 3.5] + self.assertEqual(walk(ys), 6.5) + with self.assertRaises(TypeError): + walk([1, "x"]) + + def test_iteration_sees_growth(self): + def grow(xs, limit): + n = 0 + for x in xs: + n += 1 + if len(xs) < limit: + xs.append(x + 1) + return n + + xs = [0] + self.assertEqual(grow(xs, self.N), self.N) + self.assertEqual(xs[-1], self.N - 1) + + def test_bools(self): + flags = [True, False] * 100 + self.assertEqual(walk(flags * 10), 1000) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_inline_methods.py b/tests/regrtest/test_jit_inline_methods.py new file mode 100644 index 00000000..11d5c2be --- /dev/null +++ b/tests/regrtest/test_jit_inline_methods.py @@ -0,0 +1,259 @@ +"""Compiled code runs a method that only updates a field in line. + +A method whose body is `self.n += k; return self.n` runs without a call +once the tier-2 helper has run it: each loop below runs long enough for +that, then the receiver, its class, the method or the field changes under +the compiled code, and every change must reach the generic path with the +interpreter's result. +""" + +import sys +import unittest + + +class Counter: + def __init__(self): + self.n = 0 + + def bump(self, by): + self.n += by + return self.n + + def tick(self): + self.n += 1 + return self.n + + +def bump_loop(c, n): + t = 0 + for _ in range(n): + t += c.bump(2) + return t + + +def tick_loop(c, n): + t = 0 + for _ in range(n): + t += c.tick() + return t + + +def tick_all(items, n): + for _ in range(n): + for c in items: + c.tick() + return [c.n for c in items] + + +def triangle(k, n): + # The sum of k, 2k, ..., nk. + return k * n * (n + 1) // 2 + + +class InlineMethodTests(unittest.TestCase): + N = 3000 + + def test_updates(self): + c = Counter() + self.assertEqual(bump_loop(c, self.N), triangle(2, self.N)) + self.assertEqual(c.n, 2 * self.N) + d = Counter() + self.assertEqual(tick_loop(d, self.N), triangle(1, self.N)) + self.assertEqual(tick_all([Counter() for _ in range(5)], self.N), [self.N] * 5) + + def test_instance_shadows_the_method(self): + c = Counter() + self.assertEqual(bump_loop(c, self.N), triangle(2, self.N)) + c.bump = lambda by: 1000 + self.assertEqual(bump_loop(c, 10), 10000) + self.assertEqual(c.n, 2 * self.N) + del c.bump + self.assertEqual(bump_loop(c, 1), 2 * self.N + 2) + + def test_other_instance_shadows_the_method(self): + c = Counter() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + d = Counter() + d.tick = lambda: -1 + self.assertEqual(tick_loop(d, 10), -10) + self.assertEqual(tick_loop(c, 10), 10 * self.N + triangle(1, 10)) + e = Counter() + self.assertEqual(tick_loop(e, self.N), triangle(1, self.N)) + + def test_class_rebinds_the_method(self): + class C(Counter): + pass + + c = C() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + + def tick(self): + self.n += 10 + return -self.n + + C.tick = tick + self.assertEqual(tick_loop(c, 1), -(self.N + 10)) + del C.tick + self.assertEqual(tick_loop(c, 1), self.N + 11) + + def test_code_reassignment(self): + class C: + def __init__(self): + self.n = 0 + + def tick(self): + self.n += 1 + return self.n + + def other(self): + self.n += 100 + return 0 + + c = C() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + saved = C.tick.__code__ + C.tick.__code__ = other.__code__ + self.assertEqual(tick_loop(c, 5), 0) + self.assertEqual(c.n, self.N + 500) + C.tick.__code__ = saved + self.assertEqual(tick_loop(c, 1), self.N + 501) + + def test_overflow_and_field_types(self): + c = Counter() + self.assertEqual(bump_loop(c, self.N), triangle(2, self.N)) + c.n = sys.maxsize - 3 + self.assertEqual(bump_loop(c, 4), 4 * sys.maxsize + 8) + self.assertEqual(c.n, sys.maxsize + 5) + c.n = 0.5 + self.assertEqual(bump_loop(c, 2), 7.0) + c.n = True + self.assertEqual(tick_loop(c, 3), 2 + 3 + 4) + c.n = 0 + self.assertEqual(bump_loop(c, self.N), triangle(2, self.N)) + + def test_receiver_changes_class(self): + class A: + def __init__(self): + self.n = 0 + + def tick(self): + self.n += 1 + return self.n + + class B: + def __init__(self): + self.n = 0 + + def tick(self): + self.n += 2 + return self.n + + a = A() + self.assertEqual(tick_loop(a, self.N), triangle(1, self.N)) + a.__class__ = B + self.assertEqual(tick_loop(a, 3), 3 * self.N + triangle(2, 3)) + self.assertEqual(tick_all([A(), B(), A()], self.N), [self.N, 2 * self.N, self.N]) + + def test_materialized_dict(self): + c = Counter() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + vars(c)["n"] = 10 + self.assertEqual(tick_loop(c, 3), 11 + 12 + 13) + + def test_profiler_sees_every_call(self): + c = Counter() + self.assertEqual(tick_loop(c, self.N), triangle(1, self.N)) + calls = [] + + def profile(frame, event, arg): + if event == "call" and frame.f_code.co_name == "tick": + calls.append(1) + + sys.setprofile(profile) + try: + tick_loop(c, 50) + finally: + sys.setprofile(None) + self.assertEqual(len(calls), 50) + self.assertEqual(c.n, self.N + 50) + + +class Slotted: + __slots__ = ("n", "m") + + def __init__(self): + self.n = 0 + self.m = 0 + + def tick(self): + self.n += 1 + return self.n + + def bump(self, by): + self.m += by + return self.m + + +class SlottedUpdateTests(unittest.TestCase): + N = 3000 + + def test_updates(self): + s = Slotted() + self.assertEqual(tick_loop(s, self.N), triangle(1, self.N)) + self.assertEqual(bump_loop(s, self.N), triangle(2, self.N)) + self.assertEqual((s.n, s.m), (self.N, 2 * self.N)) + + def test_deleted_slot(self): + s = Slotted() + self.assertEqual(tick_loop(s, self.N), triangle(1, self.N)) + del s.n + with self.assertRaises(AttributeError): + tick_loop(s, 1) + s.n = 5 + self.assertEqual(tick_loop(s, 2), 6 + 7) + + def test_slot_value_types(self): + s = Slotted() + self.assertEqual(bump_loop(s, self.N), triangle(2, self.N)) + s.m = 0.25 + self.assertEqual(bump_loop(s, 2), 2.25 + 4.25) + s.m = sys.maxsize + self.assertEqual(bump_loop(s, 1), sys.maxsize + 2) + s.m = "x" + with self.assertRaises(TypeError): + bump_loop(s, 1) + + def test_class_replaces_the_slot(self): + class S: + __slots__ = ("n",) + + def __init__(self): + self.n = 0 + + def tick(self): + self.n += 1 + return self.n + + s = S() + self.assertEqual(tick_loop(s, self.N), triangle(1, self.N)) + slot = S.n + seen = [] + S.n = property(lambda self: 100, lambda self, v: seen.append(v)) + self.assertEqual(tick_loop(s, 2), 200) + self.assertEqual(seen, [101, 101]) + S.n = slot + self.assertEqual(tick_loop(s, 1), self.N + 1) + + def test_subclass_with_dict(self): + class D(Slotted): + pass + + d = D() + self.assertEqual(tick_loop(d, self.N), triangle(1, self.N)) + d.tick = lambda: -1 + self.assertEqual(tick_loop(d, 3), -3) + self.assertEqual(tick_all([Slotted(), D(), Slotted()], 5), [5, 5, 5]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_inline_new_fields.py b/tests/regrtest/test_jit_inline_new_fields.py new file mode 100644 index 00000000..a4157a2c --- /dev/null +++ b/tests/regrtest/test_jit_inline_new_fields.py @@ -0,0 +1,132 @@ +"""Compiled constructors append a fresh instance's fields in line. + +An `__init__` that sets scalar fields in its class's usual order stores +each one past the previous without calling a helper. Each loop runs long +enough for the tier-2 compiler to take the constructor; then instances +arrive in other shapes, and each must end up with exactly its fields. +""" + +import unittest + + +class Point: + def __init__(self, i): + self.x = i + self.y = i * 0.5 + self.z = i > 3 + + +def build(cls, n): + out = [] + for i in range(n): + out.append(cls(i)) + return out + + +class InlineNewFieldTests(unittest.TestCase): + N = 3000 + + def check(self, p, i): + self.assertEqual(vars(p), {"x": i, "y": i * 0.5, "z": i > 3}) + + def test_fields(self): + points = build(Point, self.N) + for i in (0, 1, 4, self.N - 1): + self.check(points[i], i) + self.assertEqual(sum(p.x for p in points), self.N * (self.N - 1) // 2) + + def test_reinitialized_instance(self): + points = build(Point, self.N) + p = points[10] + Point.__init__(p, 99) + self.check(p, 99) + + def test_other_order_first(self): + class Q: + def __init__(self, i, flip=False): + if flip: + self.b = -i + self.a = i + else: + self.a = i + self.b = -i + + def make(n, flip): + return [Q(i, flip) for i in range(n)] + + qs = make(self.N, False) + self.assertEqual(vars(qs[5]), {"a": 5, "b": -5}) + flipped = make(10, True) + self.assertEqual(list(vars(flipped[5]).items()), [("b", -5), ("a", 5)]) + again = make(10, False) + self.assertEqual(list(vars(again[5]).items()), [("a", 5), ("b", -5)]) + + def test_subclass_and_extra_fields(self): + class Base: + def __init__(self, i): + self.u = i + self.v = 2 * i + + class Sub(Base): + pass + + bases = build(Base, self.N) + subs = build(Sub, 10) + self.assertEqual(vars(bases[7]), {"u": 7, "v": 14}) + self.assertEqual(vars(subs[7]), {"u": 7, "v": 14}) + b = Base(1) + b.w = 3 + self.assertEqual(vars(Base(2)), {"u": 2, "v": 4}) + self.assertEqual(vars(b), {"u": 1, "v": 2, "w": 3}) + + def test_class_attribute_becomes_a_property(self): + class R: + def __init__(self, i): + self.k = i + + rs = build(R, self.N) + self.assertEqual(rs[3].k, 3) + seen = [] + R.k = property(lambda self: 0, lambda self, v: seen.append(v)) + r = R(5) + self.assertEqual(seen, [5]) + self.assertEqual(r.k, 0) + + def test_instances_that_set_fewer_fields(self): + class S: + def __init__(self, i, full=True): + if full: + self.a = i + self.b = 2 * i + + def make(n, full): + return [S(i, full) for i in range(n)] + + full = make(self.N, True) + self.assertEqual(vars(full[3]), {"a": 3, "b": 6}) + empty = make(10, False) + self.assertEqual(vars(empty[3]), {}) + empty[3].b = 1 + empty[3].a = 2 + self.assertEqual(list(vars(empty[3]).items()), [("b", 1), ("a", 2)]) + self.assertEqual(vars(make(1, True)[0]), {"a": 0, "b": 0}) + + def test_constructor_raising_midway(self): + class E: + def __init__(self, i): + self.a = i + if i < 0: + raise ValueError(i) + self.b = i + + def make(n): + return [E(i) for i in range(n)] + + self.assertEqual(vars(make(self.N)[5]), {"a": 5, "b": 5}) + with self.assertRaises(ValueError): + E(-1) + self.assertEqual(vars(make(3)[2]), {"a": 2, "b": 2}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_ir_verifies.py b/tests/regrtest/test_jit_ir_verifies.py new file mode 100644 index 00000000..10397b49 --- /dev/null +++ b/tests/regrtest/test_jit_ir_verifies.py @@ -0,0 +1,39 @@ +"""Compiled plans and frames pass Cranelift's IR verifier. + +Release builds skip the verifier, and some invalid IR (an extension to +the type a value already has) still compiles on x86-64 but encodes an +invalid instruction on AArch64. This runs the plan-cache tests, which +exercise the in-line method, class-attribute and global caches, with +both verifiers on, and fails if any function didn't compile. +""" + +import os +import subprocess +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +class IrVerifies(unittest.TestCase): + def test_plan_caches_compile_verified(self): + script = os.path.join(HERE, "test_jit_plan_caches.py") + if not os.path.exists(script): + self.skipTest("test_jit_plan_caches.py isn't beside this test") + env = dict(os.environ) + env["WEAVEPY_PLAN_VERIFY"] = "1" + env["WEAVEPY_FRAME_JIT_VERIFY"] = "1" + proc = subprocess.run( + [sys.executable, script], + capture_output=True, + text=True, + env=env, + timeout=120, + ) + self.assertEqual(proc.returncode, 0, proc.stderr[-3000:]) + self.assertNotIn("failed to compile", proc.stderr, proc.stderr[-3000:]) + self.assertNotIn("verifier error", proc.stderr, proc.stderr[-3000:]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_items_and_isinstance.py b/tests/regrtest/test_jit_items_and_isinstance.py new file mode 100644 index 00000000..22bada75 --- /dev/null +++ b/tests/regrtest/test_jit_items_and_isinstance.py @@ -0,0 +1,118 @@ +"""Native `dict.items()` loops and `isinstance` against a tuple of classes. + +A tier-2 loop `for k, v in d.items()` takes each key and value straight +from the dict, and `isinstance(x, (A, B))` checks a flat tuple of plain +classes without the general call. Each loop here runs long enough to +compile; then the dict or the classes change, and every result must match +the interpreter's. +""" + +import unittest + +WARM = 3000 + +TABLE = {str(i): i for i in range(20)} + + +def total(): + t = 0 + for k, v in TABLE.items(): + t += v + return t + + +def keys_len(d): + n = 0 + for k, v in d.items(): + n += len(k) + return n + + +def mutate_during(d): + for k, v in d.items(): + d[k + "!"] = v + + +def churn_during(d): + for k, v in d.items(): + del d[k] + d[k + "?"] = v + + +def overwrite_during(d): + t = 0 + for k, v in d.items(): + d[k] = v + 1 + t += v + return t + + +class Base: + pass + + +class Sub(Base): + pass + + +class Masked: + @property + def __class__(self): + return Base + + +def kinds(xs, spec): + n = 0 + for x in xs: + if isinstance(x, spec): + n += 1 + return n + + +class ItemsAndIsinstanceTests(unittest.TestCase): + def test_items_loop(self): + for _ in range(WARM): + r = total() + self.assertEqual(r, sum(range(20))) + TABLE["x"] = 1.5 + try: + self.assertEqual(total(), sum(range(20)) + 1.5) + finally: + del TABLE["x"] + self.assertEqual(total(), sum(range(20))) + d = {"ab": 1, "c": 2} + for _ in range(WARM): + r = keys_len(d) + self.assertEqual(r, 3) + self.assertEqual(keys_len({}), 0) + + def test_items_mutation(self): + for _ in range(WARM): + overwrite_during({"a": 1, "b": 2}) + d = {"a": 1, "b": 2} + self.assertEqual(overwrite_during(d), 3) + self.assertEqual(d, {"a": 2, "b": 3}) + with self.assertRaises(RuntimeError): + mutate_during({"a": 1, "b": 2}) + with self.assertRaises(RuntimeError): + churn_during({"a": 1, "b": 2}) + + def test_isinstance_tuples(self): + xs = [1, "s", 2.0, Sub(), Base(), None, Masked()] + for _ in range(WARM): + r = kinds(xs, (int, str)) + self.assertEqual(r, 2) + self.assertEqual(kinds(xs, (Base,)), 3) + self.assertEqual(kinds(xs, (float, Sub)), 2) + self.assertEqual(kinds(xs, (bool, type(None))), 1) + self.assertEqual(kinds(xs, ()), 0) + self.assertEqual(kinds(xs, (int, (str, float))), 3) + import numbers + + self.assertEqual(kinds(xs, (numbers.Number, Base)), 5) + with self.assertRaises(TypeError): + kinds(xs, (int, 3)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_kw_leaf_calls.py b/tests/regrtest/test_jit_kw_leaf_calls.py new file mode 100644 index 00000000..d9bfe2f4 --- /dev/null +++ b/tests/regrtest/test_jit_kw_leaf_calls.py @@ -0,0 +1,125 @@ +"""Compiled code calls keyword-taking pure functions without a frame. + +A keyword call of a small pure function binds its values onto the +function's parameters (a fresh `**kwargs` dictionary included) and +evaluates it frameless. Each loop runs long enough for the tier-2 compiler +to take it; then the callee changes, and every call must see the change. +""" + +import unittest + + +def with_kwargs(a, **kw): + return a + kw.get("delta", 0) + + +def mixed(a, b=1, **kw): + return a * b + kw.get("c", 0) + + +def give_back(a, **kw): + return kw + + +class Holder: + def scaled(self, a, **kw): + return a * kw.get("k", 1) + + +def kw_loop(n): + t = 0 + for i in range(n): + t += with_kwargs(i, delta=2) + return t + + +def mixed_loop(n): + t = 0 + for i in range(n): + t += mixed(i, c=3, b=2) + return t + + +def fresh_dicts(n): + seen = [] + for i in range(n): + seen.append(give_back(i, x=i)) + return seen + + +def bound_loop(n): + h = Holder() + scaled = h.scaled + t = 0 + for i in range(n): + t += scaled(i, k=3) + return t + + +def triangle(n): + # The sum of 0, 1, ..., n - 1. + return n * (n - 1) // 2 + + +class KwLeafCallTests(unittest.TestCase): + N = 3000 + + def setUp(self): + self.saved = (with_kwargs.__code__, mixed.__defaults__) + + def tearDown(self): + with_kwargs.__code__, mixed.__defaults__ = self.saved + + def test_calls(self): + n = self.N + self.assertEqual(kw_loop(n), triangle(n) + 2 * n) + self.assertEqual(mixed_loop(n), 2 * triangle(n) + 3 * n) + self.assertEqual(bound_loop(n), 3 * triangle(n)) + + def test_each_call_gets_a_fresh_dict(self): + dicts = fresh_dicts(self.N) + self.assertEqual(len({id(d) for d in dicts}), self.N) + self.assertEqual(dicts[7], {"x": 7}) + dicts[7]["y"] = 1 + self.assertEqual(dicts[8], {"x": 8}) + + def test_code_reassigned(self): + def other(a, **kw): + return len(kw) * 100 + + n = self.N + self.assertEqual(kw_loop(n), triangle(n) + 2 * n) + with_kwargs.__code__ = other.__code__ + self.assertEqual(kw_loop(5), 500) + + def test_defaults_reassigned(self): + n = self.N + self.assertEqual(mixed_loop(n), 2 * triangle(n) + 3 * n) + mixed.__defaults__ = (10,) + self.assertEqual(mixed_loop(5), 2 * triangle(5) + 15) + + def call_without_b(n): + t = 0 + for i in range(n): + t += mixed(i, c=1) + return t + + self.assertEqual(call_without_b(n), 10 * triangle(n) + n) + mixed.__defaults__ = None + with self.assertRaises(TypeError): + call_without_b(1) + + def test_bad_keyword(self): + def call_bad(n): + t = 0 + for i in range(n): + t += mixed(i, b=2, c=1) + return t + + self.assertEqual(call_bad(self.N), 2 * triangle(self.N) + self.N) + with self.assertRaises(TypeError): + mixed(1, a=2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_leaf_defaults.py b/tests/regrtest/test_jit_leaf_defaults.py new file mode 100644 index 00000000..a64927a7 --- /dev/null +++ b/tests/regrtest/test_jit_leaf_defaults.py @@ -0,0 +1,135 @@ +"""Compiled code calls a compiled scalar function directly, defaults included. + +A call that leaves trailing or keyword-skipped parameters out binds the +callee's scalar defaults in compiled code. Each loop runs long enough for +the tier-2 compiler to take the callee and then the caller; then the +callee's defaults or code change, and every call must see them. +""" + +import unittest + + +def with_defaults(a, b=10, c=20): + return a + b + c + + +def scaled(x, k=1.5): + return x * k + + +def gated(a, flag=True): + if flag: + return a + return -a + + +def positional_loop(n): + t = 0 + for i in range(n): + t += with_defaults(i) + return t + + +def keyword_loop(n): + t = 0 + for i in range(n): + t += with_defaults(i, c=5) + return t + + +def middle_keyword_loop(n): + t = 0 + for i in range(n): + t += with_defaults(i, b=1) + return t + + +def float_loop(n): + t = 0.0 + for i in range(n): + t += scaled(float(i)) + return t + + +def bool_loop(n): + t = 0 + for i in range(n): + t += gated(i) + return t + + +def triangle(n): + # The sum of 0, 1, ..., n - 1. + return n * (n - 1) // 2 + + +class LeafDefaultTests(unittest.TestCase): + N = 3000 + + def setUp(self): + self.saved = ( + with_defaults.__defaults__, + with_defaults.__code__, + scaled.__defaults__, + gated.__defaults__, + ) + + def tearDown(self): + ( + with_defaults.__defaults__, + with_defaults.__code__, + scaled.__defaults__, + gated.__defaults__, + ) = self.saved + + def test_defaults_bind(self): + n = self.N + self.assertEqual(positional_loop(n), triangle(n) + 30 * n) + self.assertEqual(keyword_loop(n), triangle(n) + 15 * n) + self.assertEqual(middle_keyword_loop(n), triangle(n) + 21 * n) + self.assertEqual(float_loop(n), 1.5 * triangle(n)) + self.assertEqual(bool_loop(n), triangle(n)) + + def test_defaults_reassigned(self): + n = self.N + self.assertEqual(positional_loop(n), triangle(n) + 30 * n) + self.assertEqual(keyword_loop(n), triangle(n) + 15 * n) + with_defaults.__defaults__ = (100, 200) + self.assertEqual(positional_loop(n), triangle(n) + 300 * n) + self.assertEqual(keyword_loop(n), triangle(n) + 105 * n) + self.assertEqual(middle_keyword_loop(n), triangle(n) + 201 * n) + with_defaults.__defaults__ = (1.5, 2) + self.assertEqual(positional_loop(4), 6 + 4 * 3.5) + with_defaults.__defaults__ = (2,) + with self.assertRaises(TypeError): + positional_loop(1) + self.assertEqual(middle_keyword_loop(3), 3 + 3 * 3) + with self.assertRaises(TypeError): + keyword_loop(1) + with_defaults.__defaults__ = None + with self.assertRaises(TypeError): + keyword_loop(1) + + def test_lanes_change(self): + n = self.N + self.assertEqual(float_loop(n), 1.5 * triangle(n)) + scaled.__defaults__ = (2,) + self.assertEqual(float_loop(4), 12.0) + self.assertEqual(bool_loop(n), triangle(n)) + gated.__defaults__ = (False,) + self.assertEqual(bool_loop(4), -6) + gated.__defaults__ = (0,) + self.assertEqual(bool_loop(4), -6) + + def test_code_reassigned(self): + def other(a, b=1, c=2): + return a * b * c + + n = self.N + self.assertEqual(positional_loop(n), triangle(n) + 30 * n) + with_defaults.__code__ = other.__code__ + self.assertEqual(positional_loop(4), 10 * 20 * triangle(4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_object_compares.py b/tests/regrtest/test_jit_object_compares.py new file mode 100644 index 00000000..81af2fbf --- /dev/null +++ b/tests/regrtest/test_jit_object_compares.py @@ -0,0 +1,99 @@ +"""Compiled code compares objects with ints, and objects by identity. + +An integer compared with a value the compiler only knows as an object (a +class constant, an attribute read generically) is guarded to be an exact +machine `int`; `a is b` on two objects compares identities. Each loop +here runs long enough to compile; then the values change shape, and the +results must still match what the interpreter computes. +""" + +import unittest + +WARM = 3000 + + +class Dir: + NONE = 0 + FORWARD = 1 + + +class Edge: + def __init__(self, a, b, d): + self.a = a + self.b = b + self.d = d + + +def pick(e, n): + t = 0 + for _ in range(n): + if e.d == Dir.FORWARD: + t += 1 + else: + t -= 1 + return t + + +def ahead(e, n): + t = 0 + for i in range(n): + if Dir.FORWARD < i: + t += 1 + return t + + +def count_same(items, target): + t = 0 + for x in items: + if x is target: + t += 1 + if x is not target: + t -= 1 + return t + + +class ObjectCompareTests(unittest.TestCase): + def test_class_constant_compare(self): + e = Edge(1, 2, 1) + self.assertEqual(pick(e, WARM), WARM) + e.d = 0 + self.assertEqual(pick(e, 10), -10) + saved = Dir.FORWARD + try: + Dir.FORWARD = "fwd" + e.d = "fwd" + self.assertEqual(pick(e, 10), 10) + Dir.FORWARD = 2**70 + e.d = 2**70 + self.assertEqual(pick(e, 10), 10) + Dir.FORWARD = 1.0 + e.d = 1 + self.assertEqual(pick(e, 10), 10) + Dir.FORWARD = True + self.assertEqual(pick(e, 10), 10) + finally: + Dir.FORWARD = saved + self.assertEqual(ahead(e, WARM), WARM - 2) + Dir.FORWARD = 5 + try: + self.assertEqual(ahead(e, 10), 4) + Dir.FORWARD = None + with self.assertRaises(TypeError): + ahead(e, 10) + finally: + Dir.FORWARD = saved + + def test_identity(self): + a, b = Edge(1, 2, 0), Edge(1, 2, 0) + items = [a, b, a, None, a] + self.assertEqual(count_same(items * 600, a), 600) + self.assertEqual(count_same([b, b, None], b), 1) + self.assertEqual(count_same([None, None, a], None), 1) + self.assertEqual(count_same([[], [], a], a), -1) + lst = [] + self.assertEqual(count_same([lst, [], lst], lst), 1) + self.assertEqual(count_same(["x", "y", "x"], "x"), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_pin_limit.py b/tests/regrtest/test_jit_pin_limit.py new file mode 100644 index 00000000..14856e1f --- /dev/null +++ b/tests/regrtest/test_jit_pin_limit.py @@ -0,0 +1,64 @@ +"""A long compiled loop bounds how many dead objects its pins keep alive. + +Compiled code pins the objects it handles. A loop over objects something +else keeps alive (a list's elements) may hold many pins; a loop whose pins +are the last references must leave and release them every few thousand, +so finalizers run with a bounded delay. +""" + +import unittest + + +class Probe: + alive = 0 + + def __init__(self): + Probe.alive += 1 + + def __del__(self): + Probe.alive -= 1 + + def ping(self): + return 1 + + +def make(): + return Probe() + + +def churn(n, peaks): + t = 0 + for i in range(n): + p = make() + t += p.ping() + if i % 1000 == 0: + peaks.append(Probe.alive) + return t + + +def walk(items): + t = 0 + for p in items: + t += p.ping() + return t + + +class PinLimitTests(unittest.TestCase): + def test_dead_objects_are_released_as_the_loop_runs(self): + peaks = [] + self.assertEqual(churn(40000, peaks), 40000) + # The soft limit is a few thousand pins; never most of the loop. + self.assertLess(max(peaks), 12000) + self.assertEqual(Probe.alive, 0) + + def test_shared_objects_stay_pinned_correctly(self): + items = [Probe() for _ in range(40000)] + self.assertEqual(walk(items), 40000) + self.assertEqual(walk(items[::-1]), 40000) + self.assertEqual(Probe.alive, 40000) + del items + self.assertEqual(Probe.alive, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_plan_caches.py b/tests/regrtest/test_jit_plan_caches.py new file mode 100644 index 00000000..7449dde8 --- /dev/null +++ b/tests/regrtest/test_jit_plan_caches.py @@ -0,0 +1,189 @@ +"""Compiled leaf plans read method, class and global caches in line. + +A leaf method that calls another leaf method, reads a class constant or +reads a module global compiles (once it has run enough) to native code +that checks each cache itself. Every test defines its own methods (so its +own plans) and runs them long enough to compile; then the cached fact +changes, and each call must see it. The drivers read `locals()` on a path +they never take, which keeps the tier-2 compiler (whose calls take other +paths) off them. +""" + +import types +import unittest + +WARM = 33000 + +SCALE = 2 + + +class Val: + def __init__(self, v): + self.v = v + + +class ConstP: + FWD = 1 + + def __init__(self, x, y): + self.x = x + self.y = y + self.d = 1 + + def get(self): + return (self.y if self.d == ConstP.FWD else self.x).v + + +class ConstQ(ConstP): + pass + + +class Strength: + def __init__(self, value): + self.value = value + + @staticmethod + def weaker(a, b): + return a.value > b.value + + @staticmethod + def weakest(a, b): + return a if Strength.weaker(a, b) else b + + +class StaticC: + def __init__(self, s, t): + self.s = s + self.t = t + + def get(self): + return Strength.weakest(self.s, self.t).value + + +def drive(o, n): + if n < 0: + return locals() + t = 0 + for _ in range(n): + t += o.get() + return t + + +def set_scale(v): + global SCALE + SCALE = v + + +class PlanCacheTests(unittest.TestCase): + def test_instance_attribute_shadows_method(self): + class P: + def __init__(self, x): + self.x = x + + def pick(self): + return self.x + + def get(self): + return self.pick().v + + p = P(Val(2)) + self.assertEqual(drive(p, WARM), 2 * WARM) + p.pick = lambda: Val(10) + self.assertEqual(drive(p, 10), 100) + self.assertEqual(drive(P(Val(3)), 10), 30) + + def test_method_replaced_on_class(self): + class P: + def __init__(self, x): + self.x = x + + def pick(self): + return self.x + + def get(self): + return self.pick().v + + p = P(Val(2)) + self.assertEqual(drive(p, WARM), 2 * WARM) + P.pick = lambda self: Val(5) + self.assertEqual(drive(p, 10), 50) + + def test_subclass_receiver(self): + class P: + def __init__(self, x): + self.x = x + + def pick(self): + return self.x + + def get(self): + return self.pick().v + + class Q(P): + def pick(self): + return Val(7) + + self.assertEqual(drive(P(Val(2)), WARM), 2 * WARM) + self.assertEqual(drive(Q(Val(2)), 10), 70) + self.assertEqual(drive(P(Val(4)), 10), 40) + + def test_class_constant_rebound(self): + q = ConstQ(Val(1), Val(2)) + self.assertEqual(drive(q, WARM), 2 * WARM) + ConstQ.FWD = 0 + self.assertEqual(drive(q, 10), 20) + ConstP.FWD = 0 + self.assertEqual(drive(q, 10), 10) + ConstP.FWD = 1 + self.assertEqual(drive(q, 10), 20) + + def test_global_rebound(self): + class P: + def __init__(self, x): + self.x = x + + def get(self): + return self.x.v * SCALE + + p = P(Val(3)) + try: + self.assertEqual(drive(p, WARM), 6 * WARM) + set_scale(5) + self.assertEqual(drive(p, 10), 150) + globals()["SCALE"] = 7 + self.assertEqual(drive(p, 10), 210) + del globals()["SCALE"] + with self.assertRaises(NameError): + p.get() + globals()["SCALE"] = 11 + self.assertEqual(drive(p, 10), 330) + finally: + set_scale(2) + + def test_shared_code_other_globals(self): + class P: + def __init__(self, x): + self.x = x + + def get(self): + return self.x.v * SCALE + + p = P(Val(3)) + self.assertEqual(drive(p, WARM), 6 * WARM) + ns = dict(globals()) + ns["SCALE"] = 100 + + class Q(P): + get = types.FunctionType(P.get.__code__, ns) + + self.assertEqual(drive(Q(Val(3)), 10), 3000) + self.assertEqual(drive(p, 10), 60) + + def test_static_method_through_class(self): + c = StaticC(Strength(1), Strength(2)) + self.assertEqual(drive(c, WARM), 2 * WARM) + Strength.weaker = staticmethod(lambda x, y: x.value < y.value) + self.assertEqual(drive(c, 10), 10) + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_jit_plan_stores.py b/tests/regrtest/test_jit_plan_stores.py new file mode 100644 index 00000000..6634b9bd --- /dev/null +++ b/tests/regrtest/test_jit_plan_stores.py @@ -0,0 +1,206 @@ +"""A compiled leaf plan's last attribute store lands at once. + +An effect leaf (a method whose only side effects are attribute stores) +buffers its stores until it returns; a store nothing after it can decline +lands directly instead. Each loop here runs long enough for the plan to +compile; then the receiver changes shape, and every store must behave as +the interpreter's would. The drivers read `locals()` on a path they never +take, which keeps the tier-2 compiler (whose calls take other paths) off +them. +""" + +import unittest + +WARM = 33000 + + +class Cell: + def __init__(self): + self.v = 0 + + +class Node: + def __init__(self, x, peer): + self.x = x + self.peer = peer + + def put(self): + self.peer.v = self.x + + +def drive(o, n): + if n < 0: + return locals() + for _ in range(n): + o.put() + + +class Noisy: + deaths = 0 + + def __del__(self): + Noisy.deaths += 1 + + +class PlanStoreTests(unittest.TestCase): + def test_plain_store(self): + c = Cell() + n = Node(7, c) + drive(n, WARM) + self.assertEqual(c.v, 7) + n.x = "s" + drive(n, 3) + self.assertEqual(c.v, "s") + + def test_receiver_is_self(self): + n = Node(5, None) + n.peer = n + drive(n, WARM) + self.assertEqual(n.v, 5) + m = Node(6, None) + m.peer = m + drive(m, 1) + self.assertEqual(vars(m), {"x": 6, "peer": m, "v": 6}) + + def test_setter_and_finalizer(self): + c = Cell() + n = Node(1, c) + drive(n, WARM) + + class Watched: + def __init__(self): + self.seen = [] + + @property + def v(self): + return self.seen[-1] + + @v.setter + def v(self, value): + self.seen.append(value) + + w = Watched() + n.peer = w + drive(n, 2) + self.assertEqual(w.seen, [1, 1]) + n.peer = c + c.v = Noisy() + before = Noisy.deaths + drive(n, 1) + self.assertEqual(Noisy.deaths, before + 1) + self.assertEqual(c.v, 1) + + def test_slots_receiver(self): + class Slotted: + __slots__ = ("v",) + + c = Cell() + n = Node(3, c) + drive(n, WARM) + s = Slotted() + n.peer = s + drive(n, 1) + self.assertEqual(s.v, 3) + + def test_slots_constructor(self): + import gc + + class P: + __slots__ = ("x", "y") + + def __init__(self, x, y): + self.x = x + self.y = y + + class Q(P): + pass + + def build(n): + if n < 0: + return locals() + t = 0 + for i in range(n): + p = P(i, 2) + t += p.x + p.y + return t + + self.assertEqual(build(WARM), sum(range(WARM)) + 2 * WARM) + p = P([], "s") + self.assertEqual((p.x, p.y), ([], "s")) + self.assertTrue(gc.is_tracked(p)) + del p.x + with self.assertRaises(AttributeError): + p.x + p.__init__(1, 2) + self.assertEqual((p.x, p.y), (1, 2)) + q = Q(3, 4) + self.assertEqual((q.x, q.y), (3, 4)) + q.z = 5 + self.assertEqual(vars(q), {"z": 5}) + loop = P(0, 0) + loop.x = loop + del loop + gc.collect() + P.y = property(lambda self: "prop", lambda self, v: None) + r = P(1, 2) + self.assertEqual((r.x, r.y), (1, "prop")) + + def test_scalar_stores_in_place(self): + class Src: + def __init__(self, x): + self.x = x + + class Dst: + def __init__(self): + self.v = 0 + self.w = 1 + + class Copy: + def __init__(self, a, b): + self.a = a + self.b = b + + def put(self): + self.b.v = self.a.x + + src, dst = Src(1), Dst() + c = Copy(src, dst) + drive(c, WARM) + self.assertEqual(dst.v, 1) + for value in (2.5, True, None, -7, 2**62): + src.x = value + drive(c, 1) + self.assertIs(type(dst.v), type(value)) + self.assertEqual(dst.v, value) + # A heap value, over a scalar and then over itself. + src.x = [1] + drive(c, 2) + self.assertIs(dst.v, src.x) + src.x = 3 + drive(c, 1) + self.assertEqual(dst.v, 3) + # Another instance of the class, without the field yet. + fresh = Dst.__new__(Dst) + c.b = fresh + drive(c, 1) + self.assertEqual(vars(fresh), {"v": 3}) + # An instance whose dictionary was handed out. + other = Dst() + d = other.__dict__ + c.b = other + src.x = 9 + drive(c, 1) + self.assertEqual(d["v"], 9) + # The class changes under the cached version. + c.b = dst + Dst.extra = 1 + src.x = 11 + drive(c, 1) + self.assertEqual(dst.v, 11) + Dst.v = property(lambda self: -1, lambda self, value: None) + drive(c, 1) + self.assertEqual(dst.v, -1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_kwargs_dict_reuse.py b/tests/regrtest/test_kwargs_dict_reuse.py new file mode 100644 index 00000000..00d51245 --- /dev/null +++ b/tests/regrtest/test_kwargs_dict_reuse.py @@ -0,0 +1,83 @@ +"""A keyword leaf call's `**kwargs` dictionary may be reused across calls. + +The dictionary of a call nothing else kept is emptied and serves the next +call. Each call must still see a fresh, correct dictionary, and one that +escaped (returned, stored, captured) must stay intact and distinct. +""" + +import unittest + +WARM = 3000 + + +def total(a, **kw): + return a + kw.get("delta", 0) + + +def count(**kw): + return len(kw) + + +def echo(**kw): + return kw + + +def add_key(**kw): + kw["extra"] = 1 + return len(kw) + + +kept = [] + + +def keep(**kw): + kept.append(kw) + return 0 + + +class KwargsDictReuse(unittest.TestCase): + def test_values_per_call(self): + acc = 0 + for i in range(WARM): + acc += total(i, delta=2) + self.assertEqual(acc, sum(range(WARM)) + 2 * WARM) + for i in range(WARM): + self.assertEqual(count(a=i, b=i), 2) + self.assertEqual(count(a=i), 1) + + def test_no_keys_leak_between_calls(self): + for i in range(WARM): + self.assertEqual(add_key(a=i), 2) + self.assertEqual(count(b=i), 1) + + def test_returned_dicts_stay_distinct(self): + seen = [echo(n=i) for i in range(WARM)] + self.assertEqual([d["n"] for d in seen], list(range(WARM))) + self.assertEqual(len({id(d) for d in seen}), WARM) + + def test_stored_dicts_stay_intact(self): + kept.clear() + for i in range(WARM): + keep(n=i, m=-i) + self.assertEqual([(d["n"], d["m"]) for d in kept], [(i, -i) for i in range(WARM)]) + + def test_captured_dict(self): + def make(**kw): + return lambda: kw + + getters = [make(v=i) for i in range(WARM)] + self.assertEqual([g()["v"] for g in getters], list(range(WARM))) + + def test_identity_inside_call(self): + ids = [] + + def ident(**kw): + ids.append(id(kw)) + return kw + + held = [ident(x=i) for i in range(50)] + self.assertEqual(len(set(ids)), len(held)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_leaf_builtin_calls.py b/tests/regrtest/test_leaf_builtin_calls.py index f0d9da2e..b8802b8b 100644 --- a/tests/regrtest/test_leaf_builtin_calls.py +++ b/tests/regrtest/test_leaf_builtin_calls.py @@ -121,3 +121,14 @@ def f_floor(x): assert f_get({Key(1): 2}, Key(1)) == 2 assert Key.calls == before + 1 print("ok") + + +# A native leaf that returns a value it owns (a builtin call's fresh +# result) hands back its own reference, not a pointer into its scratch. +def f_fresh(s, a): + return s.format(a) + + +for i in range(3000): + assert f_fresh("{}!", i) == "%d!" % i +print("fresh ok") diff --git a/tests/regrtest/test_leaf_subscripts.py b/tests/regrtest/test_leaf_subscripts.py new file mode 100644 index 00000000..fba1993a --- /dev/null +++ b/tests/regrtest/test_leaf_subscripts.py @@ -0,0 +1,70 @@ +"""Subscripts in frameless leaf evaluation. + +Small functions such as `key=lambda t: t[1]` are evaluated without an +activation when native code calls them. A subscript there must give the +same result as the full call, and anything it can't settle (a miss, an +unusual container) must fall back so the full call raises or dispatches +exactly as usual. +""" + +import unittest + + +class LeafSubscripts(unittest.TestCase): + def test_sort_keys(self): + pairs = [(3, "c"), (1, "a"), (2, "b")] * 50 + self.assertEqual(sorted(pairs, key=lambda t: t[1])[:3], [(1, "a")] * 3) + self.assertEqual(sorted(pairs, key=lambda t: t[-2])[-1], (3, "c")) + rows = [[i, -i] for i in range(100)] + self.assertEqual(max(rows, key=lambda r: r[1]), [0, 0]) + words = ["delta", "alpha", "charlie", "bravo"] * 30 + self.assertEqual(sorted(words, key=lambda w: w[1])[0], "delta") + self.assertEqual(min(words, key=lambda w: w[-1]), "delta") + + def test_non_ascii_strings(self): + words = ["żółw", "ąb", "éa", "zz"] * 30 + self.assertEqual(sorted(words, key=lambda w: w[1])[0], "éa") + + def test_dicts(self): + tables = [{"k": i, 7: -i} for i in range(60)] + self.assertEqual(max(tables, key=lambda d: d["k"])["k"], 59) + self.assertEqual(max(tables, key=lambda d: d[7])["k"], 0) + + def test_misses_raise(self): + items = [(1,), (2,)] * 40 + with self.assertRaises(IndexError): + sorted(items, key=lambda t: t[1]) + dicts = [{"a": 1}] * 40 + with self.assertRaises(KeyError): + sorted(dicts, key=lambda d: d["b"]) + strs = ["a", "bb"] * 40 + with self.assertRaises(IndexError): + sorted(strs, key=lambda s: s[1]) + + def test_subclasses_dispatch(self): + class D(dict): + def __missing__(self, key): + return 42 + + class L(list): + def __getitem__(self, i): + return -super().__getitem__(i) + + ds = [D(a=i) for i in range(40)] + self.assertEqual(sorted(ds, key=lambda d: d["zz"])[0]["a"], 0) + ls = [L([i]) for i in range(40)] + self.assertEqual(max(ls, key=lambda x: x[0]), [0]) + + def test_warm_loop(self): + def second(t): + return t[1] + + data = [(i, i * i) for i in range(30)] + total = 0 + for _ in range(3000): + total += max(map(second, data)) + self.assertEqual(total, 3000 * 29 * 29) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_native_map_filter.py b/tests/regrtest/test_native_map_filter.py new file mode 100644 index 00000000..e0e24c58 --- /dev/null +++ b/tests/regrtest/test_native_map_filter.py @@ -0,0 +1,86 @@ +"""`map` and `filter` over native containers, stepped natively. + +The common shapes are built and stepped without running the lazy +classes' Python methods. Laziness, error timing, type identity and +pickling must stay exactly as with the classes' own methods. +""" + +import pickle +import unittest + + +class NativeMapFilter(unittest.TestCase): + def test_results(self): + data = list(range(20)) + self.assertEqual(list(map(str, data[:5])), ["0", "1", "2", "3", "4"]) + self.assertEqual(sum(map(abs, [-1, -2, 3])), 6) + self.assertEqual(list(map(lambda x: x * 2, (1, 2, 3))), [2, 4, 6]) + self.assertEqual("".join(map(str.upper, "abc")), "ABC") + self.assertEqual(list(map(lambda a, b: a + b, [1, 2], [10, 20, 30])), [11, 22]) + self.assertEqual(list(filter(None, [0, 1, "", "x", None])), [1, "x"]) + self.assertEqual(list(filter(bool, [0, 2, 0.0, 3])), [2, 3]) + self.assertEqual(list(filter(lambda x: x % 3 == 0, range(10))), [0, 3, 6, 9]) + total = 0 + for v in map(lambda x: x + 1, data): + total += v + self.assertEqual(total, sum(data) + len(data)) + + def test_type_identity(self): + m = map(str, [1]) + f = filter(None, [1]) + # (WeavePy's `map` and `filter` names are functions building these + # classes, so the types are compared with each other.) + self.assertIs(type(m), type(map(abs, iter([])))) + self.assertIs(type(f), type(filter(abs, iter([])))) + self.assertIs(iter(m), m) + self.assertEqual(type(m).__name__, "map") + self.assertEqual(type(f).__name__, "filter") + + def test_lazy_errors(self): + def boom(x): + if x == 2: + raise ValueError("two") + return x + + it = map(boom, [1, 2, 3]) + self.assertEqual(next(it), 1) + with self.assertRaises(ValueError): + next(it) + self.assertEqual(next(it), 3) + with self.assertRaises(StopIteration): + next(it) + fi = filter(boom, [1, 2]) + self.assertEqual(next(fi), 1) + with self.assertRaises(ValueError): + next(fi) + + def test_shared_iterator(self): + src = iter([1, 2, 3, 4]) + m = map(lambda x: x * 10, src) + self.assertEqual(next(m), 10) + self.assertEqual(next(src), 2) + self.assertEqual(list(m), [30, 40]) + + def test_pickle_round_trip(self): + m = map(abs, [-1, -2, -3]) + next(m) + self.assertEqual(list(pickle.loads(pickle.dumps(m))), [2, 3]) + f = filter(None, [0, 5, 0, 6]) + self.assertEqual(list(pickle.loads(pickle.dumps(f))), [5, 6]) + + def test_side_effect_order(self): + seen = [] + + def note(x): + seen.append(x) + return x + + m = map(note, range(5)) + self.assertEqual(seen, []) + next(m) + next(m) + self.assertEqual(seen, [0, 1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/regrtest/test_unraisable_format.py b/tests/regrtest/test_unraisable_format.py new file mode 100644 index 00000000..2d679f52 --- /dev/null +++ b/tests/regrtest/test_unraisable_format.py @@ -0,0 +1,52 @@ +"""The default unraisable report renders exceptions as `str()` does. + +Exceptions built natively don't store a separate message, so the report +must derive it the way each class's `__str__` does: an `OSError` from a +failed system call reads `[Errno N] strerror`, not its bare errno. +""" + +import errno +import os +import unittest +from test.support import captured_stderr + + +class UnraisableFormat(unittest.TestCase): + def report(self, exc_source): + class Holder: + def __del__(self): + exc_source() + + with captured_stderr() as err: + Holder() + return err.getvalue() + + def test_os_error_from_a_system_call(self): + def close_bad(): + os.close(-1) + + out = self.report(close_bad) + self.assertIn("Exception ignored", out) + self.assertIn("OSError: [Errno %d] " % errno.EBADF, out) + + def test_os_error_with_a_filename(self): + def open_missing(): + open("/nonexistent/weavepy-unraisable") + + out = self.report(open_missing) + self.assertIn("FileNotFoundError: [Errno %d] " % errno.ENOENT, out) + self.assertIn("'/nonexistent/weavepy-unraisable'", out) + + def test_key_error_and_plain_message(self): + def key(): + {}["missing"] + + def value(): + raise ValueError("bad value") + + self.assertIn("KeyError: 'missing'", self.report(key)) + self.assertIn("ValueError: bad value", self.report(value)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/bench_cold_jit.py b/tools/bench_cold_jit.py index 839b2b80..a09b074c 100644 --- a/tools/bench_cold_jit.py +++ b/tools/bench_cold_jit.py @@ -89,6 +89,8 @@ def main(): parser.add_argument("--fixture-root", type=Path, default=Path("crates/weavepy-bench/fixtures")) parser.add_argument("--samples", type=int, default=7) parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--fixtures", default="sumvm:2000000,nested_loops:120,jitloop:1000", + help="comma-separated NAME:WORK pairs to diagnose") args = parser.parse_args() if args.samples < 1: parser.error("samples must be positive") @@ -107,7 +109,9 @@ def main(): "environment": {name: os.environ.get(name) for name in ("PYTHONHASHSEED", "PYTHONDONTWRITEBYTECODE", "PYTHON_JIT", "PYTHON_GIL", "WEAVEPY_STDLIB_CACHE")}} args.out.write_text(json.dumps(report, indent=2) + "\n") - for name, work in [("sumvm", 2000000), ("nested_loops", 120), ("jitloop", 1000)]: + fixtures = [(name, int(work)) for name, work in + (item.split(":") for item in args.fixtures.split(",") if item)] + for name, work in fixtures: path = (args.fixture_root / (name + ".py")).resolve(strict=True) row = {"work": work, "fixture": str(path), "sha256": digest(path), "modes": {}} for warm in (False, True): diff --git a/tools/gen_opcode_shim.py b/tools/gen_opcode_shim.py index 5e3e7763..89ecb0f2 100644 --- a/tools/gen_opcode_shim.py +++ b/tools/gen_opcode_shim.py @@ -28,7 +28,9 @@ The stack-effect expressions in the header are plain C arithmetic over ``oparg`` (``1 + (oparg & 0xFF) + (oparg >> 8)``, ``oparg*2`` ...), which is also valid Python with identical results for the non-negative -``oparg`` values the compiler produces; they are emitted as lambdas. +``oparg`` values the compiler produces; they are emitted as lambdas, and a +count that doesn't depend on ``oparg`` as the integer itself (a module +that defines hundreds of functions is slow to import). """ from __future__ import annotations @@ -172,6 +174,15 @@ def py_expr(expr: str) -> str: return expr +def effect_value(expr: str) -> str: + """A stack-effect table value: the count itself when it doesn't depend + on ``oparg`` (most do not), else a lambda over ``oparg``.""" + expr = py_expr(expr) + if expr.isdigit(): + return expr + return "lambda oparg: %s" % expr + + def render(version: str, size: int, meta, popped, pushed, deopt) -> str: lines: list[str] = [] w = lines.append @@ -211,46 +222,75 @@ def render(version: str, size: int, meta, popped, pushed, deopt) -> str: w(" _names[%r]: %d," % (name, word)) w("}") w("") - w("# `_PyOpcode_Deopt`: specialized form -> base instruction.") - w("_DEOPT = {") - for a, b in deopt: - w(" _names[%r]: _names[%r]," % (a, b)) - w("}") + w("# The tables only `stack_effect` reads, built on its first call (most") + w("# importers of `opcode` and `dis` never compute a stack effect).") + w("_STACK_TABLES = None") w("") - w("# `_PyOpcode_num_popped(opcode, oparg)`.") - w("_POPPED = {") - for name, expr in popped: - w(" _names[%r]: lambda oparg: %s," % (name, py_expr(expr))) - w("}") w("") - w("# `_PyOpcode_num_pushed(opcode, oparg)`.") - w("_PUSHED = {") + w("def _stack_tables():") + w(" global _STACK_TABLES") + w(" if _STACK_TABLES is not None:") + w(" return _STACK_TABLES") + w(" # `_PyOpcode_Deopt`: specialized form -> base instruction.") + w(" deopt = {") + for a, b in deopt: + w(" _names[%r]: _names[%r]," % (a, b)) + w(" }") + w(" # `_PyOpcode_num_popped(opcode, oparg)`: the count, or a function of") + w(" # `oparg` for the opcodes whose count depends on it.") + w(" popped = {") + for name, expr in popped: + w(" _names[%r]: %s," % (name, effect_value(expr))) + w(" }") + w(" # `_PyOpcode_num_pushed(opcode, oparg)`, likewise.") + w(" pushed = {") for name, expr in pushed: - w(" _names[%r]: lambda oparg: %s," % (name, py_expr(expr))) - w("}") + w(" _names[%r]: %s," % (name, effect_value(expr))) + w(" }") + w(" _STACK_TABLES = (deopt, popped, pushed)") + w(" return _STACK_TABLES") + w("") w("") w("_BLOCK_PUSH = frozenset(_names[n] for n in %r)" % (BLOCK_PUSH,)) w("") + w("# The valid opcodes, and those with each flag `has_*` reports: `opcode`") + w("# (Lib/opcode.py) asks every predicate about every opcode at import, so") + w("# a valid opcode is answered by one set lookup.") + w("_VALID = frozenset(_FLAGS)") + predicates = [ + ("has_arg", "HAS_ARG_FLAG", "Return True if the opcode uses its oparg, False otherwise."), + ("has_const", "HAS_CONST_FLAG", "Return True if the opcode accesses a constant, False otherwise."), + ("has_name", "HAS_NAME_FLAG", "Return True if the opcode accesses an attribute by name, False otherwise."), + ("has_jump", "HAS_JUMP_FLAG", "Return True if the opcode has a jump target, False otherwise."), + ("has_free", "HAS_FREE_FLAG", "Return True if the opcode accesses a free variable, False otherwise."), + ("has_local", "HAS_LOCAL_FLAG", "Return True if the opcode accesses a local variable, False otherwise."), + ] + flag_bits = {name: 1 << i for i, name in enumerate(FLAG_NAMES)} + for fn, flag, _doc in predicates: + names = [name for name, word in meta if word & flag_bits[flag]] + w("_%s = frozenset({" % fn.upper()) + for name in names: + w(" _names[%r]," % name) + w("})") + w("") w("") w("def is_valid(opcode):") w(' """Return True if opcode is valid, False otherwise."""') - w(" return 0 <= opcode < _NUM_OPCODES and opcode in _FLAGS") + w(" return opcode in _VALID or (0 <= opcode < _NUM_OPCODES and opcode in _FLAGS)") w("") w("") w("def _has(opcode, flag):") w(" return is_valid(opcode) and bool(_FLAGS[opcode] & flag)") w("") w("") - for fn, flag, doc in [ - ("has_arg", "HAS_ARG_FLAG", "Return True if the opcode uses its oparg, False otherwise."), - ("has_const", "HAS_CONST_FLAG", "Return True if the opcode accesses a constant, False otherwise."), - ("has_name", "HAS_NAME_FLAG", "Return True if the opcode accesses an attribute by name, False otherwise."), - ("has_jump", "HAS_JUMP_FLAG", "Return True if the opcode has a jump target, False otherwise."), - ("has_free", "HAS_FREE_FLAG", "Return True if the opcode accesses a free variable, False otherwise."), - ("has_local", "HAS_LOCAL_FLAG", "Return True if the opcode accesses a local variable, False otherwise."), - ]: + for fn, flag, doc in predicates: w("def %s(opcode):" % fn) w(' """%s"""' % doc) + w(" if opcode in _%s:" % fn.upper()) + w(" return True") + w(" if opcode in _VALID:") + w(" return False") + w(" # An invalid opcode, or not an int (which raises as before).") w(" return _has(opcode, %s)" % flag) w("") w("") @@ -270,16 +310,17 @@ def render(version: str, size: int, meta, popped, pushed, deopt) -> str: w(" jump_int = 0") w(" else:") w(' raise ValueError("stack_effect: jump must be False, True or None")') + w(" deopt, popped_table, pushed_table = _stack_tables()") w(" # `get_stack_effects` (Python/flowgraph.c): specialized forms and") w(" # unknown opcodes have no effect entry.") - w(" if opcode < 0 or (opcode <= _MAX_REAL_OPCODE and _DEOPT.get(opcode, opcode) != opcode):") + w(" if opcode < 0 or (opcode <= _MAX_REAL_OPCODE and deopt.get(opcode, opcode) != opcode):") w(' raise ValueError("invalid opcode or oparg")') - w(" popped = _POPPED.get(opcode)") - w(" pushed = _PUSHED.get(opcode)") + w(" popped = popped_table.get(opcode)") + w(" pushed = pushed_table.get(opcode)") w(" if popped is None or pushed is None:") w(' raise ValueError("invalid opcode or oparg")') - w(" npop = popped(oparg_int)") - w(" npush = pushed(oparg_int)") + w(" npop = popped if popped.__class__ is int else popped(oparg_int)") + w(" npush = pushed if pushed.__class__ is int else pushed(oparg_int)") w(" if npop < 0 or npush < 0:") w(' raise ValueError("invalid opcode or oparg")') w(" if opcode in _BLOCK_PUSH and not jump_int:") diff --git a/vendor/lzma-sys/src/lib.rs b/vendor/lzma-sys/src/lib.rs index 38f8307b..86ad932f 100644 --- a/vendor/lzma-sys/src/lib.rs +++ b/vendor/lzma-sys/src/lib.rs @@ -2,7 +2,6 @@ #![doc(html_root_url = "https://docs.rs/lzma-sys/0.1")] use libc::{c_char, c_uchar, c_void, size_t}; -use std::u64; #[cfg(target_env = "msvc")] #[doc(hidden)]