perf: close the gap to CPython on realistic workloads

crates/weavepy-capi/src/abi314.rs

@@ -1574,13 +1574,13 @@ unsafe fn code_names(
 /// `PyCode_GetCellvars(code)` — new reference to `co_cellvars`.
 #[no_mangle]
 pub unsafe extern "C" fn PyCode_GetCellvars(code: *mut PyObject) -> *mut PyObject {
-    unsafe { code_names(code, |c| c.cellvars.clone()) }
+    unsafe { code_names(code, |c| c.cellvars.iter().map(String::from).collect()) }
 }
 
 /// `PyCode_GetFreevars(code)` — new reference to `co_freevars`.
 #[no_mangle]
 pub unsafe extern "C" fn PyCode_GetFreevars(code: *mut PyObject) -> *mut PyObject {
-    unsafe { code_names(code, |c| c.freevars.clone()) }
+    unsafe { code_names(code, |c| c.freevars.iter().map(String::from).collect()) }
 }
 
 // ---------------------------------------------------------------------------

crates/weavepy-capi/src/abstract_.rs

@@ -701,7 +701,7 @@ fn type_name(o: &Object) -> &'static str {
 fn callable_label(o: &Object) -> String {
     use Object as O;
     match o {
-        O::Function(f) => f.code().qualname.clone(),
+        O::Function(f) => f.code().qualname.to_string(),
         O::Builtin(b) => b.name.to_string(),
         O::Type(t) => format!("type:{}", t.name),
         O::BoundMethod(bm) => format!("bound:{}", callable_label(&bm.function)),

crates/weavepy-capi/src/gc_bridge.rs

@@ -144,8 +144,10 @@ fn has_clear(obj: &Object) -> bool {
 /// once from [`crate::interp::ensure_initialised`], before any extension
 /// code (hence any C GC type) can run.
 pub fn install() {
-    weavepy_vm::gc_trace::register_traverse(has_traverse, traverse);
-    weavepy_vm::gc_trace::register_clear(has_clear, clear);
+    // (Both match instances only; see `instance_type_ptr`.)
+    let kind = weavepy_vm::gc_trace::hook_kind::INSTANCE;
+    weavepy_vm::gc_trace::register_traverse_for(kind, has_traverse, traverse);
+    weavepy_vm::gc_trace::register_clear_for(kind, has_clear, clear);
 }
 
 // ====================================================================
@@ -253,8 +255,7 @@ pub unsafe extern "C" fn PyObject_GC_IsTracked(op: *mut c_void) -> c_int {
         return 0;
     }
     let obj = unsafe { crate::object::clone_object(op as *mut PyObject) };
-    let id = weavepy_vm::weakref_registry::id_of(&obj);
-    c_int::from(weavepy_vm::gc_trace::is_tracked(id))
+    c_int::from(weavepy_vm::gc_trace::is_tracked(&obj))
 }
 
 /// `PyObject_GC_Del(op)` — free a GC object's storage (a GC type's

crates/weavepy-compiler/src/cpython_code.rs

@@ -731,7 +731,7 @@ pub const CO_FAST_FREE: u8 = 0x80;
 /// `CO_FAST_LOCAL|CO_FAST_CELL` rather than getting its own entry;
 /// the argument slots carry their `CO_FAST_ARG_*` kind and hidden
 /// comprehension locals `CO_FAST_HIDDEN`.
-fn build_localsplus(code: &CodeObject) -> (Vec<String>, Vec<u8>) {
+fn build_localsplus(code: &CodeObject) -> (Vec<crate::Name>, Vec<u8>) {
     let mut names = Vec::with_capacity(code.varnames.len() + code.cellvars.len());
     let mut kinds = Vec::with_capacity(names.capacity());
     // `argvarkinds`: pos-only, pos-or-kw, kw-only, *args, **kwargs.
@@ -1053,7 +1053,7 @@ pub fn encode(code: &CodeObject) -> CpythonCode {
         co_linetable: encode_linetable(code, &ext, &mapped, &zero_width, firstlineno),
         co_exceptiontable: encode_exception_table(code, &starts),
         co_code,
-        localsplusnames,
+        localsplusnames: localsplusnames.into_iter().map(String::from).collect(),
         localspluskinds,
         stacksize: code.stacksize.unwrap_or_else(|| compute_stacksize(code)),
         firstlineno,
@@ -2061,7 +2061,7 @@ impl SlotMap {
     /// Build from a code object's variable lists (mirrors
     /// [`DerefSlots::from_code`], inverted).
     #[must_use]
-    pub fn from_code_vars(varnames: &[String], cellvars: &[String], freevars: &[String]) -> Self {
+    pub fn from_code_vars<S: PartialEq>(varnames: &[S], cellvars: &[S], freevars: &[S]) -> Self {
         let nlocals = varnames.len() as u32;
         let mut nslots = varnames.len() + freevars.len();
         for c in cellvars {
@@ -2144,16 +2144,29 @@ fn decode_instructions(
     // A comprehension whose own target is a cell announces itself with
     // `MAKE_CELL s` right after the save and keeps closure semantics.
     // Nested comprehensions re-binding the same name push the slot
-    // again, so each restore pops one entry.
+    // again, so each restore pops one entry. The restore usually
+    // follows a `SWAP`, but the optimizer may move the result's store
+    // ahead of it (`POP_ITER; STORE_FAST result; STORE_FAST s`), so a
+    // store in the run of stores and swaps right after the loop's
+    // `END_FOR`/`POP_ITER` ends the region too.
     let mut plain_shared: Vec<u32> = Vec::new();
+    let mut loop_tail = false;
     let shared_local = |slot: u32| slot < slots.nlocals && slots.is_cellish(slot);
     let end_region = |plain_shared: &mut Vec<u32>, slot: u32| {
         if let Some(p) = plain_shared.iter().rposition(|&s| s == slot) {
             plain_shared.remove(p);
         }
     };
     for (idx, r) in raws.iter().enumerate() {
-        let after_swap = idx > 0 && raws[idx - 1].cp_op == op::SWAP;
+        let in_tail = loop_tail;
+        loop_tail = match r.cp_op {
+            op::END_FOR | op::POP_ITER => true,
+            op::SWAP | op::STORE_FAST | op::STORE_FAST_STORE_FAST | op::POP_TOP | op::NOP => {
+                loop_tail
+            }
+            _ => false,
+        };
+        let after_swap = in_tail || (idx > 0 && raws[idx - 1].cp_op == op::SWAP);
         match r.cp_op {
             op::LOAD_FAST_AND_CLEAR
                 if shared_local(r.arg)
@@ -2800,7 +2813,7 @@ mod tests {
             ..CodeObject::default()
         };
         // Give a couple of locals so LOAD_FAST vs LOAD_CLOSURE disambiguates.
-        c.varnames = vec!["a".to_owned(), "b".to_owned()];
+        c.varnames = vec!["a".into(), "b".into()];
         c
     }
 
@@ -3097,9 +3110,9 @@ mod tests {
             linetable: vec![1, 2, 2, 3, 3, 4, 4].into(),
             ..CodeObject::default()
         };
-        code.varnames = vec!["a".to_owned(), "b".to_owned()];
-        code.cellvars = vec!["c".to_owned()];
-        code.freevars = vec!["f".to_owned()];
+        code.varnames = vec!["a".into(), "b".into()];
+        code.cellvars = vec!["c".into()];
+        code.freevars = vec!["f".into()];
         code.exception_table.push(ExcHandler {
             start: 1,
             end: 4,
@@ -3134,9 +3147,9 @@ mod tests {
             linetable: dc.linetable.into(),
             ..CodeObject::default()
         };
-        code2.varnames = dc.varnames;
-        code2.cellvars = dc.cellvars;
-        code2.freevars = dc.freevars;
+        code2.varnames = dc.varnames.iter().map(crate::Name::from).collect();
+        code2.cellvars = dc.cellvars.iter().map(crate::Name::from).collect();
+        code2.freevars = dc.freevars.iter().map(crate::Name::from).collect();
         code2.exception_table = dc.exception_table;
         let cp2 = encode(&code2);
         assert_eq!(cp2.co_code, cp.co_code);
@@ -3302,7 +3315,7 @@ def f():
 /// mapping pattern under-reported by one).
 #[test]
 fn stack_effects_agree_with_wire_shape() {
-    let slots = SlotMap::from_code_vars(&[], &[], &[]);
+    let slots = SlotMap::from_code_vars::<crate::Name>(&[], &[], &[]);
     // Opcodes whose WeavePy arg is not the wire arg, or whose
     // fallthrough effect legitimately differs from the wire shape
     // (jumps take their real effect on the edge; FOR_ITER/END_FOR

crates/weavepy-compiler/src/flowgraph.rs

@@ -398,19 +398,19 @@ fn is_unary_not(i: &CfgInstr) -> bool {
 pub(crate) struct BuildInput<'a> {
     /// Unconditional jumps CPython's codegen emits as the plain `JUMP`
     /// pseudo-op; every other one is `JUMP_NO_INTERRUPT`.
-    pub plain_jumps: &'a std::collections::HashSet<u32>,
+    pub plain_jumps: &'a crate::fxhash::HashSet<u32>,
     /// Unconditional jumps codegen flagged `JUMP_NO_INTERRUPT`.
-    pub no_interrupt_jumps: &'a std::collections::HashSet<u32>,
+    pub no_interrupt_jumps: &'a crate::fxhash::HashSet<u32>,
     /// `Nop`s standing in for a `SETUP_*` pseudo-op.
-    pub setup_nops: &'a std::collections::HashMap<u32, OpCode>,
+    pub setup_nops: &'a crate::fxhash::HashMap<u32, OpCode>,
     /// `Nop`s standing in for a `POP_BLOCK` pseudo-op.
-    pub popblock_nops: &'a std::collections::HashSet<u32>,
+    pub popblock_nops: &'a crate::fxhash::HashSet<u32>,
     /// Conditional jumps that, together with the `COPY 1; TO_BOOL`
     /// preceding them, stand for a `JUMP_IF_FALSE` / `JUMP_IF_TRUE`
     /// pseudo-op.
-    pub pseudo_cond_jumps: &'a std::collections::HashSet<u32>,
+    pub pseudo_cond_jumps: &'a crate::fxhash::HashSet<u32>,
     /// Conditional jumps whose `arg` is a *backward* distance.
-    pub backward_conds: &'a std::collections::HashSet<u32>,
+    pub backward_conds: &'a crate::fxhash::HashSet<u32>,
     /// The code unit carries the PEP 479 `SETUP_CLEANUP` wrap (its
     /// range is the last exception-table entry).
     pub stopiteration_wrap: bool,

crates/weavepy-compiler/src/fxhash.rs

@@ -0,0 +1,97 @@
+//! A fast, unkeyed hasher for the compiler's name sets and binding maps.
+//!
+//! Scope analysis builds many small sets of identifiers per code object.
+//! The standard library's keyed SipHash costs several times more per short
+//! string than this multiply-rotate hash (rustc's `FxHasher`), and the
+//! compiler never hashes attacker-chosen keys at a scale where hash
+//! flooding matters: a source's names are bounded by its own size.
+
+use std::hash::{BuildHasherDefault, Hasher};
+
+/// rustc's `FxHasher`: one rotate, xor and multiply per word.
+#[derive(Default, Clone, Copy)]
+pub(crate) struct FxHasher {
+    hash: u64,
+}
+
+const SEED: u64 = 0x51_7c_c1_b7_27_22_0a_95;
+
+impl FxHasher {
+    #[inline]
+    fn add(&mut self, word: u64) {
+        self.hash = (self.hash.rotate_left(5) ^ word).wrapping_mul(SEED);
+    }
+}
+
+impl Hasher for FxHasher {
+    #[inline]
+    fn write(&mut self, mut bytes: &[u8]) {
+        while let Some((head, rest)) = bytes.split_first_chunk::<8>() {
+            self.add(u64::from_le_bytes(*head));
+            bytes = rest;
+        }
+        if let Some((head, rest)) = bytes.split_first_chunk::<4>() {
+            self.add(u64::from(u32::from_le_bytes(*head)));
+            bytes = rest;
+        }
+        if let Some((head, rest)) = bytes.split_first_chunk::<2>() {
+            self.add(u64::from(u16::from_le_bytes(*head)));
+            bytes = rest;
+        }
+        if let Some(&b) = bytes.first() {
+            self.add(u64::from(b));
+        }
+    }
+
+    #[inline]
+    fn write_u8(&mut self, i: u8) {
+        self.add(u64::from(i));
+    }
+
+    #[inline]
+    fn write_u32(&mut self, i: u32) {
+        self.add(u64::from(i));
+    }
+
+    #[inline]
+    fn write_u64(&mut self, i: u64) {
+        self.add(i);
+    }
+
+    #[inline]
+    fn write_usize(&mut self, i: usize) {
+        self.add(i as u64);
+    }
+
+    #[inline]
+    fn finish(&self) -> u64 {
+        self.hash
+    }
+}
+
+pub(crate) type FxBuild = BuildHasherDefault<FxHasher>;
+
+/// A `HashSet` over [`FxHasher`]; build one with `HashSet::default()`.
+pub(crate) type HashSet<T> = std::collections::HashSet<T, FxBuild>;
+
+/// Adding a borrowed name to a set of owned names, allocating only when
+/// the name is new: scope scans see the same few names (`self`, the
+/// parameters) over and over.
+pub(crate) trait NameSet {
+    fn add_name(&mut self, name: &str);
+}
+
+impl NameSet for HashSet<String> {
+    #[inline]
+    fn add_name(&mut self, name: &str) {
+        if !self.contains(name) {
+            self.insert(name.to_owned());
+        }
+    }
+}
+
+/// A `HashMap` over [`FxHasher`]; build one with `HashMap::default()`.
+pub(crate) type HashMap<K, V> = std::collections::HashMap<K, V, FxBuild>;
+
+/// An `IndexMap` over [`FxHasher`]; build one with `IndexMap::default()`.
+pub(crate) type IndexMap<K, V> = indexmap::IndexMap<K, V, FxBuild>;

crates/weavepy-compiler/src/intern.rs

@@ -42,7 +42,7 @@ impl InternIndex {
         i
     }
 
-    pub(crate) fn intern_name(&mut self, values: &mut Vec<String>, name: &str) -> u32 {
+    pub(crate) fn intern_name(&mut self, values: &mut Vec<crate::Name>, name: &str) -> u32 {
         if values.len() < 32 {
             if let Some(i) = values.iter().position(|v| v == name) {
                 return i as u32;
@@ -55,7 +55,7 @@ impl InternIndex {
             self.next.push(self.heads.insert(key, values.len() as u32));
         }
         let i = values.len() as u32;
-        values.push(name.to_owned());
+        values.push(crate::Name::new(name));
         i
     }