From f9e8926336d4ab4363834ea85dc4a227c9d5b480 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Jun 2026 23:36:24 +0000 Subject: [PATCH 01/27] Phase 0: bit-sliced storage prototype + benchmark gate Add a standalone, self-contained prototype of bit-sliced (bit-plane) storage for F_p vectors, to validate the performance claim before the larger FqVector refactor. Not wired into the real vector types. - `bitslice_proto::BitSlicedVec`: groups of 64 elements stored across k = ceil(log2 p) bit-planes (one limb per plane per group). - Generic add/scale kernels for any prime: ripple-carry adder + single conditional subtraction of p, and double-and-add scalar multiply. Fully branch-free, no reduction tables. - Hand-written F3 fast circuit (2-plane add + plane-swap negation). - Exhaustive correctness tests against (a + c*b) % p for all small primes, plus pack/unpack roundtrip and F3-vs-generic agreement. - `benches/bitslice.rs`: compares bit-sliced add/scale vs packed FpVector across p = 3,5,7,251 and lengths 100..100k. This module and bench are temporary and will be removed in Phase 5. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/Cargo.toml | 5 + ext/crates/fp/benches/bitslice.rs | 111 ++++++++ ext/crates/fp/src/bitslice_proto.rs | 423 ++++++++++++++++++++++++++++ ext/crates/fp/src/lib.rs | 4 + 4 files changed, 543 insertions(+) create mode 100644 ext/crates/fp/benches/bitslice.rs create mode 100644 ext/crates/fp/src/bitslice_proto.rs diff --git a/ext/crates/fp/Cargo.toml b/ext/crates/fp/Cargo.toml index b0123aba05..3337006b05 100644 --- a/ext/crates/fp/Cargo.toml +++ b/ext/crates/fp/Cargo.toml @@ -47,6 +47,11 @@ harness = false name = "reduce" harness = false +# PHASE 0 PROTOTYPE bench — to be removed in Phase 5. +[[bench]] +name = "bitslice" +harness = false + [[bench]] name = "smallfq" harness = false diff --git a/ext/crates/fp/benches/bitslice.rs b/ext/crates/fp/benches/bitslice.rs new file mode 100644 index 0000000000..c7f6de92ea --- /dev/null +++ b/ext/crates/fp/benches/bitslice.rs @@ -0,0 +1,111 @@ +//! PHASE 0 PROTOTYPE bench — to be removed in Phase 5. +//! +//! Compares the bit-sliced `add`/`scale` kernels (generic, and the F3 fast path) against +//! the existing packed `FpVector` implementation, across a few representative primes and +//! vector lengths. This is the go/no-go gate for the bit-slicing project: it tells us +//! where bit-slicing actually wins before committing to the larger refactor. + +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use fp::{ + bitslice_proto::BitSlicedVec, + prime::{Prime, ValidPrime}, + vector::FpVector, +}; +use rand::Rng; + +const PRIMES: [u32; 4] = [3, 5, 7, 251]; +const LENGTHS: [usize; 4] = [100, 1000, 10_000, 100_000]; +/// A representative non-unit, non-negation scalar to exercise the full multiply path. +const SCALAR: u32 = 2; + +fn random_data(p: u32, len: usize) -> Vec { + let mut rng = rand::rng(); + (0..len).map(|_| rng.random_range(0..p)).collect() +} + +fn bench_add(c: &mut Criterion) { + for p in PRIMES { + let vp = ValidPrime::new(p); + let mut group = c.benchmark_group(format!("add_p{p}")); + for len in LENGTHS { + let a = random_data(p, len); + let b = random_data(p, len); + + // Packed reference (existing implementation). + let packed_a = FpVector::from_slice(vp, &a); + let packed_b = FpVector::from_slice(vp, &b); + group.bench_with_input(BenchmarkId::new("packed", len), &len, |bench, _| { + bench.iter_batched_ref( + || packed_a.clone(), + |va| va.add(&packed_b, SCALAR), + criterion::BatchSize::SmallInput, + ) + }); + + // Bit-sliced generic kernel. + let bs_a = BitSlicedVec::from_u32(p, &a); + let bs_b = BitSlicedVec::from_u32(p, &b); + group.bench_with_input(BenchmarkId::new("bitsliced_generic", len), &len, |bench, _| { + bench.iter_batched_ref( + || bs_a.clone(), + |va| va.add_generic(&bs_b, SCALAR), + criterion::BatchSize::SmallInput, + ) + }); + + // Bit-sliced F3 fast circuit. + if p == 3 { + group.bench_with_input(BenchmarkId::new("bitsliced_f3", len), &len, |bench, _| { + bench.iter_batched_ref( + || bs_a.clone(), + |va| va.add_f3(&bs_b, SCALAR), + criterion::BatchSize::SmallInput, + ) + }); + } + } + group.finish(); + } +} + +fn bench_scale(c: &mut Criterion) { + for p in PRIMES { + let vp = ValidPrime::new(p); + let mut group = c.benchmark_group(format!("scale_p{p}")); + for len in LENGTHS { + let a = random_data(p, len); + + let packed_a = FpVector::from_slice(vp, &a); + group.bench_with_input(BenchmarkId::new("packed", len), &len, |bench, _| { + bench.iter_batched_ref( + || packed_a.clone(), + |va| va.scale(SCALAR), + criterion::BatchSize::SmallInput, + ) + }); + + let bs_a = BitSlicedVec::from_u32(p, &a); + group.bench_with_input(BenchmarkId::new("bitsliced_generic", len), &len, |bench, _| { + bench.iter_batched_ref( + || bs_a.clone(), + |va| va.scale_generic(SCALAR), + criterion::BatchSize::SmallInput, + ) + }); + + if p == 3 { + group.bench_with_input(BenchmarkId::new("bitsliced_f3", len), &len, |bench, _| { + bench.iter_batched_ref( + || bs_a.clone(), + |va| va.scale_f3(SCALAR), + criterion::BatchSize::SmallInput, + ) + }); + } + } + group.finish(); + } +} + +criterion_group!(benches, bench_add, bench_scale); +criterion_main!(benches); diff --git a/ext/crates/fp/src/bitslice_proto.rs b/ext/crates/fp/src/bitslice_proto.rs new file mode 100644 index 0000000000..bdaf645eb1 --- /dev/null +++ b/ext/crates/fp/src/bitslice_proto.rs @@ -0,0 +1,423 @@ +//! **PHASE 0 PROTOTYPE — to be removed in Phase 5.** +//! +//! A standalone, self-contained prototype of bit-sliced (bit-plane) storage for vectors +//! over a prime field `F_p`. This is *not* wired into [`crate::vector::FqVector`]; it +//! exists only to validate the performance claim before the larger refactor (see the +//! approved plan). It deliberately re-implements the minimum needed to benchmark the +//! `add`/`scale` kernels against the existing packed representation. +//! +//! # Layout +//! +//! An element of `F_p` is represented with `k = ceil(log2 p)` bits. A *group* of 64 +//! elements occupies `k` consecutive [`Limb`]s (the *planes*): plane `j` of a group holds +//! bit `j` of all 64 elements, with element `i` living at bit `i` of each plane. A vector +//! of length `len` has `ceil(len / 64)` groups, so `k * ceil(len / 64)` limbs total. +//! +//! # Arithmetic +//! +//! - The **generic** kernels work for any prime: addition is a ripple-carry adder over the +//! `k` planes followed by a single conditional subtraction of `p` (the sum of two reduced +//! values is `< 2p`), and scalar multiplication is double-and-add with modular reduction +//! at each step. No lookup tables, fully branch-free, operating on 64 lanes at once. +//! - The **F3 fast path** uses a hand-written 2-plane circuit (addition and negation), +//! demonstrating the kind of speedup a per-prime specialization can give. + +#![allow(dead_code)] + +use crate::{constants::BITS_PER_LIMB, limb::Limb}; + +/// Maximum number of bit-planes (`k`) the prototype supports. `k = ceil(log2 p)`, so this +/// covers primes up to `2^24` — plenty for the benchmark, which only needs a handful of +/// representative primes. +const MAX_K: usize = 24; + +/// Number of field elements packed into one group. +const ENTRIES_PER_GROUP: usize = BITS_PER_LIMB; // 64 + +/// `k = ceil(log2 p)`: the number of bit-planes needed to store an element of `F_p`. +const fn bit_planes(p: u32) -> usize { + // Smallest k with 2^k >= p. + let mut k = 0; + while (1u64 << k) < p as u64 { + k += 1; + } + if k == 0 { 1 } else { k } +} + +/// A vector over `F_p` in bit-sliced layout. Prototype only. +#[derive(Clone, Debug)] +pub struct BitSlicedVec { + p: u32, + k: usize, + len: usize, + /// `k * ceil(len / 64)` limbs, group-major: group `g`'s plane `j` is `limbs[g * k + j]`. + limbs: Vec, +} + +impl BitSlicedVec { + pub fn new(p: u32, len: usize) -> Self { + let k = bit_planes(p); + let groups = len.div_ceil(ENTRIES_PER_GROUP); + Self { + p, + k, + len, + limbs: vec![0; k * groups], + } + } + + pub fn from_u32(p: u32, data: &[u32]) -> Self { + let mut v = Self::new(p, data.len()); + for (i, &value) in data.iter().enumerate() { + v.set_entry(i, value); + } + v + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + fn num_groups(&self) -> usize { + self.len.div_ceil(ENTRIES_PER_GROUP) + } + + pub fn entry(&self, index: usize) -> u32 { + debug_assert!(index < self.len); + let group = index / ENTRIES_PER_GROUP; + let lane = index % ENTRIES_PER_GROUP; + let base = group * self.k; + let mut value = 0u32; + for j in 0..self.k { + let bit = (self.limbs[base + j] >> lane) & 1; + value |= (bit as u32) << j; + } + value + } + + pub fn set_entry(&mut self, index: usize, value: u32) { + debug_assert!(index < self.len); + debug_assert!(value < self.p); + let group = index / ENTRIES_PER_GROUP; + let lane = index % ENTRIES_PER_GROUP; + let base = group * self.k; + for j in 0..self.k { + let bit = ((value >> j) & 1) as Limb; + let mask = 1 << lane; + let plane = &mut self.limbs[base + j]; + *plane = (*plane & !mask) | (bit << lane); + } + } + + pub fn to_u32(&self) -> Vec { + (0..self.len).map(|i| self.entry(i)).collect() + } + + /// Bits of `p` as full-width lane masks (`pbits[j]` is all-ones iff bit `j` of `p` is set), + /// for `j` in `0..=k`. Since `p < 2^k` (except `p = 2`), bit `k` is normally zero. + fn p_masks(&self) -> [Limb; MAX_K + 1] { + let mut masks = [0; MAX_K + 1]; + for (j, m) in masks.iter_mut().enumerate().take(self.k + 1) { + *m = if (self.p >> j) & 1 == 1 { !0 } else { 0 }; + } + masks + } + + /// `self += c * other` (mod p), generic kernel for any prime. + pub fn add_generic(&mut self, other: &Self, c: u32) { + assert_eq!(self.p, other.p); + assert_eq!(self.len, other.len); + if c == 0 { + return; + } + let k = self.k; + let p_masks = self.p_masks(); + for g in 0..self.num_groups() { + let base = g * k; + // Gather operand planes. + let mut a = [0; MAX_K]; + let mut b = [0; MAX_K]; + for j in 0..k { + a[j] = self.limbs[base + j]; + b[j] = other.limbs[base + j]; + } + // cb = c * b (mod p), then a += cb (mod p). + let cb = if c == 1 { + b + } else { + scalar_mul(&b, c, k, &p_masks) + }; + let sum = add_mod(&a, &cb, k, &p_masks); + for j in 0..k { + self.limbs[base + j] = sum[j]; + } + } + } + + /// `self *= c` (mod p), generic kernel. + pub fn scale_generic(&mut self, c: u32) { + let k = self.k; + if c == 1 { + return; + } + if c == 0 { + for limb in &mut self.limbs { + *limb = 0; + } + return; + } + let p_masks = self.p_masks(); + for g in 0..self.num_groups() { + let base = g * k; + let mut a = [0; MAX_K]; + for j in 0..k { + a[j] = self.limbs[base + j]; + } + let scaled = scalar_mul(&a, c, k, &p_masks); + for j in 0..k { + self.limbs[base + j] = scaled[j]; + } + } + } + + /// `self += c * other` (mod 3) using the hand-written F3 circuit. Requires `p == 3`. + pub fn add_f3(&mut self, other: &Self, c: u32) { + assert_eq!(self.p, 3); + assert_eq!(self.k, 2); + assert_eq!(self.len, other.len); + if c == 0 { + return; + } + for g in 0..self.num_groups() { + let base = g * 2; + let (a_lo, a_hi) = (self.limbs[base], self.limbs[base + 1]); + let (mut b_lo, mut b_hi) = (other.limbs[base], other.limbs[base + 1]); + if c == 2 { + // Multiply other by 2 = negate: in the (hi, lo) encoding, negation swaps planes. + std::mem::swap(&mut b_lo, &mut b_hi); + } + let (c_lo, c_hi) = f3_add(a_lo, a_hi, b_lo, b_hi); + self.limbs[base] = c_lo; + self.limbs[base + 1] = c_hi; + } + } + + /// `self *= c` (mod 3) using the F3 circuit. Requires `p == 3`. + pub fn scale_f3(&mut self, c: u32) { + assert_eq!(self.p, 3); + if c == 1 { + return; + } + if c == 0 { + for limb in &mut self.limbs { + *limb = 0; + } + return; + } + // c == 2: negate = swap the two planes of every group. + for g in 0..self.num_groups() { + let base = g * 2; + self.limbs.swap(base, base + 1); + } + } +} + +/// Add two reduced bit-sliced values (each `k` planes, lanes independent) mod `p`. +/// +/// Ripple-carry adder over the `k` planes gives a `(k+1)`-bit sum in `[0, 2p)`, then a +/// single conditional subtraction of `p` brings each lane back into `[0, p)`. +#[inline] +fn add_mod(a: &[Limb], b: &[Limb], k: usize, p_masks: &[Limb; MAX_K + 1]) -> [Limb; MAX_K] { + // s = a + b as a (k+1)-bit number. + let mut s = [0; MAX_K + 1]; + let mut carry: Limb = 0; + for j in 0..k { + let aj = a[j]; + let bj = b[j]; + let axb = aj ^ bj; + s[j] = axb ^ carry; + carry = (aj & bj) | (carry & axb); + } + s[k] = carry; + + // d = s - p over k+1 bits; the borrow-out marks lanes where s < p. + let mut d = [0; MAX_K + 1]; + let mut borrow: Limb = 0; + for j in 0..=k { + let sj = s[j]; + let pj = p_masks[j]; + let sxp = sj ^ pj; + d[j] = sxp ^ borrow; + borrow = (!sj & pj) | (borrow & !sxp); + } + let ge = !borrow; // lanes where s >= p + + // result = ge ? d : s, taking the low k planes (result < p < 2^k). + let mut out = [0; MAX_K]; + for j in 0..k { + out[j] = (d[j] & ge) | (s[j] & !ge); + } + out +} + +/// `c * b` (mod p) for a constant scalar `c`, via double-and-add with modular reduction. +#[inline] +fn scalar_mul(b: &[Limb], c: u32, k: usize, p_masks: &[Limb; MAX_K + 1]) -> [Limb; MAX_K] { + let mut result = [0; MAX_K]; + let mut temp = [0; MAX_K]; + temp[..k].copy_from_slice(&b[..k]); + let mut cc = c; + while cc > 0 { + if cc & 1 == 1 { + result = add_mod(&result, &temp, k, p_masks); + } + cc >>= 1; + if cc > 0 { + temp = add_mod(&temp, &temp, k, p_masks); + } + } + result +} + +/// F3 addition circuit on the `(lo, hi)` plane encoding (`value = 2*hi + lo`). +/// +/// Output `c == 1` exactly for input value pairs `{(0,1),(1,0),(2,2)}` and `c == 2` for +/// `{(0,2),(1,1),(2,0)}`; everything else is `0`. The invalid encoding `(hi,lo) = (1,1)` +/// never occurs for reduced inputs. +#[inline] +fn f3_add(a_lo: Limb, a_hi: Limb, b_lo: Limb, b_hi: Limb) -> (Limb, Limb) { + let is0_a = !(a_lo | a_hi); + let is1_a = a_lo; + let is2_a = a_hi; + let is0_b = !(b_lo | b_hi); + let is1_b = b_lo; + let is2_b = b_hi; + + let c_lo = (is0_a & is1_b) | (is1_a & is0_b) | (is2_a & is2_b); + let c_hi = (is0_a & is2_b) | (is1_a & is1_b) | (is2_a & is0_b); + (c_lo, c_hi) +} + +#[cfg(test)] +mod tests { + use super::*; + + const PRIMES: [u32; 6] = [2, 3, 5, 7, 251, 65521]; + + #[test] + fn bit_planes_correct() { + assert_eq!(bit_planes(2), 1); + assert_eq!(bit_planes(3), 2); + assert_eq!(bit_planes(5), 3); + assert_eq!(bit_planes(7), 3); + assert_eq!(bit_planes(251), 8); + assert_eq!(bit_planes(65521), 16); + } + + #[test] + fn pack_unpack_roundtrip() { + for p in PRIMES { + for len in [0, 1, 63, 64, 65, 130, 1000] { + let data: Vec = (0..len).map(|i| (i as u32 * 7 + 1) % p).collect(); + let v = BitSlicedVec::from_u32(p, &data); + assert_eq!(v.to_u32(), data, "p={p} len={len}"); + } + } + } + + /// Exhaustively check the generic add kernel against `(a + c*b) % p` for every pair of + /// field elements and every scalar. + #[test] + fn generic_add_exhaustive() { + for p in PRIMES { + // Use one lane per (a, b) so a single group covers all pairs (p <= 64 cases for + // small primes; for large primes sample instead). + let pairs: Vec<(u32, u32)> = if p * p <= 64 { + (0..p).flat_map(|a| (0..p).map(move |b| (a, b))).collect() + } else { + // Sample a spread of pairs into 64 lanes. + (0..64u32) + .map(|i| ((i.wrapping_mul(2654435761) % p), (i.wrapping_mul(40503) % p))) + .collect() + }; + for c in 0..p { + let a_data: Vec = pairs.iter().map(|&(a, _)| a).collect(); + let b_data: Vec = pairs.iter().map(|&(_, b)| b).collect(); + let mut va = BitSlicedVec::from_u32(p, &a_data); + let vb = BitSlicedVec::from_u32(p, &b_data); + va.add_generic(&vb, c); + let got = va.to_u32(); + for (idx, &(a, b)) in pairs.iter().enumerate() { + let expected = (a + c * b) % p; + assert_eq!(got[idx], expected, "p={p} a={a} b={b} c={c}"); + } + } + } + } + + #[test] + fn generic_scale_exhaustive() { + for p in PRIMES { + let data: Vec = (0..64).map(|i| (i as u32) % p).collect(); + for c in 0..p { + let mut v = BitSlicedVec::from_u32(p, &data); + v.scale_generic(c); + let got = v.to_u32(); + for (i, &x) in data.iter().enumerate() { + assert_eq!(got[i], (x * c) % p, "p={p} x={x} c={c}"); + } + } + } + } + + /// The F3 circuit must agree with `(a + c*b) % 3` for all inputs. + #[test] + fn f3_add_exhaustive() { + let p = 3; + let pairs: Vec<(u32, u32)> = (0..p).flat_map(|a| (0..p).map(move |b| (a, b))).collect(); + for c in 0..p { + let a_data: Vec = pairs.iter().map(|&(a, _)| a).collect(); + let b_data: Vec = pairs.iter().map(|&(_, b)| b).collect(); + let mut va = BitSlicedVec::from_u32(p, &a_data); + let vb = BitSlicedVec::from_u32(p, &b_data); + va.add_f3(&vb, c); + let got = va.to_u32(); + for (idx, &(a, b)) in pairs.iter().enumerate() { + assert_eq!(got[idx], (a + c * b) % p, "a={a} b={b} c={c}"); + } + } + } + + #[test] + fn f3_scale_exhaustive() { + let data: Vec = (0..64).map(|i| (i as u32) % 3).collect(); + for c in 0..3 { + let mut v = BitSlicedVec::from_u32(3, &data); + v.scale_f3(c); + let got = v.to_u32(); + for (i, &x) in data.iter().enumerate() { + assert_eq!(got[i], (x * c) % 3, "x={x} c={c}"); + } + } + } + + /// The F3 fast path and the generic kernel must produce identical results on long vectors. + #[test] + fn f3_fast_matches_generic() { + let len = 1000; + let a_data: Vec = (0..len).map(|i| (i as u32 * 2 + 1) % 3).collect(); + let b_data: Vec = (0..len).map(|i| (i as u32 * 5 + 2) % 3).collect(); + for c in 0..3 { + let mut fast = BitSlicedVec::from_u32(3, &a_data); + let mut generic = BitSlicedVec::from_u32(3, &a_data); + let b = BitSlicedVec::from_u32(3, &b_data); + fast.add_f3(&b, c); + generic.add_generic(&b, c); + assert_eq!(fast.to_u32(), generic.to_u32(), "c={c}"); + } + } +} diff --git a/ext/crates/fp/src/lib.rs b/ext/crates/fp/src/lib.rs index 8d971da2a8..75f179f7ae 100644 --- a/ext/crates/fp/src/lib.rs +++ b/ext/crates/fp/src/lib.rs @@ -10,6 +10,10 @@ pub mod matrix; pub mod prime; pub mod vector; +// PHASE 0 PROTOTYPE — to be removed in Phase 5. See module docs. +#[doc(hidden)] +pub mod bitslice_proto; + pub mod blas; pub(crate) mod simd; From 763f63b734aa619ca22cd6ae42b488d584af95b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Jun 2026 23:43:40 +0000 Subject: [PATCH 02/27] Phase 0: record bit-slice benchmark gate results Document the prototype benchmark findings. Verdict: GO. The F3 specialized circuit is ~2.5x faster than packed for add and ~1.75x for scale. The generic kernel beats packed by 3-4x for large primes (p=251) because packed's odd-prime reduce is a slow element-wise fallback for everything outside {2,3,5}. This supports the all-primes/replace strategy: specialized circuits for 3/5/7, generic kernel for the rest. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- .../fp/benches/BITSLICE_PROTO_RESULTS.md | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md diff --git a/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md b/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md new file mode 100644 index 0000000000..d3c1e2791b --- /dev/null +++ b/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md @@ -0,0 +1,59 @@ +# Phase 0 gate — bit-sliced storage prototype results + +**Verdict: GO.** Bit-slicing is worth pursuing. The decision is more favorable than the +plan predicted, because the existing *packed* path is only well-optimized for +`p ∈ {2, 3, 5}` — for every other prime its reduction step is a slow element-wise +fallback (`Fp::reduce`'s generic arm, `field/fp.rs:142`), and even an unoptimized +bit-sliced kernel beats it. + +Benchmarks: `cargo bench -p fp --bench bitslice` (median of 30 samples, short config: +warm-up 0.5s, measurement 2s). `add` is `self += 2*other`; `scale` is `self *= 2`. +"generic" = the prime-agnostic ripple-carry kernel; "f3" = the hand-written F3 circuit. + +## `add`, length 100,000 (asymptotic regime) + +| prime | packed | bitsliced generic | bitsliced F3 | best vs packed | +|------:|-------:|------------------:|-------------:|:--------------| +| 3 | 9.10 µs | 93.1 µs | **3.60 µs** | F3 **2.5× faster** | +| 5 | 19.0 µs | 102.6 µs | — | generic 5.4× slower | +| 7 | 91.6 µs | 102.4 µs | — | ~even (packed reduce is slow) | +| 251 | 528 µs | **145.6 µs** | — | generic **3.6× faster** | + +## `scale`, length 100,000 + +| prime | packed | bitsliced generic | bitsliced F3 | best vs packed | +|------:|-------:|------------------:|-------------:|:--------------| +| 3 | 2.61 µs | 82.6 µs | **1.49 µs** | F3 **1.75× faster** | +| 5 | 5.00 µs | 87.4 µs | — | generic 17× slower | +| 7 | 67.2 µs | 85.4 µs | — | generic 1.27× slower | +| 251 | 480 µs | **117.5 µs** | — | generic **4.1× faster** | + +## Reading the results + +1. **Specialized small-prime circuits win.** The F3 circuit is 2.5× faster than packed + for `add` and 1.75× for `scale`, at every length tested. This is the core validation: + replacing the madd+reduce sequence with a short branch-free circuit pays off. + +2. **The generic kernel loses for `p ∈ {3, 5}` but wins for large primes.** The packed + path has hand-tuned SWAR `reduce` only for 2/3/5; for `p = 7` and everything larger it + falls back to a per-element `pack(unpack(limb))`, which is slow. The generic bit-sliced + kernel (ripple-carry add + one conditional subtract, no tables) already beats packed by + **3.6×/4.1×** at `p = 251` and is roughly even at `p = 7` — *despite* prototype overhead + (fixed `[Limb; 24]` scratch arrays regardless of `k`, and double-and-add for `scale`). + A real implementation that sizes scratch to `k` will widen this further. + +3. **This vindicates the "all primes" + "replace" decision.** Bit-slicing helps across the + board, just via two mechanisms: specialized circuits for the tuned small primes + (3, 5, 7), and the generic kernel for the large primes where packed reduction is the + bottleneck. + +## Implications for the next phases + +- **Specialized circuits are needed for `p = 3, 5, 7`** (not just F3) to beat the tuned + packed path; F5/F7 are the same shape as F3 (3 planes instead of 2). +- **The generic kernel is sufficient for `p ≥ 11`** and for `Fp` (runtime `k`), + and is the path that makes large-prime arithmetic dramatically faster. +- The generic kernel must size its plane scratch to `k` (drop the `MAX_K` arrays) to + avoid the prototype's overhead on small `k`. +- No prime regresses badly enough to keep packed as a fallback for it — so "replace" holds + for all of `Fp`. (`SmallFq` stays packed regardless; its arithmetic is table-based.) From c792467ddca8d8a708efecc6d11a888a5588c6e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 00:14:16 +0000 Subject: [PATCH 03/27] Phase 0b: tighten bit-slice generic kernel, refresh results Replace the prototype generic kernel's fixed [Limb; 24] scratch and large by-value returns with k-sized reusable scratch written directly into the destination planes; replace double-and-add's doubling with a plane shift. The tightened generic kernel now beats the packed path for all p >= 7 (p=7: 1.3x add / 1.4x scale; p=251: 3.4x add / 4.6x scale at length 100k). Only the SWAR-tuned p in {3,5} still need specialized circuits, which the F3 circuit confirms wins (2.9x add). Exhaustive correctness tests still pass. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- .../fp/benches/BITSLICE_PROTO_RESULTS.md | 22 ++++ ext/crates/fp/src/bitslice_proto.rs | 115 +++++++++++------- 2 files changed, 94 insertions(+), 43 deletions(-) diff --git a/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md b/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md index d3c1e2791b..30fe13f4a2 100644 --- a/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md +++ b/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md @@ -47,6 +47,28 @@ warm-up 0.5s, measurement 2s). `add` is `self += 2*other`; `scale` is `self *= 2 (3, 5, 7), and the generic kernel for the large primes where packed reduction is the bottleneck. +## Phase 0b — tightened generic kernel + +After replacing the prototype's fixed `[Limb; 24]` scratch + large by-value returns with +`k`-sized reusable scratch written directly into the destination planes (and replacing +double-and-add's doubling with a plane shift), the generic kernel improved and the +crossover where it beats packed moved down to **p = 7** (numbers from one run, length 100k): + +| op | prime | packed | generic before | generic after | after vs packed | +|------:|------:|-------:|---------------:|--------------:|:----------------| +| add | 3 | 11.2 µs | 93.1 µs | 61.0 µs | 5.5× slower (use F3 circuit instead) | +| add | 7 | 95.4 µs | 102.4 µs | 73.4 µs | **1.30× faster** | +| add | 251 | 471 µs | 145.6 µs | 137.2 µs | **3.44× faster** | +| scale | 7 | 73.8 µs | — | 52.9 µs | **1.40× faster** | +| scale | 251 | 444 µs | — | 96.3 µs | **4.61× faster** | + +(The F3 specialized circuit is unchanged: add ≈ 3.8 µs / **2.9× faster** than packed, +scale ≈ 1.8 µs / **1.6× faster**, at 100k.) + +So the tightened generic kernel is now faster than packed for **all `p ≥ 7`**; only the +SWAR-tuned `p ∈ {3, 5}` still need specialized circuits to win — and F3 confirms that +specialization does win. + ## Implications for the next phases - **Specialized circuits are needed for `p = 3, 5, 7`** (not just F3) to beat the tuned diff --git a/ext/crates/fp/src/bitslice_proto.rs b/ext/crates/fp/src/bitslice_proto.rs index bdaf645eb1..18bdf11bb9 100644 --- a/ext/crates/fp/src/bitslice_proto.rs +++ b/ext/crates/fp/src/bitslice_proto.rs @@ -136,24 +136,21 @@ impl BitSlicedVec { } let k = self.k; let p_masks = self.p_masks(); + // Reusable scratch, sized exactly to the number of planes (no fixed MAX_K arrays). + let mut s = vec![0; k + 1]; + let mut d = vec![0; k + 1]; + let mut acc = vec![0; k]; + let mut temp = vec![0; k]; for g in 0..self.num_groups() { let base = g * k; - // Gather operand planes. - let mut a = [0; MAX_K]; - let mut b = [0; MAX_K]; - for j in 0..k { - a[j] = self.limbs[base + j]; - b[j] = other.limbs[base + j]; - } - // cb = c * b (mod p), then a += cb (mod p). - let cb = if c == 1 { - b + let b = &other.limbs[base..base + k]; + if c == 1 { + // dst += b + add_mod_into(&mut self.limbs[base..base + k], b, &p_masks, &mut s, &mut d); } else { - scalar_mul(&b, c, k, &p_masks) - }; - let sum = add_mod(&a, &cb, k, &p_masks); - for j in 0..k { - self.limbs[base + j] = sum[j]; + // acc = c * b, then dst += acc + scalar_mul_into(&mut acc, b, c, &p_masks, &mut temp, &mut s, &mut d); + add_mod_into(&mut self.limbs[base..base + k], &acc, &p_masks, &mut s, &mut d); } } } @@ -171,16 +168,22 @@ impl BitSlicedVec { return; } let p_masks = self.p_masks(); + let mut s = vec![0; k + 1]; + let mut d = vec![0; k + 1]; + let mut acc = vec![0; k]; + let mut temp = vec![0; k]; for g in 0..self.num_groups() { let base = g * k; - let mut a = [0; MAX_K]; - for j in 0..k { - a[j] = self.limbs[base + j]; - } - let scaled = scalar_mul(&a, c, k, &p_masks); - for j in 0..k { - self.limbs[base + j] = scaled[j]; - } + scalar_mul_into( + &mut acc, + &self.limbs[base..base + k], + c, + &p_masks, + &mut temp, + &mut s, + &mut d, + ); + self.limbs[base..base + k].copy_from_slice(&acc); } } @@ -226,17 +229,19 @@ impl BitSlicedVec { } } -/// Add two reduced bit-sliced values (each `k` planes, lanes independent) mod `p`. +/// `dst += b` (mod p), where `dst` (the augend) and `b` are each `k = dst.len()` reduced +/// planes with independent lanes. `s`/`d` are reusable `(k+1)`-limb scratch buffers. /// /// Ripple-carry adder over the `k` planes gives a `(k+1)`-bit sum in `[0, 2p)`, then a /// single conditional subtraction of `p` brings each lane back into `[0, p)`. #[inline] -fn add_mod(a: &[Limb], b: &[Limb], k: usize, p_masks: &[Limb; MAX_K + 1]) -> [Limb; MAX_K] { - // s = a + b as a (k+1)-bit number. - let mut s = [0; MAX_K + 1]; +fn add_mod_into(dst: &mut [Limb], b: &[Limb], p_masks: &[Limb], s: &mut [Limb], d: &mut [Limb]) { + let k = dst.len(); + // s = dst + b as a (k+1)-bit number. `dst` is only written in the final select pass, + // so passing `dst` as `b` (in-place doubling) is sound — handled by `double_mod_into`. let mut carry: Limb = 0; for j in 0..k { - let aj = a[j]; + let aj = dst[j]; let bj = b[j]; let axb = aj ^ bj; s[j] = axb ^ carry; @@ -244,8 +249,27 @@ fn add_mod(a: &[Limb], b: &[Limb], k: usize, p_masks: &[Limb; MAX_K + 1]) -> [Li } s[k] = carry; + conditional_subtract(dst, p_masks, s, d); +} + +/// `dst = 2 * dst` (mod p). Doubling is a one-position plane shift (`s = dst << 1`) followed +/// by the conditional subtraction of `p`. +#[inline] +fn double_mod_into(dst: &mut [Limb], p_masks: &[Limb], s: &mut [Limb], d: &mut [Limb]) { + let k = dst.len(); + s[0] = 0; + for j in 1..=k { + s[j] = dst[j - 1]; + } + conditional_subtract(dst, p_masks, s, d); +} + +/// Given a `(k+1)`-bit unreduced sum `s` in `[0, 2p)`, write `s mod p` into the `k` planes +/// of `dst`. `d` is `(k+1)`-limb scratch for the trial difference `s - p`. +#[inline] +fn conditional_subtract(dst: &mut [Limb], p_masks: &[Limb], s: &[Limb], d: &mut [Limb]) { + let k = dst.len(); // d = s - p over k+1 bits; the borrow-out marks lanes where s < p. - let mut d = [0; MAX_K + 1]; let mut borrow: Limb = 0; for j in 0..=k { let sj = s[j]; @@ -255,32 +279,37 @@ fn add_mod(a: &[Limb], b: &[Limb], k: usize, p_masks: &[Limb; MAX_K + 1]) -> [Li borrow = (!sj & pj) | (borrow & !sxp); } let ge = !borrow; // lanes where s >= p - // result = ge ? d : s, taking the low k planes (result < p < 2^k). - let mut out = [0; MAX_K]; for j in 0..k { - out[j] = (d[j] & ge) | (s[j] & !ge); + dst[j] = (d[j] & ge) | (s[j] & !ge); } - out } -/// `c * b` (mod p) for a constant scalar `c`, via double-and-add with modular reduction. +/// `acc = c * b` (mod p) for a constant scalar `c`, via double-and-add with modular +/// reduction. `temp`/`s`/`d` are reusable scratch (`temp` is `k` limbs, `s`/`d` are `k+1`). #[inline] -fn scalar_mul(b: &[Limb], c: u32, k: usize, p_masks: &[Limb; MAX_K + 1]) -> [Limb; MAX_K] { - let mut result = [0; MAX_K]; - let mut temp = [0; MAX_K]; - temp[..k].copy_from_slice(&b[..k]); +fn scalar_mul_into( + acc: &mut [Limb], + b: &[Limb], + c: u32, + p_masks: &[Limb], + temp: &mut [Limb], + s: &mut [Limb], + d: &mut [Limb], +) { + temp.copy_from_slice(b); + acc.fill(0); let mut cc = c; - while cc > 0 { + loop { if cc & 1 == 1 { - result = add_mod(&result, &temp, k, p_masks); + add_mod_into(acc, temp, p_masks, s, d); } cc >>= 1; - if cc > 0 { - temp = add_mod(&temp, &temp, k, p_masks); + if cc == 0 { + break; } + double_mod_into(temp, p_masks, s, d); } - result } /// F3 addition circuit on the `(lo, hi)` plane encoding (`value = 2*hi + lo`). From a91f37459435ea1a3163b456090ea73e60a0143f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 00:20:16 +0000 Subject: [PATCH 04/27] Phase 0b: dispatch bit-slice generic kernel to const-generic K Replace the runtime-k heap-scratch generic kernel with a dispatch on k = ceil(log2 p) to const-generic implementations (add_groups_k/scale_groups_k and the add_mod_k/double_mod_k/scalar_mul_k helpers). Each monomorphization uses exactly-K-sized stack arrays and fully-unrolled loops tuned to the prime. The dispatch covers k=1..=16 directly and falls back to the heap path for very large primes. Exhaustive correctness tests still pass. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/bitslice_proto.rs | 169 ++++++++++++++++++++++++---- 1 file changed, 149 insertions(+), 20 deletions(-) diff --git a/ext/crates/fp/src/bitslice_proto.rs b/ext/crates/fp/src/bitslice_proto.rs index 18bdf11bb9..57c05e3e4d 100644 --- a/ext/crates/fp/src/bitslice_proto.rs +++ b/ext/crates/fp/src/bitslice_proto.rs @@ -128,36 +128,33 @@ impl BitSlicedVec { } /// `self += c * other` (mod p), generic kernel for any prime. + /// + /// Dispatches on the number of planes `k` to a const-generic implementation, so the + /// per-group arithmetic uses exactly-`K`-sized stack arrays and fully-unrolled loops + /// tuned to the prime, rather than runtime-bounded loops over heap scratch. `k` only + /// takes a handful of values (`k = ceil(log2 p)`), so the dispatch covers them directly + /// and falls back to a heap path only for very large primes. pub fn add_generic(&mut self, other: &Self, c: u32) { assert_eq!(self.p, other.p); assert_eq!(self.len, other.len); if c == 0 { return; } - let k = self.k; let p_masks = self.p_masks(); - // Reusable scratch, sized exactly to the number of planes (no fixed MAX_K arrays). - let mut s = vec![0; k + 1]; - let mut d = vec![0; k + 1]; - let mut acc = vec![0; k]; - let mut temp = vec![0; k]; - for g in 0..self.num_groups() { - let base = g * k; - let b = &other.limbs[base..base + k]; - if c == 1 { - // dst += b - add_mod_into(&mut self.limbs[base..base + k], b, &p_masks, &mut s, &mut d); - } else { - // acc = c * b, then dst += acc - scalar_mul_into(&mut acc, b, c, &p_masks, &mut temp, &mut s, &mut d); - add_mod_into(&mut self.limbs[base..base + k], &acc, &p_masks, &mut s, &mut d); - } + macro_rules! dispatch { + ($($k:literal),*) => { + match self.k { + $($k => add_groups_k::<$k>(&mut self.limbs, &other.limbs, c, &p_masks),)* + _ => self.add_generic_dyn(other, c, &p_masks), + } + }; } + dispatch!(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); } - /// `self *= c` (mod p), generic kernel. + /// `self *= c` (mod p), generic kernel. See [`add_generic`](Self::add_generic) for the + /// const-generic dispatch rationale. pub fn scale_generic(&mut self, c: u32) { - let k = self.k; if c == 1 { return; } @@ -168,6 +165,39 @@ impl BitSlicedVec { return; } let p_masks = self.p_masks(); + macro_rules! dispatch { + ($($k:literal),*) => { + match self.k { + $($k => scale_groups_k::<$k>(&mut self.limbs, c, &p_masks),)* + _ => self.scale_generic_dyn(c, &p_masks), + } + }; + } + dispatch!(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); + } + + /// Heap-scratch fallback for `add_generic` when `k` exceeds the const-dispatch range. + fn add_generic_dyn(&mut self, other: &Self, c: u32, p_masks: &[Limb]) { + let k = self.k; + let mut s = vec![0; k + 1]; + let mut d = vec![0; k + 1]; + let mut acc = vec![0; k]; + let mut temp = vec![0; k]; + for g in 0..self.num_groups() { + let base = g * k; + let b = &other.limbs[base..base + k]; + if c == 1 { + add_mod_into(&mut self.limbs[base..base + k], b, p_masks, &mut s, &mut d); + } else { + scalar_mul_into(&mut acc, b, c, p_masks, &mut temp, &mut s, &mut d); + add_mod_into(&mut self.limbs[base..base + k], &acc, p_masks, &mut s, &mut d); + } + } + } + + /// Heap-scratch fallback for `scale_generic` when `k` exceeds the const-dispatch range. + fn scale_generic_dyn(&mut self, c: u32, p_masks: &[Limb]) { + let k = self.k; let mut s = vec![0; k + 1]; let mut d = vec![0; k + 1]; let mut acc = vec![0; k]; @@ -178,7 +208,7 @@ impl BitSlicedVec { &mut acc, &self.limbs[base..base + k], c, - &p_masks, + p_masks, &mut temp, &mut s, &mut d, @@ -312,6 +342,105 @@ fn scalar_mul_into( } } +// --------------------------------------------------------------------------------------- +// Const-generic kernels: `K` planes known at compile time, so every array is exactly sized +// and every loop is fully unrolled. Selected by a runtime dispatch on `k = ceil(log2 p)`. +// --------------------------------------------------------------------------------------- + +/// Reduce a `(K+1)`-bit unreduced sum (`s` low planes + `s_top`) in `[0, 2p)` to `s mod p`. +#[inline(always)] +fn cond_sub_k(s: &[Limb; K], s_top: Limb, p_masks: &[Limb]) -> [Limb; K] { + let mut d = [0 as Limb; K]; + let mut borrow: Limb = 0; + for j in 0..K { + let sj = s[j]; + let pj = p_masks[j]; + let sxp = sj ^ pj; + d[j] = sxp ^ borrow; + borrow = (!sj & pj) | (borrow & !sxp); + } + // Top bit only affects the borrow-out (the result fits in K planes since result < p). + let pj = p_masks[K]; + let sxp = s_top ^ pj; + borrow = (!s_top & pj) | (borrow & !sxp); + let ge = !borrow; + let mut out = [0 as Limb; K]; + for j in 0..K { + out[j] = (d[j] & ge) | (s[j] & !ge); + } + out +} + +/// `(a + b) mod p` over `K` planes. +#[inline(always)] +fn add_mod_k(a: &[Limb; K], b: &[Limb; K], p_masks: &[Limb]) -> [Limb; K] { + let mut s = [0 as Limb; K]; + let mut carry: Limb = 0; + for j in 0..K { + let aj = a[j]; + let bj = b[j]; + let axb = aj ^ bj; + s[j] = axb ^ carry; + carry = (aj & bj) | (carry & axb); + } + cond_sub_k::(&s, carry, p_masks) +} + +/// `(2 * a) mod p` over `K` planes (doubling is a one-position plane shift). +#[inline(always)] +fn double_mod_k(a: &[Limb; K], p_masks: &[Limb]) -> [Limb; K] { + let mut s = [0 as Limb; K]; + for j in 1..K { + s[j] = a[j - 1]; + } + let s_top = a[K - 1]; + cond_sub_k::(&s, s_top, p_masks) +} + +/// `(c * b) mod p` over `K` planes, via double-and-add. +#[inline(always)] +fn scalar_mul_k(b: &[Limb; K], c: u32, p_masks: &[Limb]) -> [Limb; K] { + let mut result = [0 as Limb; K]; + let mut temp = *b; + let mut cc = c; + loop { + if cc & 1 == 1 { + result = add_mod_k::(&result, &temp, p_masks); + } + cc >>= 1; + if cc == 0 { + break; + } + temp = double_mod_k::(&temp, p_masks); + } + result +} + +/// `dst += c * src` (mod p) over all groups, with `K` planes per group. +#[inline] +fn add_groups_k(dst: &mut [Limb], src: &[Limb], c: u32, p_masks: &[Limb]) { + for (dg, sg) in dst.chunks_exact_mut(K).zip(src.chunks_exact(K)) { + let mut a = [0 as Limb; K]; + let mut b = [0 as Limb; K]; + a.copy_from_slice(dg); + b.copy_from_slice(sg); + let addend = if c == 1 { b } else { scalar_mul_k::(&b, c, p_masks) }; + let sum = add_mod_k::(&a, &addend, p_masks); + dg.copy_from_slice(&sum); + } +} + +/// `dst *= c` (mod p) over all groups, with `K` planes per group. +#[inline] +fn scale_groups_k(dst: &mut [Limb], c: u32, p_masks: &[Limb]) { + for dg in dst.chunks_exact_mut(K) { + let mut a = [0 as Limb; K]; + a.copy_from_slice(dg); + let scaled = scalar_mul_k::(&a, c, p_masks); + dg.copy_from_slice(&scaled); + } +} + /// F3 addition circuit on the `(lo, hi)` plane encoding (`value = 2*hi + lo`). /// /// Output `c == 1` exactly for input value pairs `{(0,1),(1,0),(2,2)}` and `c == 2` for From a37a1bd568100dc1c432efafee30226eb365a7fc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 00:25:47 +0000 Subject: [PATCH 05/27] Phase 0b: record const-K dispatch benchmark results The const-generic K dispatch is the decisive win: the generic bit-slice kernel now beats packed for every prime (3.0x add at p=3, up to 24x at p=251) and is competitive with the hand-written F3 circuit. This means the const-K generic kernel can be the single code path for all primes, making per-prime specialized circuits optional polish rather than a requirement. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- .../fp/benches/BITSLICE_PROTO_RESULTS.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md b/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md index 30fe13f4a2..e8db167292 100644 --- a/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md +++ b/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md @@ -69,6 +69,33 @@ So the tightened generic kernel is now faster than packed for **all `p ≥ 7`**; SWAR-tuned `p ∈ {3, 5}` still need specialized circuits to win — and F3 confirms that specialization does win. +## Phase 0c — const-generic `K` dispatch (the decisive change) + +The heap-scratch generic kernel was replaced with a runtime dispatch on `k = ceil(log2 p)` +to **const-generic** implementations (`add_groups_k::` etc.): exactly-`K`-sized stack +arrays and fully-unrolled loops per prime, so the compiler keeps planes in registers and +auto-vectorizes the group loop. This is the single biggest win and it changes the +conclusion — the generic kernel now **beats packed for every prime tested**, and is +competitive with the hand-written F3 circuit (numbers from one run, length 100k): + +| op | prime | packed | bitsliced generic (const-K) | generic vs packed | F3 circuit | +|------:|------:|-------:|----------------------------:|:------------------|-----------:| +| add | 3 | 11.0 µs | 3.62 µs | **3.0× faster** | 3.84 µs | +| add | 5 | 21.0 µs | 6.09 µs | **3.4× faster** | — | +| add | 7 | 95.5 µs | 5.76 µs | **16.6× faster** | — | +| add | 251 | 481 µs | 19.68 µs | **24× faster** | — | +| scale | 3 | 2.75 µs | 2.50 µs | **1.1× faster** | 1.65 µs | +| scale | 5 | 5.07 µs | 3.50 µs | **1.45× faster** | — | +| scale | 7 | 74.0 µs | 3.54 µs | **21× faster** | — | +| scale | 251 | 415 µs | 11.48 µs | **36× faster** | — | + +Key consequence: **the const-K generic kernel is fast enough to be the single code path +for all primes.** It matches the F3 add circuit (3.62 vs 3.84 µs) and beats the SWAR-tuned +packed path even for `p ∈ {3, 5}`. Hand-written per-prime circuits are now *optional polish* +(F3 still wins `scale` modestly via the plane-swap negation, 1.65 vs 2.50 µs) rather than a +requirement. This substantially de-risks and simplifies Phase 2/3: implement the +const-generic kernel once; add specialized circuits later only where a measured gap remains. + ## Implications for the next phases - **Specialized circuits are needed for `p = 3, 5, 7`** (not just F3) to beat the tuned From b65ddf693d89845e1fa1d4a9608be82d6732cfe6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 00:37:42 +0000 Subject: [PATCH 06/27] Phase 1: introduce group-layout seam in FieldInternal (no-op) Add entries_per_group/limbs_per_group (packed defaults: entries_per_limb and 1), group_of/lane_of, and gather/scatter for entry-level access. Re-express number() and range() in terms of groups so a future bit-sliced layout can relocate every entry just by overriding the two layout methods. Route FqSlice::entry and FqSliceMut::set_entry through gather/scatter. Behavior is unchanged for the packed layout: full fp test suite (516 lib + 32 integration/doc tests) passes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/field_internal.rs | 72 ++++++++++++++++++--- ext/crates/fp/src/vector/impl_fqslice.rs | 11 ++-- ext/crates/fp/src/vector/impl_fqslicemut.rs | 12 ++-- 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/ext/crates/fp/src/field/field_internal.rs b/ext/crates/fp/src/field/field_internal.rs index 594bdb8d07..9b91a77fc6 100644 --- a/ext/crates/fp/src/field/field_internal.rs +++ b/ext/crates/fp/src/field/field_internal.rs @@ -133,6 +133,63 @@ pub trait FieldInternal: } } + // # Group layout + // + // The storage is organized into *groups*: a group holds [`entries_per_group`] consecutive + // entries and occupies [`limbs_per_group`] consecutive limbs. The packed layout (the + // default here) has one limb per group, so a group is exactly a limb. The bit-sliced + // layout (see [`Fp`](super::Fp)) overrides these to spread an entry's bits across several + // limbs of a group. Entry-level access goes through [`gather`]/[`scatter`], and the + // sizing helpers [`number`]/[`range`] are expressed in terms of groups so that overriding + // the two layout methods is enough to relocate every entry. + // + // [`entries_per_group`]: FieldInternal::entries_per_group + // [`limbs_per_group`]: FieldInternal::limbs_per_group + // [`gather`]: FieldInternal::gather + // [`scatter`]: FieldInternal::scatter + // [`number`]: FieldInternal::number + // [`range`]: FieldInternal::range + + /// The number of entries stored in a single group. Packed default: [`entries_per_limb`]. + /// + /// [`entries_per_limb`]: FieldInternal::entries_per_limb + fn entries_per_group(self) -> usize { + self.entries_per_limb() + } + + /// The number of limbs a single group occupies. Packed default: `1`. + fn limbs_per_group(self) -> usize { + 1 + } + + /// The index of the group containing entry `idx`. + fn group_of(self, idx: usize) -> usize { + idx / self.entries_per_group() + } + + /// The position of entry `idx` within its group, in `0..entries_per_group()`. + fn lane_of(self, idx: usize) -> usize { + idx % self.entries_per_group() + } + + /// Read entry `lane` (in `0..entries_per_group()`) out of a single group's limbs (a slice + /// of length [`limbs_per_group`](FieldInternal::limbs_per_group)). + fn gather(self, group: &[Limb], lane: usize) -> FieldElement { + // Packed default: a group is one limb; the entry is a contiguous bitfield. + let mut result = group[0] >> (lane * self.bit_length()); + result &= self.bitmask(); + self.decode(result) + } + + /// Write `value` into entry `lane` of a single group's limbs (a slice of length + /// [`limbs_per_group`](FieldInternal::limbs_per_group)). Assumes the limbs are reduced. + fn scatter(self, group: &mut [Limb], lane: usize, value: FieldElement) { + // Packed default: clear the entry's bitfield and write the encoded value. + let shift = lane * self.bit_length(); + let mask = self.bitmask() << shift; + group[0] = (group[0] & !mask) | (self.encode(value) << shift); + } + /// Check whether or not a limb is reduced. This may potentially not be faster than calling /// [`reduce`](FieldInternal::reduce) directly. fn is_reduced(self, limb: Limb) -> bool { @@ -166,17 +223,16 @@ pub trait FieldInternal: /// Return the number of limbs required to hold `dim` entries. fn number(self, dim: usize) -> usize { - if dim == 0 { - 0 - } else { - self.limb_bit_index_pair(dim - 1).limb + 1 - } + // Whole groups needed to hold `dim` entries, times the limbs in each group. For the + // packed layout (1 limb/group, `entries_per_limb` entries/group) this is `ceil(dim / + // entries_per_limb)`, matching the previous definition. + self.limbs_per_group() * dim.div_ceil(self.entries_per_group()) } - /// Return the `Range` starting at the index of the limb containing the `start`th entry, and - /// ending at the index of the limb containing the `end`th entry (including the latter). + /// Return the `Range` of limbs spanning entries `start..end`: from the first limb of + /// the group containing `start` to the last limb of the group containing `end - 1`. fn range(self, start: usize, end: usize) -> Range { - let min = self.limb_bit_index_pair(start).limb; + let min = self.group_of(start) * self.limbs_per_group(); let max = self.number(end); min..max } diff --git a/ext/crates/fp/src/vector/impl_fqslice.rs b/ext/crates/fp/src/vector/impl_fqslice.rs index e2f243f0db..b3fcfc797c 100644 --- a/ext/crates/fp/src/vector/impl_fqslice.rs +++ b/ext/crates/fp/src/vector/impl_fqslice.rs @@ -33,12 +33,11 @@ impl<'a, F: Field> FqSlice<'a, F> { index, self.len() ); - let bit_mask = self.fq().bitmask(); - let limb_index = self.fq().limb_bit_index_pair(index + self.start()); - let mut result = self.limbs()[limb_index.limb]; - result >>= limb_index.bit_index; - result &= bit_mask; - self.fq().decode(result) + let fq = self.fq(); + let idx = index + self.start(); + let lpg = fq.limbs_per_group(); + let base = fq.group_of(idx) * lpg; + fq.gather(&self.limbs()[base..base + lpg], fq.lane_of(idx)) } /// TODO: implement prime 2 version diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index d65d302471..b8e13d7bc2 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -30,12 +30,12 @@ impl<'a, F: Field> FqSliceMut<'a, F> { pub fn set_entry(&mut self, index: usize, value: FieldElement) { assert_eq!(self.fq(), value.field()); assert!(index < self.as_slice().len()); - let bit_mask = self.fq().bitmask(); - let limb_index = self.fq().limb_bit_index_pair(index + self.start()); - let mut result = self.limbs()[limb_index.limb]; - result &= !(bit_mask << limb_index.bit_index); - result |= self.fq().encode(value) << limb_index.bit_index; - self.limbs_mut()[limb_index.limb] = result; + let fq = self.fq(); + let idx = index + self.start(); + let lpg = fq.limbs_per_group(); + let base = fq.group_of(idx) * lpg; + let lane = fq.lane_of(idx); + fq.scatter(&mut self.limbs_mut()[base..base + lpg], lane, value); } fn reduce_limbs(&mut self) { From 4aa4714e51c51bb87a1de0341fbf9cd3f602b24c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 01:36:53 +0000 Subject: [PATCH 07/27] Phase 2: bit-sliced storage for all prime/extension fields Make FqVector storage uniformly bit-sliced: a group of 64 entries occupies k = ceil(log2 q) limbs (bit-planes), with entry i at bit i of each plane. - field_internal.rs: group-layout defaults (entries_per_group=64, uniform gather/scatter dispersing the encoded value across planes) and element-wise add_groups/scale_groups defaults usable by any field. - field/bitslice.rs: const-generic K-dispatched ripple-carry + conditional- subtract add/scale circuits for prime fields (heap fallback for k>16). - fp.rs: Fp overrides limbs_per_group=ceil(log2 p) and the bulk kernels with the bit-circuit. smallfq.rs: limbs_per_group=bit_length(); uses the element-wise default arithmetic (Zech-log per lane). - vector ops route bulk add/scale through add_groups/scale_groups; entry/iter/ slice/copy_from_slice/is_zero/to_owned and matrix from_vec/to_vec take a bit-sliced path (element-wise via gather/scatter for the irregular cases). - F_2 is k=1, byte-identical to the old packed layout, so its SIMD/BLAS/m4ri paths are untouched. The on-disk/serde byte format changes for odd primes (F_2 unchanged); the p3/p5 golden format tests are regenerated. All 516 fp lib proptests, row_reduce, and serde tests pass. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 274 ++++++++++++++++++++ ext/crates/fp/src/field/field_internal.rs | 97 +++++-- ext/crates/fp/src/field/fp.rs | 34 +++ ext/crates/fp/src/field/mod.rs | 1 + ext/crates/fp/src/field/smallfq.rs | 7 + ext/crates/fp/src/matrix/matrix_inner.rs | 22 ++ ext/crates/fp/src/vector/impl_fqslice.rs | 5 +- ext/crates/fp/src/vector/impl_fqslicemut.rs | 31 ++- ext/crates/fp/src/vector/impl_fqvector.rs | 47 ++-- ext/crates/fp/src/vector/iter.rs | 107 ++++++-- ext/crates/fp/tests/serde_format.rs | 7 +- 11 files changed, 566 insertions(+), 66 deletions(-) create mode 100644 ext/crates/fp/src/field/bitslice.rs diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs new file mode 100644 index 0000000000..b7c663080a --- /dev/null +++ b/ext/crates/fp/src/field/bitslice.rs @@ -0,0 +1,274 @@ +//! Bit-sliced arithmetic kernels for prime fields. +//! +//! In the bit-sliced layout, a group of [`BITS_PER_LIMB`] (64) field elements occupies +//! `k = ceil(log2 p)` consecutive limbs (the *planes*): plane `j` holds bit `j` of all 64 +//! elements, with element `i` living at bit `i` of each plane. Addition and scalar +//! multiplication then reduce to short branch-free boolean circuits over the planes that +//! act on 64 lanes at once, with no separate reduction step. +//! +//! Addition is a ripple-carry adder over the `k` planes (producing a `(k+1)`-bit sum in +//! `[0, 2p)`) followed by a single conditional subtraction of `p`. Scalar multiplication is +//! double-and-add with a modular reduction at each step. The number of planes `k` is +//! dispatched to a const-generic implementation so that, for each prime, the arrays are +//! exactly sized and the loops fully unrolled; a heap-scratch fallback covers the rare +//! primes with `k` beyond the dispatch range. + +use crate::{constants::BITS_PER_LIMB, limb::Limb}; + +/// Largest `k` that the const-generic dispatch covers directly (`p < 2^16`). Larger primes +/// fall back to the heap-scratch path. +const MAX_DISPATCH_K: usize = 16; + +/// The number of planes `k = ceil(log2 p)` needed to bit-slice an element of `F_p`. +pub(crate) fn planes(p: u32) -> usize { + debug_assert!(p >= 2); + (u32::BITS - (p - 1).leading_zeros()) as usize +} + +/// The bits of `p` as full-width lane masks: `out[j]` is all-ones iff bit `j` of `p` is set, +/// for `j` in `0..=k`. +fn p_masks(p: u32, k: usize) -> [Limb; BITS_PER_LIMB + 1] { + let mut masks = [0; BITS_PER_LIMB + 1]; + for (j, m) in masks.iter_mut().enumerate().take(k + 1) { + *m = if (p >> j) & 1 == 1 { !0 } else { 0 }; + } + masks +} + +/// `dst += c * src` (mod p) over every group, where `dst` and `src` hold the same number of +/// whole groups of `k` planes. Assumes both are reduced; the result is reduced. +pub(crate) fn add_groups(p: u32, k: usize, dst: &mut [Limb], src: &[Limb], c: u32) { + if c == 0 { + return; + } + let masks = p_masks(p, k); + macro_rules! dispatch { + ($($k:literal),*) => { + match k { + $($k => add_groups_k::<$k>(dst, src, c, &masks),)* + _ => add_groups_dyn(k, dst, src, c, &masks), + } + }; + } + dispatch!(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); +} + +/// `dst *= c` (mod p) over every group. +pub(crate) fn scale_groups(p: u32, k: usize, dst: &mut [Limb], c: u32) { + let masks = p_masks(p, k); + macro_rules! dispatch { + ($($k:literal),*) => { + match k { + $($k => scale_groups_k::<$k>(dst, c, &masks),)* + _ => scale_groups_dyn(k, dst, c, &masks), + } + }; + } + dispatch!(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); +} + +// --------------------------------------------------------------------------------------- +// Const-generic kernels: `K` planes known at compile time, so every array is exactly sized +// and every loop is fully unrolled. +// --------------------------------------------------------------------------------------- + +/// Reduce a `(K+1)`-bit unreduced sum (`s` low planes + `s_top`) in `[0, 2p)` to `s mod p`. +#[inline(always)] +fn cond_sub_k(s: &[Limb; K], s_top: Limb, masks: &[Limb]) -> [Limb; K] { + let mut d = [0 as Limb; K]; + let mut borrow: Limb = 0; + for j in 0..K { + let sj = s[j]; + let pj = masks[j]; + let sxp = sj ^ pj; + d[j] = sxp ^ borrow; + borrow = (!sj & pj) | (borrow & !sxp); + } + // Top bit only affects the borrow-out (the result fits in K planes since result < p). + let pj = masks[K]; + let sxp = s_top ^ pj; + borrow = (!s_top & pj) | (borrow & !sxp); + let ge = !borrow; + let mut out = [0 as Limb; K]; + for j in 0..K { + out[j] = (d[j] & ge) | (s[j] & !ge); + } + out +} + +/// `(a + b) mod p` over `K` planes. +#[inline(always)] +fn add_mod_k(a: &[Limb; K], b: &[Limb; K], masks: &[Limb]) -> [Limb; K] { + let mut s = [0 as Limb; K]; + let mut carry: Limb = 0; + for j in 0..K { + let aj = a[j]; + let bj = b[j]; + let axb = aj ^ bj; + s[j] = axb ^ carry; + carry = (aj & bj) | (carry & axb); + } + cond_sub_k::(&s, carry, masks) +} + +/// `(2 * a) mod p` over `K` planes (doubling is a one-position plane shift). +#[inline(always)] +fn double_mod_k(a: &[Limb; K], masks: &[Limb]) -> [Limb; K] { + let mut s = [0 as Limb; K]; + for j in 1..K { + s[j] = a[j - 1]; + } + let s_top = a[K - 1]; + cond_sub_k::(&s, s_top, masks) +} + +/// `(c * b) mod p` over `K` planes, via double-and-add. +#[inline(always)] +fn scalar_mul_k(b: &[Limb; K], c: u32, masks: &[Limb]) -> [Limb; K] { + let mut result = [0 as Limb; K]; + let mut temp = *b; + let mut cc = c; + loop { + if cc & 1 == 1 { + result = add_mod_k::(&result, &temp, masks); + } + cc >>= 1; + if cc == 0 { + break; + } + temp = double_mod_k::(&temp, masks); + } + result +} + +#[inline] +fn add_groups_k(dst: &mut [Limb], src: &[Limb], c: u32, masks: &[Limb]) { + for (dg, sg) in dst.chunks_exact_mut(K).zip(src.chunks_exact(K)) { + let mut a = [0 as Limb; K]; + let mut b = [0 as Limb; K]; + a.copy_from_slice(dg); + b.copy_from_slice(sg); + let addend = if c == 1 { b } else { scalar_mul_k::(&b, c, masks) }; + let sum = add_mod_k::(&a, &addend, masks); + dg.copy_from_slice(&sum); + } +} + +#[inline] +fn scale_groups_k(dst: &mut [Limb], c: u32, masks: &[Limb]) { + if c == 1 { + return; + } + if c == 0 { + dst.fill(0); + return; + } + for dg in dst.chunks_exact_mut(K) { + let mut a = [0 as Limb; K]; + a.copy_from_slice(dg); + let scaled = scalar_mul_k::(&a, c, masks); + dg.copy_from_slice(&scaled); + } +} + +// --------------------------------------------------------------------------------------- +// Heap-scratch fallback for `k > MAX_DISPATCH_K` (very large primes). +// --------------------------------------------------------------------------------------- + +fn cond_sub_into(dst: &mut [Limb], s: &[Limb], masks: &[Limb], d: &mut [Limb]) { + let k = dst.len(); + let mut borrow: Limb = 0; + for j in 0..=k { + let sj = s[j]; + let pj = masks[j]; + let sxp = sj ^ pj; + d[j] = sxp ^ borrow; + borrow = (!sj & pj) | (borrow & !sxp); + } + let ge = !borrow; + for j in 0..k { + dst[j] = (d[j] & ge) | (s[j] & !ge); + } +} + +fn add_mod_into(dst: &mut [Limb], b: &[Limb], masks: &[Limb], s: &mut [Limb], d: &mut [Limb]) { + let k = dst.len(); + let mut carry: Limb = 0; + for j in 0..k { + let aj = dst[j]; + let bj = b[j]; + let axb = aj ^ bj; + s[j] = axb ^ carry; + carry = (aj & bj) | (carry & axb); + } + s[k] = carry; + cond_sub_into(dst, s, masks, d); +} + +fn double_mod_into(dst: &mut [Limb], masks: &[Limb], s: &mut [Limb], d: &mut [Limb]) { + let k = dst.len(); + s[0] = 0; + for j in 1..=k { + s[j] = dst[j - 1]; + } + cond_sub_into(dst, s, masks, d); +} + +fn scalar_mul_into( + acc: &mut [Limb], + b: &[Limb], + c: u32, + masks: &[Limb], + temp: &mut [Limb], + s: &mut [Limb], + d: &mut [Limb], +) { + temp.copy_from_slice(b); + acc.fill(0); + let mut cc = c; + loop { + if cc & 1 == 1 { + add_mod_into(acc, temp, masks, s, d); + } + cc >>= 1; + if cc == 0 { + break; + } + double_mod_into(temp, masks, s, d); + } +} + +fn add_groups_dyn(k: usize, dst: &mut [Limb], src: &[Limb], c: u32, masks: &[Limb]) { + let mut s = vec![0; k + 1]; + let mut d = vec![0; k + 1]; + let mut acc = vec![0; k]; + let mut temp = vec![0; k]; + for (dg, sg) in dst.chunks_exact_mut(k).zip(src.chunks_exact(k)) { + if c == 1 { + add_mod_into(dg, sg, masks, &mut s, &mut d); + } else { + scalar_mul_into(&mut acc, sg, c, masks, &mut temp, &mut s, &mut d); + add_mod_into(dg, &acc, masks, &mut s, &mut d); + } + } +} + +fn scale_groups_dyn(k: usize, dst: &mut [Limb], c: u32, masks: &[Limb]) { + if c == 1 { + return; + } + if c == 0 { + dst.fill(0); + return; + } + let mut s = vec![0; k + 1]; + let mut d = vec![0; k + 1]; + let mut acc = vec![0; k]; + let mut temp = vec![0; k]; + for dg in dst.chunks_exact_mut(k) { + scalar_mul_into(&mut acc, dg, c, masks, &mut temp, &mut s, &mut d); + dg.copy_from_slice(&acc); + } +} + +const _: () = assert!(MAX_DISPATCH_K <= BITS_PER_LIMB); diff --git a/ext/crates/fp/src/field/field_internal.rs b/ext/crates/fp/src/field/field_internal.rs index 9b91a77fc6..7d86dfc19e 100644 --- a/ext/crates/fp/src/field/field_internal.rs +++ b/ext/crates/fp/src/field/field_internal.rs @@ -133,15 +133,16 @@ pub trait FieldInternal: } } - // # Group layout + // # Group layout (bit-sliced storage) // - // The storage is organized into *groups*: a group holds [`entries_per_group`] consecutive - // entries and occupies [`limbs_per_group`] consecutive limbs. The packed layout (the - // default here) has one limb per group, so a group is exactly a limb. The bit-sliced - // layout (see [`Fp`](super::Fp)) overrides these to spread an entry's bits across several - // limbs of a group. Entry-level access goes through [`gather`]/[`scatter`], and the - // sizing helpers [`number`]/[`range`] are expressed in terms of groups so that overriding - // the two layout methods is enough to relocate every entry. + // Storage is organized into *groups*: a group holds [`entries_per_group`] = 64 consecutive + // entries and occupies [`limbs_per_group`] = `k` consecutive limbs, the *bit-planes*. Plane + // `j` of a group holds bit `j` of all 64 entries, so entry `i` lives at bit `i` of each of + // the `k` planes. Every field uses this layout, with `k = ceil(log2 q)` (the bits needed to + // store an encoded value in `0..q`). For `q = 2` this is `k = 1`, which coincides exactly + // with the old packed layout — so `F_2` (and its SIMD / matrix machinery) is byte-identical + // and unaffected. Entry access goes through [`gather`]/[`scatter`]; the sizing helpers + // [`number`]/[`range`] are expressed in terms of groups. // // [`entries_per_group`]: FieldInternal::entries_per_group // [`limbs_per_group`]: FieldInternal::limbs_per_group @@ -150,17 +151,14 @@ pub trait FieldInternal: // [`number`]: FieldInternal::number // [`range`]: FieldInternal::range - /// The number of entries stored in a single group. Packed default: [`entries_per_limb`]. - /// - /// [`entries_per_limb`]: FieldInternal::entries_per_limb + /// The number of entries stored in a single group: one per bit of a [`Limb`]. fn entries_per_group(self) -> usize { - self.entries_per_limb() + BITS_PER_LIMB } - /// The number of limbs a single group occupies. Packed default: `1`. - fn limbs_per_group(self) -> usize { - 1 - } + /// The number of bit-planes per group, `k = ceil(log2 q)`. Each field defines this; `q = 2` + /// gives `k = 1` (packed-compatible). + fn limbs_per_group(self) -> usize; /// The index of the group containing entry `idx`. fn group_of(self, idx: usize) -> usize { @@ -172,22 +170,65 @@ pub trait FieldInternal: idx % self.entries_per_group() } - /// Read entry `lane` (in `0..entries_per_group()`) out of a single group's limbs (a slice - /// of length [`limbs_per_group`](FieldInternal::limbs_per_group)). + /// Read entry `lane` (in `0..entries_per_group()`) out of a single group's `k` planes (a + /// slice of length [`limbs_per_group`](FieldInternal::limbs_per_group)) by reassembling its + /// bit from each plane. fn gather(self, group: &[Limb], lane: usize) -> FieldElement { - // Packed default: a group is one limb; the entry is a contiguous bitfield. - let mut result = group[0] >> (lane * self.bit_length()); - result &= self.bitmask(); - self.decode(result) + let mut value: Limb = 0; + for (j, plane) in group.iter().enumerate() { + value |= ((plane >> lane) & 1) << j; + } + self.decode(value) } - /// Write `value` into entry `lane` of a single group's limbs (a slice of length - /// [`limbs_per_group`](FieldInternal::limbs_per_group)). Assumes the limbs are reduced. + /// Write `value` into entry `lane` of a single group's `k` planes, dispersing the encoded + /// value's bits one per plane. Assumes the stored value fits in `k` bits. fn scatter(self, group: &mut [Limb], lane: usize, value: FieldElement) { - // Packed default: clear the entry's bitfield and write the encoded value. - let shift = lane * self.bit_length(); - let mask = self.bitmask() << shift; - group[0] = (group[0] & !mask) | (self.encode(value) << shift); + let encoded = self.encode(value); + let lane_mask: Limb = 1 << lane; + for (j, plane) in group.iter_mut().enumerate() { + let bit = (encoded >> j) & 1; + *plane = (*plane & !lane_mask) | (bit << lane); + } + } + + /// Whether this field uses a genuinely multi-plane layout (`k > 1`). Only `F_2` has `k = 1`, + /// where the bit-sliced layout coincides with the packed one and the `F_2`-specific fast + /// paths (`offset`, `limb_masks`, SIMD, m4ri) apply. + fn is_bitsliced(self) -> bool { + self.limbs_per_group() > 1 + } + + /// `dst += coeff * src` (mod p) over a span of whole groups (`dst` and `src` have equal, + /// group-aligned length). Both are assumed reduced; the result is reduced. + /// + /// Default: element-wise over lanes via [`gather`]/[`scatter`] and the field's own + /// arithmetic — correct for any bit-sliced field (used by [`SmallFq`](super::SmallFq)). The + /// prime fields [`Fp`](super::Fp) override this with a branch-free plane circuit. + fn add_groups(self, dst: &mut [Limb], src: &[Limb], coeff: FieldElement) { + let lpg = self.limbs_per_group(); + let epg = self.entries_per_group(); + for (dgroup, sgroup) in dst.chunks_exact_mut(lpg).zip(src.chunks_exact(lpg)) { + for lane in 0..epg { + let a = self.gather(dgroup, lane); + let b = self.gather(sgroup, lane); + let result = self.add(a, self.mul(coeff.clone(), b)); + self.scatter(dgroup, lane, result); + } + } + } + + /// `dst *= coeff` (mod p) over a span of whole groups. Default: element-wise; overridden by + /// [`Fp`](super::Fp). + fn scale_groups(self, dst: &mut [Limb], coeff: FieldElement) { + let lpg = self.limbs_per_group(); + let epg = self.entries_per_group(); + for dgroup in dst.chunks_exact_mut(lpg) { + for lane in 0..epg { + let a = self.gather(dgroup, lane); + self.scatter(dgroup, lane, self.mul(a, coeff.clone())); + } + } } /// Check whether or not a limb is reduced. This may potentially not be faster than calling diff --git a/ext/crates/fp/src/field/fp.rs b/ext/crates/fp/src/field/fp.rs index 18a9d0f9d5..004730e745 100644 --- a/ext/crates/fp/src/field/fp.rs +++ b/ext/crates/fp/src/field/fp.rs @@ -142,6 +142,40 @@ impl FieldInternal for Fp

{ _ => self.pack(self.unpack(limb)), } } + + // # Bit-sliced layout + // + // Prime-field vectors are stored bit-sliced: a group of `BITS_PER_LIMB` (64) elements + // occupies `k = ceil(log2 p)` limbs, one per bit-plane. Note that for `p = 2` this is + // `k = 1`, which is byte-identical to the packed layout, so `F_2` (and all of its SIMD / + // matrix machinery) is unaffected. The packed limb helpers above are retained because + // `decode`/`encode`/`reduce` are still used by callers that construct elements; they are + // simply not used to lay out `FqVector>` storage. The uniform `gather`/`scatter` + // defaults (in `FieldInternal`) handle entry access; `Fp` only overrides the bulk kernels + // with a branch-free plane circuit. + + fn limbs_per_group(self) -> usize { + crate::field::bitslice::planes(self.characteristic().as_u32()) + } + + fn add_groups(self, dst: &mut [Limb], src: &[Limb], coeff: FieldElement) { + crate::field::bitslice::add_groups( + self.characteristic().as_u32(), + self.limbs_per_group(), + dst, + src, + self.encode(coeff) as u32, + ); + } + + fn scale_groups(self, dst: &mut [Limb], coeff: FieldElement) { + crate::field::bitslice::scale_groups( + self.characteristic().as_u32(), + self.limbs_per_group(), + dst, + self.encode(coeff) as u32, + ); + } } #[cfg(feature = "proptest")] diff --git a/ext/crates/fp/src/field/mod.rs b/ext/crates/fp/src/field/mod.rs index 714a9e735d..7348ef0475 100644 --- a/ext/crates/fp/src/field/mod.rs +++ b/ext/crates/fp/src/field/mod.rs @@ -4,6 +4,7 @@ use crate::prime::Prime; pub mod element; pub(crate) mod field_internal; +pub(crate) mod bitslice; pub mod fp; pub mod smallfq; diff --git a/ext/crates/fp/src/field/smallfq.rs b/ext/crates/fp/src/field/smallfq.rs index b1aa80115c..96f91067a6 100644 --- a/ext/crates/fp/src/field/smallfq.rs +++ b/ext/crates/fp/src/field/smallfq.rs @@ -282,6 +282,13 @@ impl FieldInternal for SmallFq

{ BITS_PER_LIMB - (self.q() - 1).leading_zeros() as usize + 1 } + fn limbs_per_group(self) -> usize { + // Bit-sliced layout: one plane per bit of the encoded value. `encode` maps a^n to the + // odd number `2n + 1` (and zero to `0`), which occupies exactly `bit_length()` bits. + // The default element-wise `add_groups`/`scale_groups` use Zech-log field arithmetic. + self.bit_length() + } + fn fma_limb(self, limb_a: Limb, limb_b: Limb, coeff: FieldElement) -> Limb { let bit_length = self.bit_length(); let mut result: Limb = 0; diff --git a/ext/crates/fp/src/matrix/matrix_inner.rs b/ext/crates/fp/src/matrix/matrix_inner.rs index 8c6dd93b38..ff58ac107d 100644 --- a/ext/crates/fp/src/matrix/matrix_inner.rs +++ b/ext/crates/fp/src/matrix/matrix_inner.rs @@ -324,6 +324,20 @@ impl Matrix { let columns = input[0].len(); let stride = fp.number(columns); let physical_rows = get_physical_rows(p, rows); + + if fp.is_bitsliced() { + // The bit-sliced layout interleaves an entry's bits across planes, so build each + // row through the (dispatching) row slice rather than by packing contiguous chunks. + let mut matrix = Self::new(p, rows, columns); + for (i, row) in input.iter().enumerate() { + let mut target = matrix.row_mut(i); + for (j, &x) in row.iter().enumerate() { + target.set_entry(j, x); + } + } + return matrix; + } + let mut data = AVec::with_capacity(0, physical_rows * stride); for row in input { for chunk in row.chunks(fp.entries_per_limb()) { @@ -352,6 +366,14 @@ impl Matrix { /// assert_eq!(Matrix::from_vec(TWO, &matrix_vec).to_vec(), matrix_vec); /// ``` pub fn to_vec(&self) -> Vec> { + if self.fp.is_bitsliced() { + return (0..self.rows()) + .map(|i| { + let row = self.row(i); + (0..self.columns()).map(|j| row.entry(j)).collect() + }) + .collect(); + } self.data .iter() .chunks(self.stride) diff --git a/ext/crates/fp/src/vector/impl_fqslice.rs b/ext/crates/fp/src/vector/impl_fqslice.rs index b3fcfc797c..19a547ea08 100644 --- a/ext/crates/fp/src/vector/impl_fqslice.rs +++ b/ext/crates/fp/src/vector/impl_fqslice.rs @@ -54,6 +54,9 @@ impl<'a, F: Field> FqSlice<'a, F> { } pub fn is_zero(&self) -> bool { + if self.fq().is_bitsliced() { + return self.first_nonzero().is_none(); + } let limb_range = self.limb_range(); if limb_range.is_empty() { return true; @@ -89,7 +92,7 @@ impl<'a, F: Field> FqSlice<'a, F> { #[must_use] pub fn to_owned(self) -> FqVector { let mut new = FqVector::new(self.fq(), self.len()); - if self.start().is_multiple_of(self.fq().entries_per_limb()) { + if !self.fq().is_bitsliced() && self.start().is_multiple_of(self.fq().entries_per_limb()) { let limb_range = self.limb_range(); new.limbs_mut()[0..limb_range.len()].copy_from_slice(&self.limbs()[limb_range]); if !new.limbs().is_empty() { diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index b8e13d7bc2..f61117e8d0 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -60,6 +60,20 @@ impl<'a, F: Field> FqSliceMut<'a, F> { return; } + if fq.is_bitsliced() { + // The packed bit-offset masking does not apply to the bit-sliced layout; scale + // each in-range entry through the layout-aware gather/scatter. + if c == fq.zero() { + self.set_to_zero(); + return; + } + for i in 0..self.as_slice().len() { + let x = self.as_slice().entry(i) * c.clone(); + self.set_entry(i, x); + } + return; + } + let limb_range = self.as_slice().limb_range(); if limb_range.is_empty() { return; @@ -85,6 +99,13 @@ impl<'a, F: Field> FqSliceMut<'a, F> { } pub fn set_to_zero(&mut self) { + if self.fq().is_bitsliced() { + let zero = self.fq().zero(); + for i in 0..self.as_slice().len() { + self.set_entry(i, zero.clone()); + } + return; + } let limb_range = self.as_slice().limb_range(); if limb_range.is_empty() { return; @@ -115,6 +136,14 @@ impl<'a, F: Field> FqSliceMut<'a, F> { Ordering::Greater => self.add_shift_right(other, self.fq().one()), }; } + } else if self.fq().is_bitsliced() { + // Bit-sliced slices: add entry-wise. The bit-shift realignment used by the packed + // paths does not apply (an entry's bits are spread across planes). + if c != self.fq().zero() { + for (i, v) in other.iter_nonzero() { + self.add_basis_element(i, v * c.clone()); + } + } } else { match self.as_slice().offset().cmp(&other.offset()) { Ordering::Equal => self.add_shift_none(other, c), @@ -153,7 +182,7 @@ impl<'a, F: Field> FqSliceMut<'a, F> { /// TODO: improve efficiency pub fn assign(&mut self, other: FqSlice<'_, F>) { assert_eq!(self.fq(), other.fq()); - if self.as_slice().offset() != other.offset() { + if self.fq().is_bitsliced() || self.as_slice().offset() != other.offset() { self.set_to_zero(); self.add(other, self.fq().one()); return; diff --git a/ext/crates/fp/src/vector/impl_fqvector.rs b/ext/crates/fp/src/vector/impl_fqvector.rs index 2a472a8b5e..dec9041212 100644 --- a/ext/crates/fp/src/vector/impl_fqvector.rs +++ b/ext/crates/fp/src/vector/impl_fqvector.rs @@ -118,12 +118,13 @@ impl FqVector { if c == fq.zero() { self.set_to_zero(); + return; } - if fq.q() != 2 { - for limb in self.limbs_mut() { - *limb = fq.reduce(fq.fma_limb(0, *limb, c.clone())); - } + if fq.q() == 2 { + // `c` is one; scaling is a no-op. + return; } + fq.scale_groups(self.limbs_mut(), c); } /// Add `other` to `self` on the assumption that the first `offset` entries of `other` are @@ -134,24 +135,21 @@ impl FqVector { assert_eq!(self.len(), other.len()); let fq = self.fq(); - let min_limb = offset / fq.entries_per_limb(); + // The first limb of the group containing `offset`. Since `other`'s entries below + // `offset` are zero, starting at the group boundary and adding whole groups is safe. + let min_limb = fq.group_of(offset) * fq.limbs_per_group(); if fq.q() == 2 { if c != fq.zero() { crate::simd::add_simd(self.limbs_mut(), other.limbs(), min_limb); } } else { - for (left, right) in self - .limbs_mut() - .iter_mut() - .zip_eq(other.limbs()) - .skip(min_limb) - { - *left = fq.fma_limb(*left, *right, c.clone()); - } - for limb in self.limbs_mut()[min_limb..].iter_mut() { - *limb = fq.reduce(*limb); - } + let end = self.limbs().len(); + fq.add_groups( + &mut self.limbs_mut()[min_limb..end], + &other.limbs()[min_limb..end], + c, + ); } } @@ -207,6 +205,20 @@ impl FqVector { assert_eq!(self.len(), slice.len()); let fq = self.fq(); + if fq.is_bitsliced() { + // The bit-sliced layout interleaves an entry's bits across planes, so we cannot + // `pack` contiguous chunks; scatter each entry into its group. + let num_limbs = fq.number(self.len()); + { + let v = self.vec_mut(); + v.clear(); + v.resize(num_limbs, 0); + } + for (i, x) in slice.iter().enumerate() { + self.set_entry(i, x.clone()); + } + return; + } self.vec_mut().clear(); self.vec_mut().extend( slice @@ -297,6 +309,9 @@ impl FqVector { /// Find the index and value of the first non-zero entry of the vector. `None` if the vector is zero. pub fn first_nonzero(&self) -> Option<(usize, FieldElement)> { + if self.fq().is_bitsliced() { + return self.as_slice().first_nonzero(); + } let entries_per_limb = self.fq().entries_per_limb(); let bit_length = self.fq().bit_length(); let bitmask = self.fq().bitmask(); diff --git a/ext/crates/fp/src/vector/iter.rs b/ext/crates/fp/src/vector/iter.rs index 3dcc00a0a4..fc2e183c11 100644 --- a/ext/crates/fp/src/vector/iter.rs +++ b/ext/crates/fp/src/vector/iter.rs @@ -4,9 +4,22 @@ use crate::{ limb::Limb, }; +/// Read entry `idx` (an absolute index into `limbs`) under the bit-sliced layout, by +/// gathering its bit from each plane of its group. +#[inline] +fn gather_at(fq: F, limbs: &[Limb], idx: usize) -> FieldElement { + let lpg = fq.limbs_per_group(); + let base = fq.group_of(idx) * lpg; + fq.gather(&limbs[base..base + lpg], fq.lane_of(idx)) +} + pub struct FqVectorIterator<'a, F> { fq: F, limbs: &'a [Limb], + // Bit-sliced path: `pos` is the absolute index of the next entry to emit. + bitsliced: bool, + pos: usize, + // Packed path state. bit_length: usize, bit_mask: Limb, entries_per_limb_m_1: usize, @@ -19,12 +32,16 @@ pub struct FqVectorIterator<'a, F> { impl<'a, F: Field> FqVectorIterator<'a, F> { pub(super) fn new(vec: FqSlice<'a, F>) -> Self { let counter = vec.len(); + let fq = vec.fq(); + let start = vec.start(); let limbs = vec.into_limbs(); if counter == 0 { return Self { - fq: vec.fq(), + fq, limbs, + bitsliced: fq.is_bitsliced(), + pos: start, bit_length: 0, entries_per_limb_m_1: 0, bit_mask: 0, @@ -34,20 +51,37 @@ impl<'a, F: Field> FqVectorIterator<'a, F> { counter, }; } - let pair = vec.fq().limb_bit_index_pair(vec.start()); - let bit_length = vec.fq().bit_length(); - let cur_limb = limbs[pair.limb] >> pair.bit_index; + if fq.is_bitsliced() { + return Self { + fq, + limbs, + bitsliced: true, + pos: start, + bit_length: 0, + entries_per_limb_m_1: 0, + bit_mask: 0, + limb_index: 0, + entries_left: 0, + cur_limb: 0, + counter, + }; + } - let entries_per_limb = vec.fq().entries_per_limb(); + let pair = fq.limb_bit_index_pair(start); + let bit_length = fq.bit_length(); + let cur_limb = limbs[pair.limb] >> pair.bit_index; + let entries_per_limb = fq.entries_per_limb(); Self { - fq: vec.fq(), + fq, limbs, + bitsliced: false, + pos: start, bit_length, entries_per_limb_m_1: entries_per_limb - 1, - bit_mask: vec.fq().bitmask(), + bit_mask: fq.bitmask(), limb_index: pair.limb, - entries_left: entries_per_limb - (vec.start() % entries_per_limb), + entries_left: entries_per_limb - (start % entries_per_limb), cur_limb, counter, } @@ -55,9 +89,15 @@ impl<'a, F: Field> FqVectorIterator<'a, F> { pub fn skip_n(&mut self, mut n: usize) { if n >= self.counter { + self.pos += self.counter; self.counter = 0; return; } + if self.bitsliced { + self.pos += n; + self.counter -= n; + return; + } let entries_per_limb = self.entries_per_limb_m_1 + 1; if n < self.entries_left { self.entries_left -= n; @@ -90,7 +130,16 @@ impl Iterator for FqVectorIterator<'_, F> { fn next(&mut self) -> Option { if self.counter == 0 { return None; - } else if self.entries_left == 0 { + } + + if self.bitsliced { + let result = gather_at(self.fq, self.limbs, self.pos); + self.pos += 1; + self.counter -= 1; + return Some(result); + } + + if self.entries_left == 0 { self.limb_index += 1; self.cur_limb = self.limbs[self.limb_index]; self.entries_left = self.entries_per_limb_m_1; @@ -117,6 +166,10 @@ impl ExactSizeIterator for FqVectorIterator<'_, F> { pub struct FqVectorNonZeroIterator<'a, F> { fq: F, limbs: &'a [Limb], + // Bit-sliced path: absolute index of the group base, and the relative cursor. + bitsliced: bool, + start: usize, + // Shared/packed path state. limb_index: usize, cur_limb_entries_left: usize, cur_limb: Limb, @@ -126,29 +179,34 @@ pub struct FqVectorNonZeroIterator<'a, F> { impl<'a, F: Field> FqVectorNonZeroIterator<'a, F> { pub(super) fn new(vec: FqSlice<'a, F>) -> Self { - let entries_per_limb = vec.fq().entries_per_limb(); - + let fq = vec.fq(); let dim = vec.len(); + let start = vec.start(); let limbs = vec.into_limbs(); - if dim == 0 { + if dim == 0 || fq.is_bitsliced() { return Self { - fq: vec.fq(), + fq, limbs, + bitsliced: fq.is_bitsliced(), + start, limb_index: 0, cur_limb_entries_left: 0, cur_limb: 0, idx: 0, - dim: 0, + dim, }; } - let min_index = vec.start(); - let pair = vec.fq().limb_bit_index_pair(min_index); + + let entries_per_limb = fq.entries_per_limb(); + let pair = fq.limb_bit_index_pair(start); let cur_limb = limbs[pair.limb] >> pair.bit_index; - let cur_limb_entries_left = entries_per_limb - (min_index % entries_per_limb); + let cur_limb_entries_left = entries_per_limb - (start % entries_per_limb); Self { - fq: vec.fq(), + fq, limbs, + bitsliced: false, + start, limb_index: pair.limb, cur_limb_entries_left, cur_limb, @@ -162,6 +220,19 @@ impl Iterator for FqVectorNonZeroIterator<'_, F> { type Item = (usize, FieldElement); fn next(&mut self) -> Option { + if self.bitsliced { + let zero = self.fq.zero(); + while self.idx < self.dim { + let value = gather_at(self.fq, self.limbs, self.start + self.idx); + let cur = self.idx; + self.idx += 1; + if value != zero { + return Some((cur, value)); + } + } + return None; + } + let bit_length: usize = self.fq.bit_length(); let bitmask: Limb = self.fq.bitmask(); let entries_per_limb: usize = self.fq.entries_per_limb(); diff --git a/ext/crates/fp/tests/serde_format.rs b/ext/crates/fp/tests/serde_format.rs index 04fa2db115..c8627a7d6a 100644 --- a/ext/crates/fp/tests/serde_format.rs +++ b/ext/crates/fp/tests/serde_format.rs @@ -249,7 +249,8 @@ fn fpvector_p3_json_format() { }, "len": 5, "limbs": [ - 5137 + 17, + 10 ] }"#]] .assert_eq(&s); @@ -266,7 +267,9 @@ fn fpvector_p5_json_format() { }, "len": 4, "limbs": [ - 98372 + 8, + 10, + 1 ] }"#]] .assert_eq(&s); From 550725d683f2ac35a8ea3b2b672ecb7119223872 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 01:47:43 +0000 Subject: [PATCH 08/27] Phase 2: fast group-kernel path for bit-sliced slice addition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route group-aligned bit-sliced slice add (both slices starting at lane 0 — the common case: whole vectors and matrix rows) through the fast plane kernel add_groups for the complete groups, with only the <64 trailing entries and non-aligned slices falling back to entry-wise addition. This makes matrix row reduction over odd primes use the bit-circuit instead of per-entry gather/scatter. All fp tests still pass (516 lib + row_reduce + serde + doc). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/vector/impl_fqslicemut.rs | 47 ++++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index f61117e8d0..98a68d6855 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -137,13 +137,7 @@ impl<'a, F: Field> FqSliceMut<'a, F> { }; } } else if self.fq().is_bitsliced() { - // Bit-sliced slices: add entry-wise. The bit-shift realignment used by the packed - // paths does not apply (an entry's bits are spread across planes). - if c != self.fq().zero() { - for (i, v) in other.iter_nonzero() { - self.add_basis_element(i, v * c.clone()); - } - } + self.add_bitsliced(other, c); } else { match self.as_slice().offset().cmp(&other.offset()) { Ordering::Equal => self.add_shift_none(other, c), @@ -153,6 +147,45 @@ impl<'a, F: Field> FqSliceMut<'a, F> { } } + /// Add `c * other` to `self` in the bit-sliced layout. When both slices begin at a group + /// boundary (lane 0 — the common case, e.g. whole vectors and matrix rows), the complete + /// groups are added with the fast plane kernel ([`add_groups`](crate::field::field_internal)); + /// the fewer-than-64 trailing entries, and any non-group-aligned slice, fall back to + /// entry-wise addition. + /// + /// [`add_groups`]: crate::field::field_internal::FieldInternal::add_groups + fn add_bitsliced(&mut self, other: FqSlice<'_, F>, c: FieldElement) { + let fq = self.fq(); + if c == fq.zero() { + return; + } + let epg = fq.entries_per_group(); + let len = self.as_slice().len(); + + if self.start() % epg == 0 && other.start() % epg == 0 { + let k = fq.limbs_per_group(); + let full_groups = len / epg; + if full_groups > 0 { + let nlimbs = full_groups * k; + let sbase = (self.start() / epg) * k; + let obase = (other.start() / epg) * k; + let src = &other.limbs()[obase..obase + nlimbs]; + fq.add_groups(&mut self.limbs_mut()[sbase..sbase + nlimbs], src, c.clone()); + } + // Trailing entries that don't fill a whole group. + for i in (full_groups * epg)..len { + let v = other.entry(i); + if v != fq.zero() { + self.add_basis_element(i, v * c.clone()); + } + } + } else { + for (i, v) in other.iter_nonzero() { + self.add_basis_element(i, v * c.clone()); + } + } + } + pub fn add_offset(&mut self, other: FqSlice<'_, F>, c: FieldElement, offset: usize) { self.slice_mut(offset, self.as_slice().len()) .add(other.restrict(offset, other.len()), c) From a79e6cbfc587c492abcc05484398f0013f9b058c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 02:01:10 +0000 Subject: [PATCH 09/27] Phase 2: bit-sliced slice add handles equal nonzero lane offsets Row reduction adds rows starting at the pivot column, so both slices share a nonzero lane offset. Use the plane kernel for the interior full groups in that case (only the partial leading/trailing groups go entry-wise), instead of falling back to fully entry-wise addition. Cuts odd-prime row reduction time by ~6x (e.g. p=3 dim 1000: 1.55s -> 235ms). row_reduce test still passes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/vector/impl_fqslicemut.rs | 56 ++++++++++++++------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index 98a68d6855..f91b341928 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -160,27 +160,49 @@ impl<'a, F: Field> FqSliceMut<'a, F> { return; } let epg = fq.entries_per_group(); + let s_start = self.start(); + let o_start = other.start(); let len = self.as_slice().len(); + if len == 0 { + return; + } - if self.start() % epg == 0 && other.start() % epg == 0 { - let k = fq.limbs_per_group(); - let full_groups = len / epg; - if full_groups > 0 { - let nlimbs = full_groups * k; - let sbase = (self.start() / epg) * k; - let obase = (other.start() / epg) * k; - let src = &other.limbs()[obase..obase + nlimbs]; - fq.add_groups(&mut self.limbs_mut()[sbase..sbase + nlimbs], src, c.clone()); + // The fast plane kernel needs the two slices to share a lane offset within their + // groups (so group `g` of one lines up with group `g` of the other). This holds for + // whole vectors and for matrix-row adds that start at the same pivot column. The + // partial leading/trailing groups (and any mismatched-offset slice) are added + // entry-wise. + let aligned = s_start % epg == o_start % epg; + let s_end = s_start + len; + let first_full = s_start.div_ceil(epg) * epg; + let last_full = (s_end / epg) * epg; + + if !aligned || first_full >= last_full { + for (i, v) in other.iter_nonzero() { + self.add_basis_element(i, v * c.clone()); } - // Trailing entries that don't fill a whole group. - for i in (full_groups * epg)..len { - let v = other.entry(i); - if v != fq.zero() { - self.add_basis_element(i, v * c.clone()); - } + return; + } + + let k = fq.limbs_per_group(); + // Leading partial group. + for i in 0..(first_full - s_start) { + let v = other.entry(i); + if v != fq.zero() { + self.add_basis_element(i, v * c.clone()); } - } else { - for (i, v) in other.iter_nonzero() { + } + // Interior full groups, via the plane kernel. + let num_full = (last_full - first_full) / epg; + let s_limb = (first_full / epg) * k; + let o_limb = ((o_start + (first_full - s_start)) / epg) * k; + let nlimbs = num_full * k; + let src = &other.limbs()[o_limb..o_limb + nlimbs]; + fq.add_groups(&mut self.limbs_mut()[s_limb..s_limb + nlimbs], src, c.clone()); + // Trailing partial group. + for i in (last_full - s_start)..len { + let v = other.entry(i); + if v != fq.zero() { self.add_basis_element(i, v * c.clone()); } } From ce635c3011c2e7ec5146301c311a7231ec7a651e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 02:14:05 +0000 Subject: [PATCH 10/27] Phase 2: masked plane-circuit add for bit-sliced slice boundaries Replace the element-wise leading/trailing partial-group handling in bit-sliced slice add with a masked plane-circuit add (add_group_masked): mask the source group to its in-range lanes and run the kernel, leaving out-of-range lanes untouched. ~2x faster odd-prime row reduction vs the element-wise version (p=3 dim 1000: 235ms -> 111ms). row_reduce test passes. Note: still ~3x slower than the packed baseline for p=3/p=5 (which have hand-tuned SWAR reduce); investigation of the remaining per-add overhead is ongoing. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 23 +++++++ ext/crates/fp/src/field/field_internal.rs | 20 ++++++ ext/crates/fp/src/field/fp.rs | 17 +++++ ext/crates/fp/src/vector/impl_fqslicemut.rs | 72 ++++++++++++++------- 4 files changed, 107 insertions(+), 25 deletions(-) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index b7c663080a..31d3b34890 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -53,6 +53,29 @@ pub(crate) fn add_groups(p: u32, k: usize, dst: &mut [Limb], src: &[Limb], c: u3 dispatch!(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); } +/// `dst += c * src` (mod p) for a single group of `k` planes, restricted to the lanes set in +/// `lane_mask`. Lanes outside the mask are unchanged. `dst` and `src` are each exactly `k` +/// limbs. +pub(crate) fn add_group_masked( + p: u32, + k: usize, + dst: &mut [Limb], + src: &[Limb], + c: u32, + lane_mask: Limb, +) { + if c == 0 { + return; + } + // Masking `src` to the in-range lanes makes the circuit a no-op elsewhere: an out-of-range + // lane adds `c * 0 = 0` to an already-reduced `dst`, leaving it unchanged. + let mut masked = [0 as Limb; BITS_PER_LIMB]; + for j in 0..k { + masked[j] = src[j] & lane_mask; + } + add_groups(p, k, dst, &masked[..k], c); +} + /// `dst *= c` (mod p) over every group. pub(crate) fn scale_groups(p: u32, k: usize, dst: &mut [Limb], c: u32) { let masks = p_masks(p, k); diff --git a/ext/crates/fp/src/field/field_internal.rs b/ext/crates/fp/src/field/field_internal.rs index 7d86dfc19e..b4e1f8b344 100644 --- a/ext/crates/fp/src/field/field_internal.rs +++ b/ext/crates/fp/src/field/field_internal.rs @@ -231,6 +231,26 @@ pub trait FieldInternal: } } + /// `dst += coeff * src` (mod p) for a single group (each `limbs_per_group()` limbs), + /// restricted to the lanes set in `lane_mask`; other lanes are unchanged. Used for the + /// partial boundary groups of a slice add. Default: element-wise; overridden by + /// [`Fp`](super::Fp) with a masked plane circuit. + fn add_group_masked( + self, + dst: &mut [Limb], + src: &[Limb], + coeff: FieldElement, + lane_mask: Limb, + ) { + for lane in 0..self.entries_per_group() { + if (lane_mask >> lane) & 1 == 1 { + let a = self.gather(dst, lane); + let b = self.gather(src, lane); + self.scatter(dst, lane, self.add(a, self.mul(coeff.clone(), b))); + } + } + } + /// Check whether or not a limb is reduced. This may potentially not be faster than calling /// [`reduce`](FieldInternal::reduce) directly. fn is_reduced(self, limb: Limb) -> bool { diff --git a/ext/crates/fp/src/field/fp.rs b/ext/crates/fp/src/field/fp.rs index 004730e745..4fa593fea3 100644 --- a/ext/crates/fp/src/field/fp.rs +++ b/ext/crates/fp/src/field/fp.rs @@ -176,6 +176,23 @@ impl FieldInternal for Fp

{ self.encode(coeff) as u32, ); } + + fn add_group_masked( + self, + dst: &mut [Limb], + src: &[Limb], + coeff: FieldElement, + lane_mask: Limb, + ) { + crate::field::bitslice::add_group_masked( + self.characteristic().as_u32(), + self.limbs_per_group(), + dst, + src, + self.encode(coeff) as u32, + lane_mask, + ); + } } #[cfg(feature = "proptest")] diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index f91b341928..52ab34e8cb 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -172,12 +172,8 @@ impl<'a, F: Field> FqSliceMut<'a, F> { // whole vectors and for matrix-row adds that start at the same pivot column. The // partial leading/trailing groups (and any mismatched-offset slice) are added // entry-wise. - let aligned = s_start % epg == o_start % epg; - let s_end = s_start + len; - let first_full = s_start.div_ceil(epg) * epg; - let last_full = (s_end / epg) * epg; - - if !aligned || first_full >= last_full { + if s_start % epg != o_start % epg { + // Mismatched lane offsets: the planes don't line up, so fall back to entry-wise. for (i, v) in other.iter_nonzero() { self.add_basis_element(i, v * c.clone()); } @@ -185,26 +181,52 @@ impl<'a, F: Field> FqSliceMut<'a, F> { } let k = fq.limbs_per_group(); - // Leading partial group. - for i in 0..(first_full - s_start) { - let v = other.entry(i); - if v != fq.zero() { - self.add_basis_element(i, v * c.clone()); - } + let s_end = s_start + len; + let first_g = s_start / epg; + let last_g = (s_end - 1) / epg; + // Group `g` of `self` lines up with group `g - first_g + o_first_g` of `other`. + let o_first_g = o_start / epg; + let group_limbs = |self_g: usize| { + let s = self_g * k; + let o = (o_first_g + (self_g - first_g)) * k; + (s, o) + }; + // Lane mask selecting bits `[lo, hi)`. + let lane_mask = |lo: usize, hi: usize| -> Limb { + let high: Limb = if hi >= epg { !0 } else { (1 << hi) - 1 }; + let low: Limb = (1 << lo) - 1; + high & !low + }; + + if first_g == last_g { + // Single (partial) group. + let (s, o) = group_limbs(first_g); + let mask = lane_mask(s_start - first_g * epg, s_end - first_g * epg); + let src = &other.limbs()[o..o + k]; + fq.add_group_masked(&mut self.limbs_mut()[s..s + k], src, c, mask); + return; } - // Interior full groups, via the plane kernel. - let num_full = (last_full - first_full) / epg; - let s_limb = (first_full / epg) * k; - let o_limb = ((o_start + (first_full - s_start)) / epg) * k; - let nlimbs = num_full * k; - let src = &other.limbs()[o_limb..o_limb + nlimbs]; - fq.add_groups(&mut self.limbs_mut()[s_limb..s_limb + nlimbs], src, c.clone()); - // Trailing partial group. - for i in (last_full - s_start)..len { - let v = other.entry(i); - if v != fq.zero() { - self.add_basis_element(i, v * c.clone()); - } + + // Leading partial group: lanes [s_start mod 64, 64). + { + let (s, o) = group_limbs(first_g); + let mask = lane_mask(s_start - first_g * epg, epg); + let src = &other.limbs()[o..o + k]; + fq.add_group_masked(&mut self.limbs_mut()[s..s + k], src, c.clone(), mask); + } + // Interior full groups, via the plane kernel in one contiguous call. + if last_g > first_g + 1 { + let (s, o) = group_limbs(first_g + 1); + let nlimbs = (last_g - first_g - 1) * k; + let src = &other.limbs()[o..o + nlimbs]; + fq.add_groups(&mut self.limbs_mut()[s..s + nlimbs], src, c.clone()); + } + // Trailing partial group: lanes [0, s_end mod 64). + { + let (s, o) = group_limbs(last_g); + let mask = lane_mask(0, s_end - last_g * epg); + let src = &other.limbs()[o..o + k]; + fq.add_group_masked(&mut self.limbs_mut()[s..s + k], src, c, mask); } } From feff5174d10779d867a1df84f2daea62cf3705ea Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 02:32:07 +0000 Subject: [PATCH 11/27] Phase 2: cut per-call overhead in bit-sliced kernels - Compute the per-plane modulus mask inline from p inside the const-K circuits instead of building a 65-element array on every add_groups call. - Dispatch add_group_masked to an exactly-K-sized masked kernel rather than zero-filling a 64-element scratch array per boundary call. Measured effect on odd-prime row reduction (dim 1000): p=3 111ms -> 66ms, p=5 205ms -> 130ms, p=7 237ms -> 159ms. All fp tests pass. Remaining: p=3/p=5 row reduction is still ~1.85x/2.9x slower than the SWAR-tuned packed path (the circuit's sequential ripple/borrow chains and double-and-add scalar multiply lose to a single packed madd+SWAR-reduce); p>=7 and large primes are faster bit-sliced. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 88 ++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 27 deletions(-) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 31d3b34890..86f3ae00e3 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -35,18 +35,23 @@ fn p_masks(p: u32, k: usize) -> [Limb; BITS_PER_LIMB + 1] { masks } +/// The full-width lane mask for bit `j` of `p`: all-ones if set, zero otherwise. +#[inline(always)] +fn pmask(p: u32, j: usize) -> Limb { + 0u64.wrapping_sub(((p >> j) & 1) as Limb) +} + /// `dst += c * src` (mod p) over every group, where `dst` and `src` hold the same number of /// whole groups of `k` planes. Assumes both are reduced; the result is reduced. pub(crate) fn add_groups(p: u32, k: usize, dst: &mut [Limb], src: &[Limb], c: u32) { if c == 0 { return; } - let masks = p_masks(p, k); macro_rules! dispatch { ($($k:literal),*) => { match k { - $($k => add_groups_k::<$k>(dst, src, c, &masks),)* - _ => add_groups_dyn(k, dst, src, c, &masks), + $($k => add_groups_k::<$k>(dst, src, c, p),)* + _ => add_groups_dyn(k, dst, src, c, &p_masks(p, k)), } }; } @@ -67,23 +72,31 @@ pub(crate) fn add_group_masked( if c == 0 { return; } - // Masking `src` to the in-range lanes makes the circuit a no-op elsewhere: an out-of-range - // lane adds `c * 0 = 0` to an already-reduced `dst`, leaving it unchanged. - let mut masked = [0 as Limb; BITS_PER_LIMB]; - for j in 0..k { - masked[j] = src[j] & lane_mask; + macro_rules! dispatch { + ($($k:literal),*) => { + match k { + $($k => add_group_masked_k::<$k>(dst, src, c, p, lane_mask),)* + _ => { + // Masking `src` to the in-range lanes makes the circuit a no-op elsewhere. + let mut masked = vec![0; k]; + for j in 0..k { + masked[j] = src[j] & lane_mask; + } + add_groups(p, k, dst, &masked, c); + } + } + }; } - add_groups(p, k, dst, &masked[..k], c); + dispatch!(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); } /// `dst *= c` (mod p) over every group. pub(crate) fn scale_groups(p: u32, k: usize, dst: &mut [Limb], c: u32) { - let masks = p_masks(p, k); macro_rules! dispatch { ($($k:literal),*) => { match k { - $($k => scale_groups_k::<$k>(dst, c, &masks),)* - _ => scale_groups_dyn(k, dst, c, &masks), + $($k => scale_groups_k::<$k>(dst, c, p),)* + _ => scale_groups_dyn(k, dst, c, &p_masks(p, k)), } }; } @@ -96,19 +109,20 @@ pub(crate) fn scale_groups(p: u32, k: usize, dst: &mut [Limb], c: u32) { // --------------------------------------------------------------------------------------- /// Reduce a `(K+1)`-bit unreduced sum (`s` low planes + `s_top`) in `[0, 2p)` to `s mod p`. +/// The per-plane mask is computed inline from `p` (no per-call mask array). #[inline(always)] -fn cond_sub_k(s: &[Limb; K], s_top: Limb, masks: &[Limb]) -> [Limb; K] { +fn cond_sub_k(s: &[Limb; K], s_top: Limb, p: u32) -> [Limb; K] { let mut d = [0 as Limb; K]; let mut borrow: Limb = 0; for j in 0..K { let sj = s[j]; - let pj = masks[j]; + let pj = pmask(p, j); let sxp = sj ^ pj; d[j] = sxp ^ borrow; borrow = (!sj & pj) | (borrow & !sxp); } // Top bit only affects the borrow-out (the result fits in K planes since result < p). - let pj = masks[K]; + let pj = pmask(p, K); let sxp = s_top ^ pj; borrow = (!s_top & pj) | (borrow & !sxp); let ge = !borrow; @@ -121,7 +135,7 @@ fn cond_sub_k(s: &[Limb; K], s_top: Limb, masks: &[Limb]) -> [Li /// `(a + b) mod p` over `K` planes. #[inline(always)] -fn add_mod_k(a: &[Limb; K], b: &[Limb; K], masks: &[Limb]) -> [Limb; K] { +fn add_mod_k(a: &[Limb; K], b: &[Limb; K], p: u32) -> [Limb; K] { let mut s = [0 as Limb; K]; let mut carry: Limb = 0; for j in 0..K { @@ -131,54 +145,74 @@ fn add_mod_k(a: &[Limb; K], b: &[Limb; K], masks: &[Limb]) -> [L s[j] = axb ^ carry; carry = (aj & bj) | (carry & axb); } - cond_sub_k::(&s, carry, masks) + cond_sub_k::(&s, carry, p) } /// `(2 * a) mod p` over `K` planes (doubling is a one-position plane shift). #[inline(always)] -fn double_mod_k(a: &[Limb; K], masks: &[Limb]) -> [Limb; K] { +fn double_mod_k(a: &[Limb; K], p: u32) -> [Limb; K] { let mut s = [0 as Limb; K]; for j in 1..K { s[j] = a[j - 1]; } let s_top = a[K - 1]; - cond_sub_k::(&s, s_top, masks) + cond_sub_k::(&s, s_top, p) } /// `(c * b) mod p` over `K` planes, via double-and-add. #[inline(always)] -fn scalar_mul_k(b: &[Limb; K], c: u32, masks: &[Limb]) -> [Limb; K] { +fn scalar_mul_k(b: &[Limb; K], c: u32, p: u32) -> [Limb; K] { let mut result = [0 as Limb; K]; let mut temp = *b; let mut cc = c; loop { if cc & 1 == 1 { - result = add_mod_k::(&result, &temp, masks); + result = add_mod_k::(&result, &temp, p); } cc >>= 1; if cc == 0 { break; } - temp = double_mod_k::(&temp, masks); + temp = double_mod_k::(&temp, p); } result } #[inline] -fn add_groups_k(dst: &mut [Limb], src: &[Limb], c: u32, masks: &[Limb]) { +fn add_groups_k(dst: &mut [Limb], src: &[Limb], c: u32, p: u32) { for (dg, sg) in dst.chunks_exact_mut(K).zip(src.chunks_exact(K)) { let mut a = [0 as Limb; K]; let mut b = [0 as Limb; K]; a.copy_from_slice(dg); b.copy_from_slice(sg); - let addend = if c == 1 { b } else { scalar_mul_k::(&b, c, masks) }; - let sum = add_mod_k::(&a, &addend, masks); + let addend = if c == 1 { b } else { scalar_mul_k::(&b, c, p) }; + let sum = add_mod_k::(&a, &addend, p); dg.copy_from_slice(&sum); } } +/// `dst += c * src` (mod p) for a single `K`-plane group, restricted to lanes in `lane_mask`. +#[inline] +fn add_group_masked_k( + dst: &mut [Limb], + src: &[Limb], + c: u32, + p: u32, + lane_mask: Limb, +) { + let mut a = [0 as Limb; K]; + let mut b = [0 as Limb; K]; + for j in 0..K { + a[j] = dst[j]; + b[j] = src[j] & lane_mask; + } + let addend = if c == 1 { b } else { scalar_mul_k::(&b, c, p) }; + let sum = add_mod_k::(&a, &addend, p); + dst[..K].copy_from_slice(&sum); +} + #[inline] -fn scale_groups_k(dst: &mut [Limb], c: u32, masks: &[Limb]) { +fn scale_groups_k(dst: &mut [Limb], c: u32, p: u32) { if c == 1 { return; } @@ -189,7 +223,7 @@ fn scale_groups_k(dst: &mut [Limb], c: u32, masks: &[Limb]) { for dg in dst.chunks_exact_mut(K) { let mut a = [0 as Limb; K]; a.copy_from_slice(dg); - let scaled = scalar_mul_k::(&a, c, masks); + let scaled = scalar_mul_k::(&a, c, p); dg.copy_from_slice(&scaled); } } From 3ce4ab69ed8e9c3240c88e17790b0f535473789d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 04:00:30 +0000 Subject: [PATCH 12/27] Phase 3: 6-gate F3 bit-slice add circuit Replace the generic ripple-carry+conditional-subtract path for p=3 with a flat 6-gate circuit (two XOR / two XOR / two AND-NOT layers, mapping to x86 andn with short dependency chains) on the (hi,lo) plane encoding; multiplication by 2 = negation is a plane swap. Circuit contributed by the user; verified exhaustively against all 9 valid input pairs and via the existing p=3 proptests. Row reduction at p=3, dim 1000: 66ms (generic bit-slice) -> 28ms, now 1.09x faster than the packed SWAR baseline (30.5ms) instead of 1.85x slower. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 77 +++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 86f3ae00e3..4ef02e4e91 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -47,6 +47,9 @@ pub(crate) fn add_groups(p: u32, k: usize, dst: &mut [Limb], src: &[Limb], c: u3 if c == 0 { return; } + if p == 3 { + return f3_add_groups(dst, src, c); + } macro_rules! dispatch { ($($k:literal),*) => { match k { @@ -72,6 +75,9 @@ pub(crate) fn add_group_masked( if c == 0 { return; } + if p == 3 { + return f3_add_group_masked(dst, src, c, lane_mask); + } macro_rules! dispatch { ($($k:literal),*) => { match k { @@ -92,6 +98,16 @@ pub(crate) fn add_group_masked( /// `dst *= c` (mod p) over every group. pub(crate) fn scale_groups(p: u32, k: usize, dst: &mut [Limb], c: u32) { + if c == 1 { + return; + } + if c == 0 { + dst.fill(0); + return; + } + if p == 3 { + return f3_scale_groups(dst, c); + } macro_rules! dispatch { ($($k:literal),*) => { match k { @@ -228,6 +244,67 @@ fn scale_groups_k(dst: &mut [Limb], c: u32, p: u32) { } } +// --------------------------------------------------------------------------------------- +// F3 specialization (k = 2). Plane 0 is the low bit, plane 1 the high bit, so an element +// `v in {0,1,2}` is stored as `(hi, lo)` with `v = 2*hi + lo`. Addition is a flat boolean +// circuit (no ripple-carry or borrow chain), and multiplication by 2 = negation just swaps +// the two planes — both avoid the sequential dependencies that make the generic circuit lose +// to the packed SWAR reduce at small primes. +// --------------------------------------------------------------------------------------- + +/// `(a + b) mod 3` as a flat 6-gate circuit on the `(lo, hi)` planes (each lane independent). +/// +/// Three parallel layers — two XORs, two XORs, two AND-NOTs — so it maps onto x86 `andn` +/// and has very short dependency chains. Verified exhaustively against the 9 valid input +/// pairs (the `(hi, lo) = (1, 1)` encoding never occurs for reduced inputs). +#[inline(always)] +fn f3_add_planes(a_lo: Limb, a_hi: Limb, b_lo: Limb, b_hi: Limb) -> (Limb, Limb) { + let t_hi = a_hi ^ b_hi; + let t_lo = a_lo ^ b_lo; + let u_hi = b_hi ^ t_lo; + let u_lo = b_lo ^ t_hi; + let r_hi = u_lo & !t_lo; + let r_lo = u_hi & !t_hi; + (r_lo, r_hi) +} + +/// Negation in F3 swaps 1 <-> 2 (and fixes 0), i.e. swaps the two planes. +#[inline(always)] +fn f3_addend(sg: &[Limb], c: u32) -> (Limb, Limb) { + // c is 1 or 2 here; c == 2 means add (-other), i.e. negate by swapping planes. + if c == 1 { + (sg[0], sg[1]) + } else { + (sg[1], sg[0]) + } +} + +fn f3_add_groups(dst: &mut [Limb], src: &[Limb], c: u32) { + for (dg, sg) in dst.chunks_exact_mut(2).zip(src.chunks_exact(2)) { + let (b_lo, b_hi) = f3_addend(sg, c); + let (r_lo, r_hi) = f3_add_planes(dg[0], dg[1], b_lo, b_hi); + dg[0] = r_lo; + dg[1] = r_hi; + } +} + +fn f3_add_group_masked(dst: &mut [Limb], src: &[Limb], c: u32, lane_mask: Limb) { + let (b_lo, b_hi) = f3_addend(src, c); + // Masking the addend to the in-range lanes makes the circuit a no-op (adds 0) elsewhere. + let (r_lo, r_hi) = f3_add_planes(dst[0], dst[1], b_lo & lane_mask, b_hi & lane_mask); + dst[0] = r_lo; + dst[1] = r_hi; +} + +fn f3_scale_groups(dst: &mut [Limb], c: u32) { + // c == 2 is negation (plane swap); c == 1 is a no-op; c == 0 is handled by the caller. + if c == 2 { + for dg in dst.chunks_exact_mut(2) { + dg.swap(0, 1); + } + } +} + // --------------------------------------------------------------------------------------- // Heap-scratch fallback for `k > MAX_DISPATCH_K` (very large primes). // --------------------------------------------------------------------------------------- From eb74407e2e733c9f348e52705f3bec7d6e4c08ef Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 04:06:41 +0000 Subject: [PATCH 13/27] Phase 3: flat F5 bit-slice add/scale circuits Replace the generic ripple-carry+conditional-subtract+double-and-add path for p=5 with flat "indicator" circuits: one-hot lane masks per operand value recombined with no carry/borrow chain, for both add and scalar multiply. This restores the instruction-level parallelism the sequential generic circuit lost. Verified via the existing p=5 proptests and row_reduce. Row reduction at p=5, dim 1000: 106ms -> 80ms. Still ~2.1x slower than the packed baseline (38ms): packed p=5 has a hand-tuned SWAR reduce and packs 12 entries/limb, so its ops-per-entry is lower than the ~70-gate bit-sliced fma. (Unlike p=7+, which has no tuned packed reduce and is faster bit-sliced.) Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 109 ++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 4ef02e4e91..5559679ea1 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -50,6 +50,9 @@ pub(crate) fn add_groups(p: u32, k: usize, dst: &mut [Limb], src: &[Limb], c: u3 if p == 3 { return f3_add_groups(dst, src, c); } + if p == 5 { + return f5_add_groups(dst, src, c); + } macro_rules! dispatch { ($($k:literal),*) => { match k { @@ -78,6 +81,9 @@ pub(crate) fn add_group_masked( if p == 3 { return f3_add_group_masked(dst, src, c, lane_mask); } + if p == 5 { + return f5_add_group_masked(dst, src, c, lane_mask); + } macro_rules! dispatch { ($($k:literal),*) => { match k { @@ -108,6 +114,9 @@ pub(crate) fn scale_groups(p: u32, k: usize, dst: &mut [Limb], c: u32) { if p == 3 { return f3_scale_groups(dst, c); } + if p == 5 { + return f5_scale_groups(dst, c); + } macro_rules! dispatch { ($($k:literal),*) => { match k { @@ -305,6 +314,106 @@ fn f3_scale_groups(dst: &mut [Limb], c: u32) { } } +// --------------------------------------------------------------------------------------- +// F5 specialization (k = 3). Planes are bits 0,1,2 of the value `v in {0,..,4}`. Both the +// add and the scalar multiply are built as flat "indicator" circuits — one-hot lane masks +// `is_v` for each operand value, recombined into the result with no carry/borrow chain — so +// they keep the wide instruction-level parallelism the sequential generic circuit loses. +// --------------------------------------------------------------------------------------- + +/// One-hot lane masks: `out[v]` has the bits of the lanes whose value is `v` (for `v in 0..5`). +#[inline(always)] +fn f5_indicators(p0: Limb, p1: Limb, p2: Limb) -> [Limb; 5] { + let n0 = !p0; + let n1 = !p1; + let n2 = !p2; + [ + n0 & n1 & n2, // 0 = 000 + p0 & n1 & n2, // 1 = 001 + n0 & p1 & n2, // 2 = 010 + p0 & p1 & n2, // 3 = 011 + n0 & n1 & p2, // 4 = 100 + ] +} + +/// Reassemble the three planes from per-value selection masks (`sel[v]` selects value `v`). +#[inline(always)] +fn f5_compose(sel: [Limb; 5]) -> (Limb, Limb, Limb) { + // bit 0 set for values {1,3}; bit 1 for {2,3}; bit 2 for {4}. + (sel[1] | sel[3], sel[2] | sel[3], sel[4]) +} + +/// `c * v mod 5` on the three planes (`c in 1..5`). +#[inline(always)] +fn f5_mul_planes(p0: Limb, p1: Limb, p2: Limb, c: u32) -> (Limb, Limb, Limb) { + let ind = f5_indicators(p0, p1, p2); + let mut sel = [0 as Limb; 5]; + for v in 0..5u32 { + sel[((c * v) % 5) as usize] |= ind[v as usize]; + } + f5_compose(sel) +} + +/// `(a + b) mod 5` on the three planes, as a flat indicator circuit. +#[inline(always)] +fn f5_add_planes( + a0: Limb, + a1: Limb, + a2: Limb, + b0: Limb, + b1: Limb, + b2: Limb, +) -> (Limb, Limb, Limb) { + let ia = f5_indicators(a0, a1, a2); + let ib = f5_indicators(b0, b1, b2); + let mut sel = [0 as Limb; 5]; + for av in 0..5usize { + for bv in 0..5usize { + sel[(av + bv) % 5] |= ia[av] & ib[bv]; + } + } + f5_compose(sel) +} + +fn f5_add_groups(dst: &mut [Limb], src: &[Limb], c: u32) { + for (dg, sg) in dst.chunks_exact_mut(3).zip(src.chunks_exact(3)) { + let (b0, b1, b2) = if c == 1 { + (sg[0], sg[1], sg[2]) + } else { + f5_mul_planes(sg[0], sg[1], sg[2], c) + }; + let (r0, r1, r2) = f5_add_planes(dg[0], dg[1], dg[2], b0, b1, b2); + dg[0] = r0; + dg[1] = r1; + dg[2] = r2; + } +} + +fn f5_add_group_masked(dst: &mut [Limb], src: &[Limb], c: u32, lane_mask: Limb) { + let (mut b0, mut b1, mut b2) = if c == 1 { + (src[0], src[1], src[2]) + } else { + f5_mul_planes(src[0], src[1], src[2], c) + }; + // Zeroing the addend outside the mask leaves those lanes unchanged (adds 0). + b0 &= lane_mask; + b1 &= lane_mask; + b2 &= lane_mask; + let (r0, r1, r2) = f5_add_planes(dst[0], dst[1], dst[2], b0, b1, b2); + dst[0] = r0; + dst[1] = r1; + dst[2] = r2; +} + +fn f5_scale_groups(dst: &mut [Limb], c: u32) { + for dg in dst.chunks_exact_mut(3) { + let (r0, r1, r2) = f5_mul_planes(dg[0], dg[1], dg[2], c); + dg[0] = r0; + dg[1] = r1; + dg[2] = r2; + } +} + // --------------------------------------------------------------------------------------- // Heap-scratch fallback for `k > MAX_DISPATCH_K` (very large primes). // --------------------------------------------------------------------------------------- From f7624e9b36063541ffe66d5dd8a4a950b364fa0c Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 04:51:04 +0000 Subject: [PATCH 14/27] Phase 2: fix bit-sliced shl_assign and augmented-matrix segment padding Two correctness bugs that the fp test suite missed but ext's odd-prime resolutions exercised: - FqSliceMut::shl_assign computed the limb shift from the packed entries_per_limb, which is meaningless in the bit-sliced layout and indexed out of bounds. Add a bit-sliced path that shifts entries down via gather/scatter (reading i+shift strictly ahead of writing i keeps it correct in place). - FpVector::padded_len, used to align AugmentedMatrix segments, computed num_limbs * entries_per_limb. For bit-sliced fields that mixes the bit-sliced limb count with the packed entries-per-limb (e.g. padded_len(3, 1) = 42), leaving segments straddling 64-entry groups. Round up to a whole number of groups (entries_per_group) instead; this equals the old value for the packed layout / F2 and correctly group-aligns bit-sliced segments. With these, the full ext test suite passes at odd primes (extend_identity, milnor_vs_adem, resolve_iterate, resolve all green). The pre-existing test_tempdir_lock failure is unrelated (it relies on file read-only bits, which root ignores in this environment; it also fails on the packed baseline). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/vector/fp_wrapper/mod.rs | 10 ++++++++-- ext/crates/fp/src/vector/impl_fqslicemut.rs | 12 ++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/ext/crates/fp/src/vector/fp_wrapper/mod.rs b/ext/crates/fp/src/vector/fp_wrapper/mod.rs index 112760002a..a132ec1ae2 100644 --- a/ext/crates/fp/src/vector/fp_wrapper/mod.rs +++ b/ext/crates/fp/src/vector/fp_wrapper/mod.rs @@ -117,9 +117,15 @@ impl FpVector { Fp::new(p).number(len) } - // Convenient for some matrix methods + // Round `len` up to a whole number of groups, so that an augmented-matrix segment of this + // length ends on a group boundary and the next segment starts on one. For the packed + // layout a group is one limb (`entries_per_group == entries_per_limb`), so this equals the + // old `num_limbs * entries_per_limb`; for the bit-sliced layout a group spans 64 entries + // across several limbs, and segments must align to those 64-entry boundaries (not to the + // packed `entries_per_limb`) or a single group would straddle two segments. pub(crate) fn padded_len(p: ValidPrime, len: usize) -> usize { - Self::num_limbs(p, len) * Fp::new(p).entries_per_limb() + let entries_per_group = Fp::new(p).entries_per_group(); + len.div_ceil(entries_per_group) * entries_per_group } } diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index 52ab34e8cb..ec0762c02e 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -294,6 +294,18 @@ impl<'a, F: Field> FqSliceMut<'a, F> { if shift == 0 { return; } + if self.fq().is_bitsliced() { + // The packed limb-move trick assumes an entry is a contiguous bitfield, which the + // bit-sliced layout breaks. Move entries down one at a time via gather/scatter: + // reading `i + shift` strictly ahead of writing `i` keeps it correct in place. + let new_len = self.as_slice().len() - shift; + for i in 0..new_len { + let v = self.as_slice().entry(i + shift); + self.set_entry(i, v); + } + *self.end_mut() -= shift; + return; + } if self.start() == 0 && shift.is_multiple_of(self.fq().entries_per_limb()) { let limb_shift = shift / self.fq().entries_per_limb(); *self.end_mut() -= shift; From 81c4351246b924d43061f8779051ac69441af390 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 05:13:04 +0000 Subject: [PATCH 15/27] Phase 5: remove dead odd-prime shift branch and orphaned num_limbs Every odd-characteristic field is now bit-sliced; only F_2 (k=1) uses the packed bit-shift realignment path (add_shift_none/left/right). Drop the unreachable odd-prime `else` branch in FqSliceMut::add (collapsing to F_2 shift vs add_bitsliced) and remove FpVector::num_limbs, which padded_len no longer calls. The add_shift_* methods are kept: they remain F_2's unaligned slice-add implementation, and routing F_2 through the element-wise bit-sliced fallback would regress the dominant prime. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/vector/fp_wrapper/mod.rs | 5 ----- ext/crates/fp/src/vector/impl_fqslicemut.rs | 11 ++++------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/ext/crates/fp/src/vector/fp_wrapper/mod.rs b/ext/crates/fp/src/vector/fp_wrapper/mod.rs index a132ec1ae2..3ae7ad1953 100644 --- a/ext/crates/fp/src/vector/fp_wrapper/mod.rs +++ b/ext/crates/fp/src/vector/fp_wrapper/mod.rs @@ -112,11 +112,6 @@ impl FpVector { v } - // Convenient for some matrix methods - pub(crate) fn num_limbs(p: ValidPrime, len: usize) -> usize { - Fp::new(p).number(len) - } - // Round `len` up to a whole number of groups, so that an augmented-matrix segment of this // length ends on a group boundary and the next segment starts on one. For the packed // layout a group is one limb (`entries_per_group == entries_per_limb`), so this equals the diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index ec0762c02e..a26b7a34f3 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -128,6 +128,8 @@ impl<'a, F: Field> FqSliceMut<'a, F> { return; } + // `F_2` is the only field that is not bit-sliced (`k = 1`, identical to the old packed + // layout); it keeps the bit-shift realignment path. Every other field is bit-sliced. if self.fq().q() == 2 { if c != self.fq().zero() { match self.as_slice().offset().cmp(&other.offset()) { @@ -136,14 +138,9 @@ impl<'a, F: Field> FqSliceMut<'a, F> { Ordering::Greater => self.add_shift_right(other, self.fq().one()), }; } - } else if self.fq().is_bitsliced() { - self.add_bitsliced(other, c); } else { - match self.as_slice().offset().cmp(&other.offset()) { - Ordering::Equal => self.add_shift_none(other, c), - Ordering::Less => self.add_shift_left(other, c), - Ordering::Greater => self.add_shift_right(other, c), - }; + debug_assert!(self.fq().is_bitsliced()); + self.add_bitsliced(other, c); } } From 339946933931e5cccf11c994c32f9dbb69647e24 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 05:43:43 +0000 Subject: [PATCH 16/27] Phase 5: unify slice add, remove add_shift_* (general per-plane shifted add) A bit-sliced vector is k independent single-bit planes; realigning two slices with different lane offsets is a per-plane bit shift. That is exactly what the F2-only add_shift_none/left/right did for one plane, so generalize it to k planes: add_bitsliced_shifted builds, per target group, the k source planes shifted into alignment and adds them via the masked group circuit. With this, FqSliceMut::add routes every field through add_bitsliced (F2 is just k=1), and the ~290 lines of add_shift_none/left/right plus AddShiftLeft/RightData are deleted. Add p==2 XOR fast paths to bitslice::add_groups/add_group_masked so F2's aligned slice add stays as cheap as the old add_shift_none. No regression (row reduction, dim 1000, same machine): F2 1.551ms -> 1.543ms (identical within noise), p3/p5/p7 unchanged. Odd-prime *unaligned* slice adds are now faster (shifted circuit instead of element-wise). All fp tests pass. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 12 + ext/crates/fp/src/vector/impl_fqslicemut.rs | 393 ++++---------------- 2 files changed, 85 insertions(+), 320 deletions(-) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 5559679ea1..01ffa814c5 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -47,6 +47,13 @@ pub(crate) fn add_groups(p: u32, k: usize, dst: &mut [Limb], src: &[Limb], c: u3 if c == 0 { return; } + if p == 2 { + // One plane (k = 1); the only nonzero scalar is 1, so addition is XOR. + for (d, s) in dst.iter_mut().zip(src) { + *d ^= *s; + } + return; + } if p == 3 { return f3_add_groups(dst, src, c); } @@ -78,6 +85,11 @@ pub(crate) fn add_group_masked( if c == 0 { return; } + if p == 2 { + // One plane (k = 1); XOR in only the in-range lanes. + dst[0] ^= src[0] & lane_mask; + return; + } if p == 3 { return f3_add_group_masked(dst, src, c, lane_mask); } diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index a26b7a34f3..4a3a9628e6 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -1,7 +1,3 @@ -use std::cmp::Ordering; - -use itertools::Itertools; - use super::inner::{FqSlice, FqSliceMut, FqVector}; use crate::{ constants, @@ -128,20 +124,9 @@ impl<'a, F: Field> FqSliceMut<'a, F> { return; } - // `F_2` is the only field that is not bit-sliced (`k = 1`, identical to the old packed - // layout); it keeps the bit-shift realignment path. Every other field is bit-sliced. - if self.fq().q() == 2 { - if c != self.fq().zero() { - match self.as_slice().offset().cmp(&other.offset()) { - Ordering::Equal => self.add_shift_none(other, self.fq().one()), - Ordering::Less => self.add_shift_left(other, self.fq().one()), - Ordering::Greater => self.add_shift_right(other, self.fq().one()), - }; - } - } else { - debug_assert!(self.fq().is_bitsliced()); - self.add_bitsliced(other, c); - } + // Every field uses the bit-sliced layout (`F_2` is just the `k = 1` case, identical to + // the old packed layout), so a single code path handles them all. + self.add_bitsliced(other, c); } /// Add `c * other` to `self` in the bit-sliced layout. When both slices begin at a group @@ -164,16 +149,12 @@ impl<'a, F: Field> FqSliceMut<'a, F> { return; } - // The fast plane kernel needs the two slices to share a lane offset within their - // groups (so group `g` of one lines up with group `g` of the other). This holds for - // whole vectors and for matrix-row adds that start at the same pivot column. The - // partial leading/trailing groups (and any mismatched-offset slice) are added - // entry-wise. + // The fast plane kernel needs the two slices to share a lane offset within their groups + // (so group `g` of one lines up with group `g` of the other). This holds for whole + // vectors and for matrix-row adds that start at the same pivot column. When the offsets + // differ, the planes must be realigned first — see `add_bitsliced_shifted`. if s_start % epg != o_start % epg { - // Mismatched lane offsets: the planes don't line up, so fall back to entry-wise. - for (i, v) in other.iter_nonzero() { - self.add_basis_element(i, v * c.clone()); - } + self.add_bitsliced_shifted(other, c); return; } @@ -227,6 +208,71 @@ impl<'a, F: Field> FqSliceMut<'a, F> { } } + /// Add `c * other` to `self` when the two slices have different lane offsets within their + /// groups, so the planes don't line up. A bit-sliced vector is `k` independent single-bit + /// planes; realigning it is a per-plane bit shift (this is exactly what the old F_2-only + /// `add_shift_*` did, generalized from one plane to `k`). For each target group we build the + /// `k` source planes shifted into alignment, then add them in with the masked group circuit. + fn add_bitsliced_shifted(&mut self, other: FqSlice<'_, F>, c: FieldElement) { + let fq = self.fq(); + let k = fq.limbs_per_group(); + let epg = fq.entries_per_group(); + let ts = self.start(); + let len = self.as_slice().len(); + // Source lane = target lane + shift (`other`'s entry i sits `shift` lanes from `self`'s). + let shift = other.start() as isize - ts as isize; + let src_limbs = other.limbs(); + + // Plane `j` of (absolute) group `g` of the source, or 0 if out of range. The whole-limb + // reads can stray outside the valid lane range, but those bits are masked off below. + let plane_limb = |g: isize, j: usize| -> Limb { + if g < 0 { + return 0; + } + let idx = g as usize * k + j; + if idx < src_limbs.len() { + src_limbs[idx] + } else { + 0 + } + }; + let lane_mask = |lo: usize, hi: usize| -> Limb { + let high: Limb = if hi >= epg { !0 } else { (1 << hi) - 1 }; + let low: Limb = (1 << lo) - 1; + high & !low + }; + + debug_assert!(k <= epg); + let mut shifted = [0 as Limb; constants::BITS_PER_LIMB]; + + let first_g = ts / epg; + let last_g = (ts + len - 1) / epg; + let epg_i = epg as isize; + for g in first_g..=last_g { + let lo = if g == first_g { ts - g * epg } else { 0 }; + let hi = if g == last_g { (ts + len) - g * epg } else { epg }; + let mask = lane_mask(lo, hi); + + // Source bit `b` of target group `g` lives at absolute source lane `g*epg + b + shift`. + let src_base = g as isize * epg_i + shift; + let sg = src_base.div_euclid(epg_i); + let bs = src_base.rem_euclid(epg_i) as u32; + for (j, s) in shifted[..k].iter_mut().enumerate() { + *s = if bs == 0 { + plane_limb(sg, j) + } else { + (plane_limb(sg, j) >> bs) | (plane_limb(sg + 1, j) << (epg as u32 - bs)) + }; + } + fq.add_group_masked( + &mut self.limbs_mut()[g * k..g * k + k], + &shifted[..k], + c.clone(), + mask, + ); + } + } + pub fn add_offset(&mut self, other: FqSlice<'_, F>, c: FieldElement, offset: usize) { self.slice_mut(offset, self.as_slice().len()) .add(other.restrict(offset, other.len()), c) @@ -315,299 +361,6 @@ impl<'a, F: Field> FqSliceMut<'a, F> { } } - /// Adds `c` * `other` to `self`. `other` must have the same length, offset, and prime as self. - pub fn add_shift_none(&mut self, other: FqSlice<'_, F>, c: FieldElement) { - assert_eq!(self.fq(), c.field()); - assert_eq!(self.fq(), other.fq()); - let fq = self.fq(); - - let target_range = self.as_slice().limb_range(); - let source_range = other.limb_range(); - - let (min_mask, max_mask) = other.limb_masks(); - - self.limbs_mut()[target_range.start] = fq.fma_limb( - self.limbs()[target_range.start], - other.limbs()[source_range.start] & min_mask, - c.clone(), - ); - self.limbs_mut()[target_range.start] = fq.reduce(self.limbs()[target_range.start]); - - let target_inner_range = self.as_slice().limb_range_inner(); - let source_inner_range = other.limb_range_inner(); - if !source_inner_range.is_empty() { - for (left, right) in self.limbs_mut()[target_inner_range] - .iter_mut() - .zip_eq(&other.limbs()[source_inner_range]) - { - *left = fq.fma_limb(*left, *right, c.clone()); - *left = fq.reduce(*left); - } - } - if source_range.len() > 1 { - // The first and last limbs are distinct, so we process the last. - self.limbs_mut()[target_range.end - 1] = fq.fma_limb( - self.limbs()[target_range.end - 1], - other.limbs()[source_range.end - 1] & max_mask, - c, - ); - self.limbs_mut()[target_range.end - 1] = fq.reduce(self.limbs()[target_range.end - 1]); - } - } - - fn add_shift_left(&mut self, other: FqSlice<'_, F>, c: FieldElement) { - struct AddShiftLeftData { - offset_shift: usize, - tail_shift: usize, - zero_bits: usize, - min_source_limb: usize, - min_target_limb: usize, - number_of_source_limbs: usize, - number_of_target_limbs: usize, - min_mask: Limb, - max_mask: Limb, - } - - impl AddShiftLeftData { - fn new(fq: F, target: FqSlice<'_, F>, source: FqSlice<'_, F>) -> Self { - debug_assert!(target.prime() == source.prime()); - debug_assert!(target.offset() <= source.offset()); - debug_assert!( - target.len() == source.len(), - "self.dim {} not equal to other.dim {}", - target.len(), - source.len() - ); - let offset_shift = source.offset() - target.offset(); - let bit_length = fq.bit_length(); - let entries_per_limb = fq.entries_per_limb(); - let usable_bits_per_limb = bit_length * entries_per_limb; - let tail_shift = usable_bits_per_limb - offset_shift; - let zero_bits = constants::BITS_PER_LIMB - usable_bits_per_limb; - let source_range = source.limb_range(); - let target_range = target.limb_range(); - let min_source_limb = source_range.start; - let min_target_limb = target_range.start; - let number_of_source_limbs = source_range.len(); - let number_of_target_limbs = target_range.len(); - let (min_mask, max_mask) = source.limb_masks(); - - Self { - offset_shift, - tail_shift, - zero_bits, - min_source_limb, - min_target_limb, - number_of_source_limbs, - number_of_target_limbs, - min_mask, - max_mask, - } - } - - fn mask_first_limb(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - (other.limbs()[i] & self.min_mask) >> self.offset_shift - } - - fn mask_middle_limb_a(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - other.limbs()[i] >> self.offset_shift - } - - fn mask_middle_limb_b(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - (other.limbs()[i] << (self.tail_shift + self.zero_bits)) >> self.zero_bits - } - - fn mask_last_limb_a(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - let source_limb_masked = other.limbs()[i] & self.max_mask; - source_limb_masked << self.tail_shift - } - - fn mask_last_limb_b(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - let source_limb_masked = other.limbs()[i] & self.max_mask; - source_limb_masked >> self.offset_shift - } - } - - let dat = AddShiftLeftData::new(self.fq(), self.as_slice(), other); - let mut i = 0; - { - self.limbs_mut()[i + dat.min_target_limb] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb], - dat.mask_first_limb(other, i + dat.min_source_limb), - c.clone(), - ); - } - for i in 1..dat.number_of_source_limbs - 1 { - self.limbs_mut()[i + dat.min_target_limb] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb], - dat.mask_middle_limb_a(other, i + dat.min_source_limb), - c.clone(), - ); - self.limbs_mut()[i + dat.min_target_limb - 1] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb - 1], - dat.mask_middle_limb_b(other, i + dat.min_source_limb), - c.clone(), - ); - self.limbs_mut()[i + dat.min_target_limb - 1] = - self.fq().reduce(self.limbs()[i + dat.min_target_limb - 1]); - } - i = dat.number_of_source_limbs - 1; - if i > 0 { - self.limbs_mut()[i + dat.min_target_limb - 1] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb - 1], - dat.mask_last_limb_a(other, i + dat.min_source_limb), - c.clone(), - ); - self.limbs_mut()[i + dat.min_target_limb - 1] = - self.fq().reduce(self.limbs()[i + dat.min_target_limb - 1]); - if dat.number_of_source_limbs == dat.number_of_target_limbs { - self.limbs_mut()[i + dat.min_target_limb] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb], - dat.mask_last_limb_b(other, i + dat.min_source_limb), - c, - ); - self.limbs_mut()[i + dat.min_target_limb] = - self.fq().reduce(self.limbs()[i + dat.min_target_limb]); - } - } else { - self.limbs_mut()[i + dat.min_target_limb] = - self.fq().reduce(self.limbs()[i + dat.min_target_limb]); - } - } - - fn add_shift_right(&mut self, other: FqSlice<'_, F>, c: FieldElement) { - struct AddShiftRightData { - offset_shift: usize, - tail_shift: usize, - zero_bits: usize, - min_source_limb: usize, - min_target_limb: usize, - number_of_source_limbs: usize, - number_of_target_limbs: usize, - min_mask: Limb, - max_mask: Limb, - } - - impl AddShiftRightData { - fn new(fq: F, target: FqSlice<'_, F>, source: FqSlice<'_, F>) -> Self { - debug_assert!(target.prime() == source.prime()); - debug_assert!(target.offset() >= source.offset()); - debug_assert!( - target.len() == source.len(), - "self.dim {} not equal to other.dim {}", - target.len(), - source.len() - ); - let offset_shift = target.offset() - source.offset(); - let bit_length = fq.bit_length(); - let entries_per_limb = fq.entries_per_limb(); - let usable_bits_per_limb = bit_length * entries_per_limb; - let tail_shift = usable_bits_per_limb - offset_shift; - let zero_bits = constants::BITS_PER_LIMB - usable_bits_per_limb; - let source_range = source.limb_range(); - let target_range = target.limb_range(); - let min_source_limb = source_range.start; - let min_target_limb = target_range.start; - let number_of_source_limbs = source_range.len(); - let number_of_target_limbs = target_range.len(); - let (min_mask, max_mask) = source.limb_masks(); - Self { - offset_shift, - tail_shift, - zero_bits, - min_source_limb, - min_target_limb, - number_of_source_limbs, - number_of_target_limbs, - min_mask, - max_mask, - } - } - - fn mask_first_limb_a(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - let source_limb_masked = other.limbs()[i] & self.min_mask; - (source_limb_masked << (self.offset_shift + self.zero_bits)) >> self.zero_bits - } - - fn mask_first_limb_b(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - let source_limb_masked = other.limbs()[i] & self.min_mask; - source_limb_masked >> self.tail_shift - } - - fn mask_middle_limb_a(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - (other.limbs()[i] << (self.offset_shift + self.zero_bits)) >> self.zero_bits - } - - fn mask_middle_limb_b(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - other.limbs()[i] >> self.tail_shift - } - - fn mask_last_limb_a(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - let source_limb_masked = other.limbs()[i] & self.max_mask; - source_limb_masked << self.offset_shift - } - - fn mask_last_limb_b(&self, other: FqSlice<'_, F>, i: usize) -> Limb { - let source_limb_masked = other.limbs()[i] & self.max_mask; - source_limb_masked >> self.tail_shift - } - } - - let dat = AddShiftRightData::new(self.fq(), self.as_slice(), other); - let mut i = 0; - { - self.limbs_mut()[i + dat.min_target_limb] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb], - dat.mask_first_limb_a(other, i + dat.min_source_limb), - c.clone(), - ); - self.limbs_mut()[i + dat.min_target_limb] = - self.fq().reduce(self.limbs()[i + dat.min_target_limb]); - if dat.number_of_target_limbs > 1 { - self.limbs_mut()[i + dat.min_target_limb + 1] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb + 1], - dat.mask_first_limb_b(other, i + dat.min_source_limb), - c.clone(), - ); - } - } - for i in 1..dat.number_of_source_limbs - 1 { - self.limbs_mut()[i + dat.min_target_limb] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb], - dat.mask_middle_limb_a(other, i + dat.min_source_limb), - c.clone(), - ); - self.limbs_mut()[i + dat.min_target_limb] = - self.fq().reduce(self.limbs()[i + dat.min_target_limb]); - self.limbs_mut()[i + dat.min_target_limb + 1] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb + 1], - dat.mask_middle_limb_b(other, i + dat.min_source_limb), - c.clone(), - ); - } - i = dat.number_of_source_limbs - 1; - if i > 0 { - self.limbs_mut()[i + dat.min_target_limb] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb], - dat.mask_last_limb_a(other, i + dat.min_source_limb), - c.clone(), - ); - self.limbs_mut()[i + dat.min_target_limb] = - self.fq().reduce(self.limbs()[i + dat.min_target_limb]); - if dat.number_of_target_limbs > dat.number_of_source_limbs { - self.limbs_mut()[i + dat.min_target_limb + 1] = self.fq().fma_limb( - self.limbs()[i + dat.min_target_limb + 1], - dat.mask_last_limb_b(other, i + dat.min_source_limb), - c.clone(), - ); - } - } - if dat.number_of_target_limbs > dat.number_of_source_limbs { - self.limbs_mut()[i + dat.min_target_limb + 1] = - self.fq().reduce(self.limbs()[i + dat.min_target_limb + 1]); - } - } - /// Given a mask v, add the `v[i]`th entry of `other` to the `i`th entry of `self`. pub fn add_masked(&mut self, other: FqSlice<'_, F>, c: FieldElement, mask: &[usize]) { // TODO: If this ends up being a bottleneck, try to use PDEP/PEXT From 64b0aba461fa0bc763d8ede8035dc5ae25fcffcd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 05:54:08 +0000 Subject: [PATCH 17/27] CI: rustfmt and fix a private intra-doc link Run cargo +nightly fmt across the touched files, and stop the bitslice_proto module docs from linking to the crate-private `Limb` (rustdoc -D warnings). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/benches/bitslice.rs | 36 +++++++++++++-------- ext/crates/fp/src/bitslice_proto.rs | 23 ++++++++++--- ext/crates/fp/src/field/bitslice.rs | 21 ++++++------ ext/crates/fp/src/vector/impl_fqslicemut.rs | 6 +++- 4 files changed, 57 insertions(+), 29 deletions(-) diff --git a/ext/crates/fp/benches/bitslice.rs b/ext/crates/fp/benches/bitslice.rs index c7f6de92ea..5b4e84ba93 100644 --- a/ext/crates/fp/benches/bitslice.rs +++ b/ext/crates/fp/benches/bitslice.rs @@ -45,13 +45,17 @@ fn bench_add(c: &mut Criterion) { // Bit-sliced generic kernel. let bs_a = BitSlicedVec::from_u32(p, &a); let bs_b = BitSlicedVec::from_u32(p, &b); - group.bench_with_input(BenchmarkId::new("bitsliced_generic", len), &len, |bench, _| { - bench.iter_batched_ref( - || bs_a.clone(), - |va| va.add_generic(&bs_b, SCALAR), - criterion::BatchSize::SmallInput, - ) - }); + group.bench_with_input( + BenchmarkId::new("bitsliced_generic", len), + &len, + |bench, _| { + bench.iter_batched_ref( + || bs_a.clone(), + |va| va.add_generic(&bs_b, SCALAR), + criterion::BatchSize::SmallInput, + ) + }, + ); // Bit-sliced F3 fast circuit. if p == 3 { @@ -85,13 +89,17 @@ fn bench_scale(c: &mut Criterion) { }); let bs_a = BitSlicedVec::from_u32(p, &a); - group.bench_with_input(BenchmarkId::new("bitsliced_generic", len), &len, |bench, _| { - bench.iter_batched_ref( - || bs_a.clone(), - |va| va.scale_generic(SCALAR), - criterion::BatchSize::SmallInput, - ) - }); + group.bench_with_input( + BenchmarkId::new("bitsliced_generic", len), + &len, + |bench, _| { + bench.iter_batched_ref( + || bs_a.clone(), + |va| va.scale_generic(SCALAR), + criterion::BatchSize::SmallInput, + ) + }, + ); if p == 3 { group.bench_with_input(BenchmarkId::new("bitsliced_f3", len), &len, |bench, _| { diff --git a/ext/crates/fp/src/bitslice_proto.rs b/ext/crates/fp/src/bitslice_proto.rs index 57c05e3e4d..5235cc4834 100644 --- a/ext/crates/fp/src/bitslice_proto.rs +++ b/ext/crates/fp/src/bitslice_proto.rs @@ -9,7 +9,7 @@ //! # Layout //! //! An element of `F_p` is represented with `k = ceil(log2 p)` bits. A *group* of 64 -//! elements occupies `k` consecutive [`Limb`]s (the *planes*): plane `j` of a group holds +//! elements occupies `k` consecutive `Limb`s (the *planes*): plane `j` of a group holds //! bit `j` of all 64 elements, with element `i` living at bit `i` of each plane. A vector //! of length `len` has `ceil(len / 64)` groups, so `k * ceil(len / 64)` limbs total. //! @@ -190,7 +190,13 @@ impl BitSlicedVec { add_mod_into(&mut self.limbs[base..base + k], b, p_masks, &mut s, &mut d); } else { scalar_mul_into(&mut acc, b, c, p_masks, &mut temp, &mut s, &mut d); - add_mod_into(&mut self.limbs[base..base + k], &acc, p_masks, &mut s, &mut d); + add_mod_into( + &mut self.limbs[base..base + k], + &acc, + p_masks, + &mut s, + &mut d, + ); } } } @@ -424,7 +430,11 @@ fn add_groups_k(dst: &mut [Limb], src: &[Limb], c: u32, p_masks: let mut b = [0 as Limb; K]; a.copy_from_slice(dg); b.copy_from_slice(sg); - let addend = if c == 1 { b } else { scalar_mul_k::(&b, c, p_masks) }; + let addend = if c == 1 { + b + } else { + scalar_mul_k::(&b, c, p_masks) + }; let sum = add_mod_k::(&a, &addend, p_masks); dg.copy_from_slice(&sum); } @@ -499,7 +509,12 @@ mod tests { } else { // Sample a spread of pairs into 64 lanes. (0..64u32) - .map(|i| ((i.wrapping_mul(2654435761) % p), (i.wrapping_mul(40503) % p))) + .map(|i| { + ( + (i.wrapping_mul(2654435761) % p), + (i.wrapping_mul(40503) % p), + ) + }) .collect() }; for c in 0..p { diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 01ffa814c5..c8a9016734 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -222,7 +222,11 @@ fn add_groups_k(dst: &mut [Limb], src: &[Limb], c: u32, p: u32) let mut b = [0 as Limb; K]; a.copy_from_slice(dg); b.copy_from_slice(sg); - let addend = if c == 1 { b } else { scalar_mul_k::(&b, c, p) }; + let addend = if c == 1 { + b + } else { + scalar_mul_k::(&b, c, p) + }; let sum = add_mod_k::(&a, &addend, p); dg.copy_from_slice(&sum); } @@ -243,7 +247,11 @@ fn add_group_masked_k( a[j] = dst[j]; b[j] = src[j] & lane_mask; } - let addend = if c == 1 { b } else { scalar_mul_k::(&b, c, p) }; + let addend = if c == 1 { + b + } else { + scalar_mul_k::(&b, c, p) + }; let sum = add_mod_k::(&a, &addend, p); dst[..K].copy_from_slice(&sum); } @@ -368,14 +376,7 @@ fn f5_mul_planes(p0: Limb, p1: Limb, p2: Limb, c: u32) -> (Limb, Limb, Limb) { /// `(a + b) mod 5` on the three planes, as a flat indicator circuit. #[inline(always)] -fn f5_add_planes( - a0: Limb, - a1: Limb, - a2: Limb, - b0: Limb, - b1: Limb, - b2: Limb, -) -> (Limb, Limb, Limb) { +fn f5_add_planes(a0: Limb, a1: Limb, a2: Limb, b0: Limb, b1: Limb, b2: Limb) -> (Limb, Limb, Limb) { let ia = f5_indicators(a0, a1, a2); let ib = f5_indicators(b0, b1, b2); let mut sel = [0 as Limb; 5]; diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index 4a3a9628e6..6ade936616 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -250,7 +250,11 @@ impl<'a, F: Field> FqSliceMut<'a, F> { let epg_i = epg as isize; for g in first_g..=last_g { let lo = if g == first_g { ts - g * epg } else { 0 }; - let hi = if g == last_g { (ts + len) - g * epg } else { epg }; + let hi = if g == last_g { + (ts + len) - g * epg + } else { + epg + }; let mask = lane_mask(lo, hi); // Source bit `b` of target group `g` lives at absolute source lane `g*epg + b + shift`. From a9e05337acfd7a8a22394921fbe1534f4ae40e2d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 06:00:24 +0000 Subject: [PATCH 18/27] Phase 5: remove Phase 0 prototype scaffolding; clippy fixes Delete the throwaway validation artifacts: src/bitslice_proto.rs, the benches/bitslice.rs comparison bench, and benches/BITSLICE_PROTO_RESULTS.md, along with their lib.rs module declaration and Cargo.toml [[bench]] entry. The real bit-sliced implementation lives in field/bitslice.rs and the vector layer. Also silence clippy under -D warnings: replace two manual plane-shift copy loops in field/bitslice.rs with copy_from_slice. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/Cargo.toml | 5 - .../fp/benches/BITSLICE_PROTO_RESULTS.md | 108 ---- ext/crates/fp/benches/bitslice.rs | 119 ---- ext/crates/fp/src/bitslice_proto.rs | 596 ------------------ ext/crates/fp/src/field/bitslice.rs | 8 +- ext/crates/fp/src/lib.rs | 4 - 6 files changed, 2 insertions(+), 838 deletions(-) delete mode 100644 ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md delete mode 100644 ext/crates/fp/benches/bitslice.rs delete mode 100644 ext/crates/fp/src/bitslice_proto.rs diff --git a/ext/crates/fp/Cargo.toml b/ext/crates/fp/Cargo.toml index 3337006b05..b0123aba05 100644 --- a/ext/crates/fp/Cargo.toml +++ b/ext/crates/fp/Cargo.toml @@ -47,11 +47,6 @@ harness = false name = "reduce" harness = false -# PHASE 0 PROTOTYPE bench — to be removed in Phase 5. -[[bench]] -name = "bitslice" -harness = false - [[bench]] name = "smallfq" harness = false diff --git a/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md b/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md deleted file mode 100644 index e8db167292..0000000000 --- a/ext/crates/fp/benches/BITSLICE_PROTO_RESULTS.md +++ /dev/null @@ -1,108 +0,0 @@ -# Phase 0 gate — bit-sliced storage prototype results - -**Verdict: GO.** Bit-slicing is worth pursuing. The decision is more favorable than the -plan predicted, because the existing *packed* path is only well-optimized for -`p ∈ {2, 3, 5}` — for every other prime its reduction step is a slow element-wise -fallback (`Fp::reduce`'s generic arm, `field/fp.rs:142`), and even an unoptimized -bit-sliced kernel beats it. - -Benchmarks: `cargo bench -p fp --bench bitslice` (median of 30 samples, short config: -warm-up 0.5s, measurement 2s). `add` is `self += 2*other`; `scale` is `self *= 2`. -"generic" = the prime-agnostic ripple-carry kernel; "f3" = the hand-written F3 circuit. - -## `add`, length 100,000 (asymptotic regime) - -| prime | packed | bitsliced generic | bitsliced F3 | best vs packed | -|------:|-------:|------------------:|-------------:|:--------------| -| 3 | 9.10 µs | 93.1 µs | **3.60 µs** | F3 **2.5× faster** | -| 5 | 19.0 µs | 102.6 µs | — | generic 5.4× slower | -| 7 | 91.6 µs | 102.4 µs | — | ~even (packed reduce is slow) | -| 251 | 528 µs | **145.6 µs** | — | generic **3.6× faster** | - -## `scale`, length 100,000 - -| prime | packed | bitsliced generic | bitsliced F3 | best vs packed | -|------:|-------:|------------------:|-------------:|:--------------| -| 3 | 2.61 µs | 82.6 µs | **1.49 µs** | F3 **1.75× faster** | -| 5 | 5.00 µs | 87.4 µs | — | generic 17× slower | -| 7 | 67.2 µs | 85.4 µs | — | generic 1.27× slower | -| 251 | 480 µs | **117.5 µs** | — | generic **4.1× faster** | - -## Reading the results - -1. **Specialized small-prime circuits win.** The F3 circuit is 2.5× faster than packed - for `add` and 1.75× for `scale`, at every length tested. This is the core validation: - replacing the madd+reduce sequence with a short branch-free circuit pays off. - -2. **The generic kernel loses for `p ∈ {3, 5}` but wins for large primes.** The packed - path has hand-tuned SWAR `reduce` only for 2/3/5; for `p = 7` and everything larger it - falls back to a per-element `pack(unpack(limb))`, which is slow. The generic bit-sliced - kernel (ripple-carry add + one conditional subtract, no tables) already beats packed by - **3.6×/4.1×** at `p = 251` and is roughly even at `p = 7` — *despite* prototype overhead - (fixed `[Limb; 24]` scratch arrays regardless of `k`, and double-and-add for `scale`). - A real implementation that sizes scratch to `k` will widen this further. - -3. **This vindicates the "all primes" + "replace" decision.** Bit-slicing helps across the - board, just via two mechanisms: specialized circuits for the tuned small primes - (3, 5, 7), and the generic kernel for the large primes where packed reduction is the - bottleneck. - -## Phase 0b — tightened generic kernel - -After replacing the prototype's fixed `[Limb; 24]` scratch + large by-value returns with -`k`-sized reusable scratch written directly into the destination planes (and replacing -double-and-add's doubling with a plane shift), the generic kernel improved and the -crossover where it beats packed moved down to **p = 7** (numbers from one run, length 100k): - -| op | prime | packed | generic before | generic after | after vs packed | -|------:|------:|-------:|---------------:|--------------:|:----------------| -| add | 3 | 11.2 µs | 93.1 µs | 61.0 µs | 5.5× slower (use F3 circuit instead) | -| add | 7 | 95.4 µs | 102.4 µs | 73.4 µs | **1.30× faster** | -| add | 251 | 471 µs | 145.6 µs | 137.2 µs | **3.44× faster** | -| scale | 7 | 73.8 µs | — | 52.9 µs | **1.40× faster** | -| scale | 251 | 444 µs | — | 96.3 µs | **4.61× faster** | - -(The F3 specialized circuit is unchanged: add ≈ 3.8 µs / **2.9× faster** than packed, -scale ≈ 1.8 µs / **1.6× faster**, at 100k.) - -So the tightened generic kernel is now faster than packed for **all `p ≥ 7`**; only the -SWAR-tuned `p ∈ {3, 5}` still need specialized circuits to win — and F3 confirms that -specialization does win. - -## Phase 0c — const-generic `K` dispatch (the decisive change) - -The heap-scratch generic kernel was replaced with a runtime dispatch on `k = ceil(log2 p)` -to **const-generic** implementations (`add_groups_k::` etc.): exactly-`K`-sized stack -arrays and fully-unrolled loops per prime, so the compiler keeps planes in registers and -auto-vectorizes the group loop. This is the single biggest win and it changes the -conclusion — the generic kernel now **beats packed for every prime tested**, and is -competitive with the hand-written F3 circuit (numbers from one run, length 100k): - -| op | prime | packed | bitsliced generic (const-K) | generic vs packed | F3 circuit | -|------:|------:|-------:|----------------------------:|:------------------|-----------:| -| add | 3 | 11.0 µs | 3.62 µs | **3.0× faster** | 3.84 µs | -| add | 5 | 21.0 µs | 6.09 µs | **3.4× faster** | — | -| add | 7 | 95.5 µs | 5.76 µs | **16.6× faster** | — | -| add | 251 | 481 µs | 19.68 µs | **24× faster** | — | -| scale | 3 | 2.75 µs | 2.50 µs | **1.1× faster** | 1.65 µs | -| scale | 5 | 5.07 µs | 3.50 µs | **1.45× faster** | — | -| scale | 7 | 74.0 µs | 3.54 µs | **21× faster** | — | -| scale | 251 | 415 µs | 11.48 µs | **36× faster** | — | - -Key consequence: **the const-K generic kernel is fast enough to be the single code path -for all primes.** It matches the F3 add circuit (3.62 vs 3.84 µs) and beats the SWAR-tuned -packed path even for `p ∈ {3, 5}`. Hand-written per-prime circuits are now *optional polish* -(F3 still wins `scale` modestly via the plane-swap negation, 1.65 vs 2.50 µs) rather than a -requirement. This substantially de-risks and simplifies Phase 2/3: implement the -const-generic kernel once; add specialized circuits later only where a measured gap remains. - -## Implications for the next phases - -- **Specialized circuits are needed for `p = 3, 5, 7`** (not just F3) to beat the tuned - packed path; F5/F7 are the same shape as F3 (3 planes instead of 2). -- **The generic kernel is sufficient for `p ≥ 11`** and for `Fp` (runtime `k`), - and is the path that makes large-prime arithmetic dramatically faster. -- The generic kernel must size its plane scratch to `k` (drop the `MAX_K` arrays) to - avoid the prototype's overhead on small `k`. -- No prime regresses badly enough to keep packed as a fallback for it — so "replace" holds - for all of `Fp`. (`SmallFq` stays packed regardless; its arithmetic is table-based.) diff --git a/ext/crates/fp/benches/bitslice.rs b/ext/crates/fp/benches/bitslice.rs deleted file mode 100644 index 5b4e84ba93..0000000000 --- a/ext/crates/fp/benches/bitslice.rs +++ /dev/null @@ -1,119 +0,0 @@ -//! PHASE 0 PROTOTYPE bench — to be removed in Phase 5. -//! -//! Compares the bit-sliced `add`/`scale` kernels (generic, and the F3 fast path) against -//! the existing packed `FpVector` implementation, across a few representative primes and -//! vector lengths. This is the go/no-go gate for the bit-slicing project: it tells us -//! where bit-slicing actually wins before committing to the larger refactor. - -use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; -use fp::{ - bitslice_proto::BitSlicedVec, - prime::{Prime, ValidPrime}, - vector::FpVector, -}; -use rand::Rng; - -const PRIMES: [u32; 4] = [3, 5, 7, 251]; -const LENGTHS: [usize; 4] = [100, 1000, 10_000, 100_000]; -/// A representative non-unit, non-negation scalar to exercise the full multiply path. -const SCALAR: u32 = 2; - -fn random_data(p: u32, len: usize) -> Vec { - let mut rng = rand::rng(); - (0..len).map(|_| rng.random_range(0..p)).collect() -} - -fn bench_add(c: &mut Criterion) { - for p in PRIMES { - let vp = ValidPrime::new(p); - let mut group = c.benchmark_group(format!("add_p{p}")); - for len in LENGTHS { - let a = random_data(p, len); - let b = random_data(p, len); - - // Packed reference (existing implementation). - let packed_a = FpVector::from_slice(vp, &a); - let packed_b = FpVector::from_slice(vp, &b); - group.bench_with_input(BenchmarkId::new("packed", len), &len, |bench, _| { - bench.iter_batched_ref( - || packed_a.clone(), - |va| va.add(&packed_b, SCALAR), - criterion::BatchSize::SmallInput, - ) - }); - - // Bit-sliced generic kernel. - let bs_a = BitSlicedVec::from_u32(p, &a); - let bs_b = BitSlicedVec::from_u32(p, &b); - group.bench_with_input( - BenchmarkId::new("bitsliced_generic", len), - &len, - |bench, _| { - bench.iter_batched_ref( - || bs_a.clone(), - |va| va.add_generic(&bs_b, SCALAR), - criterion::BatchSize::SmallInput, - ) - }, - ); - - // Bit-sliced F3 fast circuit. - if p == 3 { - group.bench_with_input(BenchmarkId::new("bitsliced_f3", len), &len, |bench, _| { - bench.iter_batched_ref( - || bs_a.clone(), - |va| va.add_f3(&bs_b, SCALAR), - criterion::BatchSize::SmallInput, - ) - }); - } - } - group.finish(); - } -} - -fn bench_scale(c: &mut Criterion) { - for p in PRIMES { - let vp = ValidPrime::new(p); - let mut group = c.benchmark_group(format!("scale_p{p}")); - for len in LENGTHS { - let a = random_data(p, len); - - let packed_a = FpVector::from_slice(vp, &a); - group.bench_with_input(BenchmarkId::new("packed", len), &len, |bench, _| { - bench.iter_batched_ref( - || packed_a.clone(), - |va| va.scale(SCALAR), - criterion::BatchSize::SmallInput, - ) - }); - - let bs_a = BitSlicedVec::from_u32(p, &a); - group.bench_with_input( - BenchmarkId::new("bitsliced_generic", len), - &len, - |bench, _| { - bench.iter_batched_ref( - || bs_a.clone(), - |va| va.scale_generic(SCALAR), - criterion::BatchSize::SmallInput, - ) - }, - ); - - if p == 3 { - group.bench_with_input(BenchmarkId::new("bitsliced_f3", len), &len, |bench, _| { - bench.iter_batched_ref( - || bs_a.clone(), - |va| va.scale_f3(SCALAR), - criterion::BatchSize::SmallInput, - ) - }); - } - } - group.finish(); - } -} - -criterion_group!(benches, bench_add, bench_scale); -criterion_main!(benches); diff --git a/ext/crates/fp/src/bitslice_proto.rs b/ext/crates/fp/src/bitslice_proto.rs deleted file mode 100644 index 5235cc4834..0000000000 --- a/ext/crates/fp/src/bitslice_proto.rs +++ /dev/null @@ -1,596 +0,0 @@ -//! **PHASE 0 PROTOTYPE — to be removed in Phase 5.** -//! -//! A standalone, self-contained prototype of bit-sliced (bit-plane) storage for vectors -//! over a prime field `F_p`. This is *not* wired into [`crate::vector::FqVector`]; it -//! exists only to validate the performance claim before the larger refactor (see the -//! approved plan). It deliberately re-implements the minimum needed to benchmark the -//! `add`/`scale` kernels against the existing packed representation. -//! -//! # Layout -//! -//! An element of `F_p` is represented with `k = ceil(log2 p)` bits. A *group* of 64 -//! elements occupies `k` consecutive `Limb`s (the *planes*): plane `j` of a group holds -//! bit `j` of all 64 elements, with element `i` living at bit `i` of each plane. A vector -//! of length `len` has `ceil(len / 64)` groups, so `k * ceil(len / 64)` limbs total. -//! -//! # Arithmetic -//! -//! - The **generic** kernels work for any prime: addition is a ripple-carry adder over the -//! `k` planes followed by a single conditional subtraction of `p` (the sum of two reduced -//! values is `< 2p`), and scalar multiplication is double-and-add with modular reduction -//! at each step. No lookup tables, fully branch-free, operating on 64 lanes at once. -//! - The **F3 fast path** uses a hand-written 2-plane circuit (addition and negation), -//! demonstrating the kind of speedup a per-prime specialization can give. - -#![allow(dead_code)] - -use crate::{constants::BITS_PER_LIMB, limb::Limb}; - -/// Maximum number of bit-planes (`k`) the prototype supports. `k = ceil(log2 p)`, so this -/// covers primes up to `2^24` — plenty for the benchmark, which only needs a handful of -/// representative primes. -const MAX_K: usize = 24; - -/// Number of field elements packed into one group. -const ENTRIES_PER_GROUP: usize = BITS_PER_LIMB; // 64 - -/// `k = ceil(log2 p)`: the number of bit-planes needed to store an element of `F_p`. -const fn bit_planes(p: u32) -> usize { - // Smallest k with 2^k >= p. - let mut k = 0; - while (1u64 << k) < p as u64 { - k += 1; - } - if k == 0 { 1 } else { k } -} - -/// A vector over `F_p` in bit-sliced layout. Prototype only. -#[derive(Clone, Debug)] -pub struct BitSlicedVec { - p: u32, - k: usize, - len: usize, - /// `k * ceil(len / 64)` limbs, group-major: group `g`'s plane `j` is `limbs[g * k + j]`. - limbs: Vec, -} - -impl BitSlicedVec { - pub fn new(p: u32, len: usize) -> Self { - let k = bit_planes(p); - let groups = len.div_ceil(ENTRIES_PER_GROUP); - Self { - p, - k, - len, - limbs: vec![0; k * groups], - } - } - - pub fn from_u32(p: u32, data: &[u32]) -> Self { - let mut v = Self::new(p, data.len()); - for (i, &value) in data.iter().enumerate() { - v.set_entry(i, value); - } - v - } - - pub fn len(&self) -> usize { - self.len - } - - pub fn is_empty(&self) -> bool { - self.len == 0 - } - - fn num_groups(&self) -> usize { - self.len.div_ceil(ENTRIES_PER_GROUP) - } - - pub fn entry(&self, index: usize) -> u32 { - debug_assert!(index < self.len); - let group = index / ENTRIES_PER_GROUP; - let lane = index % ENTRIES_PER_GROUP; - let base = group * self.k; - let mut value = 0u32; - for j in 0..self.k { - let bit = (self.limbs[base + j] >> lane) & 1; - value |= (bit as u32) << j; - } - value - } - - pub fn set_entry(&mut self, index: usize, value: u32) { - debug_assert!(index < self.len); - debug_assert!(value < self.p); - let group = index / ENTRIES_PER_GROUP; - let lane = index % ENTRIES_PER_GROUP; - let base = group * self.k; - for j in 0..self.k { - let bit = ((value >> j) & 1) as Limb; - let mask = 1 << lane; - let plane = &mut self.limbs[base + j]; - *plane = (*plane & !mask) | (bit << lane); - } - } - - pub fn to_u32(&self) -> Vec { - (0..self.len).map(|i| self.entry(i)).collect() - } - - /// Bits of `p` as full-width lane masks (`pbits[j]` is all-ones iff bit `j` of `p` is set), - /// for `j` in `0..=k`. Since `p < 2^k` (except `p = 2`), bit `k` is normally zero. - fn p_masks(&self) -> [Limb; MAX_K + 1] { - let mut masks = [0; MAX_K + 1]; - for (j, m) in masks.iter_mut().enumerate().take(self.k + 1) { - *m = if (self.p >> j) & 1 == 1 { !0 } else { 0 }; - } - masks - } - - /// `self += c * other` (mod p), generic kernel for any prime. - /// - /// Dispatches on the number of planes `k` to a const-generic implementation, so the - /// per-group arithmetic uses exactly-`K`-sized stack arrays and fully-unrolled loops - /// tuned to the prime, rather than runtime-bounded loops over heap scratch. `k` only - /// takes a handful of values (`k = ceil(log2 p)`), so the dispatch covers them directly - /// and falls back to a heap path only for very large primes. - pub fn add_generic(&mut self, other: &Self, c: u32) { - assert_eq!(self.p, other.p); - assert_eq!(self.len, other.len); - if c == 0 { - return; - } - let p_masks = self.p_masks(); - macro_rules! dispatch { - ($($k:literal),*) => { - match self.k { - $($k => add_groups_k::<$k>(&mut self.limbs, &other.limbs, c, &p_masks),)* - _ => self.add_generic_dyn(other, c, &p_masks), - } - }; - } - dispatch!(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - } - - /// `self *= c` (mod p), generic kernel. See [`add_generic`](Self::add_generic) for the - /// const-generic dispatch rationale. - pub fn scale_generic(&mut self, c: u32) { - if c == 1 { - return; - } - if c == 0 { - for limb in &mut self.limbs { - *limb = 0; - } - return; - } - let p_masks = self.p_masks(); - macro_rules! dispatch { - ($($k:literal),*) => { - match self.k { - $($k => scale_groups_k::<$k>(&mut self.limbs, c, &p_masks),)* - _ => self.scale_generic_dyn(c, &p_masks), - } - }; - } - dispatch!(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); - } - - /// Heap-scratch fallback for `add_generic` when `k` exceeds the const-dispatch range. - fn add_generic_dyn(&mut self, other: &Self, c: u32, p_masks: &[Limb]) { - let k = self.k; - let mut s = vec![0; k + 1]; - let mut d = vec![0; k + 1]; - let mut acc = vec![0; k]; - let mut temp = vec![0; k]; - for g in 0..self.num_groups() { - let base = g * k; - let b = &other.limbs[base..base + k]; - if c == 1 { - add_mod_into(&mut self.limbs[base..base + k], b, p_masks, &mut s, &mut d); - } else { - scalar_mul_into(&mut acc, b, c, p_masks, &mut temp, &mut s, &mut d); - add_mod_into( - &mut self.limbs[base..base + k], - &acc, - p_masks, - &mut s, - &mut d, - ); - } - } - } - - /// Heap-scratch fallback for `scale_generic` when `k` exceeds the const-dispatch range. - fn scale_generic_dyn(&mut self, c: u32, p_masks: &[Limb]) { - let k = self.k; - let mut s = vec![0; k + 1]; - let mut d = vec![0; k + 1]; - let mut acc = vec![0; k]; - let mut temp = vec![0; k]; - for g in 0..self.num_groups() { - let base = g * k; - scalar_mul_into( - &mut acc, - &self.limbs[base..base + k], - c, - p_masks, - &mut temp, - &mut s, - &mut d, - ); - self.limbs[base..base + k].copy_from_slice(&acc); - } - } - - /// `self += c * other` (mod 3) using the hand-written F3 circuit. Requires `p == 3`. - pub fn add_f3(&mut self, other: &Self, c: u32) { - assert_eq!(self.p, 3); - assert_eq!(self.k, 2); - assert_eq!(self.len, other.len); - if c == 0 { - return; - } - for g in 0..self.num_groups() { - let base = g * 2; - let (a_lo, a_hi) = (self.limbs[base], self.limbs[base + 1]); - let (mut b_lo, mut b_hi) = (other.limbs[base], other.limbs[base + 1]); - if c == 2 { - // Multiply other by 2 = negate: in the (hi, lo) encoding, negation swaps planes. - std::mem::swap(&mut b_lo, &mut b_hi); - } - let (c_lo, c_hi) = f3_add(a_lo, a_hi, b_lo, b_hi); - self.limbs[base] = c_lo; - self.limbs[base + 1] = c_hi; - } - } - - /// `self *= c` (mod 3) using the F3 circuit. Requires `p == 3`. - pub fn scale_f3(&mut self, c: u32) { - assert_eq!(self.p, 3); - if c == 1 { - return; - } - if c == 0 { - for limb in &mut self.limbs { - *limb = 0; - } - return; - } - // c == 2: negate = swap the two planes of every group. - for g in 0..self.num_groups() { - let base = g * 2; - self.limbs.swap(base, base + 1); - } - } -} - -/// `dst += b` (mod p), where `dst` (the augend) and `b` are each `k = dst.len()` reduced -/// planes with independent lanes. `s`/`d` are reusable `(k+1)`-limb scratch buffers. -/// -/// Ripple-carry adder over the `k` planes gives a `(k+1)`-bit sum in `[0, 2p)`, then a -/// single conditional subtraction of `p` brings each lane back into `[0, p)`. -#[inline] -fn add_mod_into(dst: &mut [Limb], b: &[Limb], p_masks: &[Limb], s: &mut [Limb], d: &mut [Limb]) { - let k = dst.len(); - // s = dst + b as a (k+1)-bit number. `dst` is only written in the final select pass, - // so passing `dst` as `b` (in-place doubling) is sound — handled by `double_mod_into`. - let mut carry: Limb = 0; - for j in 0..k { - let aj = dst[j]; - let bj = b[j]; - let axb = aj ^ bj; - s[j] = axb ^ carry; - carry = (aj & bj) | (carry & axb); - } - s[k] = carry; - - conditional_subtract(dst, p_masks, s, d); -} - -/// `dst = 2 * dst` (mod p). Doubling is a one-position plane shift (`s = dst << 1`) followed -/// by the conditional subtraction of `p`. -#[inline] -fn double_mod_into(dst: &mut [Limb], p_masks: &[Limb], s: &mut [Limb], d: &mut [Limb]) { - let k = dst.len(); - s[0] = 0; - for j in 1..=k { - s[j] = dst[j - 1]; - } - conditional_subtract(dst, p_masks, s, d); -} - -/// Given a `(k+1)`-bit unreduced sum `s` in `[0, 2p)`, write `s mod p` into the `k` planes -/// of `dst`. `d` is `(k+1)`-limb scratch for the trial difference `s - p`. -#[inline] -fn conditional_subtract(dst: &mut [Limb], p_masks: &[Limb], s: &[Limb], d: &mut [Limb]) { - let k = dst.len(); - // d = s - p over k+1 bits; the borrow-out marks lanes where s < p. - let mut borrow: Limb = 0; - for j in 0..=k { - let sj = s[j]; - let pj = p_masks[j]; - let sxp = sj ^ pj; - d[j] = sxp ^ borrow; - borrow = (!sj & pj) | (borrow & !sxp); - } - let ge = !borrow; // lanes where s >= p - // result = ge ? d : s, taking the low k planes (result < p < 2^k). - for j in 0..k { - dst[j] = (d[j] & ge) | (s[j] & !ge); - } -} - -/// `acc = c * b` (mod p) for a constant scalar `c`, via double-and-add with modular -/// reduction. `temp`/`s`/`d` are reusable scratch (`temp` is `k` limbs, `s`/`d` are `k+1`). -#[inline] -fn scalar_mul_into( - acc: &mut [Limb], - b: &[Limb], - c: u32, - p_masks: &[Limb], - temp: &mut [Limb], - s: &mut [Limb], - d: &mut [Limb], -) { - temp.copy_from_slice(b); - acc.fill(0); - let mut cc = c; - loop { - if cc & 1 == 1 { - add_mod_into(acc, temp, p_masks, s, d); - } - cc >>= 1; - if cc == 0 { - break; - } - double_mod_into(temp, p_masks, s, d); - } -} - -// --------------------------------------------------------------------------------------- -// Const-generic kernels: `K` planes known at compile time, so every array is exactly sized -// and every loop is fully unrolled. Selected by a runtime dispatch on `k = ceil(log2 p)`. -// --------------------------------------------------------------------------------------- - -/// Reduce a `(K+1)`-bit unreduced sum (`s` low planes + `s_top`) in `[0, 2p)` to `s mod p`. -#[inline(always)] -fn cond_sub_k(s: &[Limb; K], s_top: Limb, p_masks: &[Limb]) -> [Limb; K] { - let mut d = [0 as Limb; K]; - let mut borrow: Limb = 0; - for j in 0..K { - let sj = s[j]; - let pj = p_masks[j]; - let sxp = sj ^ pj; - d[j] = sxp ^ borrow; - borrow = (!sj & pj) | (borrow & !sxp); - } - // Top bit only affects the borrow-out (the result fits in K planes since result < p). - let pj = p_masks[K]; - let sxp = s_top ^ pj; - borrow = (!s_top & pj) | (borrow & !sxp); - let ge = !borrow; - let mut out = [0 as Limb; K]; - for j in 0..K { - out[j] = (d[j] & ge) | (s[j] & !ge); - } - out -} - -/// `(a + b) mod p` over `K` planes. -#[inline(always)] -fn add_mod_k(a: &[Limb; K], b: &[Limb; K], p_masks: &[Limb]) -> [Limb; K] { - let mut s = [0 as Limb; K]; - let mut carry: Limb = 0; - for j in 0..K { - let aj = a[j]; - let bj = b[j]; - let axb = aj ^ bj; - s[j] = axb ^ carry; - carry = (aj & bj) | (carry & axb); - } - cond_sub_k::(&s, carry, p_masks) -} - -/// `(2 * a) mod p` over `K` planes (doubling is a one-position plane shift). -#[inline(always)] -fn double_mod_k(a: &[Limb; K], p_masks: &[Limb]) -> [Limb; K] { - let mut s = [0 as Limb; K]; - for j in 1..K { - s[j] = a[j - 1]; - } - let s_top = a[K - 1]; - cond_sub_k::(&s, s_top, p_masks) -} - -/// `(c * b) mod p` over `K` planes, via double-and-add. -#[inline(always)] -fn scalar_mul_k(b: &[Limb; K], c: u32, p_masks: &[Limb]) -> [Limb; K] { - let mut result = [0 as Limb; K]; - let mut temp = *b; - let mut cc = c; - loop { - if cc & 1 == 1 { - result = add_mod_k::(&result, &temp, p_masks); - } - cc >>= 1; - if cc == 0 { - break; - } - temp = double_mod_k::(&temp, p_masks); - } - result -} - -/// `dst += c * src` (mod p) over all groups, with `K` planes per group. -#[inline] -fn add_groups_k(dst: &mut [Limb], src: &[Limb], c: u32, p_masks: &[Limb]) { - for (dg, sg) in dst.chunks_exact_mut(K).zip(src.chunks_exact(K)) { - let mut a = [0 as Limb; K]; - let mut b = [0 as Limb; K]; - a.copy_from_slice(dg); - b.copy_from_slice(sg); - let addend = if c == 1 { - b - } else { - scalar_mul_k::(&b, c, p_masks) - }; - let sum = add_mod_k::(&a, &addend, p_masks); - dg.copy_from_slice(&sum); - } -} - -/// `dst *= c` (mod p) over all groups, with `K` planes per group. -#[inline] -fn scale_groups_k(dst: &mut [Limb], c: u32, p_masks: &[Limb]) { - for dg in dst.chunks_exact_mut(K) { - let mut a = [0 as Limb; K]; - a.copy_from_slice(dg); - let scaled = scalar_mul_k::(&a, c, p_masks); - dg.copy_from_slice(&scaled); - } -} - -/// F3 addition circuit on the `(lo, hi)` plane encoding (`value = 2*hi + lo`). -/// -/// Output `c == 1` exactly for input value pairs `{(0,1),(1,0),(2,2)}` and `c == 2` for -/// `{(0,2),(1,1),(2,0)}`; everything else is `0`. The invalid encoding `(hi,lo) = (1,1)` -/// never occurs for reduced inputs. -#[inline] -fn f3_add(a_lo: Limb, a_hi: Limb, b_lo: Limb, b_hi: Limb) -> (Limb, Limb) { - let is0_a = !(a_lo | a_hi); - let is1_a = a_lo; - let is2_a = a_hi; - let is0_b = !(b_lo | b_hi); - let is1_b = b_lo; - let is2_b = b_hi; - - let c_lo = (is0_a & is1_b) | (is1_a & is0_b) | (is2_a & is2_b); - let c_hi = (is0_a & is2_b) | (is1_a & is1_b) | (is2_a & is0_b); - (c_lo, c_hi) -} - -#[cfg(test)] -mod tests { - use super::*; - - const PRIMES: [u32; 6] = [2, 3, 5, 7, 251, 65521]; - - #[test] - fn bit_planes_correct() { - assert_eq!(bit_planes(2), 1); - assert_eq!(bit_planes(3), 2); - assert_eq!(bit_planes(5), 3); - assert_eq!(bit_planes(7), 3); - assert_eq!(bit_planes(251), 8); - assert_eq!(bit_planes(65521), 16); - } - - #[test] - fn pack_unpack_roundtrip() { - for p in PRIMES { - for len in [0, 1, 63, 64, 65, 130, 1000] { - let data: Vec = (0..len).map(|i| (i as u32 * 7 + 1) % p).collect(); - let v = BitSlicedVec::from_u32(p, &data); - assert_eq!(v.to_u32(), data, "p={p} len={len}"); - } - } - } - - /// Exhaustively check the generic add kernel against `(a + c*b) % p` for every pair of - /// field elements and every scalar. - #[test] - fn generic_add_exhaustive() { - for p in PRIMES { - // Use one lane per (a, b) so a single group covers all pairs (p <= 64 cases for - // small primes; for large primes sample instead). - let pairs: Vec<(u32, u32)> = if p * p <= 64 { - (0..p).flat_map(|a| (0..p).map(move |b| (a, b))).collect() - } else { - // Sample a spread of pairs into 64 lanes. - (0..64u32) - .map(|i| { - ( - (i.wrapping_mul(2654435761) % p), - (i.wrapping_mul(40503) % p), - ) - }) - .collect() - }; - for c in 0..p { - let a_data: Vec = pairs.iter().map(|&(a, _)| a).collect(); - let b_data: Vec = pairs.iter().map(|&(_, b)| b).collect(); - let mut va = BitSlicedVec::from_u32(p, &a_data); - let vb = BitSlicedVec::from_u32(p, &b_data); - va.add_generic(&vb, c); - let got = va.to_u32(); - for (idx, &(a, b)) in pairs.iter().enumerate() { - let expected = (a + c * b) % p; - assert_eq!(got[idx], expected, "p={p} a={a} b={b} c={c}"); - } - } - } - } - - #[test] - fn generic_scale_exhaustive() { - for p in PRIMES { - let data: Vec = (0..64).map(|i| (i as u32) % p).collect(); - for c in 0..p { - let mut v = BitSlicedVec::from_u32(p, &data); - v.scale_generic(c); - let got = v.to_u32(); - for (i, &x) in data.iter().enumerate() { - assert_eq!(got[i], (x * c) % p, "p={p} x={x} c={c}"); - } - } - } - } - - /// The F3 circuit must agree with `(a + c*b) % 3` for all inputs. - #[test] - fn f3_add_exhaustive() { - let p = 3; - let pairs: Vec<(u32, u32)> = (0..p).flat_map(|a| (0..p).map(move |b| (a, b))).collect(); - for c in 0..p { - let a_data: Vec = pairs.iter().map(|&(a, _)| a).collect(); - let b_data: Vec = pairs.iter().map(|&(_, b)| b).collect(); - let mut va = BitSlicedVec::from_u32(p, &a_data); - let vb = BitSlicedVec::from_u32(p, &b_data); - va.add_f3(&vb, c); - let got = va.to_u32(); - for (idx, &(a, b)) in pairs.iter().enumerate() { - assert_eq!(got[idx], (a + c * b) % p, "a={a} b={b} c={c}"); - } - } - } - - #[test] - fn f3_scale_exhaustive() { - let data: Vec = (0..64).map(|i| (i as u32) % 3).collect(); - for c in 0..3 { - let mut v = BitSlicedVec::from_u32(3, &data); - v.scale_f3(c); - let got = v.to_u32(); - for (i, &x) in data.iter().enumerate() { - assert_eq!(got[i], (x * c) % 3, "x={x} c={c}"); - } - } - } - - /// The F3 fast path and the generic kernel must produce identical results on long vectors. - #[test] - fn f3_fast_matches_generic() { - let len = 1000; - let a_data: Vec = (0..len).map(|i| (i as u32 * 2 + 1) % 3).collect(); - let b_data: Vec = (0..len).map(|i| (i as u32 * 5 + 2) % 3).collect(); - for c in 0..3 { - let mut fast = BitSlicedVec::from_u32(3, &a_data); - let mut generic = BitSlicedVec::from_u32(3, &a_data); - let b = BitSlicedVec::from_u32(3, &b_data); - fast.add_f3(&b, c); - generic.add_generic(&b, c); - assert_eq!(fast.to_u32(), generic.to_u32(), "c={c}"); - } - } -} diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index c8a9016734..98a5c134bc 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -189,9 +189,7 @@ fn add_mod_k(a: &[Limb; K], b: &[Limb; K], p: u32) -> [Limb; K] #[inline(always)] fn double_mod_k(a: &[Limb; K], p: u32) -> [Limb; K] { let mut s = [0 as Limb; K]; - for j in 1..K { - s[j] = a[j - 1]; - } + s[1..K].copy_from_slice(&a[..K - 1]); let s_top = a[K - 1]; cond_sub_k::(&s, s_top, p) } @@ -464,9 +462,7 @@ fn add_mod_into(dst: &mut [Limb], b: &[Limb], masks: &[Limb], s: &mut [Limb], d: fn double_mod_into(dst: &mut [Limb], masks: &[Limb], s: &mut [Limb], d: &mut [Limb]) { let k = dst.len(); s[0] = 0; - for j in 1..=k { - s[j] = dst[j - 1]; - } + s[1..=k].copy_from_slice(&dst[..k]); cond_sub_into(dst, s, masks, d); } diff --git a/ext/crates/fp/src/lib.rs b/ext/crates/fp/src/lib.rs index 75f179f7ae..8d971da2a8 100644 --- a/ext/crates/fp/src/lib.rs +++ b/ext/crates/fp/src/lib.rs @@ -10,10 +10,6 @@ pub mod matrix; pub mod prime; pub mod vector; -// PHASE 0 PROTOTYPE — to be removed in Phase 5. See module docs. -#[doc(hidden)] -pub mod bitslice_proto; - pub mod blas; pub(crate) mod simd; From d88b76a6708627141c1dfd5df0812fcca2e29aa6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 06:46:37 +0000 Subject: [PATCH 19/27] Fix CI lint/docs and harden bit-sliced slice ops - bitslice.rs: use `as_chunks`/`as_chunks_mut` for const-size group iteration (clippy `chunks_exact_to_as_chunks`, denied under `-D warnings`). - field_internal.rs: fix broken intra-doc links to `Self::gather`/`Self::scatter`. - impl_fqslicemut.rs: assert equal slice lengths in `add`, and guard `shl_assign` against `shift >= len` underflow (empties the slice). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 28 ++++++++------------- ext/crates/fp/src/field/field_internal.rs | 2 +- ext/crates/fp/src/vector/impl_fqslicemut.rs | 9 ++++++- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 98a5c134bc..a66eac67f7 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -215,18 +215,13 @@ fn scalar_mul_k(b: &[Limb; K], c: u32, p: u32) -> [Limb; K] { #[inline] fn add_groups_k(dst: &mut [Limb], src: &[Limb], c: u32, p: u32) { - for (dg, sg) in dst.chunks_exact_mut(K).zip(src.chunks_exact(K)) { - let mut a = [0 as Limb; K]; - let mut b = [0 as Limb; K]; - a.copy_from_slice(dg); - b.copy_from_slice(sg); + for (dg, sg) in dst.as_chunks_mut::().0.iter_mut().zip(src.as_chunks::().0) { let addend = if c == 1 { - b + *sg } else { - scalar_mul_k::(&b, c, p) + scalar_mul_k::(sg, c, p) }; - let sum = add_mod_k::(&a, &addend, p); - dg.copy_from_slice(&sum); + *dg = add_mod_k::(dg, &addend, p); } } @@ -263,11 +258,8 @@ fn scale_groups_k(dst: &mut [Limb], c: u32, p: u32) { dst.fill(0); return; } - for dg in dst.chunks_exact_mut(K) { - let mut a = [0 as Limb; K]; - a.copy_from_slice(dg); - let scaled = scalar_mul_k::(&a, c, p); - dg.copy_from_slice(&scaled); + for dg in dst.as_chunks_mut::().0 { + *dg = scalar_mul_k::(dg, c, p); } } @@ -307,7 +299,7 @@ fn f3_addend(sg: &[Limb], c: u32) -> (Limb, Limb) { } fn f3_add_groups(dst: &mut [Limb], src: &[Limb], c: u32) { - for (dg, sg) in dst.chunks_exact_mut(2).zip(src.chunks_exact(2)) { + for (dg, sg) in dst.as_chunks_mut::<2>().0.iter_mut().zip(src.as_chunks::<2>().0) { let (b_lo, b_hi) = f3_addend(sg, c); let (r_lo, r_hi) = f3_add_planes(dg[0], dg[1], b_lo, b_hi); dg[0] = r_lo; @@ -326,7 +318,7 @@ fn f3_add_group_masked(dst: &mut [Limb], src: &[Limb], c: u32, lane_mask: Limb) fn f3_scale_groups(dst: &mut [Limb], c: u32) { // c == 2 is negation (plane swap); c == 1 is a no-op; c == 0 is handled by the caller. if c == 2 { - for dg in dst.chunks_exact_mut(2) { + for dg in dst.as_chunks_mut::<2>().0 { dg.swap(0, 1); } } @@ -387,7 +379,7 @@ fn f5_add_planes(a0: Limb, a1: Limb, a2: Limb, b0: Limb, b1: Limb, b2: Limb) -> } fn f5_add_groups(dst: &mut [Limb], src: &[Limb], c: u32) { - for (dg, sg) in dst.chunks_exact_mut(3).zip(src.chunks_exact(3)) { + for (dg, sg) in dst.as_chunks_mut::<3>().0.iter_mut().zip(src.as_chunks::<3>().0) { let (b0, b1, b2) = if c == 1 { (sg[0], sg[1], sg[2]) } else { @@ -417,7 +409,7 @@ fn f5_add_group_masked(dst: &mut [Limb], src: &[Limb], c: u32, lane_mask: Limb) } fn f5_scale_groups(dst: &mut [Limb], c: u32) { - for dg in dst.chunks_exact_mut(3) { + for dg in dst.as_chunks_mut::<3>().0 { let (r0, r1, r2) = f5_mul_planes(dg[0], dg[1], dg[2], c); dg[0] = r0; dg[1] = r1; diff --git a/ext/crates/fp/src/field/field_internal.rs b/ext/crates/fp/src/field/field_internal.rs index b4e1f8b344..c9beb6edbb 100644 --- a/ext/crates/fp/src/field/field_internal.rs +++ b/ext/crates/fp/src/field/field_internal.rs @@ -202,7 +202,7 @@ pub trait FieldInternal: /// `dst += coeff * src` (mod p) over a span of whole groups (`dst` and `src` have equal, /// group-aligned length). Both are assumed reduced; the result is reduced. /// - /// Default: element-wise over lanes via [`gather`]/[`scatter`] and the field's own + /// Default: element-wise over lanes via [`Self::gather`]/[`Self::scatter`] and the field's own /// arithmetic — correct for any bit-sliced field (used by [`SmallFq`](super::SmallFq)). The /// prime fields [`Fp`](super::Fp) override this with a branch-free plane circuit. fn add_groups(self, dst: &mut [Limb], src: &[Limb], coeff: FieldElement) { diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index 6ade936616..40e0e50e3b 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -119,6 +119,7 @@ impl<'a, F: Field> FqSliceMut<'a, F> { pub fn add(&mut self, other: FqSlice<'_, F>, c: FieldElement) { assert_eq!(self.fq(), c.field()); assert_eq!(self.fq(), other.fq()); + assert_eq!(self.as_slice().len(), other.len()); if self.as_slice().is_empty() { return; @@ -345,7 +346,13 @@ impl<'a, F: Field> FqSliceMut<'a, F> { // The packed limb-move trick assumes an entry is a contiguous bitfield, which the // bit-sliced layout breaks. Move entries down one at a time via gather/scatter: // reading `i + shift` strictly ahead of writing `i` keeps it correct in place. - let new_len = self.as_slice().len() - shift; + let len = self.as_slice().len(); + if shift >= len { + // Every entry shifts out; the slice becomes empty. + *self.end_mut() = self.start(); + return; + } + let new_len = len - shift; for i in 0..new_len { let v = self.as_slice().entry(i + shift); self.set_entry(i, v); From 08c9ff6d73e33a91875de18fc84f2974eb67d125 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 06:48:09 +0000 Subject: [PATCH 20/27] Run nightly rustfmt on bit-sliced group iteration The `as_chunks`/`as_chunks_mut` iterator chains exceeded the line width; apply `cargo +nightly fmt` so `cargo fmt --all -- --check` passes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index a66eac67f7..9dc877c587 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -215,7 +215,12 @@ fn scalar_mul_k(b: &[Limb; K], c: u32, p: u32) -> [Limb; K] { #[inline] fn add_groups_k(dst: &mut [Limb], src: &[Limb], c: u32, p: u32) { - for (dg, sg) in dst.as_chunks_mut::().0.iter_mut().zip(src.as_chunks::().0) { + for (dg, sg) in dst + .as_chunks_mut::() + .0 + .iter_mut() + .zip(src.as_chunks::().0) + { let addend = if c == 1 { *sg } else { @@ -299,7 +304,12 @@ fn f3_addend(sg: &[Limb], c: u32) -> (Limb, Limb) { } fn f3_add_groups(dst: &mut [Limb], src: &[Limb], c: u32) { - for (dg, sg) in dst.as_chunks_mut::<2>().0.iter_mut().zip(src.as_chunks::<2>().0) { + for (dg, sg) in dst + .as_chunks_mut::<2>() + .0 + .iter_mut() + .zip(src.as_chunks::<2>().0) + { let (b_lo, b_hi) = f3_addend(sg, c); let (r_lo, r_hi) = f3_add_planes(dg[0], dg[1], b_lo, b_hi); dg[0] = r_lo; @@ -379,7 +389,12 @@ fn f5_add_planes(a0: Limb, a1: Limb, a2: Limb, b0: Limb, b1: Limb, b2: Limb) -> } fn f5_add_groups(dst: &mut [Limb], src: &[Limb], c: u32) { - for (dg, sg) in dst.as_chunks_mut::<3>().0.iter_mut().zip(src.as_chunks::<3>().0) { + for (dg, sg) in dst + .as_chunks_mut::<3>() + .0 + .iter_mut() + .zip(src.as_chunks::<3>().0) + { let (b0, b1, b2) = if c == 1 { (sg[0], sg[1], sg[2]) } else { From c25446d74f3646d8ebe530f0b72955d1d84c71f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 19:19:07 +0000 Subject: [PATCH 21/27] Address review: bit-sliced add_truncate, empty range, ragged from_vec - add_truncate: route bit-sliced fields through the group add (their lanes reduce independently and cannot carry, so the truncation check never fails); the packed fma_limb/truncate loop would corrupt bit-planes. F2 (k=1) keeps the packed path. - FieldInternal::range: return an empty limb range for an empty entry range instead of the containing group, so is_empty() guards behave correctly. - Matrix::from_vec: assert all input rows share a width up front, so both the packed and bit-sliced storage paths reject ragged input consistently. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/field_internal.rs | 6 ++++++ ext/crates/fp/src/matrix/matrix_inner.rs | 4 ++++ ext/crates/fp/src/vector/impl_fqvector.rs | 8 ++++++++ 3 files changed, 18 insertions(+) diff --git a/ext/crates/fp/src/field/field_internal.rs b/ext/crates/fp/src/field/field_internal.rs index c9beb6edbb..12e7f77c0d 100644 --- a/ext/crates/fp/src/field/field_internal.rs +++ b/ext/crates/fp/src/field/field_internal.rs @@ -293,7 +293,13 @@ pub trait FieldInternal: /// Return the `Range` of limbs spanning entries `start..end`: from the first limb of /// the group containing `start` to the last limb of the group containing `end - 1`. fn range(self, start: usize, end: usize) -> Range { + debug_assert!(start <= end); let min = self.group_of(start) * self.limbs_per_group(); + if start == end { + // An empty entry range maps to an empty limb range; otherwise callers that guard + // on `limb_range.is_empty()` would touch the (unrelated) containing group. + return min..min; + } let max = self.number(end); min..max } diff --git a/ext/crates/fp/src/matrix/matrix_inner.rs b/ext/crates/fp/src/matrix/matrix_inner.rs index ff58ac107d..d80efa5755 100644 --- a/ext/crates/fp/src/matrix/matrix_inner.rs +++ b/ext/crates/fp/src/matrix/matrix_inner.rs @@ -322,6 +322,10 @@ impl Matrix { return Self::new(p, 0, 0); } let columns = input[0].len(); + assert!( + input.iter().all(|row| row.len() == columns), + "all rows must have the same length" + ); let stride = fp.number(columns); let physical_rows = get_physical_rows(p, rows); diff --git a/ext/crates/fp/src/vector/impl_fqvector.rs b/ext/crates/fp/src/vector/impl_fqvector.rs index dec9041212..684f5767ea 100644 --- a/ext/crates/fp/src/vector/impl_fqvector.rs +++ b/ext/crates/fp/src/vector/impl_fqvector.rs @@ -249,6 +249,14 @@ impl FqVector { pub fn add_truncate(&mut self, other: &Self, c: FieldElement) -> Option<()> { assert_eq!(self.fq(), other.fq()); let fq = self.fq(); + if fq.is_bitsliced() { + // `truncate` guards against an entry's packed sum carrying into the next entry's + // bits. The bit-sliced layout reduces each lane independently (every plane is a + // separate limb), so no such carry can occur and the addition never fails. The + // packed `fma_limb`/`truncate` loop below would instead corrupt the bit-planes. + self.add(other, c); + return Some(()); + } for (left, right) in self.limbs_mut().iter_mut().zip_eq(other.limbs()) { *left = fq.fma_limb(*left, *right, c.clone()); *left = fq.truncate(*left)?; From 8e81da34e719fef9b98c60dd5a980001d3676691 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 21:49:13 +0000 Subject: [PATCH 22/27] Rewrite p_masks loop with explicit conditional Per review: clearer as a loop setting the mask only when the bit is set, rather than a ternary. Keep `enumerate()` (not a bare index range) so clippy's needless_range_loop stays satisfied under -D warnings. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 9dc877c587..cce4ffc5c7 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -29,8 +29,10 @@ pub(crate) fn planes(p: u32) -> usize { /// for `j` in `0..=k`. fn p_masks(p: u32, k: usize) -> [Limb; BITS_PER_LIMB + 1] { let mut masks = [0; BITS_PER_LIMB + 1]; - for (j, m) in masks.iter_mut().enumerate().take(k + 1) { - *m = if (p >> j) & 1 == 1 { !0 } else { 0 }; + for (i, m) in masks.iter_mut().enumerate().take(k + 1) { + if (p >> i) & 1 == 1 { + *m = !0; + } } masks } From 4c67bbc4c774580e448158da88e6ef0fd68fe13d Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 29 Jun 2026 21:53:58 +0000 Subject: [PATCH 23/27] Simplify pmask: Limb::from + wrapping_neg Per review: replace `0u64.wrapping_sub((... ) as Limb)` with the lossless `Limb::from(bit).wrapping_neg()`, dropping the `as` cast and the literal-u64 inconsistency. Behavior is identical. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index cce4ffc5c7..6a125f299f 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -40,7 +40,9 @@ fn p_masks(p: u32, k: usize) -> [Limb; BITS_PER_LIMB + 1] { /// The full-width lane mask for bit `j` of `p`: all-ones if set, zero otherwise. #[inline(always)] fn pmask(p: u32, j: usize) -> Limb { - 0u64.wrapping_sub(((p >> j) & 1) as Limb) + // Widen bit `j` of `p` to the limb width, then broadcast it: `1 -> !0`, `0 -> 0` via + // two's-complement negation. + Limb::from((p >> j) & 1).wrapping_neg() } /// `dst += c * src` (mod p) over every group, where `dst` and `src` hold the same number of From d754faed5ee294448d4136641b86cbd445d685a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 30 Jun 2026 04:00:14 +0000 Subject: [PATCH 24/27] Credit Carl McTague for bit-slicing idea and F3 circuit The bit-sliced storage approach and the optimized 6-gate F3 addition circuit were both contributed by Carl McTague. Acknowledge this in the module docs and on f3_add_planes. Co-Authored-By: Carl McTague Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 6a125f299f..9d4084cc58 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -12,6 +12,9 @@ //! dispatched to a const-generic implementation so that, for each prime, the arrays are //! exactly sized and the loops fully unrolled; a heap-scratch fallback covers the rare //! primes with `k` beyond the dispatch range. +//! +//! The idea of bit-slicing finite-field vectors, and the optimized F3 addition circuit in +//! [`f3_add_planes`], were both contributed by Carl McTague (). use crate::{constants::BITS_PER_LIMB, limb::Limb}; @@ -285,6 +288,8 @@ fn scale_groups_k(dst: &mut [Limb], c: u32, p: u32) { /// Three parallel layers — two XORs, two XORs, two AND-NOTs — so it maps onto x86 `andn` /// and has very short dependency chains. Verified exhaustively against the 9 valid input /// pairs (the `(hi, lo) = (1, 1)` encoding never occurs for reduced inputs). +/// +/// This circuit was contributed by Carl McTague. #[inline(always)] fn f3_add_planes(a_lo: Limb, a_hi: Limb, b_lo: Limb, b_hi: Limb) -> (Limb, Limb) { let t_hi = a_hi ^ b_hi; From 781b7a7a31f8b6f4e261461ab7ea20febb476bcd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 30 Jun 2026 07:58:00 +0000 Subject: [PATCH 25/27] Fix two inaccurate comments in bit-sliced vector code Comment-only corrections from a review pass: - impl_fqslicemut.rs: add_bitsliced handles partial groups via the masked plane circuit (add_group_masked), not entry-wise as the doc claimed. - iter.rs: FqVectorNonZeroIterator's `start` is the slice's absolute start, not a group base. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/vector/impl_fqslicemut.rs | 11 ++++++----- ext/crates/fp/src/vector/iter.rs | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index 40e0e50e3b..6a2838bb18 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -130,11 +130,12 @@ impl<'a, F: Field> FqSliceMut<'a, F> { self.add_bitsliced(other, c); } - /// Add `c * other` to `self` in the bit-sliced layout. When both slices begin at a group - /// boundary (lane 0 — the common case, e.g. whole vectors and matrix rows), the complete - /// groups are added with the fast plane kernel ([`add_groups`](crate::field::field_internal)); - /// the fewer-than-64 trailing entries, and any non-group-aligned slice, fall back to - /// entry-wise addition. + /// Add `c * other` to `self` in the bit-sliced layout. Interior full groups are added with + /// the fast plane kernel ([`add_groups`](crate::field::field_internal)); the leading/trailing + /// partial groups go through the masked plane circuit + /// ([`add_group_masked`](crate::field::field_internal::FieldInternal::add_group_masked)). When + /// the two slices have different lane offsets within their groups, the planes are realigned + /// first via [`add_bitsliced_shifted`](Self::add_bitsliced_shifted). /// /// [`add_groups`]: crate::field::field_internal::FieldInternal::add_groups fn add_bitsliced(&mut self, other: FqSlice<'_, F>, c: FieldElement) { diff --git a/ext/crates/fp/src/vector/iter.rs b/ext/crates/fp/src/vector/iter.rs index fc2e183c11..aa959c48fc 100644 --- a/ext/crates/fp/src/vector/iter.rs +++ b/ext/crates/fp/src/vector/iter.rs @@ -166,7 +166,7 @@ impl ExactSizeIterator for FqVectorIterator<'_, F> { pub struct FqVectorNonZeroIterator<'a, F> { fq: F, limbs: &'a [Limb], - // Bit-sliced path: absolute index of the group base, and the relative cursor. + // Bit-sliced path: `start` is the slice's absolute start; `idx` is the relative cursor. bitsliced: bool, start: usize, // Shared/packed path state. From 12d6f02e1de8b4f1b2041655b8658013e591b3ec Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 18:14:33 +0000 Subject: [PATCH 26/27] Trim redundant comments in bit-sliced code Remove comments that restate adjacent code (F_2 masked-XOR, the single-group label, the shift-out empty guard, the scale-by-one no-op) and tighten a few verbose ones (pmask, number, the fp.rs bit-sliced-layout note, padded_len) by dropping migration/backwards-compat asides. Comment-only; no behavior change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 4 +--- ext/crates/fp/src/field/field_internal.rs | 4 +--- ext/crates/fp/src/field/fp.rs | 13 +++++-------- ext/crates/fp/src/vector/fp_wrapper/mod.rs | 8 +++----- ext/crates/fp/src/vector/impl_fqslicemut.rs | 6 ++---- ext/crates/fp/src/vector/impl_fqvector.rs | 1 - 6 files changed, 12 insertions(+), 24 deletions(-) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 9d4084cc58..33638fc604 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -43,8 +43,7 @@ fn p_masks(p: u32, k: usize) -> [Limb; BITS_PER_LIMB + 1] { /// The full-width lane mask for bit `j` of `p`: all-ones if set, zero otherwise. #[inline(always)] fn pmask(p: u32, j: usize) -> Limb { - // Widen bit `j` of `p` to the limb width, then broadcast it: `1 -> !0`, `0 -> 0` via - // two's-complement negation. + // Broadcast bit `j` across the limb via two's-complement negation: `1 -> !0`, `0 -> 0`. Limb::from((p >> j) & 1).wrapping_neg() } @@ -93,7 +92,6 @@ pub(crate) fn add_group_masked( return; } if p == 2 { - // One plane (k = 1); XOR in only the in-range lanes. dst[0] ^= src[0] & lane_mask; return; } diff --git a/ext/crates/fp/src/field/field_internal.rs b/ext/crates/fp/src/field/field_internal.rs index 12e7f77c0d..8065a5f842 100644 --- a/ext/crates/fp/src/field/field_internal.rs +++ b/ext/crates/fp/src/field/field_internal.rs @@ -284,9 +284,7 @@ pub trait FieldInternal: /// Return the number of limbs required to hold `dim` entries. fn number(self, dim: usize) -> usize { - // Whole groups needed to hold `dim` entries, times the limbs in each group. For the - // packed layout (1 limb/group, `entries_per_limb` entries/group) this is `ceil(dim / - // entries_per_limb)`, matching the previous definition. + // Whole groups needed to hold `dim` entries, times the limbs in each group. self.limbs_per_group() * dim.div_ceil(self.entries_per_group()) } diff --git a/ext/crates/fp/src/field/fp.rs b/ext/crates/fp/src/field/fp.rs index 4fa593fea3..00e6829ca4 100644 --- a/ext/crates/fp/src/field/fp.rs +++ b/ext/crates/fp/src/field/fp.rs @@ -145,14 +145,11 @@ impl FieldInternal for Fp

{ // # Bit-sliced layout // - // Prime-field vectors are stored bit-sliced: a group of `BITS_PER_LIMB` (64) elements - // occupies `k = ceil(log2 p)` limbs, one per bit-plane. Note that for `p = 2` this is - // `k = 1`, which is byte-identical to the packed layout, so `F_2` (and all of its SIMD / - // matrix machinery) is unaffected. The packed limb helpers above are retained because - // `decode`/`encode`/`reduce` are still used by callers that construct elements; they are - // simply not used to lay out `FqVector>` storage. The uniform `gather`/`scatter` - // defaults (in `FieldInternal`) handle entry access; `Fp` only overrides the bulk kernels - // with a branch-free plane circuit. + // Prime-field vectors use the bit-sliced group layout (see [`FieldInternal`]). The packed + // limb helpers above are not used to lay out `FqVector>` storage, but are retained + // because `decode`/`encode`/`reduce` are still called when constructing elements. The + // uniform `gather`/`scatter` defaults handle entry access; `Fp` only overrides the bulk + // kernels with a branch-free plane circuit. fn limbs_per_group(self) -> usize { crate::field::bitslice::planes(self.characteristic().as_u32()) diff --git a/ext/crates/fp/src/vector/fp_wrapper/mod.rs b/ext/crates/fp/src/vector/fp_wrapper/mod.rs index 3ae7ad1953..1b7810fe55 100644 --- a/ext/crates/fp/src/vector/fp_wrapper/mod.rs +++ b/ext/crates/fp/src/vector/fp_wrapper/mod.rs @@ -113,11 +113,9 @@ impl FpVector { } // Round `len` up to a whole number of groups, so that an augmented-matrix segment of this - // length ends on a group boundary and the next segment starts on one. For the packed - // layout a group is one limb (`entries_per_group == entries_per_limb`), so this equals the - // old `num_limbs * entries_per_limb`; for the bit-sliced layout a group spans 64 entries - // across several limbs, and segments must align to those 64-entry boundaries (not to the - // packed `entries_per_limb`) or a single group would straddle two segments. + // length ends on a group boundary and the next segment starts on one. A group spans 64 + // entries (across several limbs in the bit-sliced layout), and segments must align to those + // 64-entry boundaries or a single group would straddle two segments. pub(crate) fn padded_len(p: ValidPrime, len: usize) -> usize { let entries_per_group = Fp::new(p).entries_per_group(); len.div_ceil(entries_per_group) * entries_per_group diff --git a/ext/crates/fp/src/vector/impl_fqslicemut.rs b/ext/crates/fp/src/vector/impl_fqslicemut.rs index 6a2838bb18..952f7284ed 100644 --- a/ext/crates/fp/src/vector/impl_fqslicemut.rs +++ b/ext/crates/fp/src/vector/impl_fqslicemut.rs @@ -125,8 +125,8 @@ impl<'a, F: Field> FqSliceMut<'a, F> { return; } - // Every field uses the bit-sliced layout (`F_2` is just the `k = 1` case, identical to - // the old packed layout), so a single code path handles them all. + // Every field uses the bit-sliced layout (`F_2` is just the `k = 1` case), so a single + // code path handles them all. self.add_bitsliced(other, c); } @@ -179,7 +179,6 @@ impl<'a, F: Field> FqSliceMut<'a, F> { }; if first_g == last_g { - // Single (partial) group. let (s, o) = group_limbs(first_g); let mask = lane_mask(s_start - first_g * epg, s_end - first_g * epg); let src = &other.limbs()[o..o + k]; @@ -349,7 +348,6 @@ impl<'a, F: Field> FqSliceMut<'a, F> { // reading `i + shift` strictly ahead of writing `i` keeps it correct in place. let len = self.as_slice().len(); if shift >= len { - // Every entry shifts out; the slice becomes empty. *self.end_mut() = self.start(); return; } diff --git a/ext/crates/fp/src/vector/impl_fqvector.rs b/ext/crates/fp/src/vector/impl_fqvector.rs index 684f5767ea..7e2a6a1f4e 100644 --- a/ext/crates/fp/src/vector/impl_fqvector.rs +++ b/ext/crates/fp/src/vector/impl_fqvector.rs @@ -121,7 +121,6 @@ impl FqVector { return; } if fq.q() == 2 { - // `c` is one; scaling is a no-op. return; } fq.scale_groups(self.limbs_mut(), c); From 1d18af08a7dc7a6629464727971fbf7d3e7c1580 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 16 Jul 2026 09:28:30 +0000 Subject: [PATCH 27/27] Replace F5 add with Carl McTague's 17-gate circuit Swap the F5 indicator adder for a flat 17-gate boolean circuit (four layers, all-andn output). ~1.7x faster than the indicator circuit and ~1.15x faster than the earlier 21-gate version in microbenchmarks, verified exhaustively against (a + b) % 5. Scalar multiply keeps the indicator circuit. Co-Authored-By: Carl McTague Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NFZzmjYg4m8F2SHyFpVqEA --- ext/crates/fp/src/field/bitslice.rs | 47 ++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/ext/crates/fp/src/field/bitslice.rs b/ext/crates/fp/src/field/bitslice.rs index 33638fc604..0f336ba122 100644 --- a/ext/crates/fp/src/field/bitslice.rs +++ b/ext/crates/fp/src/field/bitslice.rs @@ -342,10 +342,10 @@ fn f3_scale_groups(dst: &mut [Limb], c: u32) { } // --------------------------------------------------------------------------------------- -// F5 specialization (k = 3). Planes are bits 0,1,2 of the value `v in {0,..,4}`. Both the -// add and the scalar multiply are built as flat "indicator" circuits — one-hot lane masks -// `is_v` for each operand value, recombined into the result with no carry/borrow chain — so -// they keep the wide instruction-level parallelism the sequential generic circuit loses. +// F5 specialization (k = 3). Planes are bits 0,1,2 of the value `v in {0,..,4}`. Addition is a +// flat 17-gate boolean circuit; scalar multiplication is an "indicator" circuit — one-hot lane +// masks `is_v` for each operand value, recombined into the result with no carry/borrow chain. +// Both keep the wide instruction-level parallelism the sequential generic circuit loses. // --------------------------------------------------------------------------------------- /// One-hot lane masks: `out[v]` has the bits of the lanes whose value is `v` (for `v in 0..5`). @@ -381,18 +381,37 @@ fn f5_mul_planes(p0: Limb, p1: Limb, p2: Limb, c: u32) -> (Limb, Limb, Limb) { f5_compose(sel) } -/// `(a + b) mod 5` on the three planes, as a flat indicator circuit. +/// `(a + b) mod 5` on the three planes, as a flat 17-gate circuit (planes are bits 0,1,2 of the +/// value). Four gate layers with an all-`andn` output layer, so the dependency chains stay short. +/// Verified exhaustively against `(a + b) % 5` for all reduced inputs. `andnot(x, y) = x & !y` +/// maps onto x86 `andn`. +/// +/// This circuit was contributed by Carl McTague. #[inline(always)] fn f5_add_planes(a0: Limb, a1: Limb, a2: Limb, b0: Limb, b1: Limb, b2: Limb) -> (Limb, Limb, Limb) { - let ia = f5_indicators(a0, a1, a2); - let ib = f5_indicators(b0, b1, b2); - let mut sel = [0 as Limb; 5]; - for av in 0..5usize { - for bv in 0..5usize { - sel[(av + bv) % 5] |= ia[av] & ib[bv]; - } - } - f5_compose(sel) + // Layer 1 + let g0 = a0 & b0; + let g1 = a0 | b0; + let g2 = a1 ^ b1; + let g3 = a1 | b1; + let g4 = a2 | b2; + let g5 = a2 ^ b2; + // Layer 2 + let g6 = g0 ^ g2; + let g7 = g3 | g5; + let g8 = g4 ^ g1; + let g9 = g1 & !g2; + let g10 = g4 & !g1; + // Layer 3 + let g11 = g2 ^ g7; + let g12 = g6 ^ g10; + let g13 = g6 ^ g7; + // Layer 4 + let g14 = g12 & !g11; + let g15 = g13 & !g9; + let g16 = g8 & !g13; + // m0 = g16, m1 = g14, m2 = g15 + (g16, g14, g15) } fn f5_add_groups(dst: &mut [Limb], src: &[Limb], c: u32) {