Feature/lexicon multiplier retune

.gitignore

@@ -28,9 +28,14 @@ __pycache__/
 # Step 0 exploration spikes (not part of the library crate)
 spikes/
 
-# Benchmark datasets (large, fetched manually — see benchmarks/*/README.md)
-benchmarks/*/data/
+# Benchmark datasets (large, fetched manually — see benchmarks/*/README.md).
+# Covers data/ and transformed variants like data_wh40k/.
+benchmarks/*/data*/
 
 # Embedding model cache (downloaded ONNX weights + tokenizer, ~90 MB)
 /models/
 .fastembed_cache/
+
+
+.env
+.env.local

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Changed
+
+- **Lexicon boost is now a configurable dial — `Config::lexicon_boost_clamp`.** The lexicon scorer enters ranking as a multiplier `1.0 + boost.clamp(-c, c)`; `c` was a hardcoded `2.0` (→ up to 3×), now a `Config` field defaulting to a conservative `0.05`. Rationale: fused RRF relevance scores are tightly compressed (rank-1-to-2 gap ~1–2%), so a wide bound let a query-independent importance signal overwhelm relevance — on LongMemEval, the old bound dropped single-session Recall@1 from 0.94 to 0.38 even with semantic search active; `0.05` recovers ~87% of the no-lexicon baseline. `0.0` disables lexicon influence; larger values trade relevance precision for importance weight. Note: this benchmark measures only lexicon's *cost* on factual retrieval, not its *benefit* on persona/importance workloads — the default is harm-minimizing pending a persona-style eval.
+
 ### Breaking Changes
 
 - **Lexicon scoring is now opt-in.** `ContextForge::builder(...)` no longer auto-seeds `DefaultEnglishScorer`; by default the engine ranks on relevance (BM25, plus semantic when an embedding model is set) with no lexicon layer. Enable the English importance scorer explicitly with `.with_default_english_scorer()`, and note that `.with_persona_scorer(...)` no longer implies the English layer — call both to compose them. Rationale: the always-on, query-independent importance boost was shown to degrade factual retrieval (LongMemEval Recall@1 dropped 0.71 → 0.17 with the scorer forced on). Callers that want the old behavior — persona/importance use cases, e.g. a chat assistant — add `.with_default_english_scorer()`.

Cargo.toml

@@ -23,7 +23,7 @@ exclude = [
 ]
 
 [workspace]
-members = ["benchmarks/longmemeval"]
+members = ["benchmarks/longmemeval", "benchmarks/personamem"]
 
 [lib]
 name = "context_forge"

benchmarks/personamem/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "personamem-bench"
+version = "0.0.0"
+edition = "2021"
+publish = false
+description = "PersonaMem retrieval benchmark for context-forge: measures the lexicon-BENEFIT case (surfacing the current/salient user preference) — the complement to the LongMemEval cost benchmark. Dev-only; never published."
+
+[dependencies]
+context-forge = { path = "../..", features = ["semantic", "distill-http"] }
+tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+csv = "1.4"
+anyhow = "1"

benchmarks/personamem/src/dataset.rs

@@ -0,0 +1,123 @@
+//! PersonaMem-v2 dataset: benchmark CSV + per-persona chat-history JSON.
+//!
+//! Source: HuggingFace `bowen-upenn/PersonaMem-v2` (see README). Verified
+//! 2026-07-07 against the real files: `related_conversation_snippet` is a JSON
+//! array of `{role, content}` whose `content` matches a chat-history message
+//! **exactly**, so retrieval scoring maps snippet → ingested entry by content
+//! equality with no normalization.
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+use serde::Deserialize;
+
+/// One benchmark row: a query about a persona's current preference, with the
+/// evidence turn(s) annotated. Only the columns the retrieval eval uses are
+/// deserialized (csv+serde ignores the rest by header name).
+#[derive(Debug, Deserialize)]
+pub struct Row {
+    pub persona_id: String,
+    pub chat_history_32k_link: String,
+    pub user_query: String,
+    /// JSON array string of `{role, content}` — the turn(s) where the user
+    /// implicitly stated the current preference. This is the retrieval ground
+    /// truth. Parse with [`Row::gold_contents`].
+    pub related_conversation_snippet: String,
+    /// Whether this preference supersedes an earlier one (`"True"`/`"False"`).
+    /// The `updated == true` slice is the sharpest lexicon-benefit signal
+    /// (current-vs-stale preference evolution).
+    #[serde(default)]
+    pub updated: String,
+    /// Preference category (stereotypical / anti-stereotypical / neutral /
+    /// therapy / health). Retained from the schema for future per-type slicing;
+    /// not yet read by the overall/updated scorer.
+    #[serde(default)]
+    #[allow(dead_code)]
+    pub pref_type: String,
+}
+
+impl Row {
+    /// True if this row is a preference-*update* case.
+    #[must_use]
+    pub fn is_updated(&self) -> bool {
+        self.updated.eq_ignore_ascii_case("true")
+    }
+
+    /// The gold evidence contents: the `content` field of each turn in
+    /// `related_conversation_snippet`. Returns empty if the JSON can't be parsed
+    /// (row is then excluded from scoring, like an abstention).
+    #[must_use]
+    pub fn gold_contents(&self) -> Vec<String> {
+        #[derive(Deserialize)]
+        struct Turn {
+            content: String,
+        }
+        serde_json::from_str::<Vec<Turn>>(&self.related_conversation_snippet)
+            .map(|turns| turns.into_iter().map(|t| t.content).collect())
+            .unwrap_or_default()
+    }
+}
+
+/// A chat-history file: `{ metadata, chat_history: [{role, content}, ...] }`.
+#[derive(Debug, Deserialize)]
+pub struct ChatHistory {
+    pub chat_history: Vec<Message>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct Message {
+    /// Speaker role. Present in the schema; the eval ingests content regardless
+    /// of role, so this is not currently read.
+    #[allow(dead_code)]
+    pub role: String,
+    pub content: String,
+}
+
+/// All rows for one persona, plus its (lazily loaded) chat history. Grouping by
+/// persona lets us ingest the ~189-message history **once** and run all of the
+/// persona's queries against it (queries are ~1ms; ingest is the cost).
+pub struct Persona {
+    pub persona_id: String,
+    pub chat_history_link: String,
+    pub rows: Vec<Row>,
+}
+
+/// Load `benchmark.csv` and group rows by persona, preserving first-seen order.
+pub fn load_grouped(csv_path: &Path) -> anyhow::Result<Vec<Persona>> {
+    let mut reader = csv::ReaderBuilder::new()
+        .flexible(true)
+        .from_path(csv_path)
+        .map_err(|e| anyhow::anyhow!("opening {}: {e}", csv_path.display()))?;
+
+    let mut order: Vec<String> = Vec::new();
+    let mut by_persona: HashMap<String, Persona> = HashMap::new();
+
+    for result in reader.deserialize() {
+        let row: Row = result.map_err(|e| anyhow::anyhow!("parsing benchmark.csv row: {e}"))?;
+        let pid = row.persona_id.clone();
+        let entry = by_persona.entry(pid.clone()).or_insert_with(|| {
+            order.push(pid.clone());
+            Persona {
+                persona_id: pid.clone(),
+                chat_history_link: row.chat_history_32k_link.clone(),
+                rows: Vec::new(),
+            }
+        });
+        entry.rows.push(row);
+    }
+
+    Ok(order
+        .into_iter()
+        .filter_map(|pid| by_persona.remove(&pid))
+        .collect())
+}
+
+/// Load a persona's chat history. `link` is dataset-relative (e.g.
+/// `data/chat_history_32k/...json`); `data_root` is where the dataset lives.
+pub fn load_chat_history(data_root: &Path, link: &str) -> anyhow::Result<ChatHistory> {
+    let path: PathBuf = data_root.join(link);
+    let bytes = std::fs::read(&path)
+        .map_err(|e| anyhow::anyhow!("reading chat history {}: {e}", path.display()))?;
+    serde_json::from_slice(&bytes)
+        .map_err(|e| anyhow::anyhow!("parsing chat history {}: {e}", path.display()))
+}