From 4ee49674dbdce3658bf9f0b55c5732bbd7a57096 Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 2 Aug 2026 19:53:18 -0400 Subject: [PATCH 1/2] =?UTF-8?q?feat(vocab):=20k-gram=20TF-IDF/LSA=20differ?= =?UTF-8?q?ential=20(orders=203-7)=20=E2=80=94=20confirm=20stopword=20cand?= =?UTF-8?q?idates?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The confirmation stage for the stopword deviation analysis. tools/kgram_tfidf_differential.py measures a candidate word's domain-specificity across n-gram orders 3..7: per order it builds the n-gram x domain TF-IDF matrix (n-grams over raw tokens, so phrase-embedded stopwords are captured), takes the LSA (truncated SVD) top component as the cross-domain axis, and scores the candidate by its strongest domain-specific n-gram DISCOUNTED by intrinsic unigram specificity (so 'the' can't borrow 'the state machine' specificity). The differential across orders is the discriminator: a true term PERSISTS (confirmed-term); a concentrated-but-diffuse word ('and') or a borrowed-signal word ('the') stays unconfirmed. validate-kgram-differential teeth: set/class/state confirmed across 3..7; 'and' unconfirmed; 'the' stripped by the unigram discount; orders are exactly [3,4,5,6,7]. Uses numpy (installed in the make recipe). Closes the two-stage stopword design. --- CHANGELOG.md | 1 + Makefile | 8 +- fixtures/kgram-differential/candidates.json | 9 + .../kgram-differential/domains/formal.json | 14 ++ .../kgram-differential/domains/narrative.json | 14 ++ specs/kgram-tfidf-differential.md | 30 ++++ tools/kgram_tfidf_differential.py | 156 ++++++++++++++++++ tools/validate_kgram_differential.py | 67 ++++++++ 8 files changed, 297 insertions(+), 2 deletions(-) create mode 100644 fixtures/kgram-differential/candidates.json create mode 100644 fixtures/kgram-differential/domains/formal.json create mode 100644 fixtures/kgram-differential/domains/narrative.json create mode 100644 specs/kgram-tfidf-differential.md create mode 100644 tools/kgram_tfidf_differential.py create mode 100644 tools/validate_kgram_differential.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 266a4fb..d06e31d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). Thi ### Added - Stopword deviation analysis v0.1 — the dropped words are governed vocabulary too (task #13): `tools/stopword_analysis.py` audits the loop's stoplist ACROSS domains using two signals — cross-domain deviation (concentration) AND compositional density (repeated-collocation rate) — because frequency alone can't tell a domain term from a stylistic quirk. Surfaces `term-candidate` (concentrated + compositional = a domain term hiding in the stoplist, propose un-stoplisting) vs `stylistic` (concentrated by style only) vs `noise` (uniform). `validate-stopword-analysis` teeth: domain terms surfaced; a stylistically-concentrated word ('and') is NOT wrongly promoted; a uniform word ('the') is noise. `stopword-analysis-live` audits the shipped stoplist over specs/*.md. Compositional density is the bigram floor of the k-gram TF-IDF/LSA differential (orders 3..7) to follow. +- k-gram TF-IDF/LSA differential v0.1 — confirm stopword candidates by compositional scale (task #13): `tools/kgram_tfidf_differential.py` measures a candidate word's domain-specificity across n-gram ORDERS 3..7 (TF-IDF over domains + LSA/truncated-SVD top component) and takes the differential. Signal is discounted by intrinsic unigram specificity so a stopword embedded in a domain phrase ('the state machine') can't borrow the phrase's specificity. A true term PERSISTS across orders (confirmed-term = strongest un-stoplist proposal); a concentrated-but-diffuse word ('and') or a borrowed-specificity word ('the') stays unconfirmed. `validate-kgram-differential` teeth: set/class/state confirmed across 3..7; 'and' unconfirmed; 'the' stripped by the unigram discount. Closes the two-stage design (stopword deviation -> k-gram confirmation). Uses numpy. - Agreement test v0.1 — glossary relations vs the blast-radius graph (task #13, neurosymbolic): `tools/agreement_test.py` projects the glossary's composition relations (`has-a`/`has-member`) onto the estate via `alignment.estateBinding` and compares them to a consumed blast-radius/dependency graph (GBRG owns that graph). Fail-closed on OVERCLAIM (a declared dependency with no observed edge = governance hole); reports DRIFT (an observed edge no relation names) as a remediation candidate (a proposed `has-a` relation), like the vocab-currency loop's candidate terms. `validate-agreement` teeth: aligned agrees; overclaim refused; drift surfaced as candidate. - Glossary promotion v0.1 — the governed draft→approved alignment pass (task #13, closes the vocabulary lifecycle): `tools/promote_glossary_term.py` promotes a `draft` GlossaryTerm to `approved` ONLY when the 3-method alignment holds as a fail-closed MEET — capture (`ontologyClassRef`) ∧ vector-align (`vectorLink`, pinned to the sovereign nomic/768 space AND reciprocated by the named peer) ∧ implement (`estateBinding` entity/service/action). It recomputes the meet (never trusts a flag), refuses on any missing/non-reciprocal/off-space method (term stays draft, no governance hole), and its approved output PASSES the #250 alignment drift-guard by construction. `validate-glossary-promotion` teeth: promotes on full alignment; refuses missing-implement / non-reciprocal / off-space; promoted output passes the #250 guard. - Vocab-currency loop — LIVE dogfood on the estate's own vocab (`make vocab-currency-dogfood`): `tools/dogfood_vocab_currency.py` runs the governed loop over this repo's approved `GlossaryTerm`s (fixed/LSA) vs `specs/*.md` (open/LDA). Today only a handful of terms are approved against the full spec corpus, so the governed loop escalates-human (can't reach currency in its bound), writing real draft `GlossaryTerm` proposals to `build/vocab-currency-proposals/` — the artifacts ontogenesis ingests. Reports the outcome (does not gate on it, exit 0); Markdown-cleaned + generic-word stoplisted so domain terms surface. Enforcement stays on fixtures in `validate-vocab-currency-loop`. diff --git a/Makefile b/Makefile index 77e67da..43212bb 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: validate validate-resource-contract validate-measurement validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-ingestion-pipeline-examples validate-data-acquisition-examples validate-glossary-alignment-examples validate-data-class-examples validate-table-keys validate-dag-loop validate-vocab-currency-loop vocab-currency-dogfood validate-glossary-promotion validate-agreement validate-stopword-analysis stopword-analysis-live +.PHONY: validate validate-resource-contract validate-measurement validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-ingestion-pipeline-examples validate-data-acquisition-examples validate-glossary-alignment-examples validate-data-class-examples validate-table-keys validate-dag-loop validate-vocab-currency-loop vocab-currency-dogfood validate-glossary-promotion validate-agreement validate-stopword-analysis stopword-analysis-live validate-kgram-differential -validate: validate-data-class-examples validate-glossary-alignment-examples validate-data-acquisition-examples validate-ingestion-pipeline-examples validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-resource-contract validate-table-keys validate-dag-loop validate-vocab-currency-loop validate-glossary-promotion validate-agreement validate-stopword-analysis +validate: validate-data-class-examples validate-glossary-alignment-examples validate-data-acquisition-examples validate-ingestion-pipeline-examples validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-resource-contract validate-table-keys validate-dag-loop validate-vocab-currency-loop validate-glossary-promotion validate-agreement validate-stopword-analysis validate-kgram-differential @echo "OK: validate" validate-ingestion-pipeline-examples: @@ -22,6 +22,10 @@ validate-glossary-promotion: validate-agreement: python3 tools/validate_agreement.py +validate-kgram-differential: + python3 -m pip install --user numpy >/dev/null + python3 tools/validate_kgram_differential.py + validate-stopword-analysis: python3 tools/validate_stopword_analysis.py diff --git a/fixtures/kgram-differential/candidates.json b/fixtures/kgram-differential/candidates.json new file mode 100644 index 0000000..433edd6 --- /dev/null +++ b/fixtures/kgram-differential/candidates.json @@ -0,0 +1,9 @@ +{ + "candidates": [ + "set", + "class", + "state", + "and", + "the" + ] +} diff --git a/fixtures/kgram-differential/domains/formal.json b/fixtures/kgram-differential/domains/formal.json new file mode 100644 index 0000000..478701f --- /dev/null +++ b/fixtures/kgram-differential/domains/formal.json @@ -0,0 +1,14 @@ +{ + "domain": "formal", + "documents": [ + { + "text": "the finite state machine reads the set of states. a base class defines the state of the set. the empty set of states in the state machine. the finite state machine reads the set of states. a base class defines the state of the set." + }, + { + "text": "the empty set of states in the state machine. the finite state machine reads the set of states. a base class defines the state of the set. the finite state machine reads the set of states again. the empty set of states in the state machine." + }, + { + "text": "a base class defines the state of the set. the empty set of states in the state machine. the finite state machine reads the set of states. a base class defines the state of the set once more. the finite state machine reads the set of states." + } + ] +} diff --git a/fixtures/kgram-differential/domains/narrative.json b/fixtures/kgram-differential/domains/narrative.json new file mode 100644 index 0000000..b0470b8 --- /dev/null +++ b/fixtures/kgram-differential/domains/narrative.json @@ -0,0 +1,14 @@ +{ + "domain": "narrative", + "documents": [ + { + "text": "the dog and the cat ran across the field and the sun was warm and the children laughed and the day was long and the evening came and the stars and the moon rose over the hills" + }, + { + "text": "and so the story went and the people gathered and the music played and the night was full and the wind and the rain and the memory of the summer and the sea and the distant shore" + }, + { + "text": "the morning and the mist and the river and the birds and the song and the quiet and the town and the market and the voices and the laughter and the bread and the wine and the road" + } + ] +} diff --git a/specs/kgram-tfidf-differential.md b/specs/kgram-tfidf-differential.md new file mode 100644 index 0000000..27ea39e --- /dev/null +++ b/specs/kgram-tfidf-differential.md @@ -0,0 +1,30 @@ +# k-gram TF-IDF / LSA differential (v0.1) — confirm stopword candidates by compositional scale + +The stopword deviation analysis flags a dropped word as a term-candidate when it is concentrated in +a domain AND recurs in repeated bigram collocations. That is the **bigram floor**. This is the +confirmation: it measures the candidate's domain-specificity across n-gram **orders 3..7** and takes +the **differential**. + +Per order `n`, over the domain corpora: +1. build the n-gram × domain count matrix (n-grams over RAW tokens, so a stopword that is part of a + real phrase — "held to maturity", "empty set of states" — is captured); +2. weight it **TF-IDF** (documents = domains): a phrase frequent in ONE domain and rare across the + rest scores high — it is domain-specific; +3. take the **LSA** (truncated SVD): the top singular component is the dominant axis of cross-domain + variation, and a domain-specific n-gram loads heavily on it. + +A candidate's per-order signal is the strongest domain-specific TF-IDF among the n-grams containing +it, **discounted by its intrinsic unigram specificity** — a stopword embedded in a domain phrase +("the state machine") would otherwise borrow the phrase's specificity; the discount strips that so +only intrinsically domain-specific words persist. + +**The differential across 3..7 is the discriminator:** +- a **true domain term PERSISTS** — it keeps heading domain-specific n-grams as `n` grows, so its + signal stays high across orders → **confirmed-term** (the strongest un-stoplist proposal); +- a **stylistic / noise word DECAYS or is discounted** — its longer n-grams become unique and + diffuse, or its signal was borrowed → **unconfirmed** (stays stoplisted). + +`make validate-kgram-differential` proves it: real terms (`set`/`class`/`state`) are confirmed +across all orders; a concentrated-but-diffuse word (`and`) is not; a word that only borrows phrase +specificity (`the`) is stripped by the unigram discount. This closes the two-stage design — stopword +deviation (candidates) → k-gram differential (confirmation). diff --git a/tools/kgram_tfidf_differential.py b/tools/kgram_tfidf_differential.py new file mode 100644 index 0000000..31c830c --- /dev/null +++ b/tools/kgram_tfidf_differential.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""k-gram TF-IDF / LSA differential — confirm stopword candidates by compositional scale (task #13). + +The stopword deviation analysis flags a dropped word as a term-candidate when it is concentrated in +a domain AND recurs in repeated (bigram) collocations. That is the bigram FLOOR. This tool is the +confirmation: it measures the word's domain-specificity across n-gram ORDERS 3..7 and takes the +DIFFERENTIAL. + +For each order n, over the domain corpora: + * build the n-gram x domain count matrix (n-grams over RAW tokens, so a stopword that is part of a + real phrase — "held to maturity", "empty set of states" — is captured); + * weight it TF-IDF (documents = domains): a phrase frequent in ONE domain and rare across the rest + scores high — it is domain-specific; + * take the LSA (truncated SVD) of that TF-IDF matrix: the top singular component is the dominant + axis of cross-domain variation, and a domain-specific n-gram loads heavily on it. + +A candidate word's signal at order n = the strongest domain-specific TF-IDF among the n-grams that +contain it (and its share of the LSA top-component energy). The DIFFERENTIAL across 3..7 is the +discriminator: + * a TRUE domain term PERSISTS — it keeps appearing in domain-specific n-grams as n grows (it is + the head of longer collocations), so its signal stays high across orders -> CONFIRMED-TERM; + * a stylistic / noise word DECAYS — its longer n-grams become unique and diffuse (no repeated + domain phrase), so its concentrated TF-IDF and LSA energy fall away -> UNCONFIRMED. + +Confirmed candidates are the strongest un-stoplist proposals; unconfirmed ones stay stoplisted. +""" +from __future__ import annotations + +import argparse +import json +import math +import re +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[1] +ORDERS = [3, 4, 5, 6, 7] +SIGNAL_TERMLIKE = 0.40 # per-order domain-specific-signal threshold +PERSIST_FRACTION = 0.5 # must clear the threshold on >= this share of the orders => persists + + +def raw_tokenize(text: str) -> list[str]: + return [t for t in re.findall(r"[a-z][a-z0-9-]+", text.lower()) if len(t) >= 2] + + +def ngrams(tokens: list[str], n: int) -> list[tuple[str, ...]]: + return [tuple(tokens[i:i + n]) for i in range(len(tokens) - n + 1)] + + +def order_signal(domain_tokens: dict[str, list[str]], n: int, candidates: list[str]) -> dict: + domains = list(domain_tokens) + D = len(domains) + # n-gram counts per domain + counts = {d: {} for d in domains} + vocab: set[tuple[str, ...]] = set() + for d, toks in domain_tokens.items(): + for g in ngrams(toks, n): + counts[d][g] = counts[d].get(g, 0) + 1 + vocab.add(g) + if not vocab: + return {w: {"tfidf": 0.0, "lsaEnergy": 0.0} for w in candidates} + vocab = sorted(vocab) + df = {g: sum(1 for d in domains if g in counts[d]) for g in vocab} + + # TF-IDF matrix rows=n-grams, cols=domains (smoothed idf). + M = np.zeros((len(vocab), D)) + idx = {g: i for i, g in enumerate(vocab)} + for j, d in enumerate(domains): + for g, c in counts[d].items(): + M[idx[g], j] = (1 + math.log(c)) * math.log((1 + D) / (1 + df[g])) + col_max = M.max() or 1.0 + + # LSA — top singular component = dominant axis of cross-domain variation. + try: + U, S, _ = np.linalg.svd(M, full_matrices=False) + loading = (U[:, 0] * S[0]) ** 2 if S.size else np.zeros(len(vocab)) + except np.linalg.LinAlgError: + loading = np.zeros(len(vocab)) + total_energy = loading.sum() or 1.0 + + out = {} + for w in candidates: + rows = [idx[g] for g in vocab if w in g] + tfidf = float(M[rows].max() / col_max) if rows else 0.0 # strongest domain-specific phrase w heads + energy = float(loading[rows].sum() / total_energy) if rows else 0.0 # share of latent variation from w's n-grams + out[w] = {"tfidf": round(tfidf, 3), "lsaEnergy": round(energy, 3)} + return out + + +def _unigram_specificity(domain_tokens: dict[str, list[str]], w: str) -> float: + # Intrinsic domain-specificity of w AS A UNIGRAM (the stopword-analysis concentration), mapped + # to [0,1]: uniform (1/D) -> 0, fully concentrated -> 1. A stopword embedded in a domain phrase + # ("the state machine") borrows the phrase's n-gram specificity; discounting by unigram + # specificity strips that borrowed signal, so only intrinsically domain-specific words persist. + D = len(domain_tokens) + relfreq = [domain_tokens[d].count(w) / (len(domain_tokens[d]) or 1) for d in domain_tokens] + s = sum(relfreq) or 1.0 + concentration = max(relfreq) / s + return max(0.0, min(1.0, (concentration - 1 / D) / (1 - 1 / D))) if D > 1 else 1.0 + + +def differential(domains: dict[str, str], candidates: list[str]) -> dict: + domain_tokens = {d: raw_tokenize(t) for d, t in domains.items()} + by_order = {n: order_signal(domain_tokens, n, candidates) for n in ORDERS} + uspec = {w: _unigram_specificity(domain_tokens, w) for w in candidates} + + rows = [] + for w in candidates: + # discount each order's domain-specific n-gram signal by the word's intrinsic unigram + # specificity — borrowed phrase-specificity (e.g. "the") is stripped away. + signal = {n: round(by_order[n][w]["tfidf"] * uspec[w], 3) for n in ORDERS} + lsa = {n: by_order[n][w]["lsaEnergy"] for n in ORDERS} + deltas = {f"{ORDERS[i]}->{ORDERS[i + 1]}": round(signal[ORDERS[i + 1]] - signal[ORDERS[i]], 3) + for i in range(len(ORDERS) - 1)} + cleared = sum(1 for n in ORDERS if signal[n] >= SIGNAL_TERMLIKE) + persists = cleared >= math.ceil(len(ORDERS) * PERSIST_FRACTION) + rows.append({ + "word": w, + "unigramSpecificity": round(uspec[w], 3), + "tfidfByOrder": signal, + "lsaEnergyByOrder": lsa, + "differential": deltas, + "ordersCleared": cleared, + "verdict": "confirmed-term" if persists else "unconfirmed", + }) + return {"orders": ORDERS, "domains": list(domains), + "confirmed": [r["word"] for r in rows if r["verdict"] == "confirmed-term"], + "rows": rows} + + +def _load_fixture() -> tuple[dict, list[str]]: + FIX = ROOT / "fixtures" / "kgram-differential" + domains = {d["domain"]: "\n".join(x["text"] for x in d["documents"]) + for d in (json.loads(p.read_text()) for p in sorted((FIX / "domains").glob("*.json")))} + candidates = json.loads((FIX / "candidates.json").read_text())["candidates"] + return domains, candidates + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--domains-dir", help="dir of {domain, documents:[{text}]} JSON (default: fixture)") + ap.add_argument("--candidates", nargs="*", help="candidate words (default: fixture)") + args = ap.parse_args() + if args.domains_dir: + domains = {d["domain"]: "\n".join(x["text"] for x in d["documents"]) + for d in (json.loads(p.read_text()) for p in sorted(Path(args.domains_dir).glob("*.json")))} + candidates = args.candidates or [] + else: + domains, candidates = _load_fixture() + print(json.dumps(differential(domains, candidates), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/validate_kgram_differential.py b/tools/validate_kgram_differential.py new file mode 100644 index 0000000..306f7cb --- /dev/null +++ b/tools/validate_kgram_differential.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""CI teeth for the k-gram TF-IDF/LSA differential (task #13). + +Asserts the differential confirms a stopword candidate only when its domain-specific signal PERSISTS +across n-gram orders 3..7 AND is intrinsic (unigram-backed) — so a real term is confirmed, a merely +concentrated-but-diffuse word ('and') is not, and a word that only BORROWS phrase specificity ('the') +is stripped by the unigram discount. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import kgram_tfidf_differential as K # noqa: E402 + +FAILURES: list[str] = [] +CHECKS: dict[str, bool] = {} + + +def main() -> int: + domains, candidates = K._load_fixture() + r = K.differential(domains, candidates) + confirmed = set(r["confirmed"]) + row = {x["word"]: x for x in r["rows"]} + + # 0. Orders are exactly 3..7. + if r["orders"] != [3, 4, 5, 6, 7]: + FAILURES.append(f"orders must be [3,4,5,6,7], got {r['orders']}") + else: + CHECKS["orders:3-7"] = True + + # 1. Real domain terms are confirmed (signal persists across orders). + for w in ("set", "class", "state"): + if w not in confirmed: + FAILURES.append(f"'{w}' is a domain term whose signal persists 3..7 but was not confirmed") + if not any("not confirmed" in m for m in FAILURES): + CHECKS["terms:confirmed-across-orders"] = True + + # 2. Confirmed terms actually PERSIST — cleared on a majority of orders, not one spike. + if all(row[w]["ordersCleared"] >= 3 for w in ("set", "class", "state")): + CHECKS["terms:persist-not-spike"] = True + else: + FAILURES.append("a confirmed term did not persist across a majority of orders") + + # 3. THE TRAP: 'and' is concentrated (narrative-only) but its n-grams are diffuse -> unconfirmed. + if "and" in confirmed or row["and"]["ordersCleared"] != 0: + FAILURES.append("'and' (concentrated but not compositional) must not be confirmed") + else: + CHECKS["diffuse-word:unconfirmed"] = True + + # 4. 'the' borrows phrase specificity; the unigram discount must strip it -> unconfirmed + low uspec. + if "the" in confirmed or row["the"]["unigramSpecificity"] >= 0.5: + FAILURES.append("'the' must be stripped by the unigram-specificity discount (borrowed signal)") + else: + CHECKS["borrowed-specificity:stripped"] = True + + for m in FAILURES: + print(f"FAIL: {m}", file=sys.stderr) + ok = not FAILURES and all(CHECKS.values()) + print(json.dumps({"ok": ok, "checks": CHECKS, "confirmed": sorted(confirmed)}, indent=2, sort_keys=True)) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From d0ebb139c9f5e845d7300462112c1c59b1d08ea8 Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 2 Aug 2026 20:02:11 -0400 Subject: [PATCH 2/2] =?UTF-8?q?fix(vocab):=20remediate=20Copilot=20review?= =?UTF-8?q?=20on=20#261=20=E2=80=94=20LSA=20in=20verdict,=20perf,=20<2-dom?= =?UTF-8?q?ain=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - confirmation now uses BOTH signals of the differential (docstring promised it): TF-IDF must persist across orders AND the word must load on the dominant cross-domain LSA axis (lsaParticipates). Matches 'tf-idf lsa differential'; 'and' now fails both signals. - precompute candidate->row indices in the single pass over vocab instead of rescanning the whole vocab per candidate (O(vocab) once, not O(vocab*candidates) per order). - differential() raises on <2 domains (TF-IDF/LSA are cross-domain measures — fail fast, don't emit a meaningless result). Teeth: single-domain rejected. 6 teeth total. --- tools/kgram_tfidf_differential.py | 37 ++++++++++++++++++++++------ tools/validate_kgram_differential.py | 7 ++++++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/tools/kgram_tfidf_differential.py b/tools/kgram_tfidf_differential.py index 31c830c..cbecc3d 100644 --- a/tools/kgram_tfidf_differential.py +++ b/tools/kgram_tfidf_differential.py @@ -14,13 +14,15 @@ * take the LSA (truncated SVD) of that TF-IDF matrix: the top singular component is the dominant axis of cross-domain variation, and a domain-specific n-gram loads heavily on it. -A candidate word's signal at order n = the strongest domain-specific TF-IDF among the n-grams that -contain it (and its share of the LSA top-component energy). The DIFFERENTIAL across 3..7 is the -discriminator: +A candidate word's per-order signal = the strongest domain-specific TF-IDF among the n-grams that +contain it (discounted by intrinsic unigram specificity), plus its share of the LSA top-component +energy. Confirmation uses BOTH: the DIFFERENTIAL across 3..7 discriminates: * a TRUE domain term PERSISTS — it keeps appearing in domain-specific n-grams as n grows (it is - the head of longer collocations), so its signal stays high across orders -> CONFIRMED-TERM; + the head of longer collocations), so its TF-IDF signal stays high across orders AND it loads on + the dominant cross-domain LSA axis -> CONFIRMED-TERM; * a stylistic / noise word DECAYS — its longer n-grams become unique and diffuse (no repeated - domain phrase), so its concentrated TF-IDF and LSA energy fall away -> UNCONFIRMED. + domain phrase), so its concentrated TF-IDF falls away and/or it never loads on the LSA axis + -> UNCONFIRMED. Confirmed candidates are the strongest un-stoplist proposals; unconfirmed ones stay stoplisted. """ @@ -63,9 +65,16 @@ def order_signal(domain_tokens: dict[str, list[str]], n: int, candidates: list[s vocab = sorted(vocab) df = {g: sum(1 for d in domains if g in counts[d]) for g in vocab} - # TF-IDF matrix rows=n-grams, cols=domains (smoothed idf). + # TF-IDF matrix rows=n-grams, cols=domains (smoothed idf). Precompute, in the SAME single pass + # over vocab, the row indices for each candidate word (an n-gram has <= n tokens) — so we don't + # rescan the whole vocab once per candidate. M = np.zeros((len(vocab), D)) idx = {g: i for i, g in enumerate(vocab)} + cand_set = set(candidates) + cand_rows: dict[str, list[int]] = {w: [] for w in candidates} + for g, i in idx.items(): + for tok in set(g) & cand_set: + cand_rows[tok].append(i) for j, d in enumerate(domains): for g, c in counts[d].items(): M[idx[g], j] = (1 + math.log(c)) * math.log((1 + D) / (1 + df[g])) @@ -81,7 +90,7 @@ def order_signal(domain_tokens: dict[str, list[str]], n: int, candidates: list[s out = {} for w in candidates: - rows = [idx[g] for g in vocab if w in g] + rows = cand_rows[w] tfidf = float(M[rows].max() / col_max) if rows else 0.0 # strongest domain-specific phrase w heads energy = float(loading[rows].sum() / total_energy) if rows else 0.0 # share of latent variation from w's n-grams out[w] = {"tfidf": round(tfidf, 3), "lsaEnergy": round(energy, 3)} @@ -100,7 +109,13 @@ def _unigram_specificity(domain_tokens: dict[str, list[str]], w: str) -> float: return max(0.0, min(1.0, (concentration - 1 / D) / (1 - 1 / D))) if D > 1 else 1.0 +LSA_PARTICIPATION_FLOOR = 0.03 # a confirmed term must load on the dominant cross-domain LSA axis + + def differential(domains: dict[str, str], candidates: list[str]) -> dict: + if len(domains) < 2: + raise ValueError("k-gram differential is a CROSS-domain measure — needs >= 2 domains " + "(TF-IDF and LSA are meaningless with fewer)") domain_tokens = {d: raw_tokenize(t) for d, t in domains.items()} by_order = {n: order_signal(domain_tokens, n, candidates) for n in ORDERS} uspec = {w: _unigram_specificity(domain_tokens, w) for w in candidates} @@ -115,6 +130,11 @@ def differential(domains: dict[str, str], candidates: list[str]) -> dict: for i in range(len(ORDERS) - 1)} cleared = sum(1 for n in ORDERS if signal[n] >= SIGNAL_TERMLIKE) persists = cleared >= math.ceil(len(ORDERS) * PERSIST_FRACTION) + # Confirmation uses BOTH signals of the differential: the TF-IDF domain-specificity must + # PERSIST across orders AND the word must load on the dominant cross-domain LSA axis (so a + # word with high TF-IDF but no latent-axis participation is not confirmed on TF-IDF alone). + lsa_participates = max(lsa.values()) >= LSA_PARTICIPATION_FLOOR + confirmed = persists and lsa_participates rows.append({ "word": w, "unigramSpecificity": round(uspec[w], 3), @@ -122,7 +142,8 @@ def differential(domains: dict[str, str], candidates: list[str]) -> dict: "lsaEnergyByOrder": lsa, "differential": deltas, "ordersCleared": cleared, - "verdict": "confirmed-term" if persists else "unconfirmed", + "lsaParticipates": lsa_participates, + "verdict": "confirmed-term" if confirmed else "unconfirmed", }) return {"orders": ORDERS, "domains": list(domains), "confirmed": [r["word"] for r in rows if r["verdict"] == "confirmed-term"], diff --git a/tools/validate_kgram_differential.py b/tools/validate_kgram_differential.py index 306f7cb..e19ff1f 100644 --- a/tools/validate_kgram_differential.py +++ b/tools/validate_kgram_differential.py @@ -56,6 +56,13 @@ def main() -> int: else: CHECKS["borrowed-specificity:stripped"] = True + # 5. Cross-domain measure — a single domain must fail fast, not produce a meaningless result. + try: + K.differential({"only": list(domains.values())[0]}, candidates) + FAILURES.append("differential with <2 domains must raise, not silently run") + except ValueError: + CHECKS["single-domain:rejected"] = True + for m in FAILURES: print(f"FAIL: {m}", file=sys.stderr) ok = not FAILURES and all(CHECKS.values())