From 226139bdee74eca55d70e46ad25e43420d5b8248 Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 2 Aug 2026 19:48:18 -0400 Subject: [PATCH 1/3] =?UTF-8?q?feat(vocab):=20stopword=20deviation=20analy?= =?UTF-8?q?sis=20=E2=80=94=20audit=20the=20words=20we=20DROP=20(task=20#13?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stoplist is itself ungoverned vocabulary: a word that is filler in one domain (set, class, state, required) is a real term in another. tools/stopword_analysis.py audits the dropped words across domains using TWO signals — cross-domain deviation (concentration) AND compositional density (repeated-collocation rate) — because frequency alone can't tell a domain term from a stylistic quirk (e.g. 'and' is concentrated in chatty prose but forms no repeated collocations). Surfaces term-candidate / stylistic / noise; term-candidates are un-stoplist proposals (a remediation signal like the currency loop's candidates and the agreement test's drift). validate-stopword-analysis teeth: domain terms surfaced; a stylistically-concentrated word NOT promoted (the trap frequency alone falls into); a uniform word is noise. stopword-analysis-live audits the shipped STOP over specs/*.md (currently clean: 0 candidates / 38 noise / 9 insufficient). Compositional density is the bigram floor of the k-gram TF-IDF/LSA differential (3..7) to follow. --- CHANGELOG.md | 1 + Makefile | 11 +- .../stopword-analysis/domains/formal.json | 14 ++ .../stopword-analysis/domains/narrative.json | 11 ++ fixtures/stopword-analysis/stoplist.json | 12 ++ specs/stopword-analysis.md | 29 ++++ tools/stopword_analysis.py | 124 ++++++++++++++++++ tools/validate_stopword_analysis.py | 72 ++++++++++ 8 files changed, 272 insertions(+), 2 deletions(-) create mode 100644 fixtures/stopword-analysis/domains/formal.json create mode 100644 fixtures/stopword-analysis/domains/narrative.json create mode 100644 fixtures/stopword-analysis/stoplist.json create mode 100644 specs/stopword-analysis.md create mode 100644 tools/stopword_analysis.py create mode 100644 tools/validate_stopword_analysis.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 83f7e65..266a4fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). Thi ## [Unreleased] ### Added +- Stopword deviation analysis v0.1 — the dropped words are governed vocabulary too (task #13): `tools/stopword_analysis.py` audits the loop's stoplist ACROSS domains using two signals — cross-domain deviation (concentration) AND compositional density (repeated-collocation rate) — because frequency alone can't tell a domain term from a stylistic quirk. Surfaces `term-candidate` (concentrated + compositional = a domain term hiding in the stoplist, propose un-stoplisting) vs `stylistic` (concentrated by style only) vs `noise` (uniform). `validate-stopword-analysis` teeth: domain terms surfaced; a stylistically-concentrated word ('and') is NOT wrongly promoted; a uniform word ('the') is noise. `stopword-analysis-live` audits the shipped stoplist over specs/*.md. Compositional density is the bigram floor of the k-gram TF-IDF/LSA differential (orders 3..7) to follow. - Agreement test v0.1 — glossary relations vs the blast-radius graph (task #13, neurosymbolic): `tools/agreement_test.py` projects the glossary's composition relations (`has-a`/`has-member`) onto the estate via `alignment.estateBinding` and compares them to a consumed blast-radius/dependency graph (GBRG owns that graph). Fail-closed on OVERCLAIM (a declared dependency with no observed edge = governance hole); reports DRIFT (an observed edge no relation names) as a remediation candidate (a proposed `has-a` relation), like the vocab-currency loop's candidate terms. `validate-agreement` teeth: aligned agrees; overclaim refused; drift surfaced as candidate. - Glossary promotion v0.1 — the governed draft→approved alignment pass (task #13, closes the vocabulary lifecycle): `tools/promote_glossary_term.py` promotes a `draft` GlossaryTerm to `approved` ONLY when the 3-method alignment holds as a fail-closed MEET — capture (`ontologyClassRef`) ∧ vector-align (`vectorLink`, pinned to the sovereign nomic/768 space AND reciprocated by the named peer) ∧ implement (`estateBinding` entity/service/action). It recomputes the meet (never trusts a flag), refuses on any missing/non-reciprocal/off-space method (term stays draft, no governance hole), and its approved output PASSES the #250 alignment drift-guard by construction. `validate-glossary-promotion` teeth: promotes on full alignment; refuses missing-implement / non-reciprocal / off-space; promoted output passes the #250 guard. - Vocab-currency loop — LIVE dogfood on the estate's own vocab (`make vocab-currency-dogfood`): `tools/dogfood_vocab_currency.py` runs the governed loop over this repo's approved `GlossaryTerm`s (fixed/LSA) vs `specs/*.md` (open/LDA). Today only a handful of terms are approved against the full spec corpus, so the governed loop escalates-human (can't reach currency in its bound), writing real draft `GlossaryTerm` proposals to `build/vocab-currency-proposals/` — the artifacts ontogenesis ingests. Reports the outcome (does not gate on it, exit 0); Markdown-cleaned + generic-word stoplisted so domain terms surface. Enforcement stays on fixtures in `validate-vocab-currency-loop`. diff --git a/Makefile b/Makefile index 51113ff..77e67da 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -.PHONY: validate validate-resource-contract validate-measurement validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-ingestion-pipeline-examples validate-data-acquisition-examples validate-glossary-alignment-examples validate-data-class-examples validate-table-keys validate-dag-loop validate-vocab-currency-loop vocab-currency-dogfood validate-glossary-promotion validate-agreement +.PHONY: validate validate-resource-contract validate-measurement validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-ingestion-pipeline-examples validate-data-acquisition-examples validate-glossary-alignment-examples validate-data-class-examples validate-table-keys validate-dag-loop validate-vocab-currency-loop vocab-currency-dogfood validate-glossary-promotion validate-agreement validate-stopword-analysis stopword-analysis-live -validate: validate-data-class-examples validate-glossary-alignment-examples validate-data-acquisition-examples validate-ingestion-pipeline-examples validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-resource-contract validate-table-keys validate-dag-loop validate-vocab-currency-loop validate-glossary-promotion validate-agreement +validate: validate-data-class-examples validate-glossary-alignment-examples validate-data-acquisition-examples validate-ingestion-pipeline-examples validate-control-plane-examples validate-nlboot-examples validate-lattice-data-governai-examples validate-ops-history-examples validate-runtime-observability-examples validate-interpretability-examples validate-lifecycle-boundary-examples validate-svf-contracts validate-sync-cycle-receipts validate-onboarding-examples validate-runtime-causality-examples validate-agentic-os-examples validate-triparty-examples validate-labor-market-examples validate-supply-chain-risk-examples validate-reasoning-examples validate-mpcc-event-examples validate-knowledge-nugget-examples validate-semantic-action-examples validate-epistemic-kernel-examples validate-ab-update-examples validate-device-service-examples validate-duplicate-schema-ids validate-value-type validate-source-locator validate-sourceos-repo-manifest validate-mesh-action-registry validate-lawful-dispatch-receipt validate-architectural-building-block validate-agent-passport-examples validate-seam-definition-examples validate-agent-system-vocabulary validate-genesis-inception-examples validate-measurement validate-world-model-examples validate-eval-item-examples validate-resource-contract validate-table-keys validate-dag-loop validate-vocab-currency-loop validate-glossary-promotion validate-agreement validate-stopword-analysis @echo "OK: validate" validate-ingestion-pipeline-examples: @@ -22,6 +22,13 @@ validate-glossary-promotion: validate-agreement: python3 tools/validate_agreement.py +validate-stopword-analysis: + python3 tools/validate_stopword_analysis.py + +# On-demand audit of the shipped stoplist over specs/*.md (informational). +stopword-analysis-live: + python3 tools/stopword_analysis.py --live + validate-table-keys: python3 -m pip install --user jsonschema referencing >/dev/null python3 tools/validate_key_bipartite.py diff --git a/fixtures/stopword-analysis/domains/formal.json b/fixtures/stopword-analysis/domains/formal.json new file mode 100644 index 0000000..d8b9e0c --- /dev/null +++ b/fixtures/stopword-analysis/domains/formal.json @@ -0,0 +1,14 @@ +{ + "domain": "formal", + "documents": [ + { + "text": "An empty set is a set with no elements. The empty set is unique. A finite state machine has a start state. The state machine transitions on input. A base class defines behaviour; a derived class extends the base class. Each required field is validated; a required field with no value fails. The set of states of the state machine forms a state space." + }, + { + "text": "The empty set contains nothing. A state machine has states. The base class is abstract; the base class defines the interface. A required field must be present; the required field is checked. The state machine reads the set and updates the state." + }, + { + "text": "Set theory studies the empty set and every set. The state machine and its state set. The base class and each required field. A required field maps to a state in the state machine." + } + ] +} diff --git a/fixtures/stopword-analysis/domains/narrative.json b/fixtures/stopword-analysis/domains/narrative.json new file mode 100644 index 0000000..d5b5b72 --- /dev/null +++ b/fixtures/stopword-analysis/domains/narrative.json @@ -0,0 +1,11 @@ +{ + "domain": "narrative", + "documents": [ + { + "text": "The dog and the cat ran across the field and the road, and the sun was warm and bright. The children laughed and played, and the day was long and the evening came and the stars and the moon rose over the hills and the valley and the quiet town below." + }, + { + "text": "And so the story went, and the people gathered, and the music played, and the night was full of song and the wind and the rain and the memory of the summer and the sea." + } + ] +} diff --git a/fixtures/stopword-analysis/stoplist.json b/fixtures/stopword-analysis/stoplist.json new file mode 100644 index 0000000..35b86aa --- /dev/null +++ b/fixtures/stopword-analysis/stoplist.json @@ -0,0 +1,12 @@ +{ + "stoplist": [ + "the", + "and", + "set", + "class", + "state", + "required", + "field", + "with" + ] +} diff --git a/specs/stopword-analysis.md b/specs/stopword-analysis.md new file mode 100644 index 0000000..1a1586e --- /dev/null +++ b/specs/stopword-analysis.md @@ -0,0 +1,29 @@ +# Stopword deviation analysis (v0.1) — the words we DROP are governed vocabulary too + +The vocab-currency loop drops a fixed stoplist so domain terms surface. But that list is itself an +ungoverned governance decision: a word that is filler in one domain ("set", "class", "state", +"required", "value") is a real TERM in another (math, OOP, state machines, config). Dropping it +universally erases signal where it matters. So we audit the dropped words the same way we audit the +kept ones — frequency, connections, compositional density — ACROSS domains, using TWO signals +because frequency alone lies: + +- **cross-domain deviation** (`concentration`) — is the word's relative-frequency mass concentrated + in a subset of domains? +- **compositional density** — of the word's content-neighbour adjacencies in that domain, what + fraction are REPEATED collocations? A term recurs in fixed phrases ("empty set", "state machine"); + a stylistic word ("and") glues arbitrary, unique content. + +Verdicts: **term-candidate** (concentrated AND compositional → a domain term hiding in the stoplist, +propose un-stoplisting it there), **stylistic** (concentrated by style only → keep, but flagged), +**noise** (uniform → true stopword everywhere). The stoplist becomes an auditable, per-domain +artifact instead of a hard-coded assumption; term-candidates are a remediation signal like the +currency loop's candidate terms and the agreement test's drift. + +`make validate-stopword-analysis` proves the two-signal discrimination on fixtures (real domain +terms surfaced; a stylistically-concentrated word is NOT wrongly promoted — the trap frequency alone +falls into; a uniform word is noise). `make stopword-analysis-live` audits the shipped stoplist over +`specs/*.md`. + +Compositional density here is the **bigram floor** of the deeper follow-on: the **k-gram TF-IDF/LSA +differential** over orders 3..7, which confirms a candidate by showing its signal GROWS with n-gram +order (it participates in domain-specific higher-order collocations) rather than staying diffuse. diff --git a/tools/stopword_analysis.py b/tools/stopword_analysis.py new file mode 100644 index 0000000..5cd5fe3 --- /dev/null +++ b/tools/stopword_analysis.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Stopword deviation analysis — the words we DROP are ungoverned vocabulary too (task #13). + +The vocab-currency loop drops a fixed stoplist so domain terms surface. But that list is itself an +ungoverned governance decision: a word that is noise in one domain ("set", "class", "state", +"value", "required") can be a real TERM in another (math, OOP, state machines, config). Dropping it +universally erases signal where it matters. So we evaluate the dropped words the same way we +evaluate the kept ones — their frequency, connections, and compositional density — ACROSS domains, +and use TWO signals — cross-domain deviation AND compositional density — because frequency alone +lies (a stylistic word like "and" can be concentrated in a chatty domain without being a term): + + * TERM-CANDIDATE — the word's mass is CONCENTRATED in a subset of domains (deviates) AND it recurs + in fixed collocations there ("empty set", "state machine"): a domain term hiding in the stoplist, + a candidate to UN-stoplist in that domain (a remediation signal, like the currency loop's + candidate terms and the agreement test's drift). Confirmed by the k-gram tf-idf/lsa differential. + * STYLISTIC — concentrated but its neighbours are arbitrary/unique (no repeated collocations): the + deviation is style, not signal (e.g. "and"). Keep stoplisted; flagged for the k-gram check. + * NOISE — uniformly frequent across domains: a true stopword everywhere. + +Only a governance signal, never a silent drop: the stoplist becomes an auditable, per-domain +artifact instead of a hard-coded assumption. Compositional density here is the bigram floor of the +k-gram TF-IDF/LSA differential (orders 3..7) — the deeper follow-on that confirms these candidates. +""" +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MIN_OCCURRENCES = 3 # below this we can't judge a word — verdict "insufficient-data" +CONCENTRATION_INTERESTING = 0.66 # >= this share of freq-mass in one domain => concentrated (deviates) +DENSITY_TERMLIKE = 0.30 # >= this repeated-collocation density => behaves like a term, not filler + + +def raw_tokenize(text: str) -> list[str]: + # RAW: unlike the loop, we do NOT drop stopwords — the stopwords are exactly what we analyze. + return [t for t in re.findall(r"[a-z][a-z0-9-]+", text.lower()) if len(t) >= 2] + + +def analyze(domains: dict[str, str], stoplist: set[str]) -> dict: + tokens = {d: raw_tokenize(text) for d, text in domains.items()} + totals = {d: len(t) or 1 for d, t in tokens.items()} + + interesting, noise, insufficient = [], [], [] + for w in sorted(stoplist): + occ = {d: t.count(w) for d, t in tokens.items()} + total_occ = sum(occ.values()) + if total_occ < MIN_OCCURRENCES: + insufficient.append(w) + continue + relfreq = {d: occ[d] / totals[d] for d in domains} + rf_sum = sum(relfreq.values()) or 1.0 + concentration = max(relfreq.values()) / rf_sum # 1/D (uniform) .. 1.0 (concentrated) + cand = max(relfreq, key=relfreq.get) + + # Compositional DENSITY in the concentrated domain: of all (w, content-neighbour) adjacencies, + # what fraction belong to a collocation that REPEATS (>=2)? A domain term recurs in fixed + # collocations ("empty set", "state machine"); a stylistic function word ("and") glues + # arbitrary, mostly-unique content. This is the discriminator that frequency alone can't give + # — and the light, bigram version of the k-gram TF-IDF differential (the deeper follow-on). + toks = tokens[cand] + pairs: dict[str, int] = {} + for i, tok in enumerate(toks): + if tok != w: + continue + for j in (i - 1, i + 1): + if 0 <= j < len(toks) and toks[j] not in stoplist: + pairs[toks[j]] = pairs.get(toks[j], 0) + 1 + total_adj = sum(pairs.values()) + repeated_adj = sum(c for c in pairs.values() if c >= 2) + density = round(repeated_adj / total_adj, 3) if total_adj else 0.0 + + row = {"word": w, "concentration": round(concentration, 3), "candidateDomain": cand, + "compositionalDensity": density, "occurrences": total_occ, + "perDomainFreq": {d: round(relfreq[d], 5) for d in domains}, + "topPartners": sorted(pairs, key=pairs.get, reverse=True)[:5]} + + if concentration < CONCENTRATION_INTERESTING: + row["verdict"] = "noise" # uniform across domains — a true stopword everywhere + noise.append(row) + elif density >= DENSITY_TERMLIKE: + row["verdict"] = "term-candidate" # concentrated AND recurs in fixed collocations + row["proposal"] = f"un-stoplist '{w}' in domain '{cand}' (candidate domain term)" + row["confirmWith"] = "k-gram tf-idf/lsa differential" + interesting.append(row) + else: + row["verdict"] = "stylistic" # concentrated by STYLE, not collocation (e.g. 'and') + row["note"] = "deviation is stylistic, not compositional — keep stoplisted pending k-gram check" + noise.append(row) + + return {"domains": list(domains), "stoplistSize": len(stoplist), + "termCandidates": interesting, + "noiseCount": len(noise), "insufficientDataCount": len(insufficient), + "noiseWords": [r["word"] for r in noise]} + + +def _live_domains() -> dict[str, str]: + # Each spec file is a mini-domain — cross-file deviation is a proxy for cross-domain deviation. + return {p.stem: p.read_text(encoding="utf-8") for p in sorted((ROOT / "specs").glob("*.md"))} + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--live", action="store_true", help="analyze the shipped stoplist over specs/*.md") + args = ap.parse_args() + if args.live: + import sys + sys.path.insert(0, str(ROOT / "tools")) + import vocab_currency_loop as vcl # noqa: E402 + result = analyze(_live_domains(), vcl.STOP) + else: + FIX = ROOT / "fixtures" / "stopword-analysis" + domains = {d["domain"]: "\n".join(x["text"] for x in d["documents"]) + for d in (json.loads(p.read_text()) for p in sorted((FIX / "domains").glob("*.json")))} + stoplist = set(json.loads((FIX / "stoplist.json").read_text())["stoplist"]) + result = analyze(domains, stoplist) + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/validate_stopword_analysis.py b/tools/validate_stopword_analysis.py new file mode 100644 index 0000000..b12e60f --- /dev/null +++ b/tools/validate_stopword_analysis.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""CI teeth for the stopword deviation analysis (task #13). + +Asserts the two-signal discrimination holds: a word concentrated in one domain WITH repeated +collocations is a term-candidate (a domain term hiding in the stoplist); a word concentrated only +by STYLE (no repeated collocations) is NOT promoted (the trap frequency-alone falls into); and a +word uniform across domains is noise. This is what makes the stoplist auditable rather than a silent +universal drop. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import stopword_analysis as S # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +FIX = ROOT / "fixtures" / "stopword-analysis" + +FAILURES: list[str] = [] +CHECKS: dict[str, bool] = {} + + +def main() -> int: + domains = {d["domain"]: "\n".join(x["text"] for x in d["documents"]) + for d in (json.loads(p.read_text()) for p in sorted((FIX / "domains").glob("*.json")))} + stoplist = set(json.loads((FIX / "stoplist.json").read_text())["stoplist"]) + r = S.analyze(domains, stoplist) + cand = {c["word"] for c in r["termCandidates"]} + noise = set(r["noiseWords"]) + + # 1. Domain terms hiding in the stoplist ARE surfaced as term-candidates. + for w in ("set", "class", "state", "required"): + if w not in cand: + FAILURES.append(f"'{w}' is a domain term in 'formal' but was not surfaced as a candidate") + if not FAILURES: + CHECKS["domain-terms:surfaced"] = True + + # 2. Each term-candidate is CONCENTRATED and points at the domain that uses it as a term. + if all(c["concentration"] >= S.CONCENTRATION_INTERESTING and c["candidateDomain"] == "formal" + for c in r["termCandidates"]): + CHECKS["candidates:concentrated-in-right-domain"] = True + else: + FAILURES.append("a term-candidate was not concentrated in the formal domain") + + # 3. THE TRAP: a stylistically-concentrated function word ('and') must NOT be a term-candidate + # (frequency deviation alone would wrongly promote it; compositional density saves it). + if "and" in cand: + FAILURES.append("'and' was wrongly promoted — stylistic concentration must not read as a term") + elif "and" not in noise: + FAILURES.append("'and' should be classified stylistic/noise") + else: + CHECKS["stylistic-word:not-promoted"] = True + + # 4. A uniformly-frequent function word ('the') is noise. + if "the" not in noise or "the" in cand: + FAILURES.append("'the' (uniform across domains) must be noise") + else: + CHECKS["uniform-word:noise"] = True + + for m in FAILURES: + print(f"FAIL: {m}", file=sys.stderr) + ok = not FAILURES and all(CHECKS.values()) + print(json.dumps({"ok": ok, "checks": CHECKS, + "termCandidates": sorted(cand), "noise": sorted(noise)}, indent=2, sort_keys=True)) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From e960d007b52952b00ad588568e6563454451a50e Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 2 Aug 2026 19:57:01 -0400 Subject: [PATCH 2/3] fix(vocab): remediate Copilot review on #260 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - compare UNROUNDED compositional density to the threshold (rounding could flip a boundary verdict); round only for reporting. - precompute a Counter per domain instead of t.count(w) per (word, domain) — O(tokens + words*domains). - expose the three verdicts distinctly (stylisticWords/uniformNoiseWords + counts); keep the flattened noiseWords (their union) for existing consumers. - validator: require >=1 term-candidate before the all(...) concentration check (all([]) is vacuously True and would mask an empty result). --- tools/stopword_analysis.py | 32 ++++++++++++++++++----------- tools/validate_stopword_analysis.py | 5 +++-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tools/stopword_analysis.py b/tools/stopword_analysis.py index 5cd5fe3..77eec28 100644 --- a/tools/stopword_analysis.py +++ b/tools/stopword_analysis.py @@ -26,6 +26,7 @@ import argparse import json import re +from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -41,11 +42,12 @@ def raw_tokenize(text: str) -> list[str]: def analyze(domains: dict[str, str], stoplist: set[str]) -> dict: tokens = {d: raw_tokenize(text) for d, text in domains.items()} + counters = {d: Counter(t) for d, t in tokens.items()} # count once, not per (word, domain) totals = {d: len(t) or 1 for d, t in tokens.items()} - interesting, noise, insufficient = [], [], [] + term_candidates, stylistic, uniform_noise, insufficient = [], [], [], [] for w in sorted(stoplist): - occ = {d: t.count(w) for d, t in tokens.items()} + occ = {d: counters[d].get(w, 0) for d in domains} total_occ = sum(occ.values()) if total_occ < MIN_OCCURRENCES: insufficient.append(w) @@ -69,31 +71,37 @@ def analyze(domains: dict[str, str], stoplist: set[str]) -> dict: if 0 <= j < len(toks) and toks[j] not in stoplist: pairs[toks[j]] = pairs.get(toks[j], 0) + 1 total_adj = sum(pairs.values()) - repeated_adj = sum(c for c in pairs.values() if c >= 2) - density = round(repeated_adj / total_adj, 3) if total_adj else 0.0 + # Compare the UNROUNDED density to the threshold (rounding could flip a boundary verdict); + # round only for reporting. + density_raw = repeated_adj / total_adj if total_adj else 0.0 row = {"word": w, "concentration": round(concentration, 3), "candidateDomain": cand, - "compositionalDensity": density, "occurrences": total_occ, + "compositionalDensity": round(density_raw, 3), "occurrences": total_occ, "perDomainFreq": {d: round(relfreq[d], 5) for d in domains}, "topPartners": sorted(pairs, key=pairs.get, reverse=True)[:5]} if concentration < CONCENTRATION_INTERESTING: row["verdict"] = "noise" # uniform across domains — a true stopword everywhere - noise.append(row) - elif density >= DENSITY_TERMLIKE: + uniform_noise.append(row) + elif density_raw >= DENSITY_TERMLIKE: row["verdict"] = "term-candidate" # concentrated AND recurs in fixed collocations row["proposal"] = f"un-stoplist '{w}' in domain '{cand}' (candidate domain term)" row["confirmWith"] = "k-gram tf-idf/lsa differential" - interesting.append(row) + term_candidates.append(row) else: row["verdict"] = "stylistic" # concentrated by STYLE, not collocation (e.g. 'and') row["note"] = "deviation is stylistic, not compositional — keep stoplisted pending k-gram check" - noise.append(row) + stylistic.append(row) + # Expose the three verdicts distinctly (auditable); keep the flattened noiseWords for existing + # consumers = stylistic ∪ uniform-noise (everything that stays stoplisted). + noise_words = [r["word"] for r in stylistic] + [r["word"] for r in uniform_noise] return {"domains": list(domains), "stoplistSize": len(stoplist), - "termCandidates": interesting, - "noiseCount": len(noise), "insufficientDataCount": len(insufficient), - "noiseWords": [r["word"] for r in noise]} + "termCandidates": term_candidates, + "stylisticWords": [r["word"] for r in stylistic], "stylisticCount": len(stylistic), + "uniformNoiseWords": [r["word"] for r in uniform_noise], "uniformNoiseCount": len(uniform_noise), + "noiseWords": noise_words, "noiseCount": len(noise_words), + "insufficientDataCount": len(insufficient)} def _live_domains() -> dict[str, str]: diff --git a/tools/validate_stopword_analysis.py b/tools/validate_stopword_analysis.py index b12e60f..25ce29a 100644 --- a/tools/validate_stopword_analysis.py +++ b/tools/validate_stopword_analysis.py @@ -39,8 +39,9 @@ def main() -> int: CHECKS["domain-terms:surfaced"] = True # 2. Each term-candidate is CONCENTRATED and points at the domain that uses it as a term. - if all(c["concentration"] >= S.CONCENTRATION_INTERESTING and c["candidateDomain"] == "formal" - for c in r["termCandidates"]): + # Require at least one candidate — all([]) is vacuously True and would mask an empty result. + if r["termCandidates"] and all(c["concentration"] >= S.CONCENTRATION_INTERESTING and c["candidateDomain"] == "formal" + for c in r["termCandidates"]): CHECKS["candidates:concentrated-in-right-domain"] = True else: FAILURES.append("a term-candidate was not concentrated in the formal domain") From cbc049820b824da21a20e9d9ed76c3ec04207e86 Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Sun, 2 Aug 2026 19:57:42 -0400 Subject: [PATCH 3/3] fix(vocab): restore repeated_adj line dropped in the prior remediation edit The previous commit accidentally removed the repeated_adj assignment while switching to the unrounded-density comparison, causing a NameError. Restored; all teeth + make validate green. --- tools/stopword_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/stopword_analysis.py b/tools/stopword_analysis.py index 77eec28..76e61b9 100644 --- a/tools/stopword_analysis.py +++ b/tools/stopword_analysis.py @@ -71,6 +71,7 @@ def analyze(domains: dict[str, str], stoplist: set[str]) -> dict: if 0 <= j < len(toks) and toks[j] not in stoplist: pairs[toks[j]] = pairs.get(toks[j], 0) + 1 total_adj = sum(pairs.values()) + repeated_adj = sum(c for c in pairs.values() if c >= 2) # Compare the UNROUNDED density to the threshold (rounding could flip a boundary verdict); # round only for reporting. density_raw = repeated_adj / total_adj if total_adj else 0.0