diff --git a/AGENTS.md b/AGENTS.md index 8dcf680..6153740 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -118,6 +118,8 @@ All consequential edits follow the kernel's own loop: High-impact decisions must record to `.episteme/reasoning-surface.json` before the action. See `kernel/HOOKS_MAP.md`. +**Docs follow implementation (E173).** When you edit an implementation file, the `workflow_guard` hook names the docs whose citations claim to describe that path — repair them in the same change or record why not. Query the same map directly: `episteme docs map ` (full dump with no args). The map is derived from citation edges (see `src/episteme/doc_references.py`), so it cannot rot; a doc earns edit-time surfacing by citing the code it describes. + > **Novel decisions vs. mechanical implementations.** A full Reasoning Surface fires on *novel decisions* — questions still open, alternatives still live. Mechanical implementations of an already-declared pattern produce a minimal `implementation-of` reference instead. See [`kernel/PATTERN_GOVERNANCE.md`](./kernel/PATTERN_GOVERNANCE.md) for the pattern-declaration artifact shape, the promotion gate, and the deviation-from-pattern escape hatch that promotes a "mostly mechanical" change back to novel. --- diff --git a/core/hooks/workflow_guard.py b/core/hooks/workflow_guard.py index b941390..3d26880 100644 --- a/core/hooks/workflow_guard.py +++ b/core/hooks/workflow_guard.py @@ -3,6 +3,14 @@ This is intentionally non-blocking. It nudges the agent to keep authoritative project docs in sync when editing implementation files directly. + +Event 173: when the governed project's tracked markdown cites the edited path +(exactly, or via an enclosing directory), the advisory names those docs with +their lifecycle state instead of the generic two-file reminder — the citation +walk is episteme.doc_references' (one notion of "what a doc cites"), imported +from this checkout's src/ with the E172 self-resolve pattern. Any failure on +that path — package unimportable (plugin context), project not a git repo, +no citing docs — falls back to the original generic advisory, never a crash. """ from __future__ import annotations @@ -11,6 +19,9 @@ import sys from pathlib import Path +#: Cap on docs named inline; the overflow is counted, never silently dropped. +_MAX_ADVISORY_DOCS = 6 + ALLOWED_DOC_PATHS = { "AGENTS.md", @@ -58,6 +69,70 @@ def _project_has_authoritative_docs(cwd: Path) -> bool: return any(p.exists() for p in required) +def _load_doc_references(): + """Import ``episteme.doc_references`` from this checkout's ``src/``. + + E172 self-resolve pattern: the hook file lives inside the episteme + checkout, so ``src/`` is findable from ``__file__`` regardless of the + governed project's cwd. ``None`` on ANY failure — the caller keeps the + generic advisory; a plugin install without the package must not break. + """ + try: + src = Path(__file__).resolve().parents[2] / "src" + if src.is_dir() and str(src) not in sys.path: + sys.path.insert(0, str(src)) + from episteme import doc_references # noqa: PLC0415 + + return doc_references + except Exception: + return None + + +def _relative_to_cwd(cwd: Path, target_path: str) -> str: + try: + return str(Path(target_path).resolve().relative_to(cwd.resolve())) + except (ValueError, OSError): + return Path(target_path).name + + +def _targeted_advisory(cwd: Path, target_path: str) -> "str | None": + """The doc-map advisory for ``target_path``, or ``None`` for fallback. + + ``None`` whenever the answer is unavailable (no package, no git, IO + error) or genuinely empty (positive system: no citation, no obligation). + """ + dr = _load_doc_references() + if dr is None: + return None + try: + index = dr.cached_reverse_index(cwd) + edges = dr.edges_for_path(cwd, target_path, index=index) + if not edges: + return None + docs = [e.doc for e in edges] + labels = dict(dr.annotate_docs(cwd, docs)) + rel = _relative_to_cwd(cwd, target_path) + lines = [f"DOC ADVISORY: {len(docs)} doc(s) claim to describe '{rel}' —"] + for e in edges[:_MAX_ADVISORY_DOCS]: + label = labels.get(e.doc, "") + via = f" [via {e.target}/]" if e.kind == "dir" else "" + lines.append( + f" {e.doc}" + (f" ({label})" if label else "") + via + ) + overflow = len(docs) - _MAX_ADVISORY_DOCS + if overflow > 0: + lines.append( + f" … and {overflow} more — `episteme docs map '{rel}'` lists all." + ) + lines.append( + "Verify they still tell the truth about this edit; " + "repair in the same change or record why not." + ) + return "\n".join(lines) + except Exception: + return None + + def main() -> int: raw = sys.stdin.read().strip() if not raw: @@ -85,20 +160,45 @@ def main() -> int: if not target_path: return 0 - if _is_doc_or_policy_path(target_path): + cwd = Path(payload.get("cwd") or os.getcwd()) + + # E173 repair of a latent bug: harness payloads carry ABSOLUTE file paths, + # and `lstrip("./")` strips a character set, not a prefix, so the doc-path + # suppression never matched absolute spellings — the guard nagged on doc + # edits it was designed to stay silent on. Normalize to the project-relative + # spelling before the policy test. Textual prefix-strip FIRST: this repo's + # authoritative docs are symlinks into a private tree, and resolve() would + # follow them out of the project and defeat the suppression for exactly the + # files it exists to cover; resolve() is only the fallback for spelling + # mismatches like /tmp vs /private/tmp. + policy_path = target_path + if os.path.isabs(target_path): + norm_t = os.path.normpath(target_path) + norm_c = os.path.normpath(str(cwd)) + if norm_t.startswith(norm_c + os.sep): + policy_path = norm_t[len(norm_c) + 1 :].replace(os.sep, "/") + else: + try: + policy_path = str( + Path(target_path).resolve().relative_to(cwd.resolve()) + ).replace(os.sep, "/") + except (ValueError, OSError): + policy_path = target_path + + if _is_doc_or_policy_path(policy_path): return 0 - cwd = Path(payload.get("cwd") or os.getcwd()) if not _project_has_authoritative_docs(cwd): return 0 + context = _targeted_advisory(cwd, target_path) or ( + f"WORKFLOW ADVISORY: Editing '{Path(target_path).name}' outside authoritative docs. " + "Keep docs/EVENTS.md and docs/NEXT_STEPS.md aligned with this change." + ) advisory = { "hookSpecificOutput": { "hookEventName": "PreToolUse", - "additionalContext": ( - f"WORKFLOW ADVISORY: Editing '{Path(target_path).name}' outside authoritative docs. " - "Keep docs/EVENTS.md and docs/NEXT_STEPS.md aligned with this change." - ), + "additionalContext": context, } } sys.stdout.write(json.dumps(advisory)) diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 02fe946..69fe8d4 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -1,9 +1,14 @@ - + # episteme — command reference A one-page map of every `episteme` subcommand, grouped by lifecycle phase. Run `episteme --help` for flags and arguments. +Source of truth: the argparse tree in `src/episteme/cli.py` — the +completeness of this page against it is CI-enforced both directions +(`tests/test_commands_doc_drift.py`), and naming the source here puts this +page on the reverse index so CLI edits surface it (`episteme docs map`). + Scope key: - **global** — targets `~/episteme` or `~/.claude` / `~/.hermes`, ignores CWD - **project** — takes a path arg (default `.`), acts on the current project diff --git a/docs/HOOKS.md b/docs/HOOKS.md index 0a2e127..ce76782 100644 --- a/docs/HOOKS.md +++ b/docs/HOOKS.md @@ -25,7 +25,7 @@ change; it is the actual registry, not this prose. | `reasoning_surface_guard.py` | `PreToolUse Bash\|Write\|Edit\|MultiEdit` | Blocks high-impact / irreversible ops and architectural-cascade edits that lack a valid Reasoning Surface | | `block_dangerous.py` | `PreToolUse Bash` | Blocks `rm -rf`, `git reset --hard`, `git push --force`, `sudo`, destructive SQL, and more | | `_arm_a_pre.py` | `PreToolUse Write\|Edit\|MultiEdit` | Cognitive Arm A pre-snapshot of watched profile/policy files for trajectory diffing | -| `workflow_guard.py` | `PreToolUse Write\|Edit\|MultiEdit` (balanced/strict) | Advisory nudge to keep authoritative docs (`EVENTS.md` / `NEXT_STEPS.md`) aligned with edits | +| `workflow_guard.py` | `PreToolUse Write\|Edit\|MultiEdit` (balanced/strict) | Targeted DOC ADVISORY (E173): names the docs whose citations claim to describe the edited path — the reverse index of `episteme docs map`, derived from citation edges — with lifecycle state; falls back to the generic `EVENTS.md` / `NEXT_STEPS.md` nudge when no doc cites the path (positive system) or the index is unavailable (non-git project, plugin-only install) | | `prompt_guard.py` | `PreToolUse Write\|Edit\|MultiEdit` (balanced/strict) | Advisory prompt-injection detection when writing durable context | | `format.py` | `PostToolUse Write\|Edit\|MultiEdit` (async) | Auto-runs `ruff` (Python) / `prettier` (JS/TS) after a file write | | `test_runner.py` | `PostToolUse Write\|Edit\|MultiEdit` | Runs pytest / jest when the edited file is a test | diff --git a/docs/README.md b/docs/README.md index f0a3850..96fe20f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -42,6 +42,7 @@ hand-edit between the markers; run the command and commit the result. - [`README.md`](./README.md) · living · episteme docs index - [`SETUP.md`](./SETUP.md) · living · Setup — Profile, Cognition, One-Command - [`SKILLS_AND_PERSONAS.md`](./SKILLS_AND_PERSONAS.md) · living · Skills and Agent Personas +- [`SPEC_DOC_MAP.md`](./SPEC_DOC_MAP.md) · spec-implemented · SPEC — Code→Doc Reverse Index (`docs map`) and the Targeted DOC ADVISORY - [`SPEC_REASONING_SURFACE_BRANCHING.md`](./SPEC_REASONING_SURFACE_BRANCHING.md) · spec-implemented · Spec — Reasoning Surface branching - [`SPEC_RIGOR_KNOB.md`](./SPEC_RIGOR_KNOB.md) · spec-implemented · Spec — `episteme rigor` (rigor knob) - [`SUBSTRATE_BRIDGE.md`](./SUBSTRATE_BRIDGE.md) · living · Substrate Bridge — A Generalizable Contract for External Memory Systems diff --git a/docs/SPEC_DOC_MAP.md b/docs/SPEC_DOC_MAP.md new file mode 100644 index 0000000..96d3538 --- /dev/null +++ b/docs/SPEC_DOC_MAP.md @@ -0,0 +1,107 @@ + +# SPEC — Code→Doc Reverse Index (`docs map`) and the Targeted DOC ADVISORY + +Event 173. Operator-ratified (design C of three; wiki-pages and a hand-authored +`DOC_MAP.json` were rejected — both add a hand-maintained surface that itself +rots, which is the disease under treatment). + +## Failure mode countered (anti-accretion rule 7) + +**Doc-code drift discovered late.** A code edit lands; the docs that claim to +describe that code stay unchanged; the divergence surfaces only in a later +batch sweep (the E172 class of repair: 13 findings, a full day) or in the +SessionStart staleness banner (22 living docs >15 events behind at the time of +this event). The write path — the only moment when the edit and its doc +obligations coexist in one context — carried no doc-specific signal. + +## Mechanism replaced (anti-accretion rule 7) + +`core/hooks/workflow_guard.py`'s **path-blind static advisory** ("Keep +docs/EVENTS.md and docs/NEXT_STEPS.md aligned"), which named the same two files +regardless of what was edited. The static string survives only as the fallback +tier (below). No new hook, no new queue, no new hand-maintained file. + +## Rule-shape (kernel rule 8) + +**Positive system.** An edit-time obligation exists ONLY where a live authored +doc cites the edited path — exactly, or via an enclosing directory. Uncited +code carries no obligation. Consequence: a doc earns edit-time surfacing by +citing the code it describes, so citation hygiene and map fidelity are the +same act. The obligation-citing corpus additionally excludes `archive/` +(frozen history must not generate present-tense obligations) on top of the +drift linter's fixture-tree exemptions. + +## Architecture — one notion of "what a doc cites" + +The map is the **inverse of the citation edges the drift linter already +extracts** (`src/episteme/doc_references.py::extract_references`; CI keeps them +non-dangling via `tests/test_doc_references.py`). Nothing is authored; the map +cannot rot by construction. New functions, same module: + +| Function | Contract | +|---|---| +| `build_reverse_index(root, doc_files=None)` | claimed cited path → `[DocEdge]` over the obligation corpus. Unresolved citations are indexed as claims: a doc that pre-documents a not-yet-existing file must obligate the Write that creates it, and claims-in-index keeps the index a pure function of the markdown corpus — the property the cache digest relies on (review finding, this event). Dangling claims remain `find_drift`'s findings; the mechanisms stay separate | +| `edges_for_path(root, path, index=None)` | citing edges, strongest claim first: exact file > deeper dir > shallower dir; one edge per doc; self-citations excluded | +| `docs_for_path(root, path, ...)` | the sorted doc list (thin wrapper) | +| `annotate_docs(root, docs)` | display labels: `status · reviewed_as_of · N events behind` (lifecycle marker + `docs/EVENTS.md` scan), `manifest-managed` (kernel manifest membership), else empty — every lookup degrades, never raises | +| `cached_reverse_index(root)` | the index behind an mtime+size digest cache at `.episteme/cache/doc_map.json`; ~200ms cold / ~10ms steady-state; cache written ONLY where `.episteme/` already exists; any cache failure → fresh build | + +## Consumers + +1. **`workflow_guard.py` (PreToolUse Write|Edit|MultiEdit).** On an + implementation-file edit: `DOC ADVISORY: N doc(s) claim to describe + ''` + up to 6 docs (overflow counted, never silent) + lifecycle + labels + `[via /]` tags for directory edges. Fallback ladder, each + step caught and degraded, never a crash: package unimportable (plugin-only + install) → generic advisory · project not a git repo → generic · no citing + docs → generic (positive system: no citation, no obligation). The package + imports via the E172 self-resolve pattern (`__file__ → ../../src`). + Also repaired in this event: the doc-path suppression never matched the + absolute file paths harness payloads actually carry (`lstrip("./")` strips + a character set, not a prefix) — targets are now normalized to + project-relative form before the policy test. +2. **`episteme docs map [path ...]`** — the agent/operator query surface + ("the wiki"): per-path citing docs with the same ordering and labels as the + hook; full `target ← docs` dump with no args. Informational, exit 0. +3. **`AGENTS.md` → Workflow convention** names the mechanism and the repair + duty ("repair in the same change or record why not"). + +## Drain (kernel rule 6 — no new operator-judgment queues) + +The advisory is discharged **in the same change** (repair the doc, or record +why not) — the write path is the drain. Nothing is enqueued anywhere; ignoring +the advisory leaves the corpus exactly as it was (the staleness banner and the +E172-class sweeps remain the backstop, now with a smaller tail to catch). + +## Declared limits + +- **Conceptual docs that cite no code paths are invisible to the map** (e.g. + a philosophy doc that names no files). Their alignment stays with the event + process; this mechanism owns only the mechanically derivable edges. +- Directory citations are weak claims; they rank below exact-file citations + and carry `[via …/]` so the reader can weigh them. +- The map covers the governed project's own tracked markdown only (tracked = + environment-stable; private symlinked docs are auto-excluded). + +## Verification + +`tests/test_doc_map.py`: inversion semantics (exact/dir/dangling-as-claim/ +exempt/self), query normalization (absolute, outside-root, not-yet-existing +files under a cited dir — a Write creating a hook must fire), strongest-first +ordering, cache hit/invalidation/corruption/footprint rules plus the +adversarial-review scenario (cache built while a cited target is absent, file +created later with no markdown change — the cached answer must still name the +citing doc), symlinked-doc suppression (the guard must not `resolve()` the +private-doc symlinks out of the project), two real-corpus edges this repo +relies on (`core/hooks/workflow_guard.py` → `docs/HOOKS.md`; +`src/episteme/cli.py` → `docs/COMMANDS.md` via the source-of-truth citation +added in this event), and the hook's full fallback ladder end-to-end. + +Independent review (E173, adversarial): no blockers; one should-fix (the +cache-staleness hole above — fixed by claims-in-index) and residual nits +consciously accepted: compound symlink-escape via the `resolve()` fallback +branch (reasoned, not reproducible from real harness payloads), ~40ms +steady-state hook overhead on this repo's 233-file corpus (rebuild ~180ms when +markdown changed), deleted-but-indexed markdown disabling the cache until +`git rm` (self-healing, correct answers throughout), and case-sensitive index +keys. diff --git a/kernel/MANIFEST.sha256 b/kernel/MANIFEST.sha256 index dbe90c6..33f289c 100644 --- a/kernel/MANIFEST.sha256 +++ b/kernel/MANIFEST.sha256 @@ -3,8 +3,8 @@ 133bc78628c2dda23b3b07e72977ece258dea0b68f0b9ed6075fbec0095610a0 kernel/CONSTITUTION.md 6c30b85216318af3dff1e8bde3a4074131fc0c39d986c5c2a76fd3b32b182b3a kernel/FAILURE_MODES.md -b441a08bb7fdae2a76f9edc8c0cc9f90fd235ef7895a250d08f5f41b6ed7342d kernel/HOOKS_MAP.md +7df54b8a5c73fd7e7b02bdf3d5feab92b57a14c1437b7cf56e95ef2fbb1653aa kernel/HOOKS_MAP.md bb738c912af69302139af72e5501f94c9bbd0f570c7197a0a5d58636a57fef85 kernel/OPERATOR_PROFILE_SCHEMA.md a33dd5523b45c7f327d8d90eb1c09ad75ec8e203c5c700aebd89c3e7aa1b397d kernel/README.md -75b021d8e799c59eb5fc476c7d66024487a1c5eff642abd906fb4a481a5af19b kernel/REASONING_SURFACE.md -14da782893d54fd26aa6842b037c80685b46b03ea931ed55c8806a8b469c9c8f kernel/REFERENCES.md +4a197a97b00d69a30ff5f07d3c44423835f092163ff2469bf0b10e1ff15f6956 kernel/REASONING_SURFACE.md +4ff46e527d12ce0597f4be42ebeb06d1fdc504e496b10aeef1cca9f5f573d690 kernel/REFERENCES.md diff --git a/src/episteme/cli.py b/src/episteme/cli.py index 02c9a94..12b9782 100644 --- a/src/episteme/cli.py +++ b/src/episteme/cli.py @@ -6152,6 +6152,9 @@ def build_parser() -> argparse.ArgumentParser: d_index = docs_sub.add_parser("index", help="Regenerate the docs/README.md index from lifecycle markers") d_index.add_argument("--check", action="store_true", help="Verify the committed index is up to date (CI gate); do not write") d_index.add_argument("--root", type=Path, default=None, help="Repo root to index (default: git top-level of the current working directory)") + d_map = docs_sub.add_parser("map", help="Code→doc reverse index: which docs claim to describe a code path (derived from citation edges; E173)") + d_map.add_argument("paths", nargs="*", help="Code paths to query (repo-relative or absolute); omit to dump the full target ← docs map") + d_map.add_argument("--root", type=Path, default=None, help="Repo root to map (default: git top-level of the current working directory)") kernel = sub.add_parser("kernel", help="Kernel integrity manifest + ledger maintenance operations") kernel_sub = kernel.add_subparsers(dest="kernel_action", required=True) @@ -6959,6 +6962,12 @@ def main(argv: Iterable[str] | None = None) -> int: root=getattr(args, "root", None), check=getattr(args, "check", False), ) + if args.docs_action == "map": + from . import doc_references as _dr + return _dr.run_map_cli( + root=getattr(args, "root", None), + paths=getattr(args, "paths", []), + ) return 0 if args.command == "memory": if args.memory_action == "record": diff --git a/src/episteme/doc_references.py b/src/episteme/doc_references.py index 2a353ce..ed3a256 100644 --- a/src/episteme/doc_references.py +++ b/src/episteme/doc_references.py @@ -591,6 +591,384 @@ def _format_stale_citations(findings: Sequence[StaleCitation]) -> str: ) +# --------------------------------------------------------------------------- # +# Code→doc reverse index (Event 173) # +# --------------------------------------------------------------------------- # +# +# Counters FAILURE_MODES: doc-code drift discovered late — a code edit lands, +# the docs that claim to describe that code stay unchanged, and the divergence +# is only found by a later batch sweep (the E172 class of repair). The citation +# walk above already materializes every doc→code edge, and CI keeps those edges +# non-dangling; INVERTING them answers "which docs claim to describe this code +# path" with zero hand-maintained state, so the map itself cannot rot. +# +# Rule-shape (operator's positive/negative-system rule): positive system — an +# obligation exists ONLY where a live authored doc cites the path (exact file, +# or an enclosing directory via a ``dir`` citation). Uncited code carries no +# obligation. The citing set excludes the fixture/scaffold trees +# (:data:`EXEMPT_CITING_PREFIXES`), the historical ledgers +# (:data:`EXEMPT_CITING_FILES`), and ``archive/`` — frozen history must not +# generate present-tense obligations. +# +# Replaces (anti-accretion): the path-blind static advisory in +# ``core/hooks/workflow_guard.py``. That hook now consumes +# :func:`docs_for_path` and names the actual citing docs, keeping the generic +# string only as fallback when this module is unimportable (plugin context, +# non-git project) or the edited path has no citing docs. + +#: Citing prefixes that never generate edit-time obligations. Extends the +#: linter's exemptions with ``archive/``: archived reports legitimately cite +#: code (they are validated for drift above), but a frozen point-in-time +#: artifact must not obligate present-day edits. +OBLIGATION_EXEMPT_CITING_PREFIXES: tuple[str, ...] = EXEMPT_CITING_PREFIXES + ( + "archive/", +) + + +@dataclass(frozen=True) +class DocEdge: + """One inverted citation: ``doc`` claims to describe ``target``.""" + + doc: str # citing doc, repo-root-relative + target: str # resolved cited path, repo-root-relative, no trailing slash + kind: str # "file" | "dir" + line: int # 1-based line in the citing doc + + +def _edge_target(root: Path, ref: Reference, citing: str) -> str: + """The repo-relative path this citation claims to describe. + + Mirrors :func:`resolve_exists`'s two-root policy and returns WHICH + interpretation resolved — the reverse index must key edges by the real + on-disk path (``web/src/lib/x.ts``), not the citing-relative spelling + (``src/lib/x.ts``). An UNRESOLVED citation keeps its root-relative + spelling rather than vanishing: a citation is a CLAIM, and a doc that + cites a not-yet-existing file must still obligate the Write that creates + it (E173 review finding — dropping these made the index depend on target + existence, which the markdown-only cache digest cannot observe, so the + cache served stale empty answers for pre-documented new files). Dangling + claims remain :func:`find_drift`'s findings; here they are edges. + """ + probe = root / ref.target + if probe.is_dir() if ref.kind == "dir" else probe.exists(): + return ref.target + citing_dir = os.path.dirname(citing) + if citing_dir: + alt = os.path.normpath(os.path.join(citing_dir, ref.target)).replace( + os.sep, "/" + ) + alt_probe = root / alt + if alt_probe.is_dir() if ref.kind == "dir" else alt_probe.exists(): + return alt + return ref.target + + +def build_reverse_index( + repo_root: Path, + doc_files: Optional[Sequence[str]] = None, +) -> dict: + """Map claimed cited path → list[:class:`DocEdge`] over the live corpus. + + ``doc_files`` is injectable for tests (same contract as :func:`find_drift`); + when ``None`` the tracked-markdown corpus is used. One edge per + (doc, target, kind) — repeated citations of the same path by the same doc + collapse to the first occurrence. Unresolved citations are indexed as + claims (see :func:`_edge_target`), keeping the index a pure function of + the markdown corpus — the property the cache digest relies on. + """ + root = Path(repo_root) + if doc_files is None: + doc_files = tracked_markdown(root) + index: dict = {} + for rel in doc_files: + if rel in EXEMPT_CITING_FILES or rel.startswith( + OBLIGATION_EXEMPT_CITING_PREFIXES + ): + continue + try: + text = (root / rel).read_text(encoding="utf-8", errors="replace") + except (OSError, UnicodeError): + continue + for ref in extract_references(text, rel): + target = _edge_target(root, ref, citing=rel) + edges = index.setdefault(target, []) + if not any(e.doc == rel and e.kind == ref.kind for e in edges): + edges.append( + DocEdge(doc=rel, target=target, kind=ref.kind, line=ref.line) + ) + return index + + +def _normalize_query_path(root: Path, path: str) -> Optional[str]: + """Normalize a query path to repo-root-relative form, or ``None``. + + Accepts absolute paths (hook payloads carry absolute ``file_path``) and + repo-relative spellings; a path outside ``root`` yields ``None``. + """ + norm = str(path).replace("\\", "/") + if os.path.isabs(norm): + try: + norm = str(Path(norm).resolve().relative_to(Path(root).resolve())) + except (ValueError, OSError): + return None + norm = os.path.normpath(norm).replace(os.sep, "/") + if norm in (".", "") or norm.startswith(".."): + return None + return norm + + +def docs_for_path( + repo_root: Path, + path: str, + index: Optional[dict] = None, + doc_files: Optional[Sequence[str]] = None, +) -> List[str]: + """Sorted docs that cite ``path`` — exactly, or via an enclosing dir. + + Self-citations are excluded (a doc citing itself creates no obligation). + Raises nothing extra beyond :func:`build_reverse_index`'s git requirement + when ``index``/``doc_files`` are not supplied. + """ + root = Path(repo_root) + if index is None: + index = build_reverse_index(root, doc_files=doc_files) + norm = _normalize_query_path(root, path) + if norm is None: + return [] + hits = {e.doc for e in index.get(norm, ()) if e.doc != norm} + for edges in index.values(): + for e in edges: + if ( + e.kind == "dir" + and e.doc != norm + and (norm == e.target or norm.startswith(e.target + "/")) + ): + hits.add(e.doc) + return sorted(hits) + + +def edges_for_path( + repo_root: Path, + path: str, + index: Optional[dict] = None, + doc_files: Optional[Sequence[str]] = None, +) -> List[DocEdge]: + """The citing edges for ``path``, strongest claim first. + + Exact-file citations are strong claims ("this doc describes THIS file") and + sort before directory citations (weak: the doc mentions an enclosing tree); + within dir edges, the most specific (longest) target wins. One edge per + citing doc — a doc citing both the file and its directory contributes only + its file edge. Self-citations excluded. + """ + root = Path(repo_root) + if index is None: + index = build_reverse_index(root, doc_files=doc_files) + norm = _normalize_query_path(root, path) + if norm is None: + return [] + best: dict = {} + for e in index.get(norm, ()): + if e.doc != norm: + best[e.doc] = e + for edges in index.values(): + for e in edges: + if ( + e.kind == "dir" + and e.doc != norm + and e.doc not in best + and (norm == e.target or norm.startswith(e.target + "/")) + ): + held = best.get(e.doc) + if held is None or ( + held.kind == "dir" and len(e.target) > len(held.target) + ): + best[e.doc] = e + return sorted( + best.values(), + key=lambda e: (e.kind != "file", -len(e.target), e.doc), + ) + + +#: Reverse-index cache schema version; bump when DocEdge shape or extraction +#: semantics change so stale caches self-invalidate. +_CACHE_VERSION = 1 + + +def _corpus_digest(root: Path, doc_files: Sequence[str]) -> Optional[str]: + """sha256 over (path, mtime_ns, size) of the obligation-citing corpus.""" + import hashlib + + h = hashlib.sha256() + h.update(f"v{_CACHE_VERSION}".encode()) + for rel in sorted(doc_files): + if rel in EXEMPT_CITING_FILES or rel.startswith( + OBLIGATION_EXEMPT_CITING_PREFIXES + ): + continue + try: + st = os.stat(root / rel) + except OSError: + return None + h.update(f"{rel}\0{st.st_mtime_ns}\0{st.st_size}\n".encode()) + return h.hexdigest() + + +def cached_reverse_index(repo_root: Path) -> dict: + """:func:`build_reverse_index` behind an mtime-digest cache. + + A cold build costs ~200ms on this repo's corpus — too heavy for a + per-edit PreToolUse hook — while the digest check costs ~25ms. The cache + lives at ``.episteme/cache/doc_map.json`` and is written ONLY when + ``.episteme/`` already exists (positive system: episteme caches where it + already has a footprint, never scattering dotdirs into foreign projects). + Every cache failure — unreadable, stale, malformed — degrades to a fresh + build; a wrong answer is never served to avoid a rebuild. + """ + import json as _json + + root = Path(repo_root) + doc_files = tracked_markdown(root) # raises RuntimeError without git — caller's contract + digest = _corpus_digest(root, doc_files) + cache_path = root / ".episteme" / "cache" / "doc_map.json" + + if digest is not None: + try: + payload = _json.loads(cache_path.read_text(encoding="utf-8")) + if ( + payload.get("version") == _CACHE_VERSION + and payload.get("digest") == digest + ): + return { + target: [ + DocEdge(doc=d, target=target, kind=k, line=ln) + for d, k, ln in rows + ] + for target, rows in payload.get("targets", {}).items() + } + except (OSError, ValueError, TypeError): + pass + + index = build_reverse_index(root, doc_files=doc_files) + + if digest is not None and (root / ".episteme").is_dir(): + try: + cache_path.parent.mkdir(parents=True, exist_ok=True) + # pid-suffixed: two concurrent hook processes must not interleave + # writes into one tmp file (review nit; replace() stays atomic). + tmp = cache_path.with_suffix(f".json.tmp.{os.getpid()}") + tmp.write_text( + _json.dumps( + { + "version": _CACHE_VERSION, + "digest": digest, + "targets": { + target: [[e.doc, e.kind, e.line] for e in edges] + for target, edges in index.items() + }, + } + ), + encoding="utf-8", + ) + tmp.replace(cache_path) + except OSError: + pass + return index + + +def annotate_docs(repo_root: Path, docs: Sequence[str]) -> List[tuple]: + """``(doc, label)`` pairs for display: lifecycle state, or manifest tier. + + Labels, in precedence order: a parsed lifecycle marker renders as + ``status · reviewed_as_of`` plus ``· N events behind`` when the marker is + event-tagged and ``docs/EVENTS.md`` yields a later corpus event (same + ``E`` scan as session_context's staleness banner — conscious sibling); + a manifest-managed kernel file renders as ``manifest-managed``; anything + else gets an empty label. Every lookup degrades to a weaker label rather + than raising — annotation must never break a caller's advisory. + """ + root = Path(repo_root) + latest_event: Optional[int] = None + try: + events_text = (root / "docs" / "EVENTS.md").read_text( + encoding="utf-8", errors="replace" + ) + nums = [int(m) for m in re.findall(r"\bE(\d+)\b", events_text)] + if nums: + latest_event = max(nums) + except OSError: + latest_event = None + + managed: frozenset = frozenset() + try: + from episteme.kernel_integrity import MANAGED_KERNEL_FILES + + managed = frozenset(MANAGED_KERNEL_FILES) + except Exception: + managed = frozenset() + + out: List[tuple] = [] + for rel in docs: + label = "" + marker = None + try: + from episteme import doc_lifecycle + + marker = doc_lifecycle.parse_marker(root / rel) + except Exception: + marker = None + if marker is not None and marker.status: + label = marker.status + if marker.reviewed_as_of: + label += f" · {marker.reviewed_as_of}" + ev = re.match(r"^E(\d+)$", marker.reviewed_as_of) + if ev is not None and latest_event is not None: + lag = latest_event - int(ev.group(1)) + if lag > 0: + label += f" · {lag} events behind" + elif rel in managed: + label = "manifest-managed" + out.append((rel, label)) + return out + + +def run_map_cli(root: Optional[Path] = None, paths: Sequence[str] = ()) -> int: + """``episteme docs map [path ...]`` — query or dump the reverse index. + + With paths: the citing docs (annotated) per path. Without: the full + ``target ← docs`` map. Informational — always exits 0; an empty answer is + an answer (positive system: no citation, no obligation). + """ + from episteme.doc_lifecycle import _repo_root # one notion of root resolution + + root = Path(root) if root is not None else _repo_root() + try: + index = cached_reverse_index(root) # warms the same cache the hook reads + except RuntimeError as exc: + print(f"docs map: {exc}") + return 1 + if not paths: + if not index: + print("docs map: no citation edges in tracked markdown") + return 0 + for target in sorted(index): + cited_by = sorted({e.doc for e in index[target]}) + print(f"{target} ← {', '.join(cited_by)}") + return 0 + for query in paths: + edges = edges_for_path(root, query, index=index) + if not edges: + print(f"{query}: no citing docs — no doc claims to describe this path") + continue + print(f"{query}:") + labels = dict(annotate_docs(root, [e.doc for e in edges])) + for e in edges: + label = labels.get(e.doc, "") + via = f" [via {e.target}/]" if e.kind == "dir" else "" + print(f" {e.doc}" + (f" ({label})" if label else "") + via) + return 0 + + if __name__ == "__main__": # pragma: no cover import sys diff --git a/tests/test_doc_map.py b/tests/test_doc_map.py new file mode 100644 index 0000000..a71e04d --- /dev/null +++ b/tests/test_doc_map.py @@ -0,0 +1,352 @@ +"""Event 173 — code→doc reverse index (`docs map`) + targeted DOC ADVISORY. + +The reverse index inverts the drift linter's citation walk (one notion of +"what a doc cites" — see test_doc_references.py for the extraction taxonomy), +so these tests cover only what inversion adds: + + * edge construction: exact-file vs directory citations, resolution keying, + dangling citations creating NO obligation, obligation-exempt citing sets + (archive/ + the linter's fixture trees), self-citation exclusion + * query semantics: repo-relative and absolute spellings, paths outside the + root, NEW (not-yet-existing) files under a cited directory — a Write that + creates a hook must implicate the hook docs + * ordering: strongest claim first (file > deeper dir > shallower dir), one + edge per citing doc + * the mtime-digest cache: hit, invalidation on edit, corruption tolerance, + and the positive-system write rule (cache only where .episteme/ exists) + * the real corpus: at least one live edge this repo relies on + * workflow_guard: targeted advisory on a citing project, generic fallback on + no-citations / no-git / no-package +""" + +from __future__ import annotations + +import io +import json +import subprocess +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from core.hooks import workflow_guard +from episteme import doc_references as dr + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _write(root: Path, rel: str, text: str) -> None: + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +class ReverseIndexUnitTests(unittest.TestCase): + """Pure inversion semantics on a synthetic corpus (no git needed).""" + + def setUp(self): + self._tmp = TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def test_exact_file_citation_creates_obligation(self): + _write(self.root, "core/hooks/x.py", "pass\n") + _write(self.root, "docs/HOOKS.md", "The guard is `core/hooks/x.py`.\n") + docs = dr.docs_for_path( + self.root, "core/hooks/x.py", doc_files=["docs/HOOKS.md"] + ) + self.assertEqual(docs, ["docs/HOOKS.md"]) + + def test_dir_citation_implicates_files_under_it(self): + _write(self.root, "core/hooks/y.py", "pass\n") + _write(self.root, "docs/MAP.md", "Hooks live in `core/hooks/`.\n") + docs = dr.docs_for_path( + self.root, "core/hooks/y.py", doc_files=["docs/MAP.md"] + ) + self.assertEqual(docs, ["docs/MAP.md"]) + + def test_dir_citation_implicates_new_not_yet_existing_file(self): + # A Write that CREATES a file under a cited dir must fire — a new hook + # needs documenting; this is the feature, not a resolution bug. + _write(self.root, "core/hooks/existing.py", "pass\n") + _write(self.root, "docs/MAP.md", "Hooks live in `core/hooks/`.\n") + docs = dr.docs_for_path( + self.root, "core/hooks/brand_new.py", doc_files=["docs/MAP.md"] + ) + self.assertEqual(docs, ["docs/MAP.md"]) + + def test_dangling_citation_is_indexed_as_a_claim(self): + # A citation of a not-yet-existing path stays in the index (a claim, + # not a resolved fact): the doc that pre-documents a file must + # obligate the Write that later creates it, and indexing claims keeps + # the index a pure function of the markdown corpus — the property the + # cache digest relies on (E173 review finding). Dangling claims remain + # find_drift's findings; the two mechanisms stay separate. + _write(self.root, "docs/MAP.md", "See `core/gone.py` for details.\n") + docs = dr.docs_for_path(self.root, "core/gone.py", doc_files=["docs/MAP.md"]) + self.assertEqual(docs, ["docs/MAP.md"]) + + def test_archive_and_fixture_trees_are_obligation_exempt(self): + _write(self.root, "core/hooks/x.py", "pass\n") + _write(self.root, "archive/old_report.md", "Was `core/hooks/x.py`.\n") + _write(self.root, "templates/scaffold.md", "Copies `core/hooks/x.py`.\n") + index = dr.build_reverse_index( + self.root, doc_files=["archive/old_report.md", "templates/scaffold.md"] + ) + self.assertEqual(index, {}) + + def test_self_citation_is_excluded(self): + _write(self.root, "docs/MAP.md", "This file is `docs/MAP.md`.\n") + docs = dr.docs_for_path(self.root, "docs/MAP.md", doc_files=["docs/MAP.md"]) + self.assertEqual(docs, []) + + def test_absolute_query_path_normalizes_into_root(self): + _write(self.root, "core/hooks/x.py", "pass\n") + _write(self.root, "docs/HOOKS.md", "The guard is `core/hooks/x.py`.\n") + docs = dr.docs_for_path( + self.root, + str(self.root / "core" / "hooks" / "x.py"), + doc_files=["docs/HOOKS.md"], + ) + self.assertEqual(docs, ["docs/HOOKS.md"]) + + def test_query_path_outside_root_yields_nothing(self): + _write(self.root, "docs/HOOKS.md", "The guard is `core/hooks/x.py`.\n") + self.assertEqual( + dr.docs_for_path(self.root, "/etc/passwd", doc_files=["docs/HOOKS.md"]), + [], + ) + self.assertEqual( + dr.docs_for_path(self.root, "../outside.py", doc_files=["docs/HOOKS.md"]), + [], + ) + + def test_edges_order_strongest_claim_first_one_edge_per_doc(self): + _write(self.root, "core/hooks/x.py", "pass\n") + _write(self.root, "docs/EXACT.md", "Implements `core/hooks/x.py`.\n") + _write(self.root, "docs/DEEP.md", "Hooks: `core/hooks/`.\n") + _write(self.root, "docs/SHALLOW.md", "Everything under `core/`.\n") + _write( + self.root, + "docs/BOTH.md", + "Cites dir `core/hooks/` and file `core/hooks/x.py`.\n", + ) + edges = dr.edges_for_path( + self.root, + "core/hooks/x.py", + doc_files=[ + "docs/SHALLOW.md", + "docs/DEEP.md", + "docs/EXACT.md", + "docs/BOTH.md", + ], + ) + self.assertEqual(len(edges), 4) # one edge per citing doc + kinds = [e.kind for e in edges] + self.assertEqual(kinds[:2], ["file", "file"]) # BOTH keeps its file edge + self.assertEqual( + [e.doc for e in edges], + ["docs/BOTH.md", "docs/EXACT.md", "docs/DEEP.md", "docs/SHALLOW.md"], + ) + # deeper dir target sorts before shallower + self.assertEqual(edges[2].target, "core/hooks") + self.assertEqual(edges[3].target, "core") + + +class ReverseIndexCacheTests(unittest.TestCase): + """The mtime-digest cache behind cached_reverse_index (needs a git repo).""" + + def setUp(self): + self._tmp = TemporaryDirectory() + self.root = Path(self._tmp.name).resolve() + self.addCleanup(self._tmp.cleanup) + _write(self.root, "core/hooks/x.py", "pass\n") + _write(self.root, "docs/HOOKS.md", "The guard is `core/hooks/x.py`.\n") + subprocess.run( + ["git", "-C", str(self.root), "init", "-q"], check=True + ) + subprocess.run( + ["git", "-C", str(self.root), "add", "-A"], check=True + ) + + def _cache_path(self) -> Path: + return self.root / ".episteme" / "cache" / "doc_map.json" + + def test_no_cache_written_without_episteme_dir(self): + index = dr.cached_reverse_index(self.root) + self.assertIn("core/hooks/x.py", index) + self.assertFalse(self._cache_path().exists()) + + def test_cache_written_and_hit_when_episteme_dir_exists(self): + (self.root / ".episteme").mkdir() + first = dr.cached_reverse_index(self.root) + self.assertTrue(self._cache_path().exists()) + # a cache hit must reproduce the same edges without rebuilding + with patch.object(dr, "build_reverse_index") as rebuild: + second = dr.cached_reverse_index(self.root) + rebuild.assert_not_called() + self.assertEqual( + {t: [(e.doc, e.kind) for e in es] for t, es in first.items()}, + {t: [(e.doc, e.kind) for e in es] for t, es in second.items()}, + ) + + def test_cache_invalidates_when_a_doc_changes(self): + (self.root / ".episteme").mkdir() + dr.cached_reverse_index(self.root) + # touch content AND size so the digest moves even on coarse mtimes + _write( + self.root, + "docs/HOOKS.md", + "The guard moved to `core/hooks/x.py` — expanded prose.\n", + ) + with patch.object( + dr, "build_reverse_index", wraps=dr.build_reverse_index + ) as rebuild: + dr.cached_reverse_index(self.root) + rebuild.assert_called_once() + + def test_corrupt_cache_degrades_to_fresh_build(self): + (self.root / ".episteme").mkdir() + self._cache_path().parent.mkdir(parents=True, exist_ok=True) + self._cache_path().write_text("{not json", encoding="utf-8") + index = dr.cached_reverse_index(self.root) + self.assertIn("core/hooks/x.py", index) + + def test_cached_index_sees_pre_documented_file_created_later(self): + # The reviewer's disconfirmation scenario: a doc cites a file that + # does not exist yet, the cache is built, THEN the file is created — + # no markdown changed, so the digest cannot move. The cached answer + # must still name the citing doc; claims-in-index is what makes the + # cache honest here. + (self.root / ".episteme").mkdir() + _write( + self.root, "docs/PLANNED.md", "Will be implemented in `core/new.py`.\n" + ) + subprocess.run(["git", "-C", str(self.root), "add", "-A"], check=True) + dr.cached_reverse_index(self.root) # cache built while target absent + _write(self.root, "core/new.py", "pass\n") + with patch.object(dr, "build_reverse_index") as rebuild: + index = dr.cached_reverse_index(self.root) + rebuild.assert_not_called() # digest unchanged — served from cache + self.assertIn( + "docs/PLANNED.md", + dr.docs_for_path(self.root, "core/new.py", index=index), + ) + + +class RealCorpusEdges(unittest.TestCase): + """Live edges this repo's governance actually relies on.""" + + def test_hook_edits_implicate_hooks_doc(self): + docs = dr.docs_for_path(REPO_ROOT, "core/hooks/workflow_guard.py") + self.assertIn("docs/HOOKS.md", docs) + + def test_dir_citation_from_agents_md_reaches_hooks(self): + docs = dr.docs_for_path(REPO_ROOT, "core/hooks/workflow_guard.py") + self.assertIn("AGENTS.md", docs) + + def test_cli_edits_implicate_commands_doc(self): + # The edge exists because COMMANDS.md names its source of truth — + # added in E173 so CLI edits surface the doc that documents them. + docs = dr.docs_for_path(REPO_ROOT, "src/episteme/cli.py") + self.assertIn("docs/COMMANDS.md", docs) + + +class WorkflowGuardTargetedAdvisoryTests(unittest.TestCase): + """The hook end-to-end: targeted when edges exist, generic otherwise.""" + + def _run_hook(self, payload: dict) -> str: + raw = json.dumps(payload) + with patch("sys.stdin", new=io.StringIO(raw)), patch( + "sys.stdout", new=io.StringIO() + ) as fake_out: + rc = workflow_guard.main() + self.assertEqual(rc, 0) + out = fake_out.getvalue() + if not out: + return "" + return json.loads(out)["hookSpecificOutput"]["additionalContext"] + + def _project(self, with_git: bool, with_citation: bool) -> Path: + tmp = TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name).resolve() + _write(root, "AGENTS.md", "# agents\n") # authoritative-docs trigger + _write(root, "src/app.py", "pass\n") + if with_citation: + _write(root, "docs/APP.md", "Entry point: `src/app.py`.\n") + if with_git: + subprocess.run(["git", "-C", str(root), "init", "-q"], check=True) + subprocess.run(["git", "-C", str(root), "add", "-A"], check=True) + return root + + def _payload(self, root: Path) -> dict: + return { + "tool_name": "Edit", + "tool_input": {"file_path": str(root / "src" / "app.py")}, + "session_type": "main", + "cwd": str(root), + } + + def test_targeted_advisory_names_the_citing_doc(self): + root = self._project(with_git=True, with_citation=True) + context = self._run_hook(self._payload(root)) + self.assertIn("DOC ADVISORY", context) + self.assertIn("docs/APP.md", context) + self.assertNotIn("WORKFLOW ADVISORY", context) + + def test_no_citations_falls_back_to_generic(self): + root = self._project(with_git=True, with_citation=False) + context = self._run_hook(self._payload(root)) + self.assertIn("WORKFLOW ADVISORY", context) + + def test_non_git_project_falls_back_to_generic(self): + root = self._project(with_git=False, with_citation=True) + context = self._run_hook(self._payload(root)) + self.assertIn("WORKFLOW ADVISORY", context) + + def test_unimportable_package_falls_back_to_generic(self): + root = self._project(with_git=True, with_citation=True) + with patch.object(workflow_guard, "_load_doc_references", return_value=None): + context = self._run_hook(self._payload(root)) + self.assertIn("WORKFLOW ADVISORY", context) + + def test_symlinked_doc_edit_is_suppressed_not_resolved_away(self): + # docs/NEXT_STEPS.md-style setup: the authoritative doc is a symlink + # into a tree OUTSIDE the project. resolve() would follow it out of + # cwd and defeat the doc-path suppression; the textual prefix-strip + # must keep the repo-relative spelling and stay silent. + root = self._project(with_git=True, with_citation=False) + outside = TemporaryDirectory() + self.addCleanup(outside.cleanup) + real = Path(outside.name) / "NEXT_STEPS.md" + real.write_text("# next\n", encoding="utf-8") + link = root / "docs" / "NEXT_STEPS.md" + link.parent.mkdir(parents=True, exist_ok=True) + link.symlink_to(real) + payload = { + "tool_name": "Edit", + "tool_input": {"file_path": str(link)}, + "session_type": "main", + "cwd": str(root), + } + raw = json.dumps(payload) + with patch("sys.stdin", new=io.StringIO(raw)), patch( + "sys.stdout", new=io.StringIO() + ) as fake_out: + rc = workflow_guard.main() + self.assertEqual(rc, 0) + self.assertEqual(fake_out.getvalue(), "") # suppressed, no advisory + + def test_overflow_is_counted_never_silent(self): + root = self._project(with_git=True, with_citation=True) + for i in range(8): + _write(root, f"docs/D{i}.md", "Covers `src/app.py`.\n") + subprocess.run(["git", "-C", str(root), "add", "-A"], check=True) + context = self._run_hook(self._payload(root)) + self.assertIn("DOC ADVISORY", context) + self.assertIn("more", context) # "… and N more" + + +if __name__ == "__main__": + unittest.main()