From 08ea2418b6f7ac07203c60d70c5d483a2c0c9a18 Mon Sep 17 00:00:00 2001 From: John Carpenter Date: Fri, 28 Aug 2026 10:55:41 -0600 Subject: [PATCH 1/3] Unblock ruff check and settle formatter drift `ruff check kgmd/ tests/` was already failing on main with two E501 violations that predate this branch: a long DELETE statement in kgmd/resolve.py and a long SELECT in tests/test_resolve.py. Both are wrapped as adjacent string literals, so the SQL they build is byte identical. `make format` also had pending drift in kgmd/llm.py and tests/test_docs.py. CI enforces `ruff check` but not `ruff format`, so the drift never failed a build and accumulated instead. It is applied here, isolated from feature work, to keep the pre-submit gate clean. No behaviour change. --- kgmd/llm.py | 4 +--- kgmd/resolve.py | 3 ++- tests/test_docs.py | 18 ++++-------------- tests/test_resolve.py | 9 ++++----- 4 files changed, 11 insertions(+), 23 deletions(-) diff --git a/kgmd/llm.py b/kgmd/llm.py index 0878561..36fd258 100644 --- a/kgmd/llm.py +++ b/kgmd/llm.py @@ -80,9 +80,7 @@ def call_structured( if attempt < max_retries: # Detect truncation vs malformed JSON is_truncated = isinstance(e, json.JSONDecodeError) and ( - "Unterminated" in str(e) - or "Expecting" in str(e) - or "end of" in str(e).lower() + "Unterminated" in str(e) or "Expecting" in str(e) or "end of" in str(e).lower() ) if is_truncated: correction = ( diff --git a/kgmd/resolve.py b/kgmd/resolve.py index 90fbd5c..8ba1520 100644 --- a/kgmd/resolve.py +++ b/kgmd/resolve.py @@ -289,7 +289,8 @@ def _merge_entities(conn, survivor_id: int, drop_ids: list[int], canonical_name: if drop_ids: placeholders = ",".join("?" for _ in drop_ids) conn.execute( - f"DELETE FROM relations WHERE subject_id IN ({placeholders}) OR object_id IN ({placeholders})", + f"DELETE FROM relations WHERE subject_id IN ({placeholders})" + f" OR object_id IN ({placeholders})", list(drop_ids) + list(drop_ids), ) diff --git a/tests/test_docs.py b/tests/test_docs.py index 2e26c0d..a515066 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -409,9 +409,7 @@ def test_global_option_documented(): def test_structured_output_parity(): documented = { - name - for name, body in sections(CLI_REF).items() - if "**Structured output**:" in body + name for name, body in sections(CLI_REF).items() if "**Structured output**:" in body } actual = structured_output_commands() assert documented == actual, ( @@ -449,11 +447,7 @@ def test_inert_keys_marked(): for line in page_lines(CONFIG_REF) if line.startswith("|") and line.count("|") >= 4 } - problems = [ - key - for key in INERT_CONFIG_KEYS - if INERT_MARKER not in rows.get(key, "") - ] + problems = [key for key in INERT_CONFIG_KEYS if INERT_MARKER not in rows.get(key, "")] assert not problems, f"inert keys missing the '{INERT_MARKER}' marker: {sorted(problems)}" @@ -518,9 +512,7 @@ def test_concept_terms_defined(): def test_quoted_errors_exist_in_source(): source = package_source_text() - symptoms = [ - line for line in prose_lines(TROUBLESHOOTING) if line.startswith("**Symptom**:") - ] + symptoms = [line for line in prose_lines(TROUBLESHOOTING) if line.startswith("**Symptom**:")] assert symptoms, "troubleshooting page has no **Symptom**: entries" problems = [] for line in symptoms: @@ -536,9 +528,7 @@ def test_quoted_errors_exist_in_source(): def test_walkthrough_sections(): problems = [] for path in sorted(EXAMPLES.glob("*.md")): - headings = [ - line[3:].strip() for line in prose_lines(path) if line.startswith("## ") - ] + headings = [line[3:].strip() for line in prose_lines(path) if line.startswith("## ")] for section in WALKTHROUGH_SECTIONS: if section not in headings: problems.append(f"{rel(path)}: missing '## {section}'") diff --git a/tests/test_resolve.py b/tests/test_resolve.py index 7f13033..33aa0e5 100644 --- a/tests/test_resolve.py +++ b/tests/test_resolve.py @@ -68,8 +68,7 @@ def test_resolution_merges_duplicates(initialized_corpus): conn.execute(SQL_INSERT_DOC, ("test.md", "abc", 10, 0.0, now)) conn.execute(SQL_INSERT_CHUNK) conn.execute( - "INSERT INTO extraction_runs" - " (started_at, model, status) VALUES (?, 'test', 'completed')", + "INSERT INTO extraction_runs (started_at, model, status) VALUES (?, 'test', 'completed')", (now,), ) @@ -119,8 +118,7 @@ def _seed_run_doc_chunk(conn, now): conn.execute(SQL_INSERT_DOC, ("test.md", "abc", 10, 0.0, now)) conn.execute(SQL_INSERT_CHUNK) conn.execute( - "INSERT INTO extraction_runs" - " (started_at, model, status) VALUES (?, 'test', 'completed')", + "INSERT INTO extraction_runs (started_at, model, status) VALUES (?, 'test', 'completed')", (now,), ) @@ -194,7 +192,8 @@ def test_merge_entities_relation_unique_collision(initialized_corpus): # The surviving relation is preserved. survivor_rel = conn.execute( - "SELECT COUNT(*) FROM relations WHERE subject_id = 1 AND predicate = 'runs' AND object_id = 3" + "SELECT COUNT(*) FROM relations" + " WHERE subject_id = 1 AND predicate = 'runs' AND object_id = 3" ).fetchone()[0] assert survivor_rel == 1 From af0a022d6c9cad3477e23228aa958576ba372adb Mon Sep 17 00:00:00 2001 From: John Carpenter Date: Fri, 28 Aug 2026 10:56:08 -0600 Subject: [PATCH 2/3] Add .kgmdignore exclusions and remove files that leave the corpus Closes #3. A corpus had no way to exclude anything. `corpus.include` is an allowlist of literal paths, so skipping one folder meant enumerating every other folder and keeping that list current as the corpus grew. Every indexed file is chunked and each chunk is one model call, so indexing an archive or a tree of templates is repeated spend that buys nothing and fills entity resolution with junk mentions. A `.kgmdignore` file at the corpus root now subtracts paths, with a documented gitignore subset: comments, blank lines, `*` (never crossing `/`), `?`, `**`, trailing-`/` directory rules, leading-`/` anchoring, and `!` negation resolved last-match-wins. Precedence is fixed: `corpus.include` scopes the candidates, `.kgmdignore` subtracts and negations re-add, and the dot-path rule is applied last and cannot be overridden by any pattern. An absent file changes nothing. Patterns are compiled to anchored regexes in a new stdlib-only leaf module rather than taking a dependency. `fnmatch`'s `*` crosses `/`, and `PurePath.match` is right-anchored with single-segment `**` on the supported runtimes, so neither can express these semantics; `pathspec` faithfully reproduces git's refusal to re-include a file inside an excluded directory, which is the one behaviour we deliberately diverge from, so it could not satisfy the requirement either. Exclusion alone would have been a no-op for every corpus that already has a graph, so ingest now also removes documents whose recorded path has left the resolved file set. Newly-ignored, deleted, and renamed files are one state and are handled identically, which closes the long-standing gap where a deleted note was never removed. Cascades do not cover that removal. `relations.evidence_chunk_id` is ON DELETE SET NULL, so relations bound to removed evidence are deleted explicitly rather than surviving with no provenance. The sqlite-vec tables have no foreign keys, and `chunks.id` and `entity_mentions.id` are reused after deletes while `embed_new_chunks` skips any chunk that already has a vector row -- so a leftover vector would make a future unrelated chunk look embedded and search would answer from deleted text. Both vector tables are cleared for the removed ids. Entities left with no mention and no relation are swept, scoped to the entities this removal orphaned so pre-existing orphans are not collected as a side effect. Unconditional pruning makes a stray `*` destructive, so an ignore ruleset that resolves to an empty set while the graph holds documents aborts before any write instead of emptying the graph. `kgmd build --dry-run` reports the resolved file set, the exclusion counts, and how many indexed documents would be removed. It takes no lock, calls no model, and runs ahead of `init_db` so it creates no database. `--json` gives the machine-readable form and requires `--dry-run`, because a `--json` that implied it would mean `kgmd build --json` quietly not building. The payload lives in `ingest.dry_run_report` so the substance is testable without invoking the CLI, keeping `cli.py` a thin renderer. `kgmd init` writes a starter `.kgmdignore` whose every line is a comment, so a fresh corpus indexes exactly what it did before, and an existing file is never overwritten. No new config key: `corpus.exclude` was rejected rather than shipping two mechanisms for one job, which also leaves DEFAULT_CONFIG and the configuration reference's key count untouched. Documentation is part of the change. Seven statements across the docs set became false -- the "File deleted from disk | nothing" row, the "Deleted notes are not removed from the graph" limitation, and an architecture claim that ingest.py imports nothing from kgmd among them -- and are corrected here. --- README.md | 4 + docs/contributing/architecture.md | 16 +- docs/examples/mcp-assistant.md | 6 +- docs/examples/personal-notes.md | 58 ++- docs/guides/maintenance.md | 55 ++- docs/guides/troubleshooting.md | 66 +++- docs/reference/cli.md | 47 ++- docs/reference/configuration.md | 87 ++++- kgmd/cli.py | 76 +++- kgmd/ignore.py | 184 +++++++++ kgmd/ingest.py | 196 +++++++++- tests/test_cli.py | 74 ++++ tests/test_ignore.py | 202 ++++++++++ tests/test_ingest.py | 628 ++++++++++++++++++++++++++++++ 14 files changed, 1650 insertions(+), 49 deletions(-) create mode 100644 kgmd/ignore.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_ignore.py create mode 100644 tests/test_ingest.py diff --git a/README.md b/README.md index 39c395a..b3c8c9d 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,10 @@ Full walkthrough: [docs/quickstart.md](docs/quickstart.md). All state lives in `.kgmd/graph.db`, a single SQLite file. Re-running `kgmd build` is incremental — unchanged files are skipped. See [docs/concepts.md](docs/concepts.md). +A `.kgmdignore` file at the corpus root keeps material out of the graph, and therefore out of the +paid extraction path; `kgmd build --dry-run` shows what would be indexed before you spend anything. +Syntax and precedence: [docs/reference/configuration.md](docs/reference/configuration.md). + ## MCP server `kgmd mcp` launches an MCP server over stdio exposing seven read-only tools over the graph. Setup, diff --git a/docs/contributing/architecture.md b/docs/contributing/architecture.md index cc42284..dffb932 100644 --- a/docs/contributing/architecture.md +++ b/docs/contributing/architecture.md @@ -12,7 +12,7 @@ places that own reads, state, prompts, and data contracts. | `kgmd/cli.py` | Click entry point (`main`) and every subcommand. Resolves the corpus and database path, opens connections, calls stage and query functions, and renders results as `rich` tables or as JSON. Contains no graph logic. | | `kgmd/mcp_server.py` | MCP stdio server built on `FastMCP`. Registers the tool functions, resolves its connection and configuration from the client's working directory, and delegates each tool to `kgmd/query.py`. | | `kgmd/query.py` | The read layer: `search_chunks`, `list_entities`, `get_entity`, `list_relations`, `get_neighbors`, `find_path`, `get_current_schema`. Takes a connection, returns plain `dict`/`list[dict]`. Builds a `networkx.DiGraph` in memory for traversal and pathfinding. | -| `kgmd/ingest.py` | Markdown discovery, sha256 content hashing, and chunking. `find_markdown_files` applies `corpus.include` and skips dotted paths; `chunk_markdown` splits by paragraph, heading, or fixed window; `ingest_documents` writes documents and chunks. | +| `kgmd/ingest.py` | Markdown discovery, sha256 content hashing, and chunking. `scan_corpus_files` is the single source of the resolved file set, composing `find_markdown_files` for `corpus.include` scoping, the `.kgmdignore` pass, and the dot-path rule, which is applied last and dominates both; `chunk_markdown` splits by paragraph, heading, or fixed window; `ingest_documents` calls `prune_missing_documents` to drop the rows of files that have left the corpus, then writes documents and chunks; `dry_run_report` is the read-only payload behind `kgmd build --dry-run`. | | `kgmd/extract.py` | Extraction stage. Builds the extraction prompt with type and predicate vocabulary drawn from the existing graph, calls the LLM per chunk, and upserts entities, mentions, and relations under an `extraction_runs` row. | | `kgmd/resolve.py` | Entity resolution. Clusters mention embeddings by cosine similarity, optionally asks the LLM to verify each cluster, then merges duplicate entities onto a survivor and records a `resolution_runs` row. | | `kgmd/induce.py` | Schema induction. Summarises entity and relation statistics from the graph, asks the LLM for a typed schema, and stores the result as a new `schema_versions` row. | @@ -21,6 +21,7 @@ places that own reads, state, prompts, and data contracts. | `kgmd/embed.py` | Embedding backends behind an `Embedder` protocol: `FastembedEmbedder` (local, the default) and `LitellmEmbedder` (provider API). `get_embedder` selects one from `embedding.backend`. `embed_new_chunks` and `embed_new_mentions` fill the vector tables incrementally. | | `kgmd/db.py` | Connection ownership. `get_connection` opens SQLite, sets `row_factory`, loads the `sqlite-vec` extension, and applies `journal_mode = WAL`, `foreign_keys = ON`, `synchronous = NORMAL`. `init_db` runs the DDL once and sets `PRAGMA user_version = 1`. Also `check_embedding_model` and the `build_lock` context manager. | | `kgmd/config.py` | `DEFAULT_CONFIG`, the platform-specific global config location via `platformdirs`, and `load_config`, which deep-merges built-in defaults, the global file, and the corpus `.kgmd/config.yaml`. | +| `kgmd/ignore.py` | `.kgmdignore` parsing. `load_ignore_rules` reads the corpus-root file into ordered rules with every pattern compiled to an anchored regex at parse time, `is_ignored` answers one corpus-relative path under last-match-wins ordering, and `DEFAULT_IGNORE_TEMPLATE` is the all-comments starter file `kgmd init` writes. Stdlib only; imports nothing from `kgmd` and issues no SQL. | | `kgmd/schema.py` | `SCHEMA_SQL` (all tables, indexes, and the `current_schema` view), `vec_tables_sql(dim)` for the `vec0` virtual tables, `KV_DEFAULTS`, and the pydantic models used to validate LLM output. | | `kgmd/prompts/` | Bundled prompt text assets: `extract.txt`, `resolve.txt`, `induce.txt`. Data, not code. | @@ -49,12 +50,16 @@ The rules that follow from this: job. - **Nothing imports `mcp_server.py`** except `cli.py`, from inside the `mcp` command body. - **Stage modules never import each other.** `extract.py` and `resolve.py` depend on `llm.py` and - `schema.py`; `induce.py` depends on `llm.py`; `ingest.py`, `export.py`, and `query.py` import - nothing from `kgmd` at all — they receive an open connection as their first argument. + `schema.py`; `induce.py` depends on `llm.py`; `ingest.py` imports `ignore.py` and nothing else; + `export.py` and `query.py` import nothing from `kgmd` at all — they receive an open connection as + their first argument. - **`db.py` is the only module that imports `schema.py`** for DDL purposes, and `schema.py` imports nothing internal. - **`config.py` is a leaf.** It is read by the surfaces and passed down as a plain `dict`; no stage module loads configuration for itself. +- **`ignore.py` is a leaf below `ingest.py`.** It is stdlib-only and imports nothing from `kgmd`, so + the one-way direction extends as `ingest` -> `ignore` -> nothing. It parses text and answers + questions about paths; it never sees a connection and issues no SQL. `cli.py` imports `config`, `db`, `ingest`, and `query` at module level, and imports `embed`, `extract`, `resolve`, `induce`, `export`, and `mcp_server` inside the command bodies that need them. @@ -97,7 +102,10 @@ All persistent state is one SQLite file: `.kgmd/graph.db`. `chunks`, `extraction_runs`, `entities`, `entity_mentions`, `relations`, `resolution_runs`, `schema_versions`, their indexes, and the `current_schema` view. `vec_tables_sql(dim)` defines the `sqlite-vec` virtual tables `vec_chunks` and `vec_entity_mentions`, dimensioned at creation time. - No other module issues `CREATE`, `ALTER`, or `DROP`. + No other module issues `CREATE`, `ALTER`, or `DROP`. That constrains call sites, not just + definitions: `ingest.prune_missing_documents` binds the ids it is deleting in batched `IN (...)` + clauses rather than staging them in a `TEMP TABLE`, because a temp table would be DDL outside + `kgmd/schema.py`. - **`kgmd/db.py` owns connections and their invariants.** Extension loading, pragmas, `PRAGMA user_version` as the migration marker, `KV_DEFAULTS` seeding, the embedding-model guard, and the `fcntl` exclusive lock at `.kgmd/build.lock` all live there. Stage modules receive a connection; diff --git a/docs/examples/mcp-assistant.md b/docs/examples/mcp-assistant.md index a885b23..ad99648 100644 --- a/docs/examples/mcp-assistant.md +++ b/docs/examples/mcp-assistant.md @@ -166,5 +166,7 @@ in a session pays the model load; subsequent calls in the same server process do - **One corpus per server entry.** The database path is derived from the process working directory, which the client fixes at launch. Serving several corpora means several registered entries. - **The graph is only as current as the last build.** `get_schema_tool` returns its "no schema" - string until an induction has run, and every tool reflects the state of the last `kgmd build`, - including entities extracted from notes you have since deleted. + string until an induction has run, and every tool reflects the state of the last `kgmd build` — + notes written or edited since then are invisible until you build again. That cuts both ways: a + note you deleted, renamed, or newly excluded with `.kgmdignore` before that build is gone from the + graph, so no tool will answer from it. diff --git a/docs/examples/personal-notes.md b/docs/examples/personal-notes.md index 4653bd1..eab27bb 100644 --- a/docs/examples/personal-notes.md +++ b/docs/examples/personal-notes.md @@ -40,7 +40,8 @@ then answer questions about it from the command line. Three kinds of question ar Use your own notes directory. Any layout works: kgmd walks the tree, takes every `*.md`, and always skips path components beginning with a dot (so `.kgmd/`, `.git/`, and `.claude/` are never -ingested). +ingested). A `.kgmdignore` file at the root of the notes directory subtracts further paths from +that walk; step 2 covers it. If you want a corpus that behaves the same on someone else's machine, use the seven fixture notes in a git checkout — they are small, densely cross-referential, and deliberately inconsistent about @@ -71,8 +72,10 @@ kgmd init `kgmd init` creates `.kgmd/` next to your notes containing `config.yaml` (the full default configuration, written out key by key), an empty `graph.db`, and empty `logs/` and `prompts/` -directories. It prints the corpus, database, and config paths. Run again in an initialized directory -and it prints the existing config instead of overwriting anything. +directories. It also writes a starter `.kgmdignore` — beside the notes, not inside `.kgmd/` — with +every line commented out, so a fresh corpus indexes exactly what it would have without the file. It +prints the corpus, database, and config paths. Run again in an initialized directory and it prints +the existing config instead of overwriting anything, and leaves an existing `.kgmdignore` untouched. ### 2. Scope and chunk the corpus (optional) @@ -101,6 +104,35 @@ chunking: Every key, its default, and its effect is in [configuration.md](../reference/configuration.md). +`corpus.include` only decides which part of the tree is a candidate. To subtract from that +candidate set, write a `.kgmdignore` at the corpus root — gitignore-style patterns, one per line, +`#` for comments: + +```text +archive/ +drafts/ +!archive/2024-decisions.md +``` + +A personal corpus is where this pays for itself: an `archive/` you never query and a `drafts/` full +of half-finished sentences cost one LLM call per chunk on every build that first touches them. +Patterns are read in order and the last match wins, so the negation above re-admits that one +archived note — which git will not do for a file whose parent directory is excluded. The dot-path +rule is applied last and dominates, so no pattern re-admits `.kgmd/` or `.git/`. Syntax, precedence, +and the constructs that are not supported are in +[configuration.md](../reference/configuration.md). + +Check the result before paying for a build: + +```bash +kgmd build --dry-run +``` + +It prints the files that would be indexed, how many the ignore rules and the dot-path rule dropped, +and how many already-indexed documents would be removed. It takes no build lock, makes no provider +call, and creates no database. `--json` gives the same report machine-readably and requires +`--dry-run`. + ### 3. Build the graph ```bash @@ -121,6 +153,15 @@ Re-run `kgmd build` after editing notes. Ingest compares a sha256 of the file co from the current content hash, so unchanged files cost nothing. File mtime is stored but never used to decide what to skip. `kgmd extract --force` re-extracts everything regardless. +Removal happens in the same pass, before the insert and update loop. Ingest drops every indexed +document whose path is no longer in the resolved file set — deleted, renamed, and newly excluded by +`.kgmdignore` are one state, not three — together with its chunks, its mentions, the relations whose +evidence came from those chunks, the vectors for both, and any entity the removal leaves with no +mention and no relation. `kgmd build` and `kgmd extract` both do it and both print a `Removed:` line +in the ingest summary when something was removed, and nothing extra when nothing was. If the +resolved set is empty while the graph still holds documents, ingest refuses before writing anything +rather than emptying the graph behind a pattern you did not mean. + ### 4. Query the graph Run these from the corpus directory or any subdirectory — kgmd walks upward looking for `.kgmd/`. @@ -268,11 +309,12 @@ canonical entity names — match the name `kgmd entities` shows, or find it firs - **Runs are not reproducible.** `llm.temperature` defaults to `0.0`, but identical input can still produce a different entity or relation count on a second build. Treat any count as a description of one run. -- **Deleted notes are not removed from the graph.** Ingest only inserts and updates rows for files - it finds; there is no pass that deletes documents whose file has disappeared, so entities - extracted from a note you deleted survive in the graph. To drop them, clear the graph and rebuild - — `kgmd reset --hard` (which also clears documents and chunks) or delete `.kgmd/graph.db` - outright. See [maintenance.md](../guides/maintenance.md). +- **Entities stranded by an earlier forced extraction are not swept.** Removing a note collects the + entities *that removal* leaves with no mention and no relation, and only those. An entity left + behind by an earlier `kgmd extract --force` is outside that scope: it stays in the graph and keeps + appearing in `kgmd entities`. Clearing the graph with `kgmd reset --hard` and rebuilding is the + way to drop it — see [maintenance.md](../guides/maintenance.md) for what reset does and does not + touch. - **Resolution only merges within one entity type.** Mentions clustered as `Person` never merge with mentions typed `Organization`, so a mistyped mention stays a separate entity. Raising `resolution.similarity_threshold` merges less; lowering it merges more, including things that are diff --git a/docs/guides/maintenance.md b/docs/guides/maintenance.md index bef2168..08c359a 100644 --- a/docs/guides/maintenance.md +++ b/docs/guides/maintenance.md @@ -16,10 +16,13 @@ Two sha256 content hashes gate all the expensive work. Both are digests of a fil | `documents.content_hash` | ingest, on every change | whether the file is re-read, re-chunked, re-embedded | | `documents.last_extracted_hash` | extraction, on success | whether the document is sent to the provider again | -Ingest walks the corpus root for `*.md`, drops any path with a dot-prefixed component (so `.kgmd/`, -`.git/`, and friends are never ingested), and optionally narrows the walk to `corpus.include`. For -each file it hashes the bytes and compares against the stored `content_hash`. Equal means the file is -counted as skipped and nothing else happens to it. +Ingest resolves the file set before it touches the database: it walks the corpus root for `*.md`, +narrows the walk to `corpus.include` if that key is set, subtracts the patterns in `.kgmdignore`, and +drops any path with a dot-prefixed component last of all (so `.kgmd/`, `.git/`, and friends are never +ingested, and no negated pattern can re-admit them). Every `documents` row whose recorded path is not +in the resolved set is then removed from the graph. For each file that survives, ingest hashes the +bytes and compares against the stored `content_hash`. Equal means the file is counted as skipped and +nothing else happens to it. Extraction then selects only the documents where `last_extracted_hash` is `NULL` or differs from `content_hash`. A document's watermark is written *after* the run, and only if at least one of its @@ -39,8 +42,9 @@ keys are on, so that chunk delete cascades: - `entity_mentions.chunk_id` is `ON DELETE CASCADE` — the document's mentions go with its chunks. - `relations.evidence_chunk_id` is `ON DELETE SET NULL` — relation rows **survive** with their evidence pointer cleared. -- `entities` rows are never deleted here. An entity that only ever appeared in text you removed stays - in the graph. +- `entities` rows are never deleted on this path. An entity that only ever appeared in text you + edited away stays in the graph. Removing a document is the one case that does sweep entities — + see [Change made, work repeated](#change-made-work-repeated). The practical consequence: repeated edits accumulate relation rows whose evidence is `NULL`, because the uniqueness index covers `(subject_id, predicate, object_id, evidence_chunk_id)` and the @@ -60,11 +64,17 @@ search over a heavily edited corpus is one more reason to prefer a periodic clea | Nothing | ingest skips every file, embedding finds nothing new, extraction selects no documents; resolution and induction still run in full | | One file edited | that file re-hashed and re-chunked, its chunks re-embedded, its mentions cascade-deleted, the document re-extracted; resolution and induction full | | New file added | new `documents` row, chunks created and embedded, document extracted; resolution and induction full | -| File deleted from disk | **nothing** — ingest only iterates files that exist, so the document, its chunks, and its entities stay in the graph | -| File renamed | treated as a delete plus an add: the old path's data lingers, the new path is ingested and extracted from scratch | +| File deleted from disk | its `documents` row is removed, and with it the document's chunks, their mentions, every relation whose evidence chunk belonged to it, its rows in `vec_chunks` and `vec_entity_mentions`, and any entity the removal leaves with no mention and no relation | +| File newly excluded by `.kgmdignore` | the same removal as a delete: the recorded path is no longer in the resolved file set, and ingest does not distinguish the two | +| File renamed | a removal plus an add in the same ingest pass: nothing lingers at the old path, and the new path is ingested and extracted from scratch | | `chunking.*` changed | nothing for unchanged files; their hashes still match, so old chunk boundaries persist | | `llm.model` or a prompt changed | nothing; neither is hashed. Use `kgmd extract --force` | +One guard sits in front of the removal path. If the resolved file set is empty while the graph still +holds documents, ingest raises before writing anything rather than emptying the graph — a mistyped +ignore pattern cannot silently cost you a corpus. Emptying a graph on purpose is what +[Starting over](#starting-over) is for. + Resolution and induction are not incremental at all. `kgmd resolve` re-clusters every embedded mention in the database on each run, and `kgmd induce` regenerates the schema from full aggregate statistics and appends a new `schema_versions` row, so the schema is versioned rather than mutated. @@ -83,8 +93,10 @@ Reach for it when the *inputs to extraction* changed but the *files* did not: - You suspect a bad extraction — a run where many chunks failed, or output that looks truncated. `--force` does **not** re-chunk and does **not** re-embed: it works from the chunks already in the -database, and `entities` rows left with no remaining mentions are not swept up. To change chunk -boundaries or drop orphaned entities you need a full rebuild (see [Starting over](#starting-over)). +database, and `entities` rows left with no remaining mentions are not swept up. The sweep runs only +when ingest *removes* a document, and only over the entities that removal itself orphaned — so an +entity stranded by an earlier `--force` stays in the graph until a full rebuild. To change chunk +boundaries or drop those orphans you need one (see [Starting over](#starting-over)). ## Controlling provider spend @@ -101,10 +113,19 @@ Everything else is local: ingest and chunking, all embedding while `embedding.ba including `kgmd find` — which embeds your query locally — plus `kgmd export`, `kgmd stats`, and `kgmd schema`. Setting `embedding.backend` to `litellm` moves embedding onto the provider too. -Settings that change call volume: +Extraction spend is per chunk per model call, so the cheapest saving available is not indexing text +that was never going to earn its keep: archived material, drafts, vendored documentation trees, +boilerplate templates. That is what `.kgmdignore` is for, and the division of labour between it and +config is worth fixing in your head — `corpus.include` scopes the directory walk, `.kgmdignore` +decides what actually gets indexed and therefore what costs money. Its syntax, its precedence, and +one deliberate divergence from git are in +[../reference/configuration.md](../reference/configuration.md). + +What changes call volume: -| Setting | Effect on volume | +| Lever | Effect on volume | |---|---| +| `.kgmdignore` | excluded files are never chunked, so they generate no extraction calls at all; the only lever that removes work rather than reshaping it | | `chunking.max_chars` | extraction calls scale with chunk count; larger chunks mean fewer, bigger calls | | `resolution.llm_verify_clusters` | `false` removes the resolution stage's calls entirely, at the cost of merging on cosine similarity alone | | `resolution.similarity_threshold` | a higher threshold produces fewer multi-member clusters, so fewer verification calls | @@ -118,9 +139,13 @@ Two divergences worth knowing before you tune: retry count come from the library defaults, not from `llm.max_tokens`, `llm.timeout_seconds`, or `extraction.retry_on_parse_failure`. -Trial-run before committing a large corpus. Either initialize a throwaway corpus in a small -subdirectory and build that, or set `corpus.include` to one directory so ingest only walks that -subtree, then read `kgmd stats` and the run log to project cost. Full key reference: +Trial-run before committing a large corpus. `kgmd build --dry-run` resolves the file set and reports +it — the included paths, plus counts of what the ignore rules and the dot-path rule left out and how +many indexed documents would be removed — while taking no build lock, making no provider call, and +creating no database. Read it before the first paid run and again after every ignore-rule change. +Beyond that you can initialize a throwaway corpus in a small subdirectory and build that, or set +`corpus.include` to one directory so ingest only walks that subtree, then read `kgmd stats` and the +run log to project cost. Full key reference: [../reference/configuration.md](../reference/configuration.md). ## Starting over diff --git a/docs/guides/troubleshooting.md b/docs/guides/troubleshooting.md index 5b6069e..fec14f0 100644 --- a/docs/guides/troubleshooting.md +++ b/docs/guides/troubleshooting.md @@ -133,6 +133,30 @@ cleanup is only needed for an empty lock file. Confirm no build is actually runn rm -f .kgmd/build.lock ``` +### A build refuses to run after you edited the ignore file + +**Symptom**: `Ignore rules exclude every markdown file in the corpus` + +**Cause**: Ingest prunes unconditionally — every `documents` row whose path is no longer in the +resolved file set is removed, which is how a deleted, renamed, or newly ignored file leaves the +graph. An over-broad `.kgmdignore` pattern, usually a bare `*`, resolves the file set to nothing, and +the prune would then delete the entire graph. So an empty file set against a non-empty `documents` +table is treated as a mistake rather than an instruction: the build raises before any write, and +nothing was removed. + +**Fix**: Look at what the rules actually resolve to, correct the pattern, and re-run: + +```bash +kgmd build --dry-run +``` + +A dry run is a read and the guard only protects writes, so in this state it exits 0 and simply +reports zero included files — that zero is the confirmation, not a second failure. The last matching +rule wins, so a `!` line placed after the offending pattern is often the shortest correction; the +syntax is in [../reference/configuration.md](../reference/configuration.md). If emptying the graph +really was the intent, `kgmd reset --hard` is the explicit way to ask for it — subject to the reset +bug described below. + ### The provider returns something that is not the expected JSON **Symptom**: `LLM call failed after` @@ -159,15 +183,43 @@ extraction stage applies its own lower internal default — see **Cause**: A build reaches "Build complete." even when every extraction call failed, because failures are per-chunk warnings rather than fatal errors. Induction also returns quietly with zero types when there are no entities, so `kgmd schema` reports that no schema has been induced yet. The other -possibility is that nothing was ingested at all: ingest only walks `*.md` files, skips every path -with a dot-prefixed component, and honours `corpus.include` if set. +possibility is that nothing was ingested at all: ingest only walks `*.md` files, subtracts everything +matched by `.kgmdignore`, skips every path with a dot-prefixed component, and honours +`corpus.include` if set. **Fix**: Read the stage lines the build printed. If stage 1 reported zero new or updated documents, -the problem is ingest — check the file extensions, check that the notes are not inside a -dot-prefixed directory, and check `corpus.include`. If documents and chunks exist but entities do -not, the problem is extraction: look for `Extraction failed:` on stderr and `[FAIL]` lines in -`.kgmd/logs/build.log`, then follow the credential and JSON entries above. `kgmd stats` prints -document, chunk, entity, and relation counts, which isolates the stage that produced nothing. +the problem is ingest — check the file extensions, check `.kgmdignore`, check that the notes are not +inside a dot-prefixed directory, and check `corpus.include`. If documents and chunks exist but +entities do not, the problem is extraction: look for `Extraction failed:` on stderr and `[FAIL]` +lines in `.kgmd/logs/build.log`, then follow the credential and JSON entries above. `kgmd stats` +prints document, chunk, entity, and relation counts, which isolates the stage that produced nothing. + +### A file you can see in the corpus is never indexed + +**Symptom**: `excluded as a dot-path` + +**Cause**: Three filters stand between a `*.md` file and the graph, in a fixed order. +`corpus.include`, if set, scopes the candidate set. `.kgmdignore` then subtracts, and `!` lines +re-add — the last matching rule wins, so a broad pattern further down the file overrides an earlier +exception. The dot-path rule is applied last and dominates: any path with a component starting with +`.` is dropped, and no pattern, negated or not, can re-admit it. The usual surprises are in the +patterns themselves — `*` never crosses a `/`, and a pattern with no `/` in it at all matches at any +depth, so `drafts` excludes `projects/drafts/` as well as `drafts/`. + +**Fix**: Ask kgmd what it resolved rather than re-reading the patterns: + +```bash +kgmd build --dry-run +``` + +The report lists every included path and counts what was dropped by `.kgmdignore` separately from +what was dropped by the dot-path rule, which tells you which of the two to edit; `--json` gives the +same data in a scriptable shape. Pattern syntax and precedence are in +[../reference/configuration.md](../reference/configuration.md), the flags in +[../reference/cli.md](../reference/cli.md). A dot-path exclusion cannot be worked around, so a file +under a dot-prefixed directory has to move. If the file was in the graph until recently, the +`Removed:` line a build prints accounts for it — ignoring, deleting, and renaming a file are one +state as far as ingest is concerned. ### A query cannot find an entity you know is in the notes diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 117bca9..711d43e 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -54,12 +54,17 @@ Initialize a new kgmd corpus. | `--help` | flag | off | Show usage and exit. | Creates `.kgmd/` inside the target directory containing `logs/`, `prompts/`, a `config.yaml` -written from the built-in defaults, and an initialized `graph.db`. It then prints the resolved -corpus directory, database path, and config path. +written from the built-in defaults, and an initialized `graph.db`. It also writes a starter +`.kgmdignore` at the target directory itself — beside your markdown, not inside `.kgmd/` — whose +every line is a comment, so it documents the syntax without excluding anything. An existing +`.kgmdignore` is never overwritten. It then prints the resolved corpus directory, database path, +config path, and ignore-file path. If `.kgmd/` already exists the command is a no-op: it prints `Already initialized at `, -echoes the existing `config.yaml` if there is one, and changes nothing. Re-running `kgmd init` is -therefore safe and is a quick way to view the corpus config. +echoes the existing `config.yaml` if there is one, and changes nothing — including leaving a missing +`.kgmdignore` missing. Re-running `kgmd init` is therefore safe and is a quick way to view the +corpus config. To add an ignore file to an existing corpus, write one by hand; see +[./configuration.md](./configuration.md). ```bash cd ~/notes @@ -118,6 +123,8 @@ Build the knowledge graph: extract, resolve, induce. | `PATH` | argument | `.` | Corpus directory; must exist. | | `--db` | path | `/.kgmd/graph.db` | Alternate database path. | | `--config` | path | — | **Accepted but ignored.** See the warning below. | +| `--dry-run` | flag | off | Report the files that would be indexed, then exit without building. | +| `--json` | flag | off | Output as JSON. Requires `--dry-run`. | | `--help` | flag | off | Show usage and exit. | > `--config` is parsed and then discarded: `build` always loads the corpus directory's own @@ -131,7 +138,8 @@ directory, then takes the exclusive build lock at `.kgmd/build.lock` and runs si a heading and a one-line summary for each: 1. **Ingesting documents** — scan markdown, chunk it, report new / updated / skipped / chunks - created. + created, plus a `Removed:` line when documents that are no longer in the corpus were dropped from + the graph. 2. **Embedding chunks** — verify the corpus embedding model, then embed chunks that have no vector. 3. **Extracting entities and relations** — report documents processed, entities created, relations created. @@ -146,6 +154,31 @@ second `kgmd build` over an unmodified corpus is cheap. Stages 3, 5, and 6 are t after a partial failure instead of repeating the whole pipeline — for example when extraction succeeded but induction hit a provider timeout. +`--dry-run` answers "what would this cost" before it costs anything. It resolves the file set — +`corpus.include` scoping, then `.kgmdignore`, then the dot-path rule — prints the paths that would be +indexed with counts of what each stage excluded, and exits. It takes no build lock, makes no provider +call, writes nothing, and does not even create `graph.db` for a corpus that has never been built. It +also reports how many already-indexed documents would be **removed** because their path is no longer +in the corpus. This is the way to check `.kgmdignore` patterns; see +[./configuration.md](./configuration.md) for the syntax. + +`--json` prints that same report as a single JSON object and nothing else. It requires `--dry-run` +and fails with `--json requires --dry-run.` on its own, because a `--json` that silently implied +`--dry-run` would mean `kgmd build --json` quietly not building. + +**Structured output**: with `--dry-run --json`, one object with `included`, `ignored`, and `dotpath` +arrays of corpus-relative paths, and a `counts` object holding `included`, `ignored`, `dotpath`, and +`would_remove`. Paths are sorted and always relative to the corpus root, matching the form stored in +`documents.path`. + +```bash +kgmd build --dry-run +``` + +```bash +kgmd build --dry-run --json +``` + ```bash kgmd build ``` @@ -169,7 +202,9 @@ Extract entities and relations from documents. Stage 3 of `build`, plus the ingest and embed steps it depends on. Under the build lock it ingests documents, embeds new chunks, extracts, then embeds new mentions, and prints `Extraction complete.` -It does **not** resolve duplicates or induce a schema. +It does **not** resolve duplicates or induce a schema. Because it ingests, it also drops documents +that are no longer part of the corpus — deleted, renamed, or newly excluded by `.kgmdignore` — +exactly as `build` does. By default a document is re-extracted only when its content hash differs from the hash recorded at its last extraction, so unchanged files cost nothing. `--force` ignores that check and re-extracts diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index f933f8d..4b92881 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -75,7 +75,7 @@ mapping and the second is the key inside it. | Key | Default | Accepted values | Effect | |---|---|---|---| -| `corpus.include` | unset (`null`) | list of paths relative to the corpus root, each a directory or an `.md` file | Restricts ingestion to those paths. Unset means every `.md` file under the corpus root. Paths whose components begin with a dot are always skipped, so `.kgmd/` never ingests itself. | +| `corpus.include` | unset (`null`) | list of paths relative to the corpus root, each a directory or an `.md` file | Restricts ingestion to those paths. Entries are literal paths, not globs: `notes/*.md` matches nothing and fails silently. Unset means every `.md` file under the corpus root. Paths whose components begin with a dot are always skipped, so `.kgmd/` never ingests itself. This key scopes the directory walk; [`.kgmdignore`](#the-kgmdignore-file) subtracts from whatever it selects. | | `embedding.backend` | `fastembed` | `fastembed`, `litellm` | Selects the embedder. `fastembed` runs the model locally and needs no credential; `litellm` routes embedding calls to a hosted provider. Any other value falls back to `fastembed`. | | `embedding.model` | `BAAI/bge-small-en-v1.5` | a model id the chosen backend understands | Model used to embed chunks and entity mentions. The id is recorded in the database at first build and is fixed for the life of the corpus; changing it later aborts the build. See the [maintenance guide](../guides/maintenance.md). | | `llm.model` | `openrouter/anthropic/claude-sonnet-4-5` | any litellm-routable model id | Model used for extraction, resolution cluster verification, and schema induction. | @@ -95,6 +95,91 @@ mapping and the second is the key inside it. | `induction.include_attribute_summary` | `true` | bool | **Accepted but currently has no effect.** No module reads the key; what the induced schema summarizes is decided entirely by the induction prompt. | | `induction.hierarchy_depth` | `3` | int >= 1 | Interpolated into the induction prompt as the maximum depth of the entity type hierarchy the model may produce. | +## The `.kgmdignore` file + +Exclusions are not a config key. They live in a `.kgmdignore` file at the **corpus root** — beside +your markdown, not inside `.kgmd/` — so the file can sit in version control with the notes it +describes. `kgmd init` writes a starter copy whose every line is a comment, which is why a fresh +corpus indexes exactly what it would have without one. + +This matters for cost, not tidiness. Every indexed file is chunked and each chunk is one model call, +so a vendored documentation tree or a folder of boilerplate templates is a repeated charge that buys +nothing and fills entity resolution with junk mentions. + +```text +# skip archived notes +archive/ +drafts/ + +# skip generated or boilerplate files +**/CHANGELOG.md +*-template.md + +# but keep this one +!archive/2024-decisions.md +``` + +A missing file means no exclusions, and behaves exactly as kgmd did before the file existed. A file +that is present but unreadable or not UTF-8 is an error rather than a silent skip, because treating +it as absent would index everything you meant to exclude. Only the corpus-root file is read; a +`.kgmdignore` in a subdirectory is ignored. The file is re-read on every run, so editing it takes +effect on the next `kgmd build` with no invalidation step. + +### Syntax + +Patterns match the path relative to the corpus root, always with `/` separators, and matching is +case-sensitive regardless of what the filesystem does. + +| Construct | Meaning | +|---|---| +| `# comment` | ignored, as are blank lines and surrounding whitespace | +| `*` | any run of characters, never crossing `/` | +| `?` | exactly one character, never `/` | +| `**` | any number of path segments; `a/**/b` also matches `a/b` | +| trailing `/` | directory-only: that directory and everything beneath it | +| leading `/` | anchored at the corpus root | +| any interior `/` | anchored at the corpus root | +| no `/` anywhere | matches at any depth | +| leading `!` | negation — re-includes a path an earlier rule excluded | + +A pattern without a trailing `/` matches a file with that path *and* anything beneath a directory +with that path, so `archive` and `archive/` differ only in whether a file literally named `archive` +matches. + +Character classes (`[a-z]`) and backslash escapes are **not** supported. A pattern using them is +matched literally, so it excludes nothing rather than excluding too much. + +### Precedence + +Fixed, and not configurable: + +1. `corpus.include` scopes the candidate set. +2. `.kgmdignore` rules are evaluated in file order and **the last matching rule wins**. +3. The dot-path rule is applied last and dominates. + +Each candidate file is tested against every rule, both on its own path and on each of its ancestor +directories — which is what lets a directory rule cover a whole subtree. A path with any +dot-prefixed component (`.kgmd/`, `.git/`, `.claude/`) is excluded after all rules have been +evaluated, and no pattern can re-admit it: `!.kgmd/notes.md` has no effect. + +### One difference from `.gitignore` + +git cannot re-include a file whose parent directory is excluded. `.kgmdignore` can. In the example +above, `archive/` excludes the directory and `!archive/2024-decisions.md` still brings that one file +back, because rules are evaluated per file rather than by skipping directories outright. If you are +carrying patterns over from a `.gitignore`, this is the one place the two disagree. + +### Seeing what a rule does + +`kgmd build --dry-run` prints the resolved file set and the counts excluded by `.kgmdignore` and by +the dot-path rule, without writing anything or calling a model. Use it before a build rather than +inferring rules from a bill. See the [CLI reference](./cli.md). + +A file that becomes excluded is also **removed** from an existing graph on the next +`kgmd build` or `kgmd extract`, along with everything derived from it. As a guard, ignore rules that +resolve to an empty file set abort the build instead of emptying the graph. Both behaviours are +described in the [maintenance guide](../guides/maintenance.md). + ## Full example A corpus `.kgmd/config.yaml` holding all nineteen keys at their defaults, which is what `kgmd init` diff --git a/kgmd/cli.py b/kgmd/cli.py index 47460ec..8025be4 100644 --- a/kgmd/cli.py +++ b/kgmd/cli.py @@ -12,7 +12,8 @@ from kgmd.config import load_config, write_default_config from kgmd.db import build_lock, get_connection, init_db -from kgmd.ingest import ingest_documents +from kgmd.ignore import IGNORE_FILENAME, write_default_ignore_file +from kgmd.ingest import dry_run_report, ingest_documents from kgmd.query import ( find_path, get_current_schema, @@ -48,6 +49,38 @@ def get_db_path(db: str | None, corpus_dir: Path | None = None) -> Path: return cd / ".kgmd" / "graph.db" +def _print_removals(ing_stats: dict) -> None: + """Report what left the graph, and stay silent when nothing did.""" + if not ing_stats.get("documents_removed"): + return + console.print( + f" Removed: {ing_stats['documents_removed']} document(s) no longer in the corpus" + f" — {ing_stats['chunks_removed']} chunk(s)," + f" {ing_stats['relations_removed']} relation(s)," + f" {ing_stats['entities_removed']} entity(ies)," + f" {ing_stats['vectors_removed']} vector(s)" + ) + + +def _print_dry_run(report: dict) -> None: + """Render the resolved file set for `kgmd build --dry-run`.""" + counts = report["counts"] + excluded = [] + if counts["ignored"]: + excluded.append(f"{counts['ignored']} excluded by {IGNORE_FILENAME}") + if counts["dotpath"]: + excluded.append(f"{counts['dotpath']} excluded as a dot-path") + suffix = f" ({', '.join(excluded)})" if excluded else "" + console.print(f"Resolved {counts['included']} file(s) to index{suffix}.") + for path in report["included"]: + console.print(f" {path}") + if counts["would_remove"]: + console.print( + f"Already indexed but no longer in the corpus:" + f" {counts['would_remove']} document(s) would be removed." + ) + + @click.group() @click.option("--debug", is_flag=True, help="Show full tracebacks on error.") @click.pass_context @@ -90,6 +123,11 @@ def init(path: str) -> None: cfg_path = kgmd_dir / "config.yaml" write_default_config(cfg_path) + # Starter ignore file at the corpus root. Every line is a comment, so a fresh + # corpus indexes exactly what it would have without it. + ignore_path = corpus_dir / IGNORE_FILENAME + write_default_ignore_file(ignore_path) + # Initialize database db_path = kgmd_dir / "graph.db" conn = init_db(db_path) @@ -98,6 +136,7 @@ def init(path: str) -> None: console.print(f"[green]Initialized kgmd corpus at[/green] [bold]{corpus_dir}[/bold]") console.print(f" Database: {db_path}") console.print(f" Config: {cfg_path}") + console.print(f" Ignore: {ignore_path}") @main.command() @@ -198,7 +237,19 @@ def stats(db: str | None, as_json: bool) -> None: @click.argument("path", default=".", type=click.Path(exists=True)) @click.option("--db", type=click.Path(), default=None, help="Path to graph.db") @click.option("--config", "config_path", type=click.Path(), default=None, help="Config file path") -def build(path: str, db: str | None, config_path: str | None) -> None: +@click.option( + "--dry-run", + is_flag=True, + help="Report the files that would be indexed, then exit without building.", +) +@click.option("--json", "as_json", is_flag=True, help="Output as JSON. Requires --dry-run.") +def build( + path: str, + db: str | None, + config_path: str | None, + dry_run: bool, + as_json: bool, +) -> None: """Build the knowledge graph: extract, resolve, induce.""" corpus_dir = Path(path).resolve() kgmd_dir = corpus_dir / ".kgmd" @@ -207,8 +258,27 @@ def build(path: str, db: str | None, config_path: str | None) -> None: f"Not a kgmd corpus (no .kgmd/ in {corpus_dir}). Run 'kgmd init' first." ) + if as_json and not dry_run: + raise click.ClickException("--json requires --dry-run.") + config = load_config(corpus_dir) db_path = get_db_path(db, corpus_dir) + + if dry_run: + # Read-only, and deliberately ahead of init_db: a dry run must not create + # a database for a corpus that has never been built. + conn = get_connection(db_path) if db_path.exists() else None + try: + report = dry_run_report(conn, corpus_dir, config) + finally: + if conn is not None: + conn.close() + if as_json: + click.echo(json.dumps(report, indent=2)) + else: + _print_dry_run(report) + return + conn = init_db(db_path) with build_lock(kgmd_dir): @@ -225,6 +295,7 @@ def build(path: str, db: str | None, config_path: str | None) -> None: f" New: {ing_stats['new']}, Updated: {ing_stats['updated']}, " f"Skipped: {ing_stats['skipped']}, Chunks: {ing_stats['chunks_created']}" ) + _print_removals(ing_stats) # Stage 2: Embed chunks console.print("[bold]Stage 2: Embedding chunks...[/bold]") @@ -287,6 +358,7 @@ def extract(path: str, db: str | None, force: bool) -> None: f" New: {ing_stats['new']}, Updated: {ing_stats['updated']}, " f"Skipped: {ing_stats['skipped']}" ) + _print_removals(ing_stats) # Embed chunks try: diff --git a/kgmd/ignore.py b/kgmd/ignore.py new file mode 100644 index 0000000..cb14dfa --- /dev/null +++ b/kgmd/ignore.py @@ -0,0 +1,184 @@ +"""Gitignore-style exclusion rules read from a corpus-root `.kgmdignore` file. + +Stdlib only, and imports nothing from kgmd: this is a leaf module. + +Patterns are compiled to fully anchored regexes at parse time and matched against +corpus-relative POSIX paths. The supported subset is documented in +docs/reference/configuration.md; the two deliberate departures from git are that a +negation *can* re-include a file inside an excluded directory, and that character +classes and backslash escapes are not supported (they match literally, so an +unsupported pattern excludes nothing rather than excluding too much). +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path + +IGNORE_FILENAME = ".kgmdignore" + +DEFAULT_IGNORE_TEMPLATE = """\ +# .kgmdignore — paths kgmd should not index. +# +# Every indexed file is chunked and sent to a language model, one call per chunk, +# so excluding material with no knowledge-graph value saves real money. +# +# Patterns match paths relative to this directory, always with '/' separators. +# Blank lines and lines starting with '#' are ignored. Rules are applied in order +# and the last matching rule wins. +# +# Nothing below is active — uncomment or add your own. +# +# Skip a whole directory and everything under it: +# archive/ +# drafts/ +# +# Skip files by glob. '*' does not cross '/', '**' spans any number of directories: +# **/CHANGELOG.md +# *-template.md +# +# Anchor a pattern to this directory with a leading '/': +# /README.md +# +# Re-include something an earlier rule excluded, with '!': +# !archive/2024-decisions.md +# +# Paths with a dot-prefixed component (.kgmd/, .git/) are always skipped and +# cannot be re-included with '!'. +""" + + +@dataclass(frozen=True) +class IgnoreRule: + """One parsed line of `.kgmdignore`.""" + + pattern: str + regex: re.Pattern[str] + negated: bool + dir_only: bool + line_number: int + + +def parse_ignore_rules(text: str) -> list[IgnoreRule]: + """Parse `.kgmdignore` text into ordered rules. + + Comments, blank lines, and degenerate rules yield nothing. Order is the file's + order and is significant: the last matching rule decides. + """ + rules: list[IgnoreRule] = [] + for line_number, raw in enumerate(text.splitlines(), start=1): + line = raw.strip() + if not line or line.startswith("#"): + continue + + negated = line.startswith("!") + if negated: + line = line[1:].strip() + + dir_only = line.endswith("/") + line = line.rstrip("/") + if not line: + continue + + rules.append( + IgnoreRule( + pattern=line, + regex=_compile(line), + negated=negated, + dir_only=dir_only, + line_number=line_number, + ) + ) + return rules + + +def load_ignore_rules(root: Path) -> list[IgnoreRule]: + """Read `/.kgmdignore`. Absent file yields no rules. + + A file that exists but cannot be read or decoded is an error: silently treating + it as absent would index everything the user meant to exclude. + """ + path = root / IGNORE_FILENAME + if not path.is_file(): + return [] + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise RuntimeError( + f"Could not read {IGNORE_FILENAME} at {path}: {exc}. " + f"It must be UTF-8 text. Fix or remove the file, then re-run." + ) from exc + return parse_ignore_rules(text) + + +def is_ignored(rel_path: str, rules: list[IgnoreRule]) -> bool: + """Whether a corpus-relative path is excluded by these rules. + + Each rule is tested against the path and against every ancestor directory of + the path, so a directory rule covers its whole subtree. The last matching rule + decides, which is what lets a negation re-include a file inside an excluded + directory. + """ + if not rules: + return False + + ancestors = _ancestors(rel_path) + ignored = False + for rule in rules: + targets = ancestors if rule.dir_only else (rel_path, *ancestors) + if any(rule.regex.match(target) for target in targets): + ignored = not rule.negated + return ignored + + +def write_default_ignore_file(path: Path) -> bool: + """Write the starter `.kgmdignore`, never clobbering an existing file. + + Returns True if the file was written. + """ + if path.exists(): + return False + path.write_text(DEFAULT_IGNORE_TEMPLATE, encoding="utf-8") + return True + + +def _ancestors(rel_path: str) -> tuple[str, ...]: + """Strict ancestor directories of a path, deepest first.""" + parts = rel_path.split("/")[:-1] + return tuple("/".join(parts[: i + 1]) for i in reversed(range(len(parts)))) + + +def _compile(pattern: str) -> re.Pattern[str]: + """Compile a pattern to a fully anchored regex over a '/'-separated path.""" + anchored = "/" in pattern + body = _translate(pattern.lstrip("/")) + prefix = "" if anchored else "(?:.*/)?" + return re.compile(rf"\A{prefix}{body}\Z") + + +def _translate(pattern: str) -> str: + parts = pattern.split("/") + out: list[str] = [] + for index, part in enumerate(parts): + is_last = index == len(parts) - 1 + if part == "**": + # Zero or more whole segments. As the final segment, anything at all. + out.append(".*" if is_last else "(?:[^/]+/)*") + continue + out.append(_translate_segment(part)) + if not is_last: + out.append("/") + return "".join(out) + + +def _translate_segment(segment: str) -> str: + out: list[str] = [] + for char in segment: + if char == "*": + out.append("[^/]*") + elif char == "?": + out.append("[^/]") + else: + out.append(re.escape(char)) + return "".join(out) diff --git a/kgmd/ingest.py b/kgmd/ingest.py index 6af7c32..579f51f 100644 --- a/kgmd/ingest.py +++ b/kgmd/ingest.py @@ -8,6 +8,8 @@ from datetime import datetime, timezone from pathlib import Path +from kgmd.ignore import is_ignored, load_ignore_rules + @dataclass class Chunk: @@ -128,6 +130,19 @@ def _chunk_fixed(text: str, max_chars: int, overlap_chars: int) -> list[Chunk]: return chunks +@dataclass(frozen=True) +class FileScan: + """What a corpus would index, and why everything else was left out. + + Every path is absolute and each list is sorted. The three lists are disjoint + and together they are the candidate set produced by ``corpus.include``. + """ + + included: list[Path] + ignored: list[Path] + dotpath: list[Path] + + def find_markdown_files(root: Path, include: list[str] | None = None) -> list[Path]: """Find .md files under root, optionally scoped to include paths. @@ -152,15 +167,185 @@ def _is_dotpath(filepath: Path, root: Path) -> bool: return any(part.startswith(".") for part in rel.parts) +def scan_corpus_files(root: Path, config: dict) -> FileScan: + """Resolve which markdown files the corpus would index. + + Precedence is fixed: ``corpus.include`` scopes the candidates, ``.kgmdignore`` + subtracts from them and a negation re-adds, and the dot-path rule dominates + both — no pattern, negated or otherwise, can re-admit a dotted path. + + This is the single source of the resolved file set: ingest and the dry-run + preview both read it, so a preview cannot disagree with what a build does. + """ + include = config.get("corpus", {}).get("include") + candidates = find_markdown_files(root, include=include) + rules = load_ignore_rules(root) + + included: list[Path] = [] + ignored: list[Path] = [] + dotpath: list[Path] = [] + for fpath in candidates: + if _is_dotpath(fpath, root): + dotpath.append(fpath) + elif is_ignored(fpath.relative_to(root).as_posix(), rules): + ignored.append(fpath) + else: + included.append(fpath) + + return FileScan(included=included, ignored=ignored, dotpath=dotpath) + + +def dry_run_report(conn, corpus_dir: Path, config: dict) -> dict: + """The resolved file set behind `kgmd build --dry-run`. Never writes. + + ``conn`` may be None when the corpus has no database yet, in which case + nothing can be pending removal. Paths are corpus-relative POSIX strings, the + same form stored in ``documents.path``. + """ + scan = scan_corpus_files(corpus_dir, config) + + def rel(paths: list[Path]) -> list[str]: + return [p.relative_to(corpus_dir).as_posix() for p in paths] + + included = rel(scan.included) + would_remove = 0 + if conn is not None: + indexed = {row[0] for row in conn.execute("SELECT path FROM documents").fetchall()} + would_remove = len(indexed - set(included)) + + return { + "included": included, + "ignored": rel(scan.ignored), + "dotpath": rel(scan.dotpath), + "counts": { + "included": len(scan.included), + "ignored": len(scan.ignored), + "dotpath": len(scan.dotpath), + "would_remove": would_remove, + }, + } + + +# SQLite caps host parameters per statement; bind ids in batches rather than +# creating a temp table, which would be DDL outside kgmd/schema.py. +_ID_BATCH = 500 + +REMOVAL_KEYS = ( + "documents_removed", + "chunks_removed", + "relations_removed", + "entities_removed", + "vectors_removed", +) + + +def prune_missing_documents(conn, kept_rel_paths: set[str]) -> dict: + """Remove indexed documents whose path is no longer part of the corpus. + + Newly ignored, deleted, and renamed files are the same state — the recorded + path is not in ``kept_rel_paths`` — and are handled identically. + + Cascades do not cover everything. ``entity_mentions`` follows its chunks, but + ``relations.evidence_chunk_id`` is ON DELETE SET NULL and the sqlite-vec tables + have no foreign keys at all, so relations bound to removed evidence and both + vector tables are deleted explicitly. Leaving a vector behind would be worse + than untidy: chunk and mention ids are reused, and ``embed_new_chunks`` skips + any chunk that already has a vector row, so a future chunk would silently + inherit the vector of deleted text. + """ + stats = dict.fromkeys(REMOVAL_KEYS, 0) + + rows = conn.execute("SELECT id, path FROM documents").fetchall() + if not rows: + return stats + + orphan_ids = [row["id"] for row in rows if row["path"] not in kept_rel_paths] + if not orphan_ids: + return stats + + if not kept_rel_paths: + raise RuntimeError( + "Ignore rules exclude every markdown file in the corpus, but the graph still holds " + f"{len(rows)} document(s). Refusing to empty it. Check .kgmdignore — " + f"run 'kgmd build --dry-run' to see what would be indexed — or use " + f"'kgmd reset --hard' if clearing the graph is what you meant." + ) + + # Collect ids before deleting anything: mention ids are unrecoverable once + # their chunks are gone. + chunk_ids = _ids_in(conn, "SELECT id FROM chunks WHERE document_id IN", orphan_ids) + mention_ids = _ids_in(conn, "SELECT id FROM entity_mentions WHERE chunk_id IN", chunk_ids) + entity_ids = _ids_in( + conn, "SELECT DISTINCT entity_id FROM entity_mentions WHERE chunk_id IN", chunk_ids + ) + + stats["vectors_removed"] += _delete_in( + conn, "DELETE FROM vec_entity_mentions WHERE mention_id IN", mention_ids + ) + stats["vectors_removed"] += _delete_in( + conn, "DELETE FROM vec_chunks WHERE chunk_id IN", chunk_ids + ) + stats["relations_removed"] = _delete_in( + conn, "DELETE FROM relations WHERE evidence_chunk_id IN", chunk_ids + ) + stats["chunks_removed"] = _delete_in( + conn, "DELETE FROM chunks WHERE document_id IN", orphan_ids + ) + stats["documents_removed"] = _delete_in(conn, "DELETE FROM documents WHERE id IN", orphan_ids) + stats["entities_removed"] = _sweep_orphan_entities(conn, entity_ids) + + conn.commit() + return stats + + +def _sweep_orphan_entities(conn, entity_ids: list[int]) -> int: + """Delete entities that this prune left with no mention and no relation. + + Scoped to the entities whose mentions were just removed, so pre-existing + orphans left behind by 'kgmd extract --force' are not collected here. + """ + return _delete_in( + conn, + "DELETE FROM entities WHERE id NOT IN (SELECT entity_id FROM entity_mentions)" + " AND id NOT IN (SELECT subject_id FROM relations)" + " AND id NOT IN (SELECT object_id FROM relations)" + " AND id IN", + entity_ids, + ) + + +def _batches(ids: list[int]): + for start in range(0, len(ids), _ID_BATCH): + yield ids[start : start + _ID_BATCH] + + +def _ids_in(conn, select_prefix: str, ids: list[int]) -> list[int]: + """Run a `... IN (ids)` select over batched ids and collect the first column.""" + out: list[int] = [] + for batch in _batches(ids): + placeholders = ",".join("?" * len(batch)) + rows = conn.execute(f"{select_prefix} ({placeholders})", batch).fetchall() + out.extend(row[0] for row in rows) + return out + + +def _delete_in(conn, delete_prefix: str, ids: list[int]) -> int: + """Run a `... IN (ids)` delete over batched ids and total the affected rows.""" + removed = 0 + for batch in _batches(ids): + placeholders = ",".join("?" * len(batch)) + cur = conn.execute(f"{delete_prefix} ({placeholders})", batch) + removed += cur.rowcount + return removed + + def ingest_documents(conn, corpus_dir: Path, config: dict) -> dict: """Ingest markdown files: hash-check, upsert documents, chunk. Returns a summary dict with counts. """ - include = config.get("corpus", {}).get("include") - md_files = find_markdown_files(corpus_dir, include=include) - # Exclude dotfile directories (.kgmd, .claude, .git, etc.) - md_files = [f for f in md_files if not _is_dotpath(f, corpus_dir)] + scan = scan_corpus_files(corpus_dir, config) + md_files = scan.included chunking = config.get("chunking", {}) max_chars = chunking.get("max_chars", 4000) @@ -168,6 +353,9 @@ def ingest_documents(conn, corpus_dir: Path, config: dict) -> dict: split_on = chunking.get("split_on", "paragraph") stats = {"new": 0, "updated": 0, "skipped": 0, "chunks_created": 0} + stats.update( + prune_missing_documents(conn, {p.relative_to(corpus_dir).as_posix() for p in scan.included}) + ) now = datetime.now(timezone.utc).isoformat() for fpath in md_files: diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..3d1ff1d --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,74 @@ +"""Tests for CLI-level guarantees that cannot be observed from the library. + +Most command behavior is tested through the function that backs it. Two promises +only exist at the CLI boundary: that `kgmd build --dry-run` creates no database, +and that `--json` is refused without `--dry-run`. +""" + +from __future__ import annotations + +import json + +from click.testing import CliRunner + +from kgmd.cli import main + + +def test_dry_run_creates_no_database(tmp_corpus): + """FR-017: a dry run must not create graph.db for a never-built corpus.""" + kgmd_dir = tmp_corpus / ".kgmd" + kgmd_dir.mkdir() + db_path = kgmd_dir / "graph.db" + assert not db_path.exists() + + result = CliRunner().invoke(main, ["build", str(tmp_corpus), "--dry-run"]) + + assert result.exit_code == 0, result.output + assert not db_path.exists(), "a dry run created a database" + assert "Resolved" in result.output + + +def test_dry_run_json_is_the_only_output(tmp_corpus): + (tmp_corpus / ".kgmd").mkdir() + (tmp_corpus / "archive").mkdir() + (tmp_corpus / "archive" / "old.md").write_text("# old\n") + (tmp_corpus / ".kgmdignore").write_text("archive/\n") + + result = CliRunner().invoke(main, ["build", str(tmp_corpus), "--dry-run", "--json"]) + + assert result.exit_code == 0, result.output + report = json.loads(result.output) + assert report["ignored"] == ["archive/old.md"] + assert report["counts"]["would_remove"] == 0 + + +def test_json_without_dry_run_is_refused(tmp_corpus): + """--json implying --dry-run would make `kgmd build --json` silently not build.""" + (tmp_corpus / ".kgmd").mkdir() + + result = CliRunner().invoke(main, ["build", str(tmp_corpus), "--json"]) + + assert result.exit_code != 0 + assert "--json requires --dry-run." in result.output + + +def test_init_writes_a_starter_ignore_file(tmp_path): + corpus = tmp_path / "fresh" + corpus.mkdir() + + result = CliRunner().invoke(main, ["init", "--path", str(corpus)]) + + assert result.exit_code == 0, result.output + ignore_path = corpus / ".kgmdignore" + assert ignore_path.is_file() + assert ignore_path.read_text().lstrip().startswith("#") + + +def test_init_never_clobbers_an_existing_ignore_file(tmp_path): + corpus = tmp_path / "fresh" + corpus.mkdir() + (corpus / ".kgmdignore").write_text("archive/\n") + + CliRunner().invoke(main, ["init", "--path", str(corpus)]) + + assert (corpus / ".kgmdignore").read_text() == "archive/\n" diff --git a/tests/test_ignore.py b/tests/test_ignore.py new file mode 100644 index 0000000..58db723 --- /dev/null +++ b/tests/test_ignore.py @@ -0,0 +1,202 @@ +"""Tests for .kgmdignore parsing and matching semantics. + +Pure unit tests: no filesystem, no database, no model. The contract these assert is +docs/reference/configuration.md, and every case here is a promise to users who have +written a .kgmdignore. +""" + +from __future__ import annotations + +import pytest + +from kgmd.ignore import ( + DEFAULT_IGNORE_TEMPLATE, + is_ignored, + parse_ignore_rules, +) + + +def ignored(text: str, path: str) -> bool: + return is_ignored(path, parse_ignore_rules(text)) + + +# -------------------------------------------------------------------------------------- +# Parsing +# -------------------------------------------------------------------------------------- + + +def test_comments_and_blank_lines_yield_no_rules(): + text = "# a comment\n\n \n\t\n# another\n" + assert parse_ignore_rules(text) == [] + + +def test_trailing_and_leading_whitespace_is_stripped(): + (rule,) = parse_ignore_rules(" archive/ \n") + assert rule.pattern == "archive" + assert rule.dir_only is True + + +def test_bang_sets_negation_and_slash_sets_dir_only(): + negated, plain = parse_ignore_rules("!archive/2024.md\nnotes/\n") + assert (negated.negated, negated.dir_only) == (True, False) + assert (plain.negated, plain.dir_only) == (False, True) + + +def test_degenerate_lines_yield_no_rules(): + assert parse_ignore_rules("!\n/\n!/\n//\n") == [] + + +def test_line_numbers_are_one_based_and_track_the_source_file(): + rules = parse_ignore_rules("# comment\n\narchive/\n\ndrafts/\n") + assert [r.line_number for r in rules] == [3, 5] + + +def test_rule_order_is_preserved(): + rules = parse_ignore_rules("b\na\nc\n") + assert [r.pattern for r in rules] == ["b", "a", "c"] + + +def test_starter_template_parses_to_nothing(): + assert parse_ignore_rules(DEFAULT_IGNORE_TEMPLATE) == [] + + +# -------------------------------------------------------------------------------------- +# Matching: globs +# -------------------------------------------------------------------------------------- + + +def test_no_rules_never_ignores(): + assert is_ignored("anything/at/all.md", []) is False + + +def test_star_does_not_cross_a_separator(): + # The reason fnmatch is unusable here: its '*' would match 'notes/deep'. + assert ignored("/notes/*.md", "notes/a.md") is True + assert ignored("/notes/*.md", "notes/deep/a.md") is False + + +def test_question_mark_matches_exactly_one_non_separator_character(): + assert ignored("/a?.md", "ab.md") is True + assert ignored("/a?.md", "abc.md") is False + assert ignored("/a?.md", "a/b.md") is False + + +@pytest.mark.parametrize( + "path,expected", + [ + ("CHANGELOG.md", True), + ("docs/CHANGELOG.md", True), + ("a/b/c/CHANGELOG.md", True), + ("docs/CHANGELOG.txt", False), + ], +) +def test_double_star_spans_any_number_of_segments_including_none(path, expected): + assert ignored("**/CHANGELOG.md", path) is expected + + +def test_double_star_in_the_middle_spans_several_segments(): + text = "a/**/b.md" + assert ignored(text, "a/b.md") is True + assert ignored(text, "a/x/b.md") is True + assert ignored(text, "a/x/y/z/b.md") is True + assert ignored(text, "other/b.md") is False + + +def test_trailing_double_star_covers_the_subtree(): + assert ignored("archive/**", "archive/deep/old.md") is True + assert ignored("archive/**", "notes/old.md") is False + + +# -------------------------------------------------------------------------------------- +# Matching: anchoring +# -------------------------------------------------------------------------------------- + + +def test_leading_slash_anchors_at_the_corpus_root(): + assert ignored("/README.md", "README.md") is True + assert ignored("/README.md", "notes/README.md") is False + + +def test_interior_slash_anchors_at_the_corpus_root(): + assert ignored("notes/archive/", "notes/archive/a.md") is True + assert ignored("notes/archive/", "other/notes/archive/a.md") is False + + +def test_pattern_without_a_slash_matches_at_any_depth(): + assert ignored("*-template.md", "msa-template.md") is True + assert ignored("*-template.md", "notes/deep/msa-template.md") is True + assert ignored("*-template.md", "notes/msa.md") is False + + +def test_bare_star_matches_everything(): + # The destructive case the empty-resolved-set guard exists for. + assert ignored("*", "a.md") is True + assert ignored("*", "notes/deep/a.md") is True + + +# -------------------------------------------------------------------------------------- +# Matching: directories +# -------------------------------------------------------------------------------------- + + +def test_directory_rule_covers_the_whole_subtree(): + assert ignored("archive/", "archive/old.md") is True + assert ignored("archive/", "archive/deep/old.md") is True + assert ignored("archive/", "notes/old.md") is False + + +def test_directory_rule_matches_a_directory_at_any_depth(): + assert ignored("archive/", "notes/archive/old.md") is True + + +def test_directory_only_rule_does_not_match_a_file_of_that_name(): + assert ignored("archive/", "archive") is False + assert ignored("archive", "archive") is True + + +def test_rule_without_trailing_slash_still_covers_a_subtree(): + assert ignored("archive", "archive/old.md") is True + + +# -------------------------------------------------------------------------------------- +# Matching: ordering and negation +# -------------------------------------------------------------------------------------- + + +def test_last_matching_rule_wins(): + assert ignored("*.md\n!keep.md\n", "keep.md") is False + assert ignored("!keep.md\n*.md\n", "keep.md") is True + + +def test_negation_re_includes_a_file_inside_an_excluded_directory(): + """The deliberate divergence from git, which refuses this.""" + text = "archive/\n!archive/2024-decisions.md\n" + assert ignored(text, "archive/2024-decisions.md") is False + assert ignored(text, "archive/old.md") is True + + +def test_negation_can_re_include_a_whole_directory(): + text = "archive/\n!archive/keep/\n" + assert ignored(text, "archive/keep/a.md") is False + assert ignored(text, "archive/other/a.md") is True + + +def test_negation_alone_ignores_nothing(): + assert ignored("!keep.md\n", "keep.md") is False + assert ignored("!keep.md\n", "other.md") is False + + +# -------------------------------------------------------------------------------------- +# Unsupported constructs fail toward inclusion +# -------------------------------------------------------------------------------------- + + +def test_character_class_is_treated_literally_and_excludes_nothing(): + assert ignored("[a-z].md", "a.md") is False + assert ignored("[a-z].md", "[a-z].md") is True + + +def test_regex_metacharacters_in_a_pattern_are_escaped(): + assert ignored("a+b.md", "aab.md") is False + assert ignored("a+b.md", "a+b.md") is True + assert ignored("notes.md", "notesXmd") is False diff --git a/tests/test_ingest.py b/tests/test_ingest.py new file mode 100644 index 0000000..1c6ddf9 --- /dev/null +++ b/tests/test_ingest.py @@ -0,0 +1,628 @@ +"""Tests for corpus file discovery, .kgmdignore exclusion, and orphan pruning.""" + +from __future__ import annotations + +import struct +from datetime import datetime, timezone + +import pytest + +from kgmd.ingest import ( + dry_run_report, + ingest_documents, + prune_missing_documents, + scan_corpus_files, +) + + +def _rel(corpus, paths): + return sorted(p.relative_to(corpus).as_posix() for p in paths) + + +# -------------------------------------------------------------------------------------- +# Discovery: the resolved file set (Foundational) +# -------------------------------------------------------------------------------------- + + +def test_scan_matches_plain_rglob_when_no_ignore_file(tmp_corpus): + """With no .kgmdignore, the resolved set is every .md file minus dot-paths.""" + expected = sorted( + p.relative_to(tmp_corpus).as_posix() + for p in tmp_corpus.rglob("*.md") + if not any(part.startswith(".") for part in p.relative_to(tmp_corpus).parts) + ) + + scan = scan_corpus_files(tmp_corpus, {}) + + assert _rel(tmp_corpus, scan.included) == expected + assert scan.ignored == [] + assert expected, "fixture corpus should contain markdown files" + + +def test_scan_groups_are_disjoint(initialized_corpus): + """A path appears in exactly one group, and .kgmd/ lands in dotpath.""" + (initialized_corpus / ".kgmd" / "notes.md").write_text("# hidden\n") + (initialized_corpus / "archive").mkdir() + (initialized_corpus / "archive" / "old.md").write_text("# old\n") + (initialized_corpus / ".kgmdignore").write_text("archive/\n") + + scan = scan_corpus_files(initialized_corpus, {}) + + groups = [set(scan.included), set(scan.ignored), set(scan.dotpath)] + for i, left in enumerate(groups): + for right in groups[i + 1 :]: + assert not left & right + + assert _rel(initialized_corpus, scan.ignored) == ["archive/old.md"] + assert ".kgmd/notes.md" in _rel(initialized_corpus, scan.dotpath) + + +def test_scan_honours_corpus_include(tmp_corpus): + """corpus.include still scopes the candidate set to literal paths.""" + (tmp_corpus / "notes").mkdir() + (tmp_corpus / "notes" / "a.md").write_text("# a\n") + + scan = scan_corpus_files(tmp_corpus, {"corpus": {"include": ["notes"]}}) + + assert _rel(tmp_corpus, scan.included) == ["notes/a.md"] + + +# -------------------------------------------------------------------------------------- +# Discovery: .kgmdignore interaction (US1) +# -------------------------------------------------------------------------------------- + + +def _corpus_with_worked_example(root): + for name in ("notes", "archive", "drafts", "docs"): + (root / name).mkdir() + (root / "notes" / "today.md").write_text("# today\n") + (root / "notes" / "msa-template.md").write_text("# template\n") + (root / "archive" / "old.md").write_text("# old\n") + (root / "archive" / "2024-decisions.md").write_text("# decisions\n") + (root / "drafts" / "idea.md").write_text("# idea\n") + (root / "docs" / "CHANGELOG.md").write_text("# changelog\n") + (root / "CHANGELOG.md").write_text("# root changelog\n") + (root / ".kgmdignore").write_text( + "# skip archived notes\n" + "archive/\n" + "drafts/\n" + "\n" + "# skip generated or boilerplate files\n" + "**/CHANGELOG.md\n" + "*-template.md\n" + "\n" + "# but keep this one\n" + "!archive/2024-decisions.md\n" + ) + + +def test_worked_example_resolves_as_documented(tmp_path): + """The example from contracts/kgmdignore-format.md, asserted path by path.""" + corpus = tmp_path / "corpus" + corpus.mkdir() + _corpus_with_worked_example(corpus) + + scan = scan_corpus_files(corpus, {}) + + assert _rel(corpus, scan.included) == [ + "archive/2024-decisions.md", + "notes/today.md", + ] + assert _rel(corpus, scan.ignored) == [ + "CHANGELOG.md", + "archive/old.md", + "docs/CHANGELOG.md", + "drafts/idea.md", + "notes/msa-template.md", + ] + + +def test_ignore_subtracts_from_corpus_include(tmp_path): + """Both mechanisms present: include scopes, ignore subtracts from it.""" + corpus = tmp_path / "corpus" + (corpus / "notes" / "archive").mkdir(parents=True) + (corpus / "notes" / "keep.md").write_text("# keep\n") + (corpus / "notes" / "archive" / "drop.md").write_text("# drop\n") + (corpus / "outside.md").write_text("# outside\n") + (corpus / ".kgmdignore").write_text("notes/archive/\n") + + scan = scan_corpus_files(corpus, {"corpus": {"include": ["notes"]}}) + + assert _rel(corpus, scan.included) == ["notes/keep.md"] + assert _rel(corpus, scan.ignored) == ["notes/archive/drop.md"] + # outside.md was never a candidate; it is not reported as ignored. + assert "outside.md" not in _rel(corpus, scan.ignored) + + +def test_negation_cannot_readmit_a_dotpath(initialized_corpus): + """The dot-path rule is applied last and is not overridable (FR-008).""" + (initialized_corpus / ".kgmd" / "notes.md").write_text("# hidden\n") + (initialized_corpus / ".kgmdignore").write_text("!.kgmd/notes.md\n") + + scan = scan_corpus_files(initialized_corpus, {}) + + assert ".kgmd/notes.md" not in _rel(initialized_corpus, scan.included) + assert ".kgmd/notes.md" in _rel(initialized_corpus, scan.dotpath) + + +def test_comments_only_ignore_file_excludes_nothing(tmp_corpus): + """An empty or all-comment file behaves exactly like no file at all.""" + baseline = scan_corpus_files(tmp_corpus, {}) + (tmp_corpus / ".kgmdignore").write_text("# just a comment\n\n \n") + + scan = scan_corpus_files(tmp_corpus, {}) + + assert scan.included == baseline.included + assert scan.ignored == [] + + +def test_starter_template_excludes_nothing(tmp_corpus): + """kgmd init's starter file must not change what a fresh corpus indexes (FR-019).""" + from kgmd.ignore import DEFAULT_IGNORE_TEMPLATE, parse_ignore_rules + + assert parse_ignore_rules(DEFAULT_IGNORE_TEMPLATE) == [] + + baseline = scan_corpus_files(tmp_corpus, {}) + (tmp_corpus / ".kgmdignore").write_text(DEFAULT_IGNORE_TEMPLATE) + + assert scan_corpus_files(tmp_corpus, {}).included == baseline.included + + +def test_undecodable_ignore_file_fails_loudly(tmp_corpus): + """A present-but-unreadable file must not be silently treated as absent (FR-006).""" + (tmp_corpus / ".kgmdignore").write_bytes(b"archive/\n\xff\xfe invalid \x80\n") + + with pytest.raises(RuntimeError, match=r"\.kgmdignore"): + scan_corpus_files(tmp_corpus, {}) + + +# -------------------------------------------------------------------------------------- +# Pruning helpers (US2) +# -------------------------------------------------------------------------------------- + + +def _seed_document(conn, path, *, chunk_count=1): + """Insert a document with chunks, one mention per chunk, and hand-packed vectors.""" + now = datetime.now(timezone.utc).isoformat() + cur = conn.execute( + "INSERT INTO documents (path, content_hash, size_bytes, mtime, ingested_at," + " last_extracted_hash) VALUES (?, ?, ?, ?, ?, ?)", + (path, f"hash-{path}", 10, 0.0, now, f"hash-{path}"), + ) + doc_id = cur.lastrowid + chunk_ids = [] + for index in range(chunk_count): + cur = conn.execute( + "INSERT INTO chunks (document_id, chunk_index, content, char_start, char_end," + " token_count) VALUES (?, ?, ?, ?, ?, ?)", + (doc_id, index, f"text of {path} #{index}", 0, 10, 3), + ) + chunk_ids.append(cur.lastrowid) + return doc_id, chunk_ids + + +def _vector(dim=384, fill=0.1): + return struct.pack(f"{dim}f", *([fill] * dim)) + + +def _seed_extraction_run(conn): + now = datetime.now(timezone.utc).isoformat() + cur = conn.execute( + "INSERT INTO extraction_runs (started_at, model, status) VALUES (?, ?, ?)", + (now, "test-model", "completed"), + ) + return cur.lastrowid + + +def _seed_entity(conn, name, entity_type="Person"): + now = datetime.now(timezone.utc).isoformat() + cur = conn.execute( + "INSERT INTO entities (canonical_name, entity_type, created_at, updated_at)" + " VALUES (?, ?, ?, ?)", + (name, entity_type, now, now), + ) + return cur.lastrowid + + +def _seed_mention(conn, entity_id, chunk_id, run_id, surface="Someone"): + cur = conn.execute( + "INSERT INTO entity_mentions (entity_id, surface_form, chunk_id, extraction_run_id)" + " VALUES (?, ?, ?, ?)", + (entity_id, surface, chunk_id, run_id), + ) + return cur.lastrowid + + +def _seed_relation(conn, subject_id, object_id, run_id, evidence_chunk_id, predicate="knows"): + now = datetime.now(timezone.utc).isoformat() + cur = conn.execute( + "INSERT INTO relations (subject_id, predicate, object_id, evidence_chunk_id," + " extraction_run_id, created_at) VALUES (?, ?, ?, ?, ?, ?)", + (subject_id, predicate, object_id, evidence_chunk_id, run_id, now), + ) + return cur.lastrowid + + +def _counts(conn): + tables = ( + "documents", + "chunks", + "entity_mentions", + "relations", + "entities", + "vec_chunks", + "vec_entity_mentions", + ) + return {t: conn.execute(f"SELECT count(*) FROM {t}").fetchone()[0] for t in tables} + + +@pytest.fixture +def graph_with_two_documents(db_conn): + """Two documents, each with a chunk, a mention, a vector, and a relation. + + Returns the ids needed to assert that removing one leaves the other intact. + """ + conn = db_conn + run_id = _seed_extraction_run(conn) + + doomed_id, doomed_chunks = _seed_document(conn, "archive/old.md") + kept_id, kept_chunks = _seed_document(conn, "notes/today.md") + + only_in_doomed = _seed_entity(conn, "Ghost") + in_both = _seed_entity(conn, "Sarah Chen") + + doomed_mention = _seed_mention(conn, only_in_doomed, doomed_chunks[0], run_id) + shared_mention = _seed_mention(conn, in_both, doomed_chunks[0], run_id) + kept_mention = _seed_mention(conn, in_both, kept_chunks[0], run_id) + + doomed_relation = _seed_relation( + conn, only_in_doomed, in_both, run_id, doomed_chunks[0], predicate="worked_with" + ) + kept_relation = _seed_relation( + conn, in_both, in_both, run_id, kept_chunks[0], predicate="self_ref" + ) + + for chunk_id in doomed_chunks + kept_chunks: + conn.execute( + "INSERT INTO vec_chunks (chunk_id, embedding) VALUES (?, ?)", + (chunk_id, _vector()), + ) + for mention_id in (doomed_mention, shared_mention, kept_mention): + conn.execute( + "INSERT INTO vec_entity_mentions (mention_id, embedding) VALUES (?, ?)", + (mention_id, _vector()), + ) + conn.commit() + + return { + "conn": conn, + "doomed_doc": doomed_id, + "kept_doc": kept_id, + "doomed_chunk": doomed_chunks[0], + "kept_chunk": kept_chunks[0], + "only_in_doomed": only_in_doomed, + "in_both": in_both, + "doomed_mention": doomed_mention, + "kept_mention": kept_mention, + "doomed_relation": doomed_relation, + "kept_relation": kept_relation, + } + + +# -------------------------------------------------------------------------------------- +# Pruning (US2) +# -------------------------------------------------------------------------------------- + + +def test_prune_removes_the_document_and_its_derived_rows(graph_with_two_documents): + g = graph_with_two_documents + conn = g["conn"] + + stats = prune_missing_documents(conn, {"notes/today.md"}) + + assert stats["documents_removed"] == 1 + assert stats["chunks_removed"] == 1 + + paths = [r[0] for r in conn.execute("SELECT path FROM documents").fetchall()] + assert paths == ["notes/today.md"] + assert ( + conn.execute( + "SELECT count(*) FROM chunks WHERE document_id = ?", (g["doomed_doc"],) + ).fetchone()[0] + == 0 + ) + assert ( + conn.execute( + "SELECT count(*) FROM entity_mentions WHERE chunk_id = ?", (g["doomed_chunk"],) + ).fetchone()[0] + == 0 + ) + + # The surviving document keeps everything of its own. + assert ( + conn.execute( + "SELECT count(*) FROM chunks WHERE document_id = ?", (g["kept_doc"],) + ).fetchone()[0] + == 1 + ) + assert ( + conn.execute( + "SELECT count(*) FROM entity_mentions WHERE id = ?", (g["kept_mention"],) + ).fetchone()[0] + == 1 + ) + + +def test_prune_deletes_relations_whose_evidence_is_gone(graph_with_two_documents): + """relations.evidence_chunk_id is ON DELETE SET NULL, so this must be explicit.""" + g = graph_with_two_documents + conn = g["conn"] + + stats = prune_missing_documents(conn, {"notes/today.md"}) + + assert stats["relations_removed"] == 1 + surviving = conn.execute("SELECT id, evidence_chunk_id FROM relations").fetchall() + assert [r[0] for r in surviving] == [g["kept_relation"]] + assert all(r[1] is not None for r in surviving), "no relation may survive with null evidence" + + +def test_prune_clears_stale_vectors(graph_with_two_documents): + """A leftover vector row would bind to a reused id and poison later search. + + embed_new_chunks selects chunks WHERE id NOT IN (SELECT chunk_id FROM vec_chunks), + and chunks.id is a plain INTEGER PRIMARY KEY, so ids are reused after deletes. + """ + g = graph_with_two_documents + conn = g["conn"] + + stats = prune_missing_documents(conn, {"notes/today.md"}) + + assert ( + conn.execute( + "SELECT count(*) FROM vec_chunks WHERE chunk_id = ?", (g["doomed_chunk"],) + ).fetchone()[0] + == 0 + ) + assert ( + conn.execute( + "SELECT count(*) FROM vec_entity_mentions WHERE mention_id = ?", (g["doomed_mention"],) + ).fetchone()[0] + == 0 + ) + + # The survivors' vectors are untouched. + assert ( + conn.execute( + "SELECT count(*) FROM vec_chunks WHERE chunk_id = ?", (g["kept_chunk"],) + ).fetchone()[0] + == 1 + ) + assert ( + conn.execute( + "SELECT count(*) FROM vec_entity_mentions WHERE mention_id = ?", (g["kept_mention"],) + ).fetchone()[0] + == 1 + ) + + # No vector row may outlive the row it describes. + assert ( + conn.execute( + "SELECT count(*) FROM vec_chunks WHERE chunk_id NOT IN (SELECT id FROM chunks)" + ).fetchone()[0] + == 0 + ) + # One chunk vector plus the two mention vectors on that chunk. + assert stats["vectors_removed"] == 3 + + +def test_prune_sweeps_only_entities_left_with_nothing(graph_with_two_documents): + """An entity extracted only from a removed note must stop answering queries.""" + g = graph_with_two_documents + conn = g["conn"] + + stats = prune_missing_documents(conn, {"notes/today.md"}) + + names = [r[0] for r in conn.execute("SELECT canonical_name FROM entities").fetchall()] + assert names == ["Sarah Chen"], "the entity mentioned only in the removed note should be gone" + assert stats["entities_removed"] == 1 + + +def test_prune_keeps_an_entity_that_still_holds_a_relation(db_conn): + """No mentions left but still party to a relation: not an orphan, so not swept.""" + conn = db_conn + run_id = _seed_extraction_run(conn) + _, chunks = _seed_document(conn, "archive/old.md") + _, kept_chunks = _seed_document(conn, "notes/today.md") + + lonely = _seed_entity(conn, "Lonely") + anchor = _seed_entity(conn, "Anchor") + _seed_mention(conn, lonely, chunks[0], run_id) + _seed_mention(conn, anchor, kept_chunks[0], run_id) + # Evidence lives in the surviving document, so this relation is not removed. + _seed_relation(conn, lonely, anchor, run_id, kept_chunks[0]) + conn.commit() + + prune_missing_documents(conn, {"notes/today.md"}) + + names = sorted(r[0] for r in conn.execute("SELECT canonical_name FROM entities").fetchall()) + assert names == ["Anchor", "Lonely"] + + +def test_prune_is_idempotent(graph_with_two_documents): + """Re-running over an unchanged corpus removes nothing and reports nothing.""" + g = graph_with_two_documents + conn = g["conn"] + + prune_missing_documents(conn, {"notes/today.md"}) + before = _counts(conn) + watermark = conn.execute( + "SELECT last_extracted_hash FROM documents WHERE path = 'notes/today.md'" + ).fetchone()[0] + + stats = prune_missing_documents(conn, {"notes/today.md"}) + + assert all(value == 0 for value in stats.values()), stats + assert _counts(conn) == before + assert ( + conn.execute( + "SELECT last_extracted_hash FROM documents WHERE path = 'notes/today.md'" + ).fetchone()[0] + == watermark + ), "pruning must not disturb the extraction watermark" + + +def test_prune_refuses_to_empty_the_graph(graph_with_two_documents): + """A stray '*' pattern must not silently delete an entire corpus (FR-015).""" + g = graph_with_two_documents + conn = g["conn"] + before = _counts(conn) + + with pytest.raises(RuntimeError, match="exclude every"): + prune_missing_documents(conn, set()) + + assert _counts(conn) == before, "the guard must fire before any write" + + +def test_prune_on_an_empty_graph_is_a_no_op(db_conn): + """The guard is about protecting existing work, not about refusing empty corpora.""" + stats = prune_missing_documents(db_conn, set()) + assert stats["documents_removed"] == 0 + + +# -------------------------------------------------------------------------------------- +# Pruning through ingest_documents: deleted, renamed, and newly-ignored files (US2) +# -------------------------------------------------------------------------------------- + + +def test_ingest_removes_a_deleted_file(initialized_corpus, db_conn): + (initialized_corpus / "extra.md").write_text("# extra\n\nSome text.\n") + ingest_documents(db_conn, initialized_corpus, {}) + assert _indexed_paths(db_conn) and "extra.md" in _indexed_paths(db_conn) + + (initialized_corpus / "extra.md").unlink() + stats = ingest_documents(db_conn, initialized_corpus, {}) + + assert stats["documents_removed"] == 1 + assert "extra.md" not in _indexed_paths(db_conn) + + +def test_ingest_treats_a_rename_as_one_removal_and_one_insert(initialized_corpus, db_conn): + (initialized_corpus / "before.md").write_text("# note\n\nStable text.\n") + ingest_documents(db_conn, initialized_corpus, {}) + + (initialized_corpus / "before.md").rename(initialized_corpus / "after.md") + stats = ingest_documents(db_conn, initialized_corpus, {}) + + assert stats["documents_removed"] == 1 + assert stats["new"] == 1 + paths = _indexed_paths(db_conn) + assert "after.md" in paths and "before.md" not in paths + + +def test_ingest_removes_a_newly_ignored_file(initialized_corpus, db_conn): + """The case that makes .kgmdignore work for a corpus that already has a graph.""" + (initialized_corpus / "archive").mkdir() + (initialized_corpus / "archive" / "old.md").write_text("# old\n\nSuperseded.\n") + ingest_documents(db_conn, initialized_corpus, {}) + assert "archive/old.md" in _indexed_paths(db_conn) + + (initialized_corpus / ".kgmdignore").write_text("archive/\n") + stats = ingest_documents(db_conn, initialized_corpus, {}) + + assert stats["documents_removed"] == 1 + assert "archive/old.md" not in _indexed_paths(db_conn) + assert conn_has_no_orphan_chunks(db_conn) + + +def test_ingest_is_idempotent_after_a_removal(initialized_corpus, db_conn): + (initialized_corpus / "archive").mkdir() + (initialized_corpus / "archive" / "old.md").write_text("# old\n\nSuperseded.\n") + ingest_documents(db_conn, initialized_corpus, {}) + (initialized_corpus / ".kgmdignore").write_text("archive/\n") + ingest_documents(db_conn, initialized_corpus, {}) + + before = _counts(db_conn) + stats = ingest_documents(db_conn, initialized_corpus, {}) + + assert stats["documents_removed"] == 0 + assert stats["new"] == 0 + assert stats["updated"] == 0 + assert _counts(db_conn) == before + + +def test_ingest_refuses_to_empty_an_existing_graph(initialized_corpus, db_conn): + ingest_documents(db_conn, initialized_corpus, {}) + before = _counts(db_conn) + + (initialized_corpus / ".kgmdignore").write_text("*\n") + + with pytest.raises(RuntimeError, match="exclude every"): + ingest_documents(db_conn, initialized_corpus, {}) + + assert _counts(db_conn) == before + + +def _indexed_paths(conn): + return {r[0] for r in conn.execute("SELECT path FROM documents").fetchall()} + + +def conn_has_no_orphan_chunks(conn): + orphans = conn.execute( + "SELECT count(*) FROM chunks WHERE document_id NOT IN (SELECT id FROM documents)" + ).fetchone()[0] + stale_vectors = conn.execute( + "SELECT count(*) FROM vec_chunks WHERE chunk_id NOT IN (SELECT id FROM chunks)" + ).fetchone()[0] + return orphans == 0 and stale_vectors == 0 + + +# -------------------------------------------------------------------------------------- +# Dry-run report (US3) +# -------------------------------------------------------------------------------------- + + +def test_dry_run_report_shape_and_paths(initialized_corpus, db_conn): + (initialized_corpus / "archive").mkdir() + (initialized_corpus / "archive" / "old.md").write_text("# old\n") + (initialized_corpus / ".kgmd" / "notes.md").write_text("# hidden\n") + (initialized_corpus / ".kgmdignore").write_text("archive/\n") + + report = dry_run_report(db_conn, initialized_corpus, {}) + + assert set(report) == {"included", "ignored", "dotpath", "counts"} + assert set(report["counts"]) == {"included", "ignored", "dotpath", "would_remove"} + assert report["ignored"] == ["archive/old.md"] + assert ".kgmd/notes.md" in report["dotpath"] + assert report["counts"]["included"] == len(report["included"]) + assert report["included"] == sorted(report["included"]) + assert all(not p.startswith("/") for p in report["included"]), "paths must be corpus-relative" + + +def test_dry_run_report_counts_pending_removals(initialized_corpus, db_conn): + (initialized_corpus / "archive").mkdir() + (initialized_corpus / "archive" / "old.md").write_text("# old\n\nSuperseded.\n") + ingest_documents(db_conn, initialized_corpus, {}) + (initialized_corpus / ".kgmdignore").write_text("archive/\n") + + report = dry_run_report(db_conn, initialized_corpus, {}) + + assert report["counts"]["would_remove"] == 1 + assert "archive/old.md" not in report["included"] + + +def test_dry_run_report_writes_nothing(initialized_corpus, db_conn): + ingest_documents(db_conn, initialized_corpus, {}) + (initialized_corpus / ".kgmdignore").write_text("*\n") + before = _counts(db_conn) + + # Even the case that a real build refuses is only reported, never acted on. + report = dry_run_report(db_conn, initialized_corpus, {}) + + assert report["included"] == [] + assert _counts(db_conn) == before + + +def test_dry_run_report_without_a_database(tmp_corpus): + report = dry_run_report(None, tmp_corpus, {}) + assert report["counts"]["would_remove"] == 0 + assert report["counts"]["included"] > 0 From 80bbb310289911be0e49e258e5de29fe0739ab72 Mon Sep 17 00:00:00 2001 From: John Carpenter Date: Fri, 28 Aug 2026 10:56:24 -0600 Subject: [PATCH 3/3] Add Spec Kit planning artifacts for .kgmdignore Matches the convention set by specs/001-project-documentation: the spec, the Phase 0 research record, the Phase 1 design artifacts, and the task breakdown are committed alongside the change they describe. research.md is the part worth keeping. Every decision is recorded with the evidence that settled it, including the probe output showing fnmatch's `*` crosses `/` and PurePath.match's `**` spans one segment, the verification that real git refuses to re-include a file inside an excluded directory, and R11's line-by-line list of documented statements this change falsifies. --- .../checklists/requirements.md | 70 ++++ .../contracts/cli-build-dry-run.md | 103 ++++++ .../contracts/kgmdignore-format.md | 126 +++++++ .../contracts/library-api.md | 150 +++++++++ specs/002-kgmdignore-exclusions/data-model.md | 171 ++++++++++ specs/002-kgmdignore-exclusions/plan.md | 220 ++++++++++++ specs/002-kgmdignore-exclusions/quickstart.md | 133 ++++++++ specs/002-kgmdignore-exclusions/research.md | 313 ++++++++++++++++++ specs/002-kgmdignore-exclusions/spec.md | 294 ++++++++++++++++ specs/002-kgmdignore-exclusions/tasks.md | 302 +++++++++++++++++ 10 files changed, 1882 insertions(+) create mode 100644 specs/002-kgmdignore-exclusions/checklists/requirements.md create mode 100644 specs/002-kgmdignore-exclusions/contracts/cli-build-dry-run.md create mode 100644 specs/002-kgmdignore-exclusions/contracts/kgmdignore-format.md create mode 100644 specs/002-kgmdignore-exclusions/contracts/library-api.md create mode 100644 specs/002-kgmdignore-exclusions/data-model.md create mode 100644 specs/002-kgmdignore-exclusions/plan.md create mode 100644 specs/002-kgmdignore-exclusions/quickstart.md create mode 100644 specs/002-kgmdignore-exclusions/research.md create mode 100644 specs/002-kgmdignore-exclusions/spec.md create mode 100644 specs/002-kgmdignore-exclusions/tasks.md diff --git a/specs/002-kgmdignore-exclusions/checklists/requirements.md b/specs/002-kgmdignore-exclusions/checklists/requirements.md new file mode 100644 index 0000000..cd27170 --- /dev/null +++ b/specs/002-kgmdignore-exclusions/checklists/requirements.md @@ -0,0 +1,70 @@ +# Specification Quality Checklist: Corpus Exclusions via `.kgmdignore` + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-08-28 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Validation Notes + +Iteration 1 findings and resolutions: + +1. **Implementation leak in Dependencies** — the original text named storage mechanics ("cascade + behavior", "evidence links are nulled", "vector rows are not cascaded"). Rewritten in behavioral + terms: discarding a note's text discards its entity mentions but leaves its search vectors and + unsupported relations behind. The load-bearing risk is preserved without naming schema mechanics. + +2. **Zero clarification markers** — the source issue leaves five points open. Each was resolved to + the simplest correct option and recorded in Assumptions rather than deferred to the user: + - *Negation* — in scope. The motivating example in the issue uses it and last-match-wins ordering + needs no new dependency. + - *Precedence* — allowlist scopes, ignore subtracts, negation re-admits, dot-path exclusion last + and non-overridable (FR-007, FR-008). + - *Inline config key* — rejected (FR-020). Two mechanisms for one job, and it would add a + configuration key requiring its own documentation entry. + - *Reusing `.gitignore`* — out of scope; conflates two concerns. + - *Pruning split* — not split. Comparing the resolved file set against recorded document paths + covers newly-ignored, deleted, and renamed files with one mechanism; an ignore-specific variant + would be more work for less coverage. The issue's "document the limitation instead" escape is + therefore not used. + - *Preview / dry-run* — in scope as P3. Ignore rules are otherwise unverifiable before the spend + they exist to prevent. Surfaced on the existing build path since there is no standalone ingest + command. + +3. **Terms retained deliberately** — `.kgmdignore`, pattern syntax, and the preview are user-facing + surfaces named in the issue's acceptance criteria, not implementation choices. Domain nouns + (document, chunk, entity mention, relation, search vector) appear only in Key Entities and + Dependencies, where the template calls for the data the feature touches. + +4. **Safety requirement added beyond the issue** — FR-015: an ignore ruleset that resolves to an + empty file set must not silently empty an existing graph. Unified pruning makes a stray `*` + pattern destructive, so this guard is required by the pruning decision above. + +## Notes + +- All items pass. Spec is ready for `/speckit.plan`. +- Items marked incomplete would require spec updates before `/speckit.clarify` or `/speckit.plan`. diff --git a/specs/002-kgmdignore-exclusions/contracts/cli-build-dry-run.md b/specs/002-kgmdignore-exclusions/contracts/cli-build-dry-run.md new file mode 100644 index 0000000..93c5134 --- /dev/null +++ b/specs/002-kgmdignore-exclusions/contracts/cli-build-dry-run.md @@ -0,0 +1,103 @@ +# Contract: `kgmd build --dry-run` + +**Feature**: [../spec.md](../spec.md) | Satisfies FR-017, FR-018 + +## Signature + +```text +kgmd build [OPTIONS] [PATH] + + PATH argument default "." Corpus directory; must exist. + --db path /.kgmd/graph.db + --config path — Accepted but ignored (pre-existing). + --dry-run flag off Report the resolved file set and exit. NEW + --json flag off Machine-readable output. Requires --dry-run. NEW + --help flag off +``` + +## Behavior + +`--dry-run` resolves the file set and reports it. It performs **no** writes: + +- no `documents`, `chunks`, `entity_mentions`, `relations`, `entities`, or vector rows created, + updated, or deleted +- no language-model call, no embedding call +- no `graph.db` created — the dry-run branch runs **before** `init_db`, so a dry run against a corpus + that has never been built leaves the filesystem untouched apart from reads +- no build lock taken (it is a read, consistent with query commands) + +Removal counts are computed by comparing recorded `documents.path` values against the resolved set. +When `graph.db` does not exist, `would_remove` is `0`. + +`--json` without `--dry-run` fails: `--json requires --dry-run.` Exit 1, nothing else printed. This +is deliberate — the alternative, `--json` implying `--dry-run`, would make `kgmd build --json` +silently not build. + +## Human output + +```text +$ kgmd build --dry-run +Resolved 3 files to index (2 excluded by .kgmdignore, 1 excluded as a dot-path). + notes/today.md + notes/projects/atlas.md + archive/2024-decisions.md +Already indexed but no longer in the corpus: 1 document would be removed. +``` + +Ordering is the sorted corpus-relative path order, identical to ingest order. With no `.kgmdignore` +present, the exclusion clause names only the dot-path count. + +## Structured output + +`kgmd build --dry-run --json` writes one JSON object to stdout and nothing else: + +```json +{ + "included": ["archive/2024-decisions.md", "notes/projects/atlas.md", "notes/today.md"], + "ignored": ["archive/old.md", "drafts/idea.md"], + "dotpath": [".kgmd/notes.md"], + "counts": { + "included": 3, + "ignored": 2, + "dotpath": 1, + "would_remove": 1 + } +} +``` + +| Field | Type | Meaning | +|---|---|---| +| `included` | `list[str]` | corpus-relative paths that would be indexed, sorted | +| `ignored` | `list[str]` | paths excluded by `.kgmdignore`, sorted | +| `dotpath` | `list[str]` | paths excluded by the dot-path rule, sorted | +| `counts.would_remove` | `int` | indexed documents whose path is absent from `included` | + +Every path is corpus-relative POSIX, matching the form stored in `documents.path`. Absolute paths +never appear, so output is stable across machines and safe to commit in a test fixture. + +## Errors + +| Condition | Behavior | +|---|---| +| `PATH` has no `.kgmd/` | `Not a kgmd corpus (no .kgmd/ in ). Run 'kgmd init' first.` (pre-existing) | +| `--json` without `--dry-run` | `--json requires --dry-run.`, exit 1 | +| `.kgmdignore` unreadable or not UTF-8 | error naming the file, exit 1, no output | +| resolved set empty, graph non-empty | dry run **reports** it and exits 0; it is not a write, so the FR-015 guard does not fire here. A real build fails. | + +## Documentation gate consequences + +Both are obligations, not options — `tests/test_docs.py` enforces them: + +- `test_all_parameters_documented` requires `` `--dry-run` `` and `` `--json` `` as inline code spans + in the `### build` section of `docs/reference/cli.md`. +- `test_structured_output_parity` compares the set of commands documented with a + `**Structured output**:` block against the set of commands having an `as_json` parameter. Adding + `--json` to `build` puts it in the second set, so the block becomes mandatory. + +## Non-goals + +- No MCP tool. The preview answers a filesystem-and-config question about work not yet done, for an + operator about to spend money; MCP tools read committed graph state for an assistant. Stated here + because Principle III requires a reason when new capability reaches only one surface. +- No `kgmd/query.py` function. The resolved file set is not graph state; `query.py` remains the sole + read layer for the graph, and discovery stays in `ingest.py`. diff --git a/specs/002-kgmdignore-exclusions/contracts/kgmdignore-format.md b/specs/002-kgmdignore-exclusions/contracts/kgmdignore-format.md new file mode 100644 index 0000000..f0c53fa --- /dev/null +++ b/specs/002-kgmdignore-exclusions/contracts/kgmdignore-format.md @@ -0,0 +1,126 @@ +# Contract: `.kgmdignore` file format + +**Feature**: [../spec.md](../spec.md) | **Status**: authoritative for implementation and docs + +This is a user-authored file format, so it is a public contract: once shipped, a corpus's +`.kgmdignore` must keep meaning what it meant. Everything below is normative. + +## Location and discovery + +- Exactly one file is read: `/.kgmdignore`. +- Absent file → empty ruleset → discovery behaves exactly as it did before this feature. +- Present but unreadable or not valid UTF-8 → hard error naming the file. Never silently ignored. +- Files named `.kgmdignore` in subdirectories are **not** read. +- The file is read fresh on every run. No caching, no invalidation step. + +## Line grammar + +```text +line := blank | comment | rule +blank := WS* +comment := WS* "#" ANY* +rule := "!"? pattern "/"? +pattern := segment ( "/" segment )* +segment := ( literal | "*" | "?" | "**" )+ +``` + +Processing order per line: + +1. Strip trailing whitespace. +2. If the result is empty, or its first non-whitespace character is `#`, produce no rule. +3. A leading `!` sets `negated` and is removed. +4. A trailing `/` sets `dir_only` and is removed. +5. If nothing remains, produce no rule. + +## Matching + +Patterns match the **corpus-relative path** with `/` separators on every platform. Matching is +**case-sensitive** regardless of the host filesystem. + +| Construct | Matches | +|---|---| +| `*` | any run of characters, never crossing `/` | +| `?` | exactly one character, never `/` | +| `**` | any number of path segments; `a/**/b` also matches `a/b` | +| leading `/` | anchors the pattern at the corpus root | +| any interior `/` | anchors the pattern at the corpus root | +| no `/` anywhere | matches at any depth | +| trailing `/` | directory-only: the named directory and everything beneath it | + +A pattern without a trailing `/` matches a file with that path **and** any path beneath a directory +with that path. So `archive` and `archive/` differ only in whether a *file* named `archive` matches. + +**Not supported** (a pattern using these matches literally, so it excludes nothing rather than too +much): + +- character classes — `[a-z]`, `[!abc]` +- backslash escapes for a literal `#`, `!`, or trailing space + +## Precedence + +Fixed, and not configurable: + +```text +corpus.include scopes the candidate set (unchanged behavior) + v +.kgmdignore rules evaluated in file order, LAST MATCH WINS + v +dot-path rule applied last, NEVER overridable +``` + +For each candidate file, every rule is tested against the file's relative path and against each of +its ancestor directory prefixes. The last rule that matches decides: a negation re-admits, a normal +rule excludes. + +A path with any dot-prefixed component (`.kgmd/`, `.git/`, `.claude/`) is excluded after all rule +evaluation. No pattern, negated or not, can re-admit it. + +## Divergence from git — negation inside an excluded directory + +git states: "It is not possible to re-include a file if a parent directory of that file is +excluded." **`.kgmdignore` does not have that limitation.** This works: + +```text +archive/ +!archive/2024-decisions.md +``` + +Result: `archive/2024-decisions.md` is indexed; everything else under `archive/` is not. + +This is the motivating example from the source issue, and it is why rules are evaluated per file +against all ancestor prefixes rather than by pruning the directory walk. + +## Worked example + +```text +# skip archived notes +archive/ +drafts/ + +# skip generated or boilerplate files +**/CHANGELOG.md +*-template.md + +# but keep this one +!archive/2024-decisions.md +``` + +Against a corpus containing: + +| Path | Result | Deciding rule | +|---|---|---| +| `notes/today.md` | indexed | no rule matches | +| `archive/old.md` | excluded | `archive/` | +| `archive/2024-decisions.md` | indexed | `!archive/2024-decisions.md` (last match) | +| `drafts/idea.md` | excluded | `drafts/` | +| `docs/CHANGELOG.md` | excluded | `**/CHANGELOG.md` | +| `CHANGELOG.md` | excluded | `**/CHANGELOG.md` (`a/**/b` also matches `a/b`) | +| `notes/msa-template.md` | excluded | `*-template.md` (unanchored, any depth) | +| `.kgmd/notes.md` | excluded | dot-path rule, after all rules | + +## Guarantees a change to this contract must preserve + +1. An absent `.kgmdignore` never changes behavior. +2. The dot-path rule is never overridable. +3. Ordering semantics are last-match-wins, in file order. +4. An unsupported construct fails toward including files, never toward excluding them. diff --git a/specs/002-kgmdignore-exclusions/contracts/library-api.md b/specs/002-kgmdignore-exclusions/contracts/library-api.md new file mode 100644 index 0000000..c49e753 --- /dev/null +++ b/specs/002-kgmdignore-exclusions/contracts/library-api.md @@ -0,0 +1,150 @@ +# Contract: library API + +**Feature**: [../spec.md](../spec.md) | Modules: `kgmd/ignore.py` (new), `kgmd/ingest.py` (changed) + +Public functions carry type hints; the connection and the config dict are passed as arguments, never +reached for. No module-level mutable state. + +## `kgmd/ignore.py` — new leaf module + +Imports stdlib only (`dataclasses`, `pathlib`, `re`). Imports nothing from `kgmd`, so the +constitution's one-way dependency direction is preserved: `ingest` → `ignore` → nothing. + +```python +@dataclass(frozen=True) +class IgnoreRule: + pattern: str + regex: re.Pattern[str] + negated: bool + dir_only: bool + line_number: int + + +def parse_ignore_rules(text: str) -> list[IgnoreRule]: + """Parse .kgmdignore text into ordered rules. Comments and blanks yield nothing.""" + + +def load_ignore_rules(root: Path) -> list[IgnoreRule]: + """Read /.kgmdignore. Returns [] if absent; raises if present and undecodable.""" + + +def is_ignored(rel_path: str, rules: list[IgnoreRule]) -> bool: + """True if rel_path (corpus-relative, '/' separators) is excluded. + + Every rule is tested against rel_path and each of its ancestor prefixes; + the last matching rule decides. Empty rules -> False. + """ + + +DEFAULT_IGNORE_TEMPLATE: str # every line a comment or blank + + +def write_default_ignore_file(path: Path) -> None: + """Write the starter .kgmdignore. Never overwrites an existing file.""" +``` + +**Guarantees** + +- `is_ignored([], …)` is `False` for every path — an absent ignore file cannot change behavior. +- `parse_ignore_rules` never raises on content; any line is either a rule or skipped. Only I/O and + decoding fail, and only in `load_ignore_rules`. +- Rule order is preserved exactly; rules are never sorted or deduplicated. +- `is_ignored` is pure and does not touch the filesystem, so it is unit-testable without `tmp_path`. +- `write_default_ignore_file` is a no-op when the target exists — a user's file is never clobbered. + +## `kgmd/ingest.py` — changed + +```python +@dataclass(frozen=True) +class FileScan: + included: list[Path] # absolute, sorted — will be indexed + ignored: list[Path] # absolute, sorted — excluded by .kgmdignore + dotpath: list[Path] # absolute, sorted — excluded by the dot-path rule + + +def scan_corpus_files(root: Path, config: dict) -> FileScan: + """Resolve what the corpus would index. + + corpus.include scopes candidates, .kgmdignore subtracts (negations re-add), + the dot-path rule is applied last and is not overridable. + """ + + +def prune_missing_documents(conn, kept_rel_paths: set[str]) -> dict: + """Remove indexed documents whose path is absent from kept_rel_paths. + + Covers newly-ignored, deleted, and renamed files identically. Clears chunks, + mentions, evidence-bound relations, both vector tables, and entities left with + no mention and no relation. Raises if kept_rel_paths is empty while documents + exist, before any write. + + Returns counts: documents_removed, chunks_removed, relations_removed, + entities_removed, vectors_removed. + """ + + +def dry_run_report(conn, corpus_dir: Path, config: dict) -> dict: + """The payload behind `kgmd build --dry-run`. Read-only; never writes. + + `conn` may be None when no database exists yet, in which case + counts.would_remove is 0. Paths in the returned dict are corpus-relative + POSIX strings, sorted — the same form stored in documents.path. + """ +``` + +**Unchanged signatures** — `find_markdown_files(root, include=None)`, `_is_dotpath(filepath, root)`, +`chunk_markdown`, `hash_content`, and `ingest_documents(conn, corpus_dir, config)` all keep their +current shape. `scan_corpus_files` composes the first two rather than replacing them, which is what +makes "no `.kgmdignore` behaves exactly as today" hold by construction. + +**`ingest_documents` behavior change**: it calls `scan_corpus_files` instead of inlining the +discovery-and-dot-path filter, then calls `prune_missing_documents` with the resolved relative paths +before its insert/update loop. Its return dict gains the five removal keys; existing keys (`new`, +`updated`, `skipped`, `chunks_created`) keep their names and meanings, so both current call sites +(`kgmd/cli.py` `build` and `extract`) keep working unchanged. + +Pruning runs before the insert/update loop so a rename is a removal plus an insert within one +transaction, never a window where both paths exist. + +**Why `dry_run_report` is a library function rather than CLI code**: the repository has no +CLI-invocation test convention — `tests/test_mcp.py` tests "the query layer that backs the MCP +tool", and no test uses `click.testing.CliRunner`. Keeping the payload in `ingest.py` means the +preview's substance (resolved set, counts, would-remove) is testable the same way everything else +is, and `cli.py` stays the thin renderer that Principle III requires. The two genuinely CLI-level +guarantees — that a dry run creates no `graph.db`, and that `--json` without `--dry-run` fails — do +need `CliRunner`, and establishing that convention in `tests/test_cli.py` is a deliberate, scoped +addition rather than an accident. + +## Error contract + +| Condition | Raised by | Type | +|---|---|---| +| `.kgmdignore` present but undecodable | `load_ignore_rules` | `RuntimeError`, message names the file | +| resolved set empty while `documents` non-empty | `prune_missing_documents` | `RuntimeError`, before any write | + +`RuntimeError` matches the house style for library-layer failures (`check_embedding_model`, +`build_lock` in `kgmd/db.py`). The CLI's exception hook renders it as a single `Error: ` +line; `--debug` shows the traceback. + +The empty-set message must be quotable in `docs/guides/troubleshooting.md`: +`test_quoted_errors_exist_in_source` requires the single code span on each `**Symptom**:` line to +appear verbatim in `kgmd/**/*.py`. So the interpolated document count goes in a later fragment and +the leading fragment stays one contiguous string literal that the doc quotes exactly. + +## Call graph after the change + +```text +cli.build / cli.extract + └── ingest.ingest_documents + ├── ingest.scan_corpus_files + │ ├── ingest.find_markdown_files (corpus.include scoping, unchanged) + │ ├── ignore.load_ignore_rules + is_ignored + │ └── ingest._is_dotpath (last, non-overridable) + └── ingest.prune_missing_documents + +cli.build --dry-run + └── ingest.scan_corpus_files (+ a read-only documents.path count; no writes) + +cli.init + └── ignore.write_default_ignore_file +``` diff --git a/specs/002-kgmdignore-exclusions/data-model.md b/specs/002-kgmdignore-exclusions/data-model.md new file mode 100644 index 0000000..1fc51cb --- /dev/null +++ b/specs/002-kgmdignore-exclusions/data-model.md @@ -0,0 +1,171 @@ +# Phase 1 Data Model: Corpus Exclusions via `.kgmdignore` + +**Feature**: [spec.md](./spec.md) | **Research**: [research.md](./research.md) + +Two kinds of data appear here: **new in-memory structures** that carry ignore rules and the resolved +file set, and **existing database tables** whose rows this feature removes. No DDL changes, no schema +version bump, no new configuration key. + +--- + +## 1. New in-memory structures + +### `IgnoreRule` (frozen `@dataclass`, `kgmd/ignore.py`) + +One parsed line of `.kgmdignore`. + +| Field | Type | Meaning | +|---|---|---| +| `pattern` | `str` | the original text of the line, after stripping whitespace and any `!` prefix; kept for error messages and preview output | +| `regex` | `re.Pattern[str]` | compiled once at parse time; matched against a corpus-relative POSIX path | +| `negated` | `bool` | line began with `!`; a match re-admits the path | +| `dir_only` | `bool` | line ended with `/`; matches a directory and everything beneath it | +| `line_number` | `int` | 1-based line in `.kgmdignore`, for diagnostics | + +**Validation rules** + +- A line whose first non-whitespace character is `#` produces no rule. +- A blank or whitespace-only line produces no rule. +- Trailing whitespace is stripped before parsing; a lone `!` or `/` after stripping produces no rule. +- `regex` is fully anchored (`\A…\Z`). Anchoring at the corpus root vs. matching at any depth is + baked into the compiled pattern, not decided at match time. +- Rule order is significant and is the file's order. `IgnoreRule` instances are never reordered or + deduplicated. + +### `Ignore ruleset` — `list[IgnoreRule]` + +A plain list, not a wrapper class: order is the only invariant and a list already expresses it. An +absent `.kgmdignore` yields `[]`, which is what makes spec FR-009 (absent file behaves exactly as +today) hold by construction. + +**State transitions**: none. The list is built from file text on every run and never mutated or +cached. Editing `.kgmdignore` between builds therefore takes effect on the next build with no +invalidation step. + +### `FileScan` (frozen `@dataclass`, `kgmd/ingest.py`) + +The resolved file set plus the reasons for every exclusion. This is the single value both ingest and +the preview consume, so the preview cannot drift from what a build would do. + +| Field | Type | Meaning | +|---|---|---| +| `included` | `list[Path]` | absolute paths that will be indexed, sorted | +| `ignored` | `list[Path]` | absolute paths excluded by `.kgmdignore`, sorted | +| `dotpath` | `list[Path]` | absolute paths excluded by the dot-path rule, sorted | + +**Validation rules** + +- The three lists are disjoint, and their union is the candidate set produced by `corpus.include` + scoping. +- Precedence is fixed and encoded in construction order (spec FR-007): `corpus.include` scopes the + candidates → `.kgmdignore` moves paths into `ignored` → the dot-path rule moves paths into + `dotpath`. Because the dot-path pass runs last and reads no rule state, no negation can move a + path out of `dotpath` (FR-008). +- Paths are absolute; callers derive the corpus-relative form for display and for `documents.path` + comparison, matching what `ingest_documents` already does. + +--- + +## 2. Existing tables this feature removes rows from + +No column, index, constraint, or `PRAGMA user_version` changes. The tables are listed with the +cascade behavior that decides whether a delete must be explicit. + +| Table | Removal trigger | Cascade behavior | Explicit delete needed? | +|---|---|---|---| +| `documents` | recorded `path` is absent from `FileScan.included` | — | **Yes** — the orphan set is defined here | +| `chunks` | `document_id` in the orphan set | `ON DELETE CASCADE` from `documents` | Yes, issued before `documents` so ids can be collected first | +| `entity_mentions` | `chunk_id` in the removed chunks | `ON DELETE CASCADE` from `chunks` | No — the chunk delete covers it | +| `relations` | `evidence_chunk_id` in the removed chunks | `ON DELETE SET NULL` — **rows survive** | **Yes** (FR-013) | +| `entities` | lost their last mention in this prune and hold no relations | none — never cascaded | **Yes** (required by SC-005) | +| `vec_chunks` | `chunk_id` in the removed chunks | none — virtual table, no foreign keys | **Yes** (FR-012) | +| `vec_entity_mentions` | `mention_id` in the removed mentions | none — virtual table, no foreign keys | **Yes** (FR-012) | + +### Why the two vec tables are the dangerous ones + +`chunks.id` and `entity_mentions.id` are plain `INTEGER PRIMARY KEY`, so SQLite reuses their values +after deletes. `embed_new_chunks` selects `chunks` rows `WHERE c.id NOT IN (SELECT chunk_id FROM +vec_chunks)`. A stale `vec_chunks` row therefore makes a *future, unrelated* chunk look +already-embedded, and search then answers from the vector of text that no longer exists. Leaving +vectors behind would make this feature actively harmful to notes the user never excluded. + +### Why relation rows are deleted rather than left with null evidence + +`ON DELETE SET NULL` would keep a relation with no provenance, which contradicts the product's +provenance promise and leaves `kgmd relations` asserting something no chunk supports. The unique +index `(subject_id, predicate, object_id, evidence_chunk_id)` means a relation independently +extracted from a surviving chunk is a *different row*, so deleting the evidence-bound row never +loses an independently supported assertion. + +### Why the entity sweep is scoped + +Only entities whose mentions were removed by this prune are candidates, and only those left with no +mentions and no relations are deleted. A global sweep would also collect the pre-existing orphan +entities that `kgmd extract --force` leaves behind — separate recorded debt in +`docs/guides/maintenance.md`, and removing it here would be an unrequested behavior change. + +--- + +## 3. Orphan removal: ordering and states + +**Orphan** — an indexed document whose recorded `path` is absent from `FileScan.included`. The cause +is not recorded and does not matter: newly ignored, deleted, and renamed are the same state. + +**Ordered sequence** (single transaction, inside the existing build lock): + +```text +1. orphan_ids <- documents whose path is not in included_relative_paths +2. if not orphan_ids -> return zero counts, no writes +3. if included is empty and documents exist -> RAISE, no writes (FR-015) +4. chunk_ids <- chunks.id WHERE document_id IN orphan_ids +5. mention_ids <- entity_mentions.id WHERE chunk_id IN chunk_ids +6. entity_ids <- entity_mentions.entity_id WHERE chunk_id IN chunk_ids +7. DELETE vec_entity_mentions WHERE mention_id IN mention_ids +8. DELETE vec_chunks WHERE chunk_id IN chunk_ids +9. DELETE relations WHERE evidence_chunk_id IN chunk_ids +10. DELETE chunks WHERE document_id IN orphan_ids (cascades mentions) +11. DELETE documents WHERE id IN orphan_ids +12. DELETE entities WHERE id IN entity_ids + AND no remaining mention AND no remaining relation +``` + +**Invariants** + +- Steps 4–6 collect ids **before** any delete; after step 10 the mention ids are unrecoverable. +- Step 3 precedes every write, so the guard cannot fire mid-prune and leave a partial graph. +- Steps 2 and 3 are the only early exits; both leave the database untouched. +- Re-running over an unchanged corpus takes the step-2 exit, which is what makes the operation + idempotent (FR-014). +- Id lists are bound in batches of 500 to stay under SQLite's variable limit. A `TEMP TABLE` would + be simpler but issues DDL outside `kgmd/schema.py`, which Principle I prohibits. + +--- + +## 4. Reported counts + +`ingest_documents` already returns `{"new", "updated", "skipped", "chunks_created"}`. Pruning adds +five keys, so the build summary can state what left the graph (FR-016): + +| Key | Meaning | +|---|---| +| `documents_removed` | orphan `documents` rows deleted | +| `chunks_removed` | `chunks` rows deleted | +| `relations_removed` | `relations` rows deleted for missing evidence | +| `entities_removed` | entities swept for having no mention and no relation left | +| `vectors_removed` | `vec_chunks` + `vec_entity_mentions` rows deleted | + +Existing keys keep their names and meanings, so the two current call sites +(`kgmd/cli.py` `build` and `extract`) continue to read the same fields. + +--- + +## 5. What is deliberately not modelled + +- **No `corpus.exclude` config key** (FR-020). `DEFAULT_CONFIG` is untouched, so the configuration + reference's key count and its bidirectional coverage test are unaffected. +- **No cache of parsed rules.** Re-parsing a file of a few dozen lines once per run is free, and a + cache would need an invalidation story for a file the user edits between runs. +- **No record of why a file was excluded** in the database. Exclusion is derived from the corpus + directory plus config on every run; storing it would create corpus state outside `graph.db`. +- **No `mtime` or timestamp involvement.** Skip decisions stay content-hash driven (Principle IV); + removal is driven by set difference, not by time. diff --git a/specs/002-kgmdignore-exclusions/plan.md b/specs/002-kgmdignore-exclusions/plan.md new file mode 100644 index 0000000..8c99346 --- /dev/null +++ b/specs/002-kgmdignore-exclusions/plan.md @@ -0,0 +1,220 @@ +# Implementation Plan: Corpus Exclusions via `.kgmdignore` + +**Branch**: `002-kgmdignore-exclusions` | **Date**: 2026-08-28 | **Spec**: [spec.md](./spec.md) + +**Input**: Feature specification from `/specs/002-kgmdignore-exclusions/spec.md` + +## Summary + +Add a gitignore-style `.kgmdignore` file at the corpus root that subtracts paths from ingest, and +make the file set shrinking actually shrink the graph. + +Three parts, in dependency order: + +1. **`kgmd/ignore.py`** — a new stdlib-only leaf module that parses `.kgmdignore` into ordered rules + and answers `is_ignored(rel_path, rules)`. Patterns are compiled to `re.Pattern` at parse time + because both stdlib candidates are provably wrong: `fnmatch`'s `*` crosses `/`, and + `PurePath.match` is right-anchored with single-segment `**` on the supported runtimes + (`full_match` is 3.13+, the floor is 3.10). Verified by probe — see [research.md](./research.md) + §R1. No new runtime dependency. +2. **`ingest.scan_corpus_files`** — one function that resolves the file set with fixed precedence + (`corpus.include` scopes → `.kgmdignore` subtracts, negations re-add → dot-path rule last and + non-overridable) and returns the included, ignored, and dot-path groups. Both ingest and the + preview read it, so the preview cannot drift from what a build does. +3. **`ingest.prune_missing_documents`** — removes indexed documents whose path is absent from the + resolved set. One mechanism covers newly-ignored, deleted, and renamed files. It must clear the + two `sqlite-vec` tables explicitly: `chunks.id` and `entity_mentions.id` are plain + `INTEGER PRIMARY KEY` so ids are reused, and `embed_new_chunks` selects + `WHERE c.id NOT IN (SELECT chunk_id FROM vec_chunks)` — a stale vector row makes a *future, + unrelated* chunk look already-embedded and search then answers from deleted text. It must also + delete relations bound to removed evidence, because `relations.evidence_chunk_id` is + `ON DELETE SET NULL` and those rows would otherwise survive with no provenance. + +Surfaces: `kgmd build --dry-run [--json]` previews the resolved set without touching the graph; +`kgmd init` writes a fully-commented starter `.kgmdignore`. No new config key — the ignore file is +the only mechanism, which also keeps `DEFAULT_CONFIG` and the configuration reference's key count +untouched. + +Guard added beyond the issue: an ignore ruleset that resolves to an empty set while the graph holds +documents raises before any write. Unconditional pruning makes a stray `*` destructive, and silently +emptying a graph is a worse failure than the one being fixed. + +## Technical Context + +**Language/Version**: Python `>=3.10`; supported matrix 3.10 / 3.11 / 3.12 / 3.13. No language +feature newer than 3.10 — which is precisely why `PurePath.full_match` (3.13) is unavailable. + +**Primary Dependencies**: none added. Existing runtime set unchanged (`click`, `rich`, `pyyaml`, +`platformdirs`, `litellm`, `fastembed`, `sqlite-vec`, `networkx`, `fastmcp`). `pathspec` was +considered and rejected — [research.md](./research.md) §R1. + +**Storage**: the single SQLite artifact `.kgmd/graph.db`. **No DDL change, no +`PRAGMA user_version` bump** — this feature only deletes rows from existing tables. New user-authored +input file `/.kgmdignore`, which is configuration rather than corpus state. + +**Testing**: `pytest`, function style, `tests/conftest.py` fixture chain +(`tmp_corpus` → `initialized_corpus` → `db_conn` / `seeded_db`). Two new modules: +`tests/test_ignore.py` (pure pattern semantics, no filesystem) and `tests/test_ingest.py` (discovery, +pruning, guard, stale-vector regression). Vectors are hand-packed `struct.pack` rows; no embedding +backend and no network. + +**Target Platform**: local CLI plus the MCP stdio server, on macOS and Linux. Patterns always use `/` +regardless of host separator. + +**Project Type**: single flat Python package `kgmd/` with a `click` CLI. One new module, no new +subpackage. + +**Performance Goals**: SC-004 — the resolved file set is reportable in under 5 seconds on a 1,000-file +corpus. Rule matching is a compiled regex per rule per candidate path; discovery cost stays the +existing `rglob` walk. + +**Constraints**: no new runtime dependency; offline-deterministic tests; documentation in the same +change (the docs gate is bidirectional and there is no follow-up window); `ruff` `line-length = 100`, +lint select `E,F,I,W`. + +**Scale/Scope**: ~5 source files touched (`kgmd/ignore.py` new, `kgmd/ingest.py`, `kgmd/cli.py`, plus +2 new test modules), and 6 documentation pages — 5 of which contain statements this change +*falsifies*, enumerated in [research.md](./research.md) §R11. + +## Constitution Check + +*GATE: evaluated before Phase 0, re-evaluated after Phase 1. Both passes recorded.* + +### Pass 1 — before Phase 0 research + +| Principle | Verdict | Reasoning | +|---|---|---| +| **I. Single durable artifact, versioned schema** | PASS with a question to resolve | No DDL, no new table, no sidecar store; only `DELETE` on existing tables. `.kgmdignore` is a new file outside `graph.db` — must confirm it is *input*, not *state*. Deletes must go through `db.py::get_connection` and run under `build_lock`. | +| **II. Deterministic, mockable LLM boundary** | PASS | Ingest and discovery make no provider call. Nothing added near `call_structured`; no prompt asset changes. | +| **III. Dual-surface parity over one query layer** | OPEN | A new user-visible read (the preview) needs either both surfaces or a stated exclusion, and Principle III ties CLI query commands to `kgmd/query.py`. Resolve in Phase 0. | +| **IV. Content-hash incrementality, idempotent re-runs** | OPEN | Pruning is a new mutation on every build. Must not disturb hash-based skip decisions, must be idempotent, must not be timestamp-driven, and needs an idempotency test in the shape of `test_extraction_idempotent`. | +| **V. Offline-deterministic test gate** | PASS | All new behavior is filesystem and SQL; testable with `tmp_path` and hand-packed vectors. New capability ships tests in the same change. | +| **Tech constraints** | OPEN | New runtime dependency requires justification — decide `pathspec` vs stdlib in Phase 0. New module must not break the one-way dependency direction and must not create a subpackage. Every new config key must be consumed and documented — avoidable only by adding none. | +| **Docs as contract** | PASS, with work | New CLI options and behavior changes must land with docs, verified bidirectionally by `tests/test_docs.py`. | + +No violation. Two OPEN items (III, IV) and one dependency question routed to Phase 0. + +### Pass 2 — after Phase 1 design + +| Principle | Verdict | Evidence | +|---|---|---| +| **I. Single durable artifact** | PASS | No DDL; `user_version` untouched. Pruning runs on the existing connection inside `ingest_documents`, which both call sites already wrap in `build_lock` (`kgmd/cli.py:214` for `build`, `:282` for `extract`). `.kgmdignore` is user-authored *input*, the same class as `.kgmd/config.yaml`, so no corpus state is added outside `graph.db`. Id batching uses chunked `IN (...)` lists specifically to avoid a `TEMP TABLE`, which would be DDL outside `kgmd/schema.py`. | +| **II. Deterministic LLM boundary** | PASS | Unchanged. No module in this change imports `litellm` or `kgmd/llm.py`. `kgmd/induce.py` — the recorded deviation — is not touched. | +| **III. Dual-surface parity** | PASS with stated exclusion | The preview is CLI-only, and the reason is recorded in [contracts/cli-build-dry-run.md](./contracts/cli-build-dry-run.md): the resolved file set is **not graph state**, so `query.py` (the sole read layer for the *graph*) gains no function and discovery stays in `ingest.py`; and the preview answers a pre-spend operator question, whereas MCP tools serve assistants reading committed graph state. `--json` satisfies the machine-output rule. Failures use `click.ClickException` with remediation, human output to `console`, diagnostics to `err_console`. | +| **IV. Incrementality and idempotency** | PASS | Removal is driven by set difference on `documents.path`, never by `mtime`. The hash-based skip path is untouched: an unchanged, un-ignored document still takes the `content_hash` match branch. Pruning takes an early zero-write exit when no orphans exist, which is the idempotency property; `tests/test_ingest.py` asserts a second run removes nothing and re-extracts nothing. Downstream state is invalidated explicitly rather than left mixed-generation — exactly the rule that forces the vector and relation deletes. | +| **V. Offline-deterministic tests** | PASS | Two new hermetic test modules; `tests/test_ignore.py` needs no filesystem at all. Vectors are hand-packed `struct.pack` rows. `tests/fixtures/*.md` is not edited, so no exact-count assertion moves. | +| **Tech constraints** | PASS | Zero new runtime dependencies — the stdlib decision is evidence-backed in [research.md](./research.md) §R1. `kgmd/ignore.py` is a module, not a subpackage, and imports nothing from `kgmd`, so the one-way direction holds as `ingest` → `ignore` → nothing. Connection and config stay parameters; no module-level mutable state. `FileScan` and `IgnoreRule` are `@dataclass`, matching `ingest.Chunk`; pydantic stays reserved for the LLM boundary. **No config key added**, so the "every tunable must be consumed and documented" rule is satisfied vacuously and the reference's key count is unchanged. | +| **Docs as contract** | PASS, with work enumerated | [research.md](./research.md) §R11 lists 7 statements this change falsifies and 5 additions, including the two doc-gate obligations that adding `--json` to `build` creates (`test_all_parameters_documented`, `test_structured_output_parity`) and the exact-literal requirement for the troubleshooting quote (`test_quoted_errors_exist_in_source`). | + +**Result: PASS. Complexity Tracking table stays empty — no principle is violated.** + +Two recorded constitutional deviations are checked for contact: + +- `kgmd/induce.py` bypassing `call_structured` — not touched. +- `llm.max_tokens` split between `config.py` (16384) and `extract.py` (4096) — neither module is + touched by this change, so the "reconcile when next touched" obligation is not triggered. Left as + recorded debt rather than silently widening scope. + +One adjacent defect is deliberately **not** fixed: `kgmd reset` and `reset --hard` also fail to clear +the vec tables (`kgmd/cli.py:674-688`), and `reset` is separately broken by `VACUUM` inside a +transaction. Both are documented in `docs/guides/maintenance.md` and neither is in this feature's +scope; fixing `reset` here would be unrequested scope. The prune path introduced here does clear +vectors, so this change does not add to that debt. + +## Project Structure + +### Documentation (this feature) + +```text +specs/002-kgmdignore-exclusions/ +├── plan.md # This file +├── spec.md # Feature specification +├── research.md # Phase 0 — 12 decisions, probe-verified +├── data-model.md # Phase 1 — structures, tables, removal ordering +├── quickstart.md # Phase 1 — runnable validation scenarios +├── contracts/ +│ ├── kgmdignore-format.md # The user-authored file format (public contract) +│ ├── cli-build-dry-run.md # New CLI flags + JSON shape +│ └── library-api.md # Function signatures and error contract +├── checklists/ +│ └── requirements.md # Spec quality checklist (16/16) +└── tasks.md # Phase 2 — created by /speckit.tasks, NOT here +``` + +### Source Code (repository root) + +```text +kgmd/ +├── ignore.py # NEW — parse/compile/match .kgmdignore; starter template + writer +├── ingest.py # CHANGED — FileScan, scan_corpus_files, prune_missing_documents +├── cli.py # CHANGED — build --dry-run/--json; init writes starter .kgmdignore +├── config.py # UNCHANGED — no new config key (FR-020) +├── schema.py # UNCHANGED — no DDL change +├── embed.py # UNCHANGED — but its NOT IN (SELECT chunk_id FROM vec_chunks) +│ # query is why vector cleanup is mandatory +├── db.py · query.py · extract.py · resolve.py · induce.py · export.py · mcp_server.py +└── prompts/ # UNCHANGED + +tests/ +├── test_ignore.py # NEW — pattern semantics, pure unit tests +├── test_ingest.py # NEW — discovery, pruning, guard, stale-vector regression +├── conftest.py # UNCHANGED — existing fixture chain is sufficient +├── fixtures/*.md # UNCHANGED — golden corpus, alias variants preserved +└── test_docs.py # UNCHANGED — but gates every doc edit below + +docs/ +├── reference/cli.md # build: --dry-run, --json, Structured output block; init: starter file +├── reference/configuration.md # NEW .kgmdignore section; corpus.include interaction +├── guides/maintenance.md # spend control; the deleted/renamed rows that are now wrong +├── guides/troubleshooting.md # empty-resolved-set guard entry +├── examples/personal-notes.md # "Deleted notes are not removed" limitation is gone +├── examples/mcp-assistant.md # stale claim about deleted notes +└── contributing/architecture.md # ingest.py row: name the ignore pass and scan_corpus_files +``` + +**Structure Decision**: the existing flat `kgmd/` package with one added leaf module. `kgmd/ignore.py` +holds pattern parsing, matching, and the starter-file template — ~60 lines of translation logic with +a dense unit-test surface that would otherwise be half of `ingest.py`. It imports stdlib only, so the +constitution's dependency direction extends cleanly as `ingest` → `ignore` → nothing. No subpackage +is created (prohibited for code), and `.kgmdignore` lives at the corpus root beside the user's notes +because it is version-controllable input, not derived state. + +## Phase Outputs + +| Phase | Status | Artifacts | +|---|---|---| +| 0 — Outline & Research | Complete | [research.md](./research.md) — R1…R12, all decisions probe- or source-verified; zero unresolved unknowns | +| 1 — Design & Contracts | Complete | [data-model.md](./data-model.md), [contracts/](./contracts/) ×3, [quickstart.md](./quickstart.md) | +| 2 — Tasks | Not started | `tasks.md` — produced by `/speckit.tasks` | + +### Suggested implementation order + +Derived from the dependency graph, not from the story priorities — story P1 needs items 1–3, P2 needs +4, P3 needs 5: + +1. `kgmd/ignore.py` + `tests/test_ignore.py` — pure, no dependants yet. +2. `ingest.scan_corpus_files` + `FileScan`, composing the unchanged `find_markdown_files` and + `_is_dotpath`; point `ingest_documents` at it. Assert the no-ignore-file path is unchanged first. +3. `tests/test_ingest.py` discovery cases, including the `corpus.include` interaction and dot-path + non-overridability. +4. `ingest.prune_missing_documents` + the empty-set guard, wired into `ingest_documents` before its + insert/update loop; pruning, idempotency, and stale-vector tests. +5. `kgmd build --dry-run [--json]` before `init_db`; `kgmd init` starter file. +6. Documentation — the 5 falsified statements first, then the 5 additions. Run `make test` to let + `tests/test_docs.py` prove both directions. + +## Risks + +| Risk | Mitigation | +|---|---| +| Pruning silently deletes a graph on a bad pattern | FR-015 guard raises before any write; dry run makes rules inspectable first | +| Stale vectors bind to reused ids | Explicit deletes from both vec tables before the chunk delete; regression test asserts the row is gone | +| Preview drifts from real build behavior | Single `scan_corpus_files` feeds both; no second code path exists | +| Entity sweep removes more than intended | Scoped to entities whose mentions this prune removed; pre-existing orphans left as recorded debt | +| Doc gate fails late in the change | The falsified statements are enumerated with line numbers in research.md §R11 rather than discovered by a red suite | + +## Complexity Tracking + +> Fill ONLY if Constitution Check has violations that must be justified. + +No violations. Table intentionally empty. diff --git a/specs/002-kgmdignore-exclusions/quickstart.md b/specs/002-kgmdignore-exclusions/quickstart.md new file mode 100644 index 0000000..6746b61 --- /dev/null +++ b/specs/002-kgmdignore-exclusions/quickstart.md @@ -0,0 +1,133 @@ +# Quickstart: validating `.kgmdignore` + +**Feature**: [spec.md](./spec.md) | **Contracts**: [contracts/](./contracts/) + +Runnable checks that prove the feature works end to end. Every scenario is offline — no provider +credential, no model download — except Scenario 5, which is explicitly marked as the only one that +spends money and is optional. + +## Prerequisites + +```bash +make install # pip install -e ".[dev]" +``` + +No `OPENROUTER_API_KEY` is needed for Scenarios 1–4 and 6: ingest, discovery, and pruning never +reach the language-model boundary. + +## Scenario 1 — Exclusion works (FR-001 … FR-005) + +```bash +mkdir -p /tmp/kgmd-demo/{notes,archive,drafts} +cd /tmp/kgmd-demo +printf '# Today\n\nSarah Chen leads Atlas.\n' > notes/today.md +printf '# Old\n\nSuperseded.\n' > archive/old.md +printf '# Decisions\n\nKeep this one.\n' > archive/2024-decisions.md +printf '# Draft\n\nHalf an idea.\n' > drafts/idea.md +printf '# Changelog\n\n- thing\n' > notes/CHANGELOG.md + +kgmd init +cat >> .kgmdignore <<'EOF' +archive/ +drafts/ +**/CHANGELOG.md +!archive/2024-decisions.md +EOF + +kgmd build --dry-run +``` + +**Expected**: `notes/today.md` and `archive/2024-decisions.md` are listed; `archive/old.md`, +`drafts/idea.md`, and `notes/CHANGELOG.md` are counted as ignored. The negation re-admitting a file +inside an excluded directory is the deliberate divergence from git — see +[contracts/kgmdignore-format.md](./contracts/kgmdignore-format.md). + +## Scenario 2 — The preview is machine-readable and writes nothing (FR-017, FR-018) + +```bash +rm -f .kgmd/graph.db # prove a dry run does not create it +kgmd build --dry-run --json | python -m json.tool +test -f .kgmd/graph.db && echo "FAIL: dry run created the database" || echo "OK: no database created" +kgmd build --json ; echo "exit=$?" +``` + +**Expected**: a single JSON object matching the shape in +[contracts/cli-build-dry-run.md](./contracts/cli-build-dry-run.md), with `counts.would_remove` of +`0`; no `graph.db`; and `kgmd build --json` alone failing with `--json requires --dry-run.` and +`exit=1`. + +## Scenario 3 — Absent `.kgmdignore` changes nothing (FR-009) + +```bash +mv .kgmdignore /tmp/kgmd-demo-ignore-backup +kgmd build --dry-run --json | python -c 'import json,sys; d=json.load(sys.stdin); print(d["counts"])' +mv /tmp/kgmd-demo-ignore-backup .kgmdignore +``` + +**Expected**: `ignored` is `0` and `included` is every `.md` file in the tree. This is the regression +guard for SC-003 — the same assertion exists as a test over `tests/fixtures/`. + +## Scenario 4 — A newly excluded note leaves an existing graph (FR-011 … FR-016) + +The offline version of this is the authoritative check, because it needs no provider: + +```bash +python -m pytest tests/test_ingest.py -v -k "prune or ignored" +``` + +**Expected**: passing assertions that after adding an ignore rule covering an already-indexed +document and re-running ingest, the `documents` row, its `chunks`, its `entity_mentions`, its +evidence-bound `relations`, its `vec_chunks` / `vec_entity_mentions` rows, and any entity left with +no mention and no relation are all gone — while untouched documents and their derived rows are +unchanged. Ordering and cascade reasoning is in [data-model.md](./data-model.md) §3. + +The stale-vector case is the one to read first: it asserts the `vec_chunks` row for a removed chunk +is deleted, because `chunks.id` values are reused and `embed_new_chunks` skips any chunk that already +has a vector row. + +## Scenario 5 — End-to-end with a real provider (optional, costs money) + +```bash +cd /tmp/kgmd-demo +export OPENROUTER_API_KEY=... # not read, stored, or logged by kgmd +kgmd build # indexes 2 files, not 5 +kgmd find "Atlas" # hits notes/today.md +kgmd stats + +printf 'archive/2024-decisions.md\n' >> .kgmdignore +kgmd build # reports 1 document removed +kgmd find "Keep this one" # no hit from the excluded note +``` + +**Expected**: the first build's summary reports 2 new documents; after the second build the ingest +line reports `Removed: 1`, and the excluded note no longer answers `kgmd find` (SC-005). + +## Scenario 6 — Guard against emptying a graph (FR-015) + +```bash +printf '*\n' >> .kgmdignore +kgmd build ; echo "exit=$?" +kgmd stats # unchanged +``` + +**Expected**: the build fails with a message saying the ignore rules exclude every file while the +graph still holds documents, exit 1, and `kgmd stats` shows the graph untouched. `kgmd build +--dry-run` in the same state exits 0 and simply reports zero included files — a dry run is a read, +so the guard does not apply. + +## Cleanup + +```bash +rm -rf /tmp/kgmd-demo +``` + +## Full gate before proposing the change + +```bash +make format && make lint && make test +``` + +`make test` includes `tests/test_docs.py`, which fails if any new user-facing surface is +undocumented or if a documented surface names something that does not exist — in both directions. The +documentation edits listed in [research.md](./research.md) §R11 are part of this change, not a +follow-up. diff --git a/specs/002-kgmdignore-exclusions/research.md b/specs/002-kgmdignore-exclusions/research.md new file mode 100644 index 0000000..072f864 --- /dev/null +++ b/specs/002-kgmdignore-exclusions/research.md @@ -0,0 +1,313 @@ +# Phase 0 Research: Corpus Exclusions via `.kgmdignore` + +**Feature**: [spec.md](./spec.md) | **Plan**: [plan.md](./plan.md) | **Date**: 2026-08-28 + +Every decision below was verified against the code or a live probe, not assumed. Probe transcripts +are reproduced where the result decided the design. + +--- + +## R1: Pattern matching without a new runtime dependency + +**Decision**: Hand-roll a gitignore-subset pattern compiler in a new leaf module `kgmd/ignore.py` +that translates each pattern to a `re.Pattern` once at parse time. No new runtime dependency. + +**Rationale**: Both stdlib candidates are provably wrong for this job. + +`fnmatch` treats the path as an opaque string — its `*` crosses `/`: + +```text +fnmatch.translate("*.md") -> (?s:.*\.md)\Z # `.*`, not `[^/]*` +fnmatch.fnmatch("a/b.md", "*.md") -> True # must be False for `/`-scoped globs +``` + +`PurePath.match` is right-anchored and its `**` is single-segment on the supported runtimes +(`full_match` arrived in 3.13; the floor is 3.10): + +```text +PurePath("a/b.md").match("*.md") -> True # cannot express "only at root" +PurePath("a/b/d/c.md").match("a/**/c.md") -> False # `**` spans one segment only +PurePath("b.md").match("/b.md") -> False # leading-`/` anchoring unavailable +hasattr(PurePath, "full_match") -> False # on 3.12; absent on 3.10–3.12 +``` + +So neither can express root anchoring, and neither can express recursive `**`. A translator is +~60 lines of `re.escape` plus four substitutions, and it is exactly the part that needs dense unit +tests anyway. + +**Alternatives considered**: + +- **`pathspec`** — gives real gitignore semantics for free. Rejected: the constitution requires + justification for any new runtime dependency, `sqlite-vec` and `fastembed` already make installs + fragile, and spec FR-022 plus the issue's acceptance criteria call for no new dependency. The + semantics we need are a documented subset, not the full grammar. +- **`fnmatch` with a pre-split path** — matching segment lists instead of strings. Rejected: it + collapses under `**`, which must match a variable number of segments, and the bookkeeping ends up + longer than the regex translator. +- **`glob.glob` per pattern against the filesystem** — rejected: it re-walks the tree once per + pattern, cannot express negation ordering, and makes matching depend on what exists on disk rather + than on the path string. + +## R2: Which gitignore semantics to implement, and where to diverge + +**Decision**: Implement this subset, matched against the corpus-relative POSIX path: + +| Construct | Behavior | +|---|---| +| `# comment`, blank line | skipped | +| trailing whitespace | stripped | +| `!pattern` | negation; re-admits a path an earlier rule excluded | +| trailing `/` | directory-only: matches a directory and everything beneath it | +| leading `/` | anchored at the corpus root | +| any interior `/` | anchored at the corpus root | +| no `/` at all | matches at any depth | +| `*` | any run of characters except `/` | +| `?` | one character except `/` | +| `**` | any number of path segments; `a/**/b` also matches `a/b` | +| ordering | last matching rule wins | + +**Deliberate divergence from git**: git cannot re-include a file whose parent directory is excluded +("It is not possible to re-include a file if a parent directory of that file is excluded"). Spec +Acceptance Scenario 4 — `archive/` followed by `!archive/2024-decisions.md` — requires re-inclusion, +and that example comes from the issue itself. Rules are therefore evaluated **per candidate file**: +every rule is tested against the file's relative path and against each of its ancestor prefixes, and +the last rule that matches decides. A negation naming the file wins over an earlier directory rule. + +Verified against real git rather than taken from the documentation — with `archive/` and +`!archive/2024-decisions.md` in `.gitignore`, `git status --untracked-files=all` reports only +`.gitignore`, so the negated file stays invisible: + +```text +.gitignore: archive/ + !archive/2024-decisions.md +git sees: ['.gitignore'] # archive/2024-decisions.md NOT re-included +``` + +This has a second consequence that settles R1 independently: **`pathspec` could not satisfy spec +Acceptance Scenario 4 either**, because it faithfully reproduces git's limitation. The new dependency +would have bought semantics we must diverge from anyway. + +The divergence is a semantic promise, not an accident, so it is documented in +[contracts/kgmdignore-format.md](./contracts/kgmdignore-format.md) and in the configuration +reference. + +**Out of the subset** (documented as such, per spec Assumptions): character classes (`[a-z]`), +backslash escapes for literal `#`, `!`, or trailing space. A pattern using them matches literally, +which is the conservative failure — it excludes nothing rather than excluding too much. + +**Alternatives considered**: full gitignore parity. Rejected — it drags in escape handling and the +re-inclusion limitation we specifically do not want, for constructs no motivating example uses. + +## R3: Whether to prune the directory walk + +**Decision**: Keep the existing `Path.rglob("*.md")` walk. Ignore rules filter the discovered list; +they do not prune traversal. + +**Rationale**: Pruning traversal and supporting re-inclusion are mutually exclusive — if `archive/` +is never descended into, `!archive/2024-decisions.md` can never match. R2 chose re-inclusion because +the spec requires it. The cost being controlled here is provider spend (one model call per chunk), +not walk time; a directory walk is local, free, and already crosses dot-directories today, so this +is not a regression. `corpus.include` remains the tool for scoping the *walk* on a huge tree, and +`.kgmdignore` is the tool for controlling *spend*. That division is worth stating in the docs +because it is the question a user with a vendored `node_modules` will ask. + +**Alternatives considered**: `os.walk` with in-place `dirnames` pruning and git's re-inclusion +limitation. Rejected: it trades a spec requirement for a saving on an operation that costs nothing. + +## R4: Where the resolved file set is computed + +**Decision**: One new function `ingest.scan_corpus_files(root, config) -> FileScan` composes the +three existing filters in a fixed order and is the single source of the resolved set. Both the +ingest path and the preview call it. `find_markdown_files` and `_is_dotpath` keep their current +signatures and roles. + +Order (spec FR-007): `corpus.include` scopes candidates → `.kgmdignore` subtracts, negations re-add +→ dot-path exclusion applied last. + +**Rationale**: Two code paths computing "what will be indexed" is how a preview drifts from reality. +`FileScan` returns the three groups (`included`, `ignored`, `dotpath`) so the preview can report +counts without recomputing anything. Composing rather than rewriting keeps the diff to +`find_markdown_files` at zero, which matters because FR-009 demands byte-identical behavior when no +ignore file exists. + +**Alternatives considered**: threading ignore rules into `find_markdown_files` as a parameter. +Rejected: it would have to return the excluded groups too, so the include-scoping helper would grow +a second responsibility and every caller would pay for it. + +## R5: Why the dot-path rule cannot be re-enabled by a negation + +**Decision**: The dot-path filter stays a separate pass applied *after* ignore evaluation, exactly +as today (`ingest_documents` currently filters with `_is_dotpath` after discovery). + +**Rationale**: Non-overridability (spec FR-008) is structural rather than a check to remember — +there is no code path by which a negation result reaches the dot-path decision. A single combined +rule list would make `!.kgmd/notes.md` a live risk, and `.kgmd/` contains the database and the +prompt overrides. + +## R6: Clearing stale vectors is a correctness fix, not tidiness + +**Decision**: Pruning MUST delete `vec_chunks` and `vec_entity_mentions` rows for the removed ids, +before deleting the `chunks` rows. + +**Rationale**: This is a demonstrable data-corruption path, verified in source: + +```python +# kgmd/embed.py:89-92 +rows = conn.execute( + """SELECT c.id, c.content FROM chunks c + WHERE c.id NOT IN (SELECT chunk_id FROM vec_chunks)""" +).fetchall() +``` + +`chunks.id` and `entity_mentions.id` are plain `INTEGER PRIMARY KEY` (`kgmd/schema.py:36`, `:75`), +so SQLite reuses ids after deletes. The vec0 tables have no foreign key to cascade from. A stale +`vec_chunks` row therefore makes a *future, unrelated* chunk look already-embedded, and semantic +search then answers from the vector of text that no longer exists. `docs/guides/maintenance.md` +already documents this exact hazard for the `reset` path. + +Deleting from a vec0 virtual table by primary key works, including with a subselect — probed against +`sqlite-vec` on this machine: + +```text +DELETE FROM vec_chunks WHERE chunk_id IN (1, 3) -> remaining: [(2,)] +DELETE FROM vec_chunks WHERE chunk_id IN (SELECT id FROM chunks) -> count: 0 +``` + +**Alternatives considered**: leaving vectors and relying on a future clean rebuild. Rejected: it +makes the feature actively harmful — turning on an ignore rule would poison search for unrelated +notes. + +## R7: What else must be removed, and what the cascades do not cover + +**Decision**: Remove in this order, inside the existing ingest transaction: + +1. Resolve orphan `documents.id` set (recorded path not in the resolved set). +2. Collect their `chunks.id`, then the `entity_mentions.id` on those chunks, and the `entity_id` + values those mentions point at (needed in step 7). +3. `DELETE FROM vec_entity_mentions WHERE mention_id IN (...)`. +4. `DELETE FROM vec_chunks WHERE chunk_id IN (...)`. +5. `DELETE FROM relations WHERE evidence_chunk_id IN (...)` — **explicit**, because + `relations.evidence_chunk_id` is `ON DELETE SET NULL` (`kgmd/schema.py:95`), so the rows would + otherwise survive with null evidence and keep answering `kgmd relations` with no provenance + (spec FR-013). The unique index includes `evidence_chunk_id`, so a relation independently + extracted from a surviving chunk is a different row and is untouched. +6. `DELETE FROM chunks WHERE document_id IN (...)` — cascades to `entity_mentions` + (`ON DELETE CASCADE`, `kgmd/schema.py:78`) — then `DELETE FROM documents`. +7. Sweep entities collected in step 2 that now have no mentions and no relations. + +**Why step 7 exists**: spec SC-005 says an excluded note "appears in no query output". An entity +extracted only from an excluded note would otherwise still be listed by `kgmd entities` and +`kgmd find`. The sweep is scoped to entities touched by this prune, so it never collects the +pre-existing orphan entities that `extract --force` leaves behind — that is separate recorded debt +in `docs/guides/maintenance.md` and widening the blast radius here would be an unrequested behavior +change. + +**Binding**: chunked `IN (...)` lists (batch 500) against SQLite's variable limit. A `TEMP TABLE` +would be simpler but issues DDL outside `kgmd/schema.py`, which Principle I prohibits. + +## R8: The empty-resolved-set guard + +**Decision**: Before any delete, if the resolved set is empty and `documents` is non-empty, raise +and change nothing. + +**Rationale**: R7 makes pruning unconditional, so a single stray `*` in `.kgmdignore` would silently +delete an entire graph — a worse failure than the one this feature fixes. Fail fast with a +remediation hint, in the house style of `check_embedding_model` (`kgmd/db.py:50-54`). + +The message must be quotable in `docs/guides/troubleshooting.md`: `test_quoted_errors_exist_in_source` +requires the single code span on each `**Symptom**:` line to appear verbatim in `kgmd/**/*.py`. So +the interpolated count goes in a later fragment and the leading fragment stays one contiguous string +literal that the doc can quote. + +## R9: The preview surface + +**Decision**: `kgmd build --dry-run`, with `--json` for machine output. `--json` without `--dry-run` +is rejected with a `ClickException`. Handled before `init_db`, so a dry run on a corpus with no +database creates nothing. + +**Rationale**: There is no standalone `ingest` command (ingest runs inside `build` at +`kgmd/cli.py:223` and inside `extract` at `:285`), so the preview belongs on the command that does +the spending — "what would build do" is self-documenting and sits where the user is already typing. +Ordering before `init_db` matters: `init_db` creates `graph.db`, which would violate FR-017 for a +dry run on a fresh corpus. When the database is absent the removal count is reported as 0. + +Doc-gate consequences, both satisfied by writing the docs: adding an `as_json` parameter puts `build` +into `structured_output_commands()`, so `test_structured_output_parity` requires a +`**Structured output**:` block in the `### build` section of `docs/reference/cli.md`, and +`test_all_parameters_documented` requires `` `--dry-run` `` and `` `--json` `` as code spans there. + +**Alternatives considered**: + +- **A new `kgmd files` command.** Cleaner as a read command, and `--json` needs no gating. Rejected: + it adds a top-level surface, a `### files` doc section, and a Principle III question about whether + a discovery command belongs in `kgmd/query.py` — for output that only ever matters immediately + before a build. +- **`--json` implying `--dry-run`.** Rejected: `kgmd build --json` would then silently not build, + which is the kind of surprise the constitution's fail-fast rule exists to prevent. + +**MCP exclusion** (Principle III requires stating it): the preview is not added to +`kgmd/mcp_server.py`. It answers a filesystem-and-config question about work not yet done, for an +operator about to spend money; MCP tools read committed graph state for an assistant. The resolved +file set is not graph state, so it also gets no `kgmd/query.py` function — `query.py` remains the +sole read layer for the *graph*, and discovery stays in `ingest.py` where it already lives. + +## R10: Module placement and the starter file + +**Decision**: New leaf module `kgmd/ignore.py` — stdlib only, imported by `ingest.py` and `cli.py`, +importing nothing from the package. It also owns the starter-file template and its writer, so all +`.kgmdignore` knowledge lives in one place. + +`kgmd init` writes `.kgmdignore` at the **corpus root** only when no such file exists, and every +line is a comment or blank so a fresh corpus indexes exactly what it indexes today (spec FR-019). + +**Rationale**: Pattern compilation is a self-contained concern with a large unit-test surface; inside +`ingest.py` it would be half the module. Placing it as a leaf keeps the constitution's one-way +dependency direction intact (`ingest` → `ignore`, and `ignore` → nothing). No new subpackage is +created, which the constitution prohibits. + +`.kgmdignore` sits at the corpus root rather than in `.kgmd/` because it is user-authored input that +users will want in version control beside their notes, and because the issue specifies that +location. It is configuration, not corpus state, so Principle I is not engaged — the same reasoning +that puts `.kgmd/config.yaml` outside `graph.db`. + +**Alternatives considered**: keeping the writer next to `write_default_config` in `kgmd/config.py`. +Rejected: it would split `.kgmdignore` knowledge across two modules for one 12-line constant. + +## R11: Documentation statements this change falsifies + +**Decision**: Treat the following as part of the change, not follow-up. Each is a documented claim +that becomes false, and the docs gate gives no follow-up window. + +| Location | Current claim | Why it breaks | +|---|---|---| +| `docs/guides/maintenance.md:63` | "File deleted from disk \| **nothing**" | Pruning now removes it | +| `docs/guides/maintenance.md:64` | "File renamed … the old path's data lingers" | No longer lingers | +| `docs/guides/maintenance.md:86-87` | orphaned entities need a full rebuild | Now swept on removal (scoped per R7) | +| `docs/examples/personal-notes.md:271-275` | "**Deleted notes are not removed from the graph.**" | Limitation is gone | +| `docs/examples/mcp-assistant.md:170` | "including entities extracted from notes you have since deleted" | No longer true | +| `docs/contributing/architecture.md:15` | `find_markdown_files` applies `corpus.include` and skips dotted paths | Must name the ignore pass and `scan_corpus_files` | +| `docs/reference/configuration.md:78` | `corpus.include` row | Must state the ignore interaction | + +Additions: a `.kgmdignore` section in `docs/reference/configuration.md`; a spend-control mention +under `docs/guides/maintenance.md` → "Controlling provider spend"; `--dry-run` / `--json` rows plus a +`**Structured output**:` block in the `### build` section and the starter file in `### init` of +`docs/reference/cli.md`; a troubleshooting entry for the empty-resolved-set guard. + +No new config key is introduced, so `DEFAULT_CONFIG` is untouched and the "Nineteen keys" count in +`docs/reference/configuration.md` stays correct — a direct benefit of rejecting `corpus.exclude`. + +## R12: Test strategy under Principle V + +**Decision**: Two new test modules, both hermetic. + +- `tests/test_ignore.py` — pattern semantics as pure unit tests: comments, blanks, whitespace, + anchoring, `*` not crossing `/`, `**` across depths, directory rules, negation ordering, + re-inclusion inside an excluded directory, and unsupported constructs matching literally. +- `tests/test_ingest.py` — discovery and pruning over `tmp_path` / `tmp_corpus`: resolved set with + and without an ignore file, `corpus.include` interaction, dot-path non-overridability, prune + removes document/chunks/mentions/relations/vectors, prune idempotency, the empty-set guard, and + the stale-vector regression (hand-packed `struct.pack` vectors per Principle V, asserting the + `vec_chunks` row for a removed chunk is gone so a reused id cannot inherit it). + +No model call is involved in either module — ingest and discovery do not touch the LLM boundary, so +no `litellm` patching is needed except where an existing fixture already provides it. diff --git a/specs/002-kgmdignore-exclusions/spec.md b/specs/002-kgmdignore-exclusions/spec.md new file mode 100644 index 0000000..b5c3e3e --- /dev/null +++ b/specs/002-kgmdignore-exclusions/spec.md @@ -0,0 +1,294 @@ +# Feature Specification: Corpus Exclusions via `.kgmdignore` + +**Feature Branch**: `002-kgmdignore-exclusions` + +**Created**: 2026-08-28 + +**Status**: Draft + +**Input**: User description: "gh issue 3 - add a kgmdignore capability" — GitHub issue #3, "Add a .kgmdignore file to exclude paths from indexing" + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Exclude a folder from indexing (Priority: P1) + +A user keeps a notes directory that contains material with no knowledge-graph value: an `archive/` +folder of superseded notes, a `drafts/` folder of half-written thoughts, a vendored documentation +tree, and a set of `*-template.md` boilerplate files. Today the only way to keep those out of the +graph is to list *every other* directory in the `corpus.include` allowlist and keep that list in +sync as the corpus grows — and a new folder added outside the list is silently never indexed. + +The user instead creates a `.kgmdignore` file at the corpus root, writes the paths and patterns to +skip, and runs a build. The excluded material is not read, not chunked, not sent to the language +model, and does not appear in the graph. Everything else is indexed exactly as before. + +**Why this priority**: This is the feature. Every indexed file costs money — one model call per +chunk — so indexing an archive or a template folder is direct, repeated spend that buys nothing and +pollutes entity resolution with junk mentions. Without this story there is no way to subtract +anything from a corpus. + +**Independent Test**: Create a temporary corpus with files inside and outside the ignored paths, add +a `.kgmdignore`, and assert the resolved set of files to be indexed contains exactly the expected +paths. Fully testable with no model access and no network. + +**Acceptance Scenarios**: + +1. **Given** a corpus containing `notes/a.md` and `archive/b.md`, **When** `.kgmdignore` contains + `archive/`, **Then** the resolved file set is exactly `notes/a.md`. +2. **Given** a corpus containing `docs/CHANGELOG.md` and `docs/guide.md`, **When** `.kgmdignore` + contains `**/CHANGELOG.md`, **Then** `docs/guide.md` is indexed and `docs/CHANGELOG.md` is not. +3. **Given** a `.kgmdignore` whose lines include blank lines, lines starting with `#`, and lines + with surrounding whitespace, **When** a build runs, **Then** comments and blank lines are ignored + and the remaining patterns take effect. +4. **Given** `.kgmdignore` containing `archive/` followed by `!archive/2024-decisions.md`, + **When** a build runs, **Then** `archive/2024-decisions.md` is indexed and every other file + under `archive/` is not. +5. **Given** no `.kgmdignore` file at the corpus root, **When** a build runs, **Then** the resolved + file set is identical to the set produced before this feature existed. +6. **Given** a corpus with `corpus.include: ["notes"]` in configuration and `.kgmdignore` + containing `notes/archive/`, **When** a build runs, **Then** the allowlist scopes the scan to + `notes` and the ignore rules subtract `notes/archive/` from it. +7. **Given** `.kgmdignore` containing `!.kgmd/graph-notes.md` or `!.git/README.md`, **When** a build + runs, **Then** dot-prefixed paths remain excluded — the negation cannot re-admit them. + +--- + +### User Story 2 - Newly-excluded material leaves the graph (Priority: P2) + +A user has already built a graph over a corpus. They now add a `.kgmdignore` that excludes +`archive/`. They expect those notes to stop answering searches. The same user has also deleted and +renamed notes over time and expects the graph to reflect that. + +After the next build, documents that are no longer part of the resolved file set — because they were +newly ignored, deleted, or renamed — are gone from the graph, along with everything derived from +them: their text chunks, the entity mentions found in them, and their search vectors. + +**Why this priority**: Without this, the feature is a no-op for every corpus that already exists. +The ignored notes, their chunks, their mentions, and their embeddings stay in the single durable +artifact and keep answering queries, so the user sees no behavioral change and reasonably concludes +the ignore file does not work. It is P2 rather than P1 only because Story 1 alone delivers value for +a corpus built after the ignore file exists. + +**Independent Test**: Build a graph over a seeded corpus, add an ignore rule covering one document, +re-build, and assert that document, its chunks, its mentions, and its vector rows are absent while +the untouched documents and their derived rows are unchanged. + +**Acceptance Scenarios**: + +1. **Given** an existing graph containing `archive/old.md` and its derived chunks, mentions, and + vectors, **When** `.kgmdignore` gains `archive/` and a build runs, **Then** the document row, its + chunks, its entity mentions, and its vector rows are all removed. +2. **Given** an existing graph, **When** a note is deleted or renamed on disk and a build runs, + **Then** the record for the old path and all state derived from it is removed. +3. **Given** a relation whose supporting evidence came from a chunk of a now-removed document, + **When** the removal completes, **Then** no relation is left pointing at evidence that no longer + exists. +4. **Given** a corpus where removal has already happened, **When** the build is re-run with no + changes, **Then** nothing further is removed and the reported counts show no work done. +5. **Given** an existing non-empty graph, **When** the ignore rules would resolve to an empty file + set (for example a stray `*` pattern), **Then** the build stops with an actionable message and + removes nothing. +6. **Given** an existing graph and a file whose content is unchanged and which remains un-ignored, + **When** a build runs, **Then** that document is skipped exactly as before and no re-extraction + is triggered. + +--- + +### User Story 3 - Preview what will be indexed before spending (Priority: P3) + +A user has written several ignore patterns and wants to know whether they did it right before paying +for a build. They ask for a preview and see the exact list of files that would be indexed, the count +of files, and the count excluded — with nothing written to the graph and no model calls made. + +**Why this priority**: Ignore rules are otherwise unverifiable guesswork; a wrong pattern is +discovered only after the spend it was supposed to prevent. It is P3 because Stories 1 and 2 are +correct and useful without it. + +**Independent Test**: Run the preview against a temporary corpus with an ignore file and assert the +printed/returned file list matches expectation and that the graph is byte-for-byte unmodified. + +**Acceptance Scenarios**: + +1. **Given** a corpus with a `.kgmdignore`, **When** the user requests a preview, **Then** the + resolved file set is reported and no document, chunk, mention, or vector is created, updated, or + removed. +2. **Given** a preview request, **When** it completes, **Then** no language-model call is made. +3. **Given** a preview request, **When** the user asks for machine-readable output, **Then** the + resolved file set and counts are available as structured data. + +--- + +### Edge Cases + +- **No ignore file**: behavior is exactly as before this feature — no warning, no new output. +- **Empty ignore file, or one containing only comments and blank lines**: treated as no rules. +- **Rules exclude everything**: on an existing non-empty graph the build stops with an actionable + message and removes nothing, rather than silently emptying the graph. On an empty graph it reports + zero files to index. +- **Negation that re-admits a dot-prefixed path**: refused; the dot-path rule is applied last and is + not overridable. +- **Ignore file placed in a subdirectory**: only the corpus-root file is read; nested ignore files + are not consulted and this is documented. +- **Unreadable or non-UTF-8 ignore file**: build fails fast with a message naming the file and the + remediation, rather than silently indexing everything. +- **Pattern written with a leading separator** (`/archive/`): anchored to the corpus root, not + matched at every depth. +- **Pattern that matches a directory name mid-path** (`node_modules/`): excludes the whole subtree + wherever it appears, unless anchored. +- **Ignore file edited between builds**: re-evaluated from disk on every run; no cached rule state. +- **A file both allowed by `corpus.include` and matched by an ignore rule**: excluded — ignore + subtracts from the allowlist. +- **A file matched by an ignore rule but outside `corpus.include`**: already excluded; no error, no + duplicate reporting. +- **Trailing whitespace and CRLF line endings in the ignore file**: tolerated. + +## Requirements *(mandatory)* + +### Functional Requirements + +**Rule source and syntax** + +- **FR-001**: The system MUST read exclusion rules from a single `.kgmdignore` file at the corpus + root, matching patterns against corpus-relative paths using `/` as the separator regardless of + host platform. +- **FR-002**: The system MUST ignore blank lines, lines whose first non-whitespace character is `#`, + and surrounding whitespace on each rule. +- **FR-003**: The system MUST support directory-only rules, written with a trailing `/`, which + exclude the entire subtree beneath the matched directory. +- **FR-004**: The system MUST support glob rules covering single-segment wildcards (`*`, `?`), + multi-segment wildcards (`**`), and root-anchoring via a leading `/`. An unanchored pattern with no + separator MUST match at any depth. +- **FR-005**: The system MUST support negation rules prefixed with `!` that re-admit a path excluded + by an earlier rule, resolved in file order with the last matching rule winning. +- **FR-006**: The system MUST fail with an actionable message naming the file when `.kgmdignore` + exists but cannot be read or decoded, rather than proceeding as if it were absent. + +**Precedence and compatibility** + +- **FR-007**: The system MUST apply exclusion in this order: the `corpus.include` allowlist scopes + the candidate set (behavior unchanged), then `.kgmdignore` rules subtract from it with negations + re-adding, then the existing dot-prefixed-path exclusion is applied last. +- **FR-008**: The system MUST keep the dot-prefixed-path exclusion non-overridable: no ignore or + negation rule may cause a path containing a dot-prefixed component to be indexed. +- **FR-009**: The system MUST produce, when no `.kgmdignore` exists, exactly the file set it + produced before this feature — no change in discovery, ordering, counts, or output. +- **FR-010**: The system MUST document and cover by test the combined case where both + `corpus.include` and `.kgmdignore` are present. + +**Removing material that is no longer part of the corpus** + +- **FR-011**: The system MUST remove from the graph every document whose path is no longer in the + resolved file set — whether it became ignored, was deleted, or was renamed. +- **FR-012**: Removal MUST also clear all state derived from that document: its chunks, the entity + mentions in those chunks, and the search vectors for both, leaving no vector row that could later + bind to an unrelated record. +- **FR-013**: Removal MUST NOT leave a relation asserting evidence from a chunk that no longer + exists. +- **FR-014**: Re-running a build after removal MUST remove nothing further and MUST report no work, + and MUST NOT re-extract documents whose content is unchanged. +- **FR-015**: When the resolved file set is empty and the graph is not, the system MUST stop with an + actionable message and MUST NOT remove anything. +- **FR-016**: Removal counts MUST be reported in the build summary so the user can see what left the + graph. + +**Preview** + +- **FR-017**: Users MUST be able to preview the resolved file set — the paths that would be indexed, + plus counts of included and excluded files — without creating, updating, or removing any graph + state and without any language-model call. +- **FR-018**: The preview MUST offer machine-readable output alongside its human-readable rendering. + +**Starter file and documentation** + +- **FR-019**: Corpus initialization MUST write a starter `.kgmdignore` whose every line is a comment + or blank, so a freshly initialized corpus indexes exactly what it does today. +- **FR-020**: The system MUST NOT introduce a second mechanism for the same job: no + `corpus.exclude` configuration key is added. +- **FR-021**: Documentation MUST be updated in the same change: a `.kgmdignore` reference section + covering syntax, precedence, and the removal behavior; a spend-control mention in the maintenance + guide; and command reference entries for any new user-facing option. The documentation coverage + gate MUST pass in both directions. +- **FR-022**: The feature MUST NOT add a runtime dependency, or MUST justify one per the project's + dependency rule. + +### Key Entities + +- **Ignore ruleset**: the ordered list of rules parsed from the corpus-root `.kgmdignore`. Ordered, + because the last matching rule decides. Absent file means an empty ruleset. +- **Ignore rule**: one line — a pattern, whether it is negated, whether it is directory-only, and + whether it is root-anchored. +- **Resolved file set**: the corpus-relative paths that will be indexed, after allowlist scoping, + ignore subtraction, negation re-admission, and the dot-path exclusion. This is the value the + preview reports and the value pruning compares against. +- **Orphan**: an indexed document whose path is absent from the resolved file set. Orphans and + everything derived from them are removed. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A user excludes a subtree by adding one line to one file; the indexed file count drops + by exactly the number of markdown files in that subtree and no other file's indexing status + changes. +- **SC-002**: Excluded files generate zero language-model calls and zero spend on every subsequent + build, measured by call count attributable to those files being 0. +- **SC-003**: A corpus with no `.kgmdignore` produces an identical indexed file set and identical + build counts before and after this change — verified as a regression check, not by inspection. +- **SC-004**: A user can see the complete list of files a build would index, before any spend, in + under 5 seconds on a 1,000-file corpus. +- **SC-005**: A note that becomes excluded returns zero search results and appears in no query + output after the next build. +- **SC-006**: Every new user-facing surface introduced by this change has a documentation entry, and + every documentation entry names something that exists — both directions enforced by the test gate. +- **SC-007**: The exclusion behavior is covered by tests that run with no network access and no model + downloads, and pass on every supported runtime version. + +## Assumptions + +- **Gitignore-familiar semantics are what users expect.** The file name promises them, so the + supported syntax is the common subset: comments, blank lines, `*`/`?`/`**` globs, trailing-`/` + directory rules, leading-`/` anchoring, and `!` negation with last-match-wins ordering. Rarely used + gitignore corners (character classes such as `[a-z]`, escaped literal `#`/`!`/spaces) are treated + as out of scope for this change and documented as such. +- **Negation is in scope.** The issue permits declaring it out of scope, but the motivating example + uses it and last-match-wins ordering is implementable without a new dependency. +- **One root-level file only.** Nested per-directory ignore files are out of scope; the corpus root + is the single source of rules. +- **Pattern matching is case-sensitive**, matching gitignore behavior on a case-sensitive + filesystem, regardless of the host filesystem's own case sensitivity. +- **Pruning is unified rather than ignore-specific.** Comparing the resolved file set against + recorded document paths handles newly-ignored, deleted, and renamed files with one mechanism. + Detecting "newly ignored" specifically would be strictly more work for strictly less coverage, so + the issue's optional split is not taken and the "document the limitation instead" escape is not + used. +- **The ignore file is the only exclusion mechanism.** No inline configuration key is added, so no + new configuration key needs documenting and there is only one place to look when a file is missing + from the graph. +- **`corpus.include` keeps its current literal-path behavior.** Its lack of glob support is a + separate defect and is not fixed here. +- **Reusing `.gitignore` is out of scope.** A toggle to honor the repository's own ignore file + conflates two concerns and belongs in a follow-up. +- **Only markdown files are candidates**, unchanged from today; ignore rules narrow that set and + never widen it. +- **The preview surfaces on the existing build path** rather than as a new top-level command, since + there is no standalone ingest command today. + +## Dependencies + +- The existing graph store's removal behavior: discarding a note's text automatically discards the + entity mentions found in it, but it does not clear that note's search vectors and it leaves + relations asserting evidence that no longer exists. Removal must therefore clear those explicitly, + or stale search results and unsupported relations survive the removal. +- The documentation coverage gate, which fails the test suite if a new user-facing surface is + undocumented or a documented surface does not exist. +- The project's offline test conventions — temporary-directory corpora, a mocked model boundary, and + pre-computed search vectors — which this feature's tests must follow. + +## Out of Scope + +- A `corpus.exclude` configuration key or any second exclusion mechanism. +- Honoring `.gitignore`, or any toggle to do so. +- Glob support for `corpus.include`. +- Per-directory (nested) ignore files. +- Gitignore character classes and escape sequences. +- Following symbolic links during discovery; current behavior is unchanged. diff --git a/specs/002-kgmdignore-exclusions/tasks.md b/specs/002-kgmdignore-exclusions/tasks.md new file mode 100644 index 0000000..5774b08 --- /dev/null +++ b/specs/002-kgmdignore-exclusions/tasks.md @@ -0,0 +1,302 @@ +--- + +description: "Task list for Corpus Exclusions via .kgmdignore" +--- + +# Tasks: Corpus Exclusions via `.kgmdignore` + +**Input**: Design documents from `/specs/002-kgmdignore-exclusions/` + +**Prerequisites**: [plan.md](./plan.md), [spec.md](./spec.md), [research.md](./research.md), [data-model.md](./data-model.md), [contracts/](./contracts/) + +**Tests**: REQUIRED, not optional. Constitution Principle V ("Every new pipeline stage, query +function, export format, MCP tool, or CLI command MUST ship tests in the same change"), spec SC-007, +and the source issue's acceptance criteria ("verified by a test asserting the discovered file set") +all demand them. Repo convention is TDD — test red, then implement green. + +**Organization**: Tasks are grouped by user story so each story is independently implementable and +independently testable. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel — different files, no dependency on an incomplete task +- **[Story]**: US1 / US2 / US3, mapping to the prioritized stories in [spec.md](./spec.md) +- Exact file paths are in every task + +## Path Conventions + +Single flat Python package at the repository root: `kgmd/` for source, `tests/` for tests, `docs/` +for the authoritative documentation surface. No `src/` directory. Per +[plan.md](./plan.md) → Project Structure. + +## Critical constraints that shape these tasks + +1. **Documentation is gated per story, not deferred to Polish.** `tests/test_docs.py` fails the whole + suite in both directions, so a story whose docs are missing leaves a red suite. Each story phase + therefore carries its own documentation tasks and each checkpoint is genuinely green. +2. **Same-file tasks are never `[P]`.** `kgmd/ingest.py`, `kgmd/cli.py`, and `tests/test_ingest.py` + are each touched by several phases; those tasks are sequential by construction. +3. **Collect ids before deleting.** In pruning, mention ids are unrecoverable once chunks are gone — + see [data-model.md](./data-model.md) §3 for the exact 12-step order. +4. **No new runtime dependency, no DDL, no new config key.** If a task seems to need one, stop: the + design says it does not. + +--- + +## Phase 1: Setup + +**Purpose**: Establish an attributable baseline before touching anything. + +- [X] T001 Run `make format && make lint && make test` on the unmodified tree and record the passing test count, so every later failure is attributable to this change rather than to pre-existing state + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Introduce the single resolved-file-set function that all three stories read. This phase +deliberately adds **no** new behavior — it is a pure refactor whose success criterion is that nothing +changes. + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete. + +- [X] T002 [P] Write failing test in `tests/test_ingest.py` asserting `scan_corpus_files(corpus, config)` returns a `FileScan` whose `included` equals the current `sorted(root.rglob("*.md"))` minus dot-paths over `tests/fixtures/`, whose `ignored` is empty with no `.kgmdignore` present, and whose three lists are disjoint (spec FR-009, SC-003) +- [X] T003 Add the frozen `FileScan` dataclass (`included`, `ignored`, `dotpath`: `list[Path]`) and `scan_corpus_files(root, config) -> FileScan` to `kgmd/ingest.py`, composing the **unchanged** `find_markdown_files` and `_is_dotpath` with an empty ignore stage, per [contracts/library-api.md](./contracts/library-api.md) +- [X] T004 Replace the inline discovery-and-dot-path filter in `ingest_documents` (`kgmd/ingest.py`, currently lines 160-163) with a `scan_corpus_files` call, leaving the insert/update loop and the returned `new`/`updated`/`skipped`/`chunks_created` keys untouched +- [X] T005 Verify no behavior change: run `python -m pytest tests/ -v` and confirm the same passing count as T001, with `tests/test_extract.py` (which calls `ingest_documents` three times) green + +**Checkpoint**: One function now answers "what would this corpus index". All three stories can begin. + +--- + +## Phase 3: User Story 1 - Exclude a folder from indexing (Priority: P1) 🎯 MVP + +**Goal**: A `.kgmdignore` file at the corpus root subtracts paths from ingest, with comments, blank +lines, directory rules, globs, and negation — and the dot-path rule still cannot be overridden. + +**Independent Test**: Create a temporary corpus with files inside and outside the ignored paths, add +a `.kgmdignore`, and assert the resolved file set is exactly the expected paths. No model access, no +network, no database needed. + +### Tests for User Story 1 + +> Write these first and confirm they fail — `kgmd/ignore.py` does not exist yet. + +- [X] T006 [P] [US1] Create `tests/test_ignore.py` covering parsing: `#` comments, blank and whitespace-only lines, trailing-whitespace stripping, `!` setting negation, trailing `/` setting directory-only, a lone `!` or `/` producing no rule, and `line_number` being 1-based +- [X] T007 [P] [US1] Extend `tests/test_ignore.py` with matching semantics from [contracts/kgmdignore-format.md](./contracts/kgmdignore-format.md): `*` never crossing `/`, `?` matching one non-`/` character, `**` spanning several segments with `a/**/b` also matching `a/b`, leading-`/` anchoring, interior-`/` anchoring, no-`/` matching at any depth, directory rules covering the whole subtree, last-match-wins ordering, negation re-including a file inside an excluded directory, `is_ignored(path, [])` always `False`, and an unsupported `[a-z]` class matching literally rather than excluding +- [X] T008 [P] [US1] Add ignore-integration cases to `tests/test_ingest.py`: the worked example from the format contract resolving exactly as tabulated, `corpus.include` plus `.kgmdignore` together (ignore subtracts from the allowlist), a file matched by a rule but already outside `corpus.include` causing no error, `!.kgmd/notes.md` failing to re-admit a dot-path (FR-008), an empty/comments-only `.kgmdignore` behaving as no rules, and an undecodable `.kgmdignore` raising with the filename in the message + +### Implementation for User Story 1 + +- [X] T009 [US1] Create `kgmd/ignore.py` with the frozen `IgnoreRule` dataclass and `parse_ignore_rules(text) -> list[IgnoreRule]`, compiling each pattern to a fully anchored `re.Pattern` at parse time — stdlib only (`dataclasses`, `pathlib`, `re`), importing nothing from `kgmd` +- [X] T010 [US1] Add `load_ignore_rules(root)` (returns `[]` when `.kgmdignore` is absent; raises `RuntimeError` naming the file when present and undecodable) and `is_ignored(rel_path, rules)` (tests every rule against the path and each ancestor prefix, last match wins) to `kgmd/ignore.py` +- [X] T011 [US1] Wire the ignore stage into `scan_corpus_files` in `kgmd/ingest.py`: load rules once per call, partition candidates into `ignored` and survivors, then apply `_is_dotpath` **last** so no negation can reach the dot-path decision +- [X] T012 [US1] Add `DEFAULT_IGNORE_TEMPLATE` (every line a comment or blank, with worked examples for directory, glob, and negation rules) and `write_default_ignore_file(path)` (no-op when the file exists, so a user's file is never clobbered) to `kgmd/ignore.py` +- [X] T013 [US1] Call `write_default_ignore_file(corpus_dir / ".kgmdignore")` from `init` in `kgmd/cli.py` (after `write_default_config`, around line 91) and print the path alongside the existing Database/Config lines +- [X] T014 [US1] Add a test to `tests/test_ingest.py` asserting the starter template parses to zero rules, so a freshly initialized corpus indexes exactly what it indexes today (FR-019) + +### Documentation for User Story 1 + +- [X] T015 [P] [US1] Add a `.kgmdignore` section to `docs/reference/configuration.md` covering location, line grammar, the supported constructs table, precedence, the deliberate git divergence on negation inside an excluded directory, and the unsupported constructs — sourced from [contracts/kgmdignore-format.md](./contracts/kgmdignore-format.md) +- [X] T016 [P] [US1] Update the `corpus.include` row in `docs/reference/configuration.md` (line 78) to state the interaction: the allowlist scopes the walk, `.kgmdignore` subtracts from it, and the dot-path rule applies last — without adding a config key, so the page's "Nineteen keys" count stays correct +- [X] T017 [P] [US1] Add `.kgmdignore` to `docs/guides/maintenance.md` under "Controlling provider spend" (line 89), stating that excluded files cost nothing because spend is per chunk, and that `corpus.include` scopes the walk while `.kgmdignore` controls spend +- [X] T018 [P] [US1] Note the starter `.kgmdignore` in the `### init` section of `docs/reference/cli.md` (line 45), including that an existing file is never overwritten + +**Checkpoint**: Exclusion works end to end. `make test` is green, docs are consistent, and this is a +shippable MVP for any corpus built after the ignore file exists. + +--- + +## Phase 4: User Story 2 - Newly-excluded material leaves the graph (Priority: P2) + +**Goal**: Documents absent from the resolved set — newly ignored, deleted, or renamed — are removed +from the graph along with their chunks, mentions, evidence-bound relations, vectors, and any entity +left with no mention and no relation. + +**Independent Test**: Build a graph over a seeded corpus, add an ignore rule covering one document, +re-run ingest, and assert that document and everything derived from it is gone while untouched +documents are unchanged. Works without US1 as well, by deleting a file instead of ignoring it. + +### Tests for User Story 2 + +> Write these first. Use hand-packed `struct.pack` vectors per Principle V — no embedding backend. + +- [X] T019 [US2] Add prune tests to `tests/test_ingest.py`: after a document's path leaves the resolved set, its `documents` row, its `chunks`, its `entity_mentions`, and its `relations` rows bound to those chunks as evidence are all gone, while a sibling document's rows are untouched +- [X] T020 [US2] Add the stale-vector regression to `tests/test_ingest.py`: insert hand-packed rows into `vec_chunks` and `vec_entity_mentions` for a document's chunks and mentions, remove the document, and assert both vector tables have no row for the removed ids — the failure mode this prevents is `embed_new_chunks` selecting `WHERE c.id NOT IN (SELECT chunk_id FROM vec_chunks)` and treating a reused id as already embedded (`kgmd/embed.py:89-92`) +- [X] T021 [US2] Add the entity-sweep test to `tests/test_ingest.py`: an entity mentioned only in the removed document is deleted, an entity also mentioned elsewhere survives, and an entity that still holds a relation survives (SC-005) +- [X] T022 [US2] Add the idempotency test to `tests/test_ingest.py`, in the shape of `tests/test_extract.py::test_extraction_idempotent`: a second ingest over an unchanged corpus removes nothing, reports zero removal counts, and leaves `documents.last_extracted_hash` intact so no re-extraction is triggered (FR-014, Principle IV) +- [X] T023 [US2] Add the guard test to `tests/test_ingest.py`: with a non-empty graph and an ignore rule matching everything, ingest raises and the `documents`, `chunks`, and vector tables are byte-for-byte unchanged (FR-015) — plus the deleted-file and renamed-file cases, asserting a rename resolves to one removal and one insert in a single transaction + +### Implementation for User Story 2 + +- [X] T024 [US2] Implement `prune_missing_documents(conn, kept_rel_paths) -> dict` in `kgmd/ingest.py` following the ordered sequence in [data-model.md](./data-model.md) §3: resolve orphan document ids, collect chunk ids then mention ids then their entity ids **before** any delete, and take the zero-write early exit when there are no orphans +- [X] T025 [US2] Add the empty-set guard to `prune_missing_documents` before any write: when `kept_rel_paths` is empty and `documents` is non-empty, raise `RuntimeError` in the style of `check_embedding_model` (`kgmd/db.py:50-54`), keeping the leading fragment a single contiguous string literal so `docs/guides/troubleshooting.md` can quote it verbatim, with the document count in a later interpolated fragment +- [X] T026 [US2] Add the delete sequence to `prune_missing_documents`: `vec_entity_mentions` by mention id, `vec_chunks` by chunk id, `relations` by `evidence_chunk_id` (explicit — the column is `ON DELETE SET NULL`, so those rows would otherwise survive with no provenance), `chunks` by document id (cascades mentions), `documents` by id, then the scoped entity sweep; bind ids in batches of 500 and use no `TEMP TABLE`, which would be DDL outside `kgmd/schema.py` +- [X] T027 [US2] Call `prune_missing_documents` from `ingest_documents` in `kgmd/ingest.py` with the resolved corpus-relative paths, before the insert/update loop, and merge the five removal counts into the returned stats dict without renaming any existing key +- [X] T028 [US2] Report the removal counts in `kgmd/cli.py` in both ingest summaries — `build` (line 224) and `extract` (line 286) — so the user sees what left the graph (FR-016) + +### Documentation for User Story 2 + +- [X] T029 [P] [US2] Correct the "Change made, work repeated" table in `docs/guides/maintenance.md` (lines 63-64): "File deleted from disk" now removes the document, its chunks, its mentions, its evidence-bound relations, and its vectors; "File renamed" is now a removal plus an insert with nothing lingering at the old path +- [X] T030 [P] [US2] Update `docs/guides/maintenance.md` lines 42-43 and 86-87 so the orphan-entity statements stay accurate: entities are still never swept on an **edit**, but a document **removal** now sweeps entities it leaves with no mention and no relation; keep the `extract --force` orphan case documented as an open limitation +- [X] T031 [P] [US2] Remove the now-false "**Deleted notes are not removed from the graph.**" limitation from `docs/examples/personal-notes.md` (lines 271-275) and replace it with the new behavior, including the empty-resolved-set guard +- [X] T032 [P] [US2] Fix the stale claim in `docs/examples/mcp-assistant.md` (line 170) that tools reflect "entities extracted from notes you have since deleted" +- [X] T033 [P] [US2] Add a troubleshooting entry to `docs/guides/troubleshooting.md` for the empty-resolved-set guard, with exactly one code span on the `**Symptom**:` line quoting a literal that appears verbatim in `kgmd/ingest.py` — `test_quoted_errors_exist_in_source` enforces this + +**Checkpoint**: The file set shrinking now shrinks the graph. US1 and US2 both work independently and +`make test` is green. + +--- + +## Phase 5: User Story 3 - Preview what will be indexed (Priority: P3) + +**Goal**: `kgmd build --dry-run` reports the resolved file set, the exclusion counts, and how many +indexed documents would be removed — writing nothing and calling no model. `--json` gives the +machine-readable form. + +**Independent Test**: Run the preview against a temporary corpus with an ignore file; assert the +reported set matches expectation, no `graph.db` is created, and the JSON shape matches the contract. + +### Tests for User Story 3 + +- [X] T034 [US3] Add `dry_run_report` tests to `tests/test_ingest.py`: the returned dict matches the shape in [contracts/cli-build-dry-run.md](./contracts/cli-build-dry-run.md), all paths are corpus-relative POSIX strings sorted identically to ingest order, `counts.would_remove` reflects indexed documents absent from `included`, `conn=None` yields `would_remove` of 0, and calling it leaves every table unchanged +- [X] T035 [P] [US3] Create `tests/test_cli.py` using `click.testing.CliRunner` for the two guarantees that only exist at the CLI level: `kgmd build --dry-run` on a corpus with no database creates no `.kgmd/graph.db`, and `kgmd build --json` without `--dry-run` exits non-zero with `--json requires --dry-run.` — this deliberately establishes the CliRunner convention, since no existing test invokes the CLI + +### Implementation for User Story 3 + +- [X] T036 [US3] Implement `dry_run_report(conn, corpus_dir, config) -> dict` in `kgmd/ingest.py`: call `scan_corpus_files`, convert paths to sorted corpus-relative POSIX strings, and count indexed documents absent from `included` by reading `documents.path` (read-only; tolerate `conn=None`) +- [X] T037 [US3] Add `--dry-run` and `--json` (dest `as_json`) options to `build` in `kgmd/cli.py`, rejecting `--json` without `--dry-run` via `click.ClickException`, and handle the dry-run branch **before** `init_db` so no `graph.db` is created — passing a connection only when the database already exists +- [X] T038 [US3] Render the preview in `kgmd/cli.py`: the human form on `console` per the contract's sample output, and `json.dumps` of the report as the only stdout content under `--json` + +### Documentation for User Story 3 + +- [X] T039 [US3] Add `` `--dry-run` `` and `` `--json` `` rows to the parameter table in the `### build` section of `docs/reference/cli.md` (lines 116-121) — `test_all_parameters_documented` requires both as inline code spans in that section +- [X] T040 [US3] Add a `**Structured output**:` block to the `### build` section of `docs/reference/cli.md` documenting the JSON shape — mandatory now that `build` has an `as_json` parameter, because `test_structured_output_parity` compares documented blocks against commands having one +- [X] T041 [US3] Document the dry-run workflow in the `### build` prose of `docs/reference/cli.md`: it takes no build lock, makes no provider call, creates no database, and is the way to check `.kgmdignore` rules before spending + +**Checkpoint**: All three stories independently functional; ignore rules are debuggable instead of +guesswork. + +--- + +## Phase 6: Polish & Cross-Cutting Concerns + +- [X] T042 [P] Update the `kgmd/ingest.py` row in `docs/contributing/architecture.md` (line 15) to name `scan_corpus_files` as the resolved-file-set entry point, the `.kgmdignore` pass, and `prune_missing_documents`, and add a `kgmd/ignore.py` row placing it as a stdlib-only leaf +- [X] T043 [P] Add one orientation line to `README.md` under Quickstart pointing at `.kgmdignore` and `kgmd build --dry-run` as the spend-control path, linking to `docs/reference/configuration.md` rather than duplicating detail +- [X] T044 Run `make format && make lint` and fix any `ruff` finding (`line-length = 100`, select `E,F,I,W`); do not add a competing formatter config +- [X] T045 Run `make test` and confirm every `tests/test_docs.py` check passes in both directions — no undocumented surface, no documented surface that does not exist +- [X] T046 Walk [quickstart.md](./quickstart.md) Scenarios 1, 2, 3, 4, and 6 (all offline) and confirm each expected outcome; Scenario 5 is optional and costs money +- [X] T047 Confirm the Success Criteria in [spec.md](./spec.md) hold, naming the artifact that proves each: SC-001 and SC-003 via the discovery assertions in `tests/test_ingest.py`, SC-004 by timing `dry_run_report` over a 1,000-file corpus generated in `tmp_path` (throwaway, never committed), SC-005 via the entity-sweep and search assertions in `tests/test_ingest.py`, SC-006 via `tests/test_docs.py`, and SC-007 by running `make test` with no network access + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: no dependencies +- **Foundational (Phase 2)**: depends on Setup — **blocks all three stories**, because every story + reads `scan_corpus_files` +- **User Story 1 (Phase 3)**: depends on Foundational only +- **User Story 2 (Phase 4)**: depends on Foundational only. It does **not** depend on US1 — pruning is + driven by set difference, so it is demonstrable with a deleted file and needs no ignore rules +- **User Story 3 (Phase 5)**: depends on Foundational only. Independent of US1 and US2; with neither + present it previews the include/dot-path resolution and reports `would_remove` from the existing + document set +- **Polish (Phase 6)**: depends on every story that is being shipped + +### Within Each User Story + +- Tests are written first and must fail before implementation (repo TDD convention) +- `kgmd/ignore.py` before its `kgmd/ingest.py` wiring; `kgmd/ingest.py` before its `kgmd/cli.py` + rendering +- Documentation tasks last within the story, but **inside** the story — not deferred to Polish, or + the story's checkpoint leaves a red suite + +### Sequential by shared file (never `[P]` together) + +| File | Tasks | +|---|---| +| `kgmd/ingest.py` | T003, T004, T011, T024, T025, T026, T027, T036 | +| `kgmd/ignore.py` | T009, T010, T012 | +| `kgmd/cli.py` | T013, T028, T037, T038 | +| `tests/test_ingest.py` | T002, T008, T014, T019, T020, T021, T022, T023, T034 | +| `tests/test_ignore.py` | T006, T007 | +| `docs/reference/cli.md` | T018, T039, T040, T041 | +| `docs/reference/configuration.md` | T015, T016 | +| `docs/guides/maintenance.md` | T017, T029, T030 | + +### Parallel Opportunities + +- T006, T007, T008 — three test files at the start of US1 (`test_ignore.py` twice is sequential; T008 + is a different file, so T006+T008 or T007+T008 pair) +- T015, T016, T017, T018 — four documentation files in US1, all independent +- T029, T030, T031, T032, T033 — five documentation files in US2, all independent +- T034 and T035 — different test files in US3 +- T042 and T043 — different documentation files in Polish +- With multiple developers, all three stories can run concurrently once Phase 2 is done; the shared + files above are the only contention points + +--- + +## Parallel Example: User Story 2 documentation + +```bash +# Five independent files, one agent each: +Task: "Correct the deleted/renamed rows in docs/guides/maintenance.md" +Task: "Update the orphan-entity statements in docs/guides/maintenance.md" # same file — NOT parallel +Task: "Remove the stale limitation from docs/examples/personal-notes.md" +Task: "Fix the deleted-notes claim in docs/examples/mcp-assistant.md" +Task: "Add the guard entry to docs/guides/troubleshooting.md" +``` + +The two `maintenance.md` tasks (T029, T030) must run in sequence; the other three are genuinely +parallel. + +## Parallel Example: User Story 1 tests + +```bash +Task: "Parsing tests in tests/test_ignore.py" # T006 +Task: "Ignore-integration cases in tests/test_ingest.py" # T008 — different file, parallel +# T007 extends tests/test_ignore.py, so it follows T006 +``` + +--- + +## Implementation Strategy + +### MVP First (Foundational + User Story 1) + +1. Phase 1 — baseline recorded +2. Phase 2 — `scan_corpus_files` in place, proven to change nothing +3. Phase 3 — `.kgmdignore` excludes paths, with docs +4. **STOP and VALIDATE**: quickstart Scenarios 1 and 3; `make test` green + +This is a coherent, shippable increment: exclusion works for any corpus built after the ignore file +exists. Its honest limitation is that an already-indexed file that becomes ignored stays in the graph +— which is exactly what US2 fixes, and why US2 is not optional for a corpus that already has a graph. + +### Incremental Delivery + +1. Foundational → nothing observable changes +2. + US1 → exclusion works (MVP) +3. + US2 → the graph shrinks with the file set; deleted and renamed notes stop lingering +4. + US3 → rules become inspectable before spending +5. Polish → contributor docs, README orientation, full gate + +### Recommended full-feature order + +US1 → US2 → US3. US2 carries the correctness risk (stale vectors, dangling relations, the +destructive-pattern guard), so it should land while the design reasoning is fresh rather than after +the lower-risk preview work. + +--- + +## Notes + +- `[P]` means different files and no dependency on an incomplete task +- Every user-story task carries its story label for traceability +- Verify tests fail before implementing +- Commit after each task or logical group, with a capitalized imperative subject describing the why +- Never use `--no-verify`; never skip or delete a failing test in place of fixing it +- Do not fix `kgmd reset`'s missing vector cleanup or its `VACUUM`-in-transaction defect here — both + are pre-existing, documented, and out of scope for this feature