From 96f00cf5f27d806da3c52bc4d02c1741e831efc0 Mon Sep 17 00:00:00 2001 From: santxkoz Date: Wed, 1 Jul 2026 06:34:23 +0700 Subject: [PATCH] Add personal knowledge base workflow --- .gitignore | 5 +- .../knowledge-system-architecture.md | 164 +++++++ .../knowledge-ingestion-workflow.md | 162 +++++++ .../knowledge-api-and-graphql-plan.md | 181 ++++++++ docs/UET_Documentation_Details/README.md | 17 + docs/knowledge_base/README.md | 30 +- docs/knowledge_base/personal_kb.py | 406 ++++++++++++++++++ 7 files changed, 963 insertions(+), 2 deletions(-) create mode 100644 docs/UET_Documentation_Details/01_Introduction/knowledge-system-architecture.md create mode 100644 docs/UET_Documentation_Details/04_User_Guides/knowledge-ingestion-workflow.md create mode 100644 docs/UET_Documentation_Details/05_API_Reference/knowledge-api-and-graphql-plan.md create mode 100644 docs/knowledge_base/personal_kb.py diff --git a/.gitignore b/.gitignore index 575d899c9..b6696dcec 100644 --- a/.gitignore +++ b/.gitignore @@ -79,7 +79,10 @@ node_modules/ .obsidian/ /uet_history/ /uet_knowledge_state.json - +docs/knowledge_base/personal_index.sqlite3 +docs/knowledge_base/personal_index.sqlite3-* +docs/knowledge_base/vectors/vectors.db +docs/knowledge_base/vectors/vectors.db-* # Local Cache and States uet_knowledge_state.json uet_miner/target/ diff --git a/docs/UET_Documentation_Details/01_Introduction/knowledge-system-architecture.md b/docs/UET_Documentation_Details/01_Introduction/knowledge-system-architecture.md new file mode 100644 index 000000000..31b4b62e8 --- /dev/null +++ b/docs/UET_Documentation_Details/01_Introduction/knowledge-system-architecture.md @@ -0,0 +1,164 @@ +# Knowledge System Architecture + +This document defines the intended knowledge architecture for the UET research +platform. + +The goal is to let humans, applications, and AI agents search the project without +turning the search index into a new source of truth. UET research changes over +time, so the knowledge system must support incremental updates instead of manual +full re-embedding after every research edit. + +## Core principle + +The research files remain canonical. + +The knowledge base is a searchable copy of selected project material. It exists +to make retrieval faster and cheaper, not to replace the documents, metadata, +verifier artifacts, or topic standards that define the actual project state. + +## System roles + +| Layer | Role | Source-of-truth status | +| :-- | :-- | :-- | +| `docs/` | Main documentation and research codebase | Canonical for written project content | +| `docs/topics/` | Topic workspaces and research packages | Canonical for topic-local evidence | +| `docs/topics/For Work/` | Research workflow standard | Canonical for research operating rules | +| `docs/meta/` | Machine-readable project status and release metadata | Canonical for repo-wide status summaries | +| `docs/knowledge_base/` | Legacy and utility layer for local indexing/search experiments | Not canonical; should not be treated as current truth without service alignment | +| `services_and_experiments/uet_kb/` | Knowledge-base service and MCP-oriented access layer | Service implementation layer | +| `services_and_experiments/uet_api/` | Platform API, auth, quota, and retrieval endpoints | Application service layer | +| PostgreSQL + pgvector | Canonical deployed vector store once selected | Search index, not research truth | +| MCP tools | AI-agent interface for simple retrieval actions | Access layer | +| GraphQL API | Structured query/admin layer for apps and humans | Access/control layer | + +## Intended data flow + +```mermaid +flowchart TD + ResearchDocs["Research docs and topic packages"] --> ChangeDetector["Change detector"] + MetaDocs["docs/meta status records"] --> StructuredAPI["GraphQL/API layer"] + ChangeDetector --> IngestQueue["Ingest queue"] + IngestQueue --> Chunker["Chunking + metadata extraction"] + Chunker --> HashCache["File hash + chunk hash cache"] + HashCache --> EmbedWorker["Embedding worker"] + EmbedWorker --> VectorStore["PostgreSQL + pgvector"] + VectorStore --> MCP["MCP tools for AI agents"] + VectorStore --> StructuredAPI + StructuredAPI --> WebApp["Web app / installer / dashboard"] + MCP --> Agents["AI agents"] +``` + +## Boundaries + +### Canonical state + +Use the repository documents and metadata for truth: + +- topic status comes from `docs/topics/README.md`, `docs/meta/`, local topic + documents, verifier artifacts, gates, manifests, and update logs +- research workflow rules come from `docs/topics/For Work/` +- formula, data, claim, and result readiness must be reconstructed from the + relevant topic package and artifacts + +### Searchable state + +Use the knowledge base for retrieval: + +- semantic search +- topic-aware document lookup +- chunk-level recall +- AI context retrieval +- app-facing document search + +Search results should point back to canonical files. A search hit is not proof +that a claim is current, validated, or publication-ready. + +### Access layers + +MCP and GraphQL should not compete. + +MCP should expose small AI-friendly tools such as: + +- `search_knowledge_base` +- `get_document` +- `list_topics` +- `get_topic_status` +- `search_physics` + +GraphQL should expose structured project navigation and admin/control surfaces +such as: + +- topics +- documents +- chunks +- ingest jobs +- stale documents +- index versions +- search results with filters + +## Why incremental indexing is required + +UET research changes continuously. A manual full re-embedding process creates +three problems: + +1. it wastes compute on unchanged files +2. it makes the index easy to forget or leave stale +3. it hides whether a search answer came from current or outdated content + +The intended system should store file and chunk hashes so only changed content is +embedded again. + +## Recommended canonical backend + +The platform should converge on one deployed vector backend: + +- PostgreSQL for document/index metadata +- pgvector for embeddings +- local embedding generation as the default path +- optional higher-cost embedding providers only when explicitly configured + +SQLite, LanceDB, or local experimental stores may remain useful for migration or +development experiments, but they should not be treated as the canonical platform +index unless the architecture is deliberately changed. + +## Current migration posture + +The repository currently contains multiple historical approaches to knowledge +search. Until they are aligned: + +- `docs/knowledge_base/` should be treated as legacy/utility code +- `services_and_experiments/uet_kb/` should be treated as the likely MCP service + direction +- `services_and_experiments/uet_api/` should be treated as the application API + direction +- PostgreSQL + pgvector should be treated as the preferred target store + +Before implementation, verify the current database schema, service routes, and +indexing scripts so the system does not keep multiple incompatible stores alive. +## Personal-first implementation note + +The first working version is for personal research use, not for a public +platform. + +Near-term work should stay deliberately small: + +1. index local files +2. detect changed files by hash +3. provide local search +4. point search results back to canonical files +5. prepare a clean path for later embeddings and MCP access + +Web UI, GraphQL, installer flows, auth, quota, and public API surfaces are later +platform work. They should not block the personal research memory layer. + +The current local helper is: + +```text +python -m docs.knowledge_base.personal_kb status +python -m docs.knowledge_base.personal_kb ingest --dry-run +python -m docs.knowledge_base.personal_kb ingest +python -m docs.knowledge_base.personal_kb search "claim evidence" +``` + +This is not the final vector system. It is the small base layer that makes file +changes and local recall visible before adding embeddings, MCP, or GraphQL. diff --git a/docs/UET_Documentation_Details/04_User_Guides/knowledge-ingestion-workflow.md b/docs/UET_Documentation_Details/04_User_Guides/knowledge-ingestion-workflow.md new file mode 100644 index 000000000..e4da721e8 --- /dev/null +++ b/docs/UET_Documentation_Details/04_User_Guides/knowledge-ingestion-workflow.md @@ -0,0 +1,162 @@ +# Knowledge Ingestion Workflow + +This document defines the intended workflow for keeping the UET knowledge base +current while research continues to change. + +The goal is not to re-embed the whole repository manually. The goal is to detect +what changed, update only the affected chunks, and make stale index state visible. + +## Workflow summary + +```mermaid +flowchart TD + Files["Tracked documentation and research files"] --> Detect["Detect changed, moved, or deleted files"] + Detect --> Hash["Compute file hash"] + Hash --> Skip{"File unchanged?"} + Skip -->|Yes| Done["Skip embedding"] + Skip -->|No| Chunk["Chunk file"] + Chunk --> ChunkHash["Compute chunk hashes"] + ChunkHash --> Reuse{"Chunk already embedded with same model?"} + Reuse -->|Yes| Link["Reuse embedding"] + Reuse -->|No| Embed["Embed changed chunk"] + Embed --> Store["Upsert document + chunk rows"] + Link --> Store + Store --> Mark["Mark removed chunks stale or deleted"] + Mark --> Report["Write ingest run report"] +``` + +## Files to index + +The first pass should focus on high-value documentation and research sources: + +- `docs/UET_Documentation_Details/` +- `docs/topics/` +- `docs/meta/` +- `docs/core/` +- `docs/knowledge_base/` documentation only, if it explains retrieval behavior +- `thailand_proposals/`, if public project/policy material should be searchable +- selected `uet_history/` material, only when marked as historical context + +Generated outputs, caches, build artifacts, and temporary reports should be +excluded unless a specific result artifact is meant to be searchable. + +## Required metadata + +Each indexed document should store: + +| Field | Purpose | +| :-- | :-- | +| `source_path` | Repo-relative source file path | +| `source_kind` | Documentation, topic doc, metadata, result artifact, architecture doc, etc. | +| `topic_id` | Topic identifier when applicable | +| `file_hash` | Hash of full file content | +| `git_commit` | Commit or working-tree marker used during ingest | +| `indexed_at` | Ingest timestamp | +| `status` | Active, stale, deleted, ignored, or failed | +| `parser_version` | Version of the chunking/parser rule | + +Each indexed chunk should store: + +| Field | Purpose | +| :-- | :-- | +| `chunk_hash` | Hash of normalized chunk content | +| `chunk_index` | Stable order within the source file | +| `heading_path` | Markdown heading context when available | +| `text` | Chunk text used for retrieval | +| `embedding` | Vector generated by the configured model | +| `embedding_model` | Model identity such as local BGE-M3/FastEmbed | +| `embedding_dim` | Vector dimension | +| `token_count` | Approximate chunk size | + +## Ingest modes + +### First-run ingest + +Use this when a new installation is created or the index is missing. + +Expected behavior: + +1. scan the configured source paths +2. create document and chunk records +3. generate embeddings for all eligible chunks +4. write an ingest report +5. expose the index version through API/MCP/GraphQL + +### Incremental ingest + +Use this during normal research work. + +Expected behavior: + +1. compare current files against stored `file_hash` records +2. skip unchanged files +3. re-chunk changed files +4. reuse unchanged chunk embeddings when `chunk_hash` and `embedding_model` match +5. embed only new or modified chunks +6. mark missing files and removed chunks as stale/deleted +7. write a compact ingest report + +### Forced reindex + +Use this only when the chunking rule, parser version, embedding model, or vector +dimension changes. + +A forced reindex should create a visible new index version rather than silently +overwriting old state. + +## Staleness rules + +The system should make stale state explicit: + +- if a source file changed but ingest failed, the previous chunks should be + marked stale +- if a file was deleted or moved, old chunks should not continue appearing as + normal active search results +- if an embedding model changes, old chunks should remain linked to their model + and not be mixed invisibly with the new model +- if `docs/meta/` says a topic status changed, search results should still point + users back to the current metadata rather than inferring status from old prose + +## Local development command shape + +The exact command can change during implementation, but the user-facing behavior +should converge toward: + +```text +kb ingest --all +kb ingest --changed +kb ingest --paths docs/topics/0.20_Atomic_Physics +kb watch +kb status +``` + +`kb watch` should be optional. The reliable base feature is `kb ingest --changed`. + +## Verification checklist + +An ingest implementation is not ready until it can show: + +- unchanged files are skipped +- modified files update only affected chunks +- deleted files become stale/deleted +- the active embedding model is recorded +- the ingest report lists failures +- search results return source paths and chunk context +- AI answers can link back to canonical files +## Current personal workflow + +The first implementation is intentionally small and local. It should help the +researcher and AI agent work inside the repo before any public platform work is +attempted. + +Use: + +```text +python -m docs.knowledge_base.personal_kb status +python -m docs.knowledge_base.personal_kb ingest --dry-run +python -m docs.knowledge_base.personal_kb ingest +python -m docs.knowledge_base.personal_kb search "formula audit" +``` + +The helper tracks hashes and builds a text index. Embeddings can be added later +once the changed-file workflow is stable. diff --git a/docs/UET_Documentation_Details/05_API_Reference/knowledge-api-and-graphql-plan.md b/docs/UET_Documentation_Details/05_API_Reference/knowledge-api-and-graphql-plan.md new file mode 100644 index 000000000..ecc994f5b --- /dev/null +++ b/docs/UET_Documentation_Details/05_API_Reference/knowledge-api-and-graphql-plan.md @@ -0,0 +1,181 @@ +# Knowledge API and GraphQL Plan + +This document defines the planned interface split for the UET knowledge system. + +GraphQL should be added as a structured query and admin layer. It should not +replace the vector store, the embedding worker, the canonical research +documents, or MCP tools. + +## Interface split + +| Interface | Primary user | Best for | Should not do | +| :-- | :-- | :-- | :-- | +| REST | Web app and simple clients | Auth, quota, health checks, simple search calls | Become the only structured knowledge browser | +| MCP | AI agents | Simple tool calls for retrieval and topic lookup | Expose overly complex admin workflows | +| GraphQL | Web app, installer, admin UI, power users | Structured topic/document/index queries | Become the source of truth for research claims | +| Direct database access | Services only | Storage and internal queries | Be required for normal users or AI agents | + +## GraphQL responsibilities + +GraphQL should make the knowledge system inspectable: + +- which documents are indexed +- which topics have indexed content +- which files are stale +- which ingest jobs ran +- which embedding model and index version are active +- which source file a search result came from +- whether a result is from current, stale, deleted, or historical content + +## Initial schema sketch + +This is a planning sketch, not a locked implementation contract. + +```graphql +type Topic { + id: ID! + title: String + status: String + readiness: String + controllingBlocker: String + documents: [Document!]! +} + +type Document { + id: ID! + sourcePath: String! + sourceKind: String! + topicId: String + fileHash: String + status: DocumentStatus! + indexedAt: String + chunkCount: Int! + chunks(limit: Int = 20): [Chunk!]! +} + +type Chunk { + id: ID! + documentId: ID! + chunkIndex: Int! + headingPath: String + text: String! + chunkHash: String! + embeddingModel: String + embeddingDim: Int +} + +type IngestRun { + id: ID! + mode: String! + startedAt: String! + finishedAt: String + status: IngestStatus! + changedFiles: Int! + embeddedChunks: Int! + reusedChunks: Int! + failedFiles: Int! +} + +type SearchResult { + document: Document! + chunk: Chunk! + score: Float + snippet: String! +} + +enum DocumentStatus { + ACTIVE + STALE + DELETED + IGNORED + FAILED +} + +enum IngestStatus { + RUNNING + SUCCEEDED + PARTIAL + FAILED +} +``` + +## Initial queries + +```graphql +type Query { + topics(status: String, readiness: String): [Topic!]! + topic(id: ID!): Topic + documents(topicId: String, status: DocumentStatus): [Document!]! + document(path: String!): Document + staleDocuments: [Document!]! + ingestRuns(limit: Int = 20): [IngestRun!]! + searchKnowledgeBase(query: String!, topK: Int = 8, topicId: String): [SearchResult!]! +} +``` + +## Initial mutations + +```graphql +type Mutation { + enqueueIngest(paths: [String!]!): IngestRun! + enqueueChangedIngest: IngestRun! + forceReindex(paths: [String!]!, reason: String!): IngestRun! + markDocumentIgnored(path: String!, reason: String!): Document! +} +``` + +Forced reindex operations should require a reason because they can invalidate +large parts of the index. + +## Optional subscriptions + +Subscriptions are useful later, but they are not required for the first stable +version. + +```graphql +type Subscription { + ingestRunUpdated(id: ID!): IngestRun! + knowledgeIndexChanged: IngestRun! +} +``` + +## MCP tool shape + +MCP should stay small and practical. Candidate tools: + +| Tool | Purpose | +| :-- | :-- | +| `search_knowledge_base` | Semantic search across indexed UET content | +| `search_physics` | Physics/math-focused search when the index supports it | +| `get_document` | Retrieve a source document or selected chunks | +| `list_topics` | List indexed topic identifiers and titles | +| `get_topic_status` | Return status reconstructed from canonical metadata, not from search prose alone | +| `kb_status` | Report index version, model, stale count, and last ingest run | + +## Safety rules + +- GraphQL search results must include source paths. +- Search snippets must not be treated as claim verification. +- Topic status should come from `docs/meta/`, `docs/topics/README.md`, and local + topic evidence, not from vector similarity alone. +- Stale documents must be visible to users and AI agents. +- The API should distinguish historical context from current project state. +- The API should expose the embedding model and index version used for a result. + +## Implementation order + +1. Confirm one canonical database schema for documents, chunks, ingest runs, and + index versions. +2. Implement or repair incremental ingestion against PostgreSQL + pgvector. +3. Expose minimal REST/MCP status and search endpoints. +4. Add GraphQL read queries for topics, documents, stale documents, and ingest + runs. +5. Add GraphQL ingest mutations after the ingest queue is stable. +6. Add subscriptions only when a real UI needs live ingest progress. +## Current priority + +GraphQL is parked until the personal knowledge-base layer is useful. + +The first implementation should focus on local status, changed-file ingest, and +search. GraphQL becomes useful later when a web app, installer, dashboard, or +external admin surface actually needs structured queries. diff --git a/docs/UET_Documentation_Details/README.md b/docs/UET_Documentation_Details/README.md index 49c4b6d24..647791471 100644 --- a/docs/UET_Documentation_Details/README.md +++ b/docs/UET_Documentation_Details/README.md @@ -29,6 +29,12 @@ This documentation tree exists to make the project readable in the correct order 7. [03_Core_Theory/correspondence-and-reduction.md](./03_Core_Theory/correspondence-and-reduction.md) 8. [06_Evidence_and_Research/validation-reports.md](./06_Evidence_and_Research/validation-reports.md) +### If you want the knowledge-base / AI search architecture + +1. [01_Introduction/knowledge-system-architecture.md](./01_Introduction/knowledge-system-architecture.md) +2. [04_User_Guides/knowledge-ingestion-workflow.md](./04_User_Guides/knowledge-ingestion-workflow.md) +3. [05_API_Reference/knowledge-api-and-graphql-plan.md](./05_API_Reference/knowledge-api-and-graphql-plan.md) + ### If you want the theory backstory 1. [01_Introduction/origin-and-development.md](./01_Introduction/origin-and-development.md) @@ -83,6 +89,17 @@ understand the theory correctly: - [06_Evidence_and_Research/criticism-and-collaboration.md](./06_Evidence_and_Research/criticism-and-collaboration.md) - [legacy-promotion-map.md](./legacy-promotion-map.md) +## Knowledge system planning + +The project also contains a planned knowledge-base layer for AI retrieval, +incremental embedding, MCP access, and future GraphQL inspection. + +Start with: + +- [01_Introduction/knowledge-system-architecture.md](./01_Introduction/knowledge-system-architecture.md) +- [04_User_Guides/knowledge-ingestion-workflow.md](./04_User_Guides/knowledge-ingestion-workflow.md) +- [05_API_Reference/knowledge-api-and-graphql-plan.md](./05_API_Reference/knowledge-api-and-graphql-plan.md) + ## Archive note `LEGACY_REPORTS` still matters for provenance and historical context. diff --git a/docs/knowledge_base/README.md b/docs/knowledge_base/README.md index 394c596b9..de3134895 100644 --- a/docs/knowledge_base/README.md +++ b/docs/knowledge_base/README.md @@ -1,4 +1,32 @@ -# 🧠 UET Knowledge Base Client (`docs.knowledge_base`) +# UET Knowledge Base Client (`docs.knowledge_base`) + +## Current local-first status + +This folder contains older knowledge-base experiments and bridge code. Some of +that code still describes the larger MCP/Postgres/vector-search direction, but +it should not be treated as the current working path without verification. + +For day-to-day personal research use, start with the small local helper: + +```bash +python -m docs.knowledge_base.personal_kb status +python -m docs.knowledge_base.personal_kb ingest --dry-run +python -m docs.knowledge_base.personal_kb ingest +python -m docs.knowledge_base.personal_kb search "claim evidence" +``` + +This helper builds a local SQLite text index and tracks file hashes so changed +files can be identified before heavier embedding infrastructure is repaired. It +does not replace canonical project documents, `docs/meta/`, topic packages, or +the future MCP/Postgres knowledge service. + +The larger service plan is documented in: + +- `docs/UET_Documentation_Details/01_Introduction/knowledge-system-architecture.md` +- `docs/UET_Documentation_Details/04_User_Guides/knowledge-ingestion-workflow.md` +- `docs/UET_Documentation_Details/05_API_Reference/knowledge-api-and-graphql-plan.md` + +--- ![Status](https://img.shields.io/badge/Status-ACTIVE-brightgreen) ![Client](https://img.shields.io/badge/Client-Python-blue) diff --git a/docs/knowledge_base/personal_kb.py b/docs/knowledge_base/personal_kb.py new file mode 100644 index 000000000..fd38cc78b --- /dev/null +++ b/docs/knowledge_base/personal_kb.py @@ -0,0 +1,406 @@ +"""Personal UET knowledge-base helper. + +This is the small local-first layer for day-to-day research work. It does not +replace the larger MCP/Postgres/GraphQL plan. It gives the repo a cheap way to +answer three questions before heavier infrastructure exists: + +- what files are currently indexed? +- what changed since the last ingest? +- where does a term appear in the local research corpus? +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sqlite3 +import sys +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Iterable + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_DB = REPO_ROOT / "docs" / "knowledge_base" / "personal_index.sqlite3" +DEFAULT_PATHS = ( + "docs/UET_Documentation_Details", + "docs/topics/For Work", + "docs/topics/README.md", + "docs/meta", + "docs/core", +) +TEXT_EXTENSIONS = {".md", ".txt", ".py", ".json", ".toml", ".yaml", ".yml"} +SKIP_PARTS = { + ".git", + ".venv", + "__pycache__", + "node_modules", + "vectors", + "media", + "target", + "dist", + "build", +} + + +@dataclass(frozen=True) +class SourceFile: + path: Path + rel_path: str + content: str + file_hash: str + + +def now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def connect(db_path: Path) -> sqlite3.Connection: + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + init_db(conn) + return conn + + +def init_db(conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS documents ( + source_path TEXT PRIMARY KEY, + file_hash TEXT NOT NULL, + status TEXT NOT NULL, + indexed_at TEXT NOT NULL, + chunk_count INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS chunks ( + source_path TEXT NOT NULL, + chunk_index INTEGER NOT NULL, + heading_path TEXT, + text TEXT NOT NULL, + chunk_hash TEXT NOT NULL, + PRIMARY KEY (source_path, chunk_index) + ); + + CREATE TABLE IF NOT EXISTS ingest_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + mode TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT NOT NULL, + scanned_files INTEGER NOT NULL, + changed_files INTEGER NOT NULL, + skipped_files INTEGER NOT NULL, + deleted_files INTEGER NOT NULL, + chunks_written INTEGER NOT NULL, + dry_run INTEGER NOT NULL + ); + """ + ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + """ + ) + fts_version = conn.execute( + "SELECT value FROM schema_meta WHERE key = 'fts_schema_version'" + ).fetchone() + if fts_version is None or fts_version["value"] != "2": + conn.execute("DROP TABLE IF EXISTS chunks_fts") + try: + conn.execute( + """ + CREATE VIRTUAL TABLE chunks_fts + USING fts5(source_path UNINDEXED, heading_path, text); + """ + ) + except sqlite3.OperationalError: + conn.execute( + """ + CREATE TABLE chunks_fts ( + source_path TEXT NOT NULL, + heading_path TEXT, + text TEXT NOT NULL + ); + """ + ) + conn.execute( + """ + INSERT INTO schema_meta (key, value) + VALUES ('fts_schema_version', '2') + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """ + ) + conn.commit() + + +def normalize_rel(path: Path) -> str: + return path.resolve().relative_to(REPO_ROOT).as_posix() + + +def should_skip(path: Path) -> bool: + parts = set(path.parts) + return bool(parts & SKIP_PARTS) or path.suffix.lower() not in TEXT_EXTENSIONS + + +def iter_files(paths: Iterable[str]) -> Iterable[Path]: + for item in paths: + root = (REPO_ROOT / item).resolve() + if root.is_file(): + if not should_skip(root): + yield root + continue + if not root.exists(): + continue + for path in root.rglob("*"): + if path.is_file() and not should_skip(path): + yield path + + +def read_source(path: Path) -> SourceFile | None: + try: + raw = path.read_bytes() + except OSError: + return None + if b"\x00" in raw: + return None + text = raw.decode("utf-8", errors="replace") + file_hash = hashlib.sha256(raw).hexdigest() + return SourceFile(path=path, rel_path=normalize_rel(path), content=text, file_hash=file_hash) + + +def chunk_markdownish(text: str) -> list[tuple[str, str]]: + chunks: list[tuple[str, str]] = [] + heading_stack: list[str] = [] + current_lines: list[str] = [] + current_heading = "" + + def flush() -> None: + nonlocal current_lines, current_heading + body = "\n".join(line.rstrip() for line in current_lines).strip() + if body: + chunks.append((current_heading, body)) + current_lines = [] + + for line in text.splitlines(): + match = re.match(r"^(#{1,6})\s+(.+?)\s*$", line) + if match: + flush() + level = len(match.group(1)) + title = match.group(2).strip() + heading_stack[:] = heading_stack[: level - 1] + heading_stack.append(title) + current_heading = " > ".join(heading_stack) + current_lines.append(line) + + flush() + if not chunks and text.strip(): + chunks.append(("", text.strip())) + return chunks + + +def existing_hashes(conn: sqlite3.Connection) -> dict[str, str]: + rows = conn.execute("SELECT source_path, file_hash FROM documents WHERE status = 'active'").fetchall() + return {row["source_path"]: row["file_hash"] for row in rows} + + +def clear_document(conn: sqlite3.Connection, rel_path: str) -> None: + conn.execute("DELETE FROM chunks WHERE source_path = ?", (rel_path,)) + conn.execute("DELETE FROM chunks_fts WHERE source_path = ?", (rel_path,)) + + +def upsert_source(conn: sqlite3.Connection, source: SourceFile) -> int: + chunks = chunk_markdownish(source.content) + clear_document(conn, source.rel_path) + for index, (heading, text) in enumerate(chunks): + chunk_hash = hashlib.sha256(text.encode("utf-8")).hexdigest() + conn.execute( + """ + INSERT INTO chunks (source_path, chunk_index, heading_path, text, chunk_hash) + VALUES (?, ?, ?, ?, ?) + """, + (source.rel_path, index, heading, text, chunk_hash), + ) + conn.execute( + "INSERT INTO chunks_fts (source_path, heading_path, text) VALUES (?, ?, ?)", + (source.rel_path, heading, text), + ) + conn.execute( + """ + INSERT INTO documents (source_path, file_hash, status, indexed_at, chunk_count) + VALUES (?, ?, 'active', ?, ?) + ON CONFLICT(source_path) DO UPDATE SET + file_hash = excluded.file_hash, + status = 'active', + indexed_at = excluded.indexed_at, + chunk_count = excluded.chunk_count + """, + (source.rel_path, source.file_hash, now_iso(), len(chunks)), + ) + return len(chunks) + + +def cmd_ingest(args: argparse.Namespace) -> int: + conn = connect(args.db) + started_at = now_iso() + known = existing_hashes(conn) + seen: set[str] = set() + scanned = changed = skipped = chunks_written = 0 + + mode = "all" if args.all else "changed" + for path in iter_files(args.paths): + source = read_source(path) + if source is None: + continue + scanned += 1 + seen.add(source.rel_path) + unchanged = known.get(source.rel_path) == source.file_hash + if unchanged and not args.all: + skipped += 1 + continue + changed += 1 + if not args.dry_run: + chunks_written += upsert_source(conn, source) + + deleted_paths = sorted(set(known) - seen) + if not args.dry_run: + for rel_path in deleted_paths: + conn.execute( + "UPDATE documents SET status = 'deleted', indexed_at = ? WHERE source_path = ?", + (now_iso(), rel_path), + ) + conn.execute( + """ + INSERT INTO ingest_runs ( + mode, started_at, finished_at, scanned_files, changed_files, + skipped_files, deleted_files, chunks_written, dry_run + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + mode, + started_at, + now_iso(), + scanned, + changed, + skipped, + len(deleted_paths), + chunks_written, + int(args.dry_run), + ), + ) + conn.commit() + + result = { + "mode": mode, + "dry_run": args.dry_run, + "scanned_files": scanned, + "changed_files": changed, + "skipped_files": skipped, + "deleted_files": len(deleted_paths), + "chunks_written": chunks_written, + "db": normalize_rel(args.db) if args.db.is_relative_to(REPO_ROOT) else str(args.db), + } + print(json.dumps(result, indent=2, ensure_ascii=False)) + return 0 + + +def cmd_status(args: argparse.Namespace) -> int: + conn = connect(args.db) + doc_counts = conn.execute( + "SELECT status, COUNT(*) AS count FROM documents GROUP BY status ORDER BY status" + ).fetchall() + chunk_count = conn.execute("SELECT COUNT(*) AS count FROM chunks").fetchone()["count"] + last_run = conn.execute( + "SELECT * FROM ingest_runs ORDER BY id DESC LIMIT 1" + ).fetchone() + result = { + "db": normalize_rel(args.db) if args.db.is_relative_to(REPO_ROOT) else str(args.db), + "documents": {row["status"]: row["count"] for row in doc_counts}, + "chunks": chunk_count, + "last_ingest_run": dict(last_run) if last_run else None, + } + print(json.dumps(result, indent=2, ensure_ascii=False)) + return 0 + + +def cmd_search(args: argparse.Namespace) -> int: + conn = connect(args.db) + query = args.query.strip() + try: + rows = conn.execute( + """ + SELECT source_path, heading_path, text + FROM chunks_fts + WHERE chunks_fts MATCH ? + LIMIT ? + """, + (query, args.limit), + ).fetchall() + except sqlite3.OperationalError: + like = f"%{query}%" + rows = conn.execute( + """ + SELECT source_path, heading_path, text + FROM chunks + WHERE text LIKE ? + LIMIT ? + """, + (like, args.limit), + ).fetchall() + + for row in rows: + snippet = " ".join(row["text"].split())[: args.snippet_chars] + print( + json.dumps( + { + "source_path": row["source_path"], + "heading_path": row["heading_path"], + "snippet": snippet, + }, + ensure_ascii=False, + ) + ) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Personal UET knowledge-base helper") + parser.add_argument("--db", type=Path, default=DEFAULT_DB) + sub = parser.add_subparsers(dest="command", required=True) + + ingest = sub.add_parser("ingest", help="Index changed local research files") + ingest.add_argument("--all", action="store_true", help="Reindex all scanned files") + ingest.add_argument("--dry-run", action="store_true", help="Report changes without writing") + ingest.add_argument("paths", nargs="*", default=list(DEFAULT_PATHS)) + ingest.set_defaults(func=cmd_ingest) + + status = sub.add_parser("status", help="Show local index status") + status.set_defaults(func=cmd_status) + + search = sub.add_parser("search", help="Search the local text index") + search.add_argument("query") + search.add_argument("--limit", type=int, default=8) + search.add_argument("--snippet-chars", type=int, default=320) + search.set_defaults(func=cmd_search) + + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + args.db = args.db.resolve() + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main())