From c1859a84fb062b1c25d10174a10fc04768e8e12e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C3=BAl=20G=C3=B3mez=20Jim=C3=A9nez?= Date: Mon, 25 May 2026 18:36:55 +0200 Subject: [PATCH 1/5] feat(docs): sync knowledge base from remote docs API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an optional remote sync path so the bot can index the docs published by the website instead of the local snapshot under bot/docs/knowledge/. Selected via DOCS_USE_REMOTE: - local (default): keep current behavior, no remote calls. - remote: load knowledge from the remote manifest/body API. - shadow: load from local but also fetch the remote and log a per-slug diff (local_only / remote_only / changed) — useful to validate parity before flipping to remote. What changes: - New bot/docs_client.py — typed async client over httpx for GET /api/docs/manifest.json and GET /api/docs/{slug}.md, with on-disk cache under DOCS_CACHE_DIR (manifest + per-slug body) for fallback when the remote is unreachable. - New load_documentation_from_remote() in bot/knowledge.py — diffs per-entry sha256, only re-chunks/embeds what changed, drops sources that disappeared from the manifest, falls back to cache and then to local docs if both fail. - bot/base.py: background refresh task (DOCS_REFRESH_INTERVAL, 60s floor) that calls _refresh_docs_once, embeds new chunks and reports docs_last_sync / docs_last_sync_ok on /health. The !docs command now lists tracked sources from the vector store, not the local filesystem. - main.py: cold start branches on DOCS_USE_REMOTE; shadow mode also exercises fetch_body to surface remote errors early. - config: docs_base_url, docs_refresh_interval, docs_use_remote, docs_cache_dir, docs_http_timeout. - pyproject: add httpx>=0.27. The legacy local loader (bot/docs/knowledge/*.md + load_documentation) is intentionally left untouched so deploys can keep using local until remote sync is validated in prod; cleanup is a follow-up. Co-Authored-By: Claude Opus 4.7 --- .env.example | 7 ++ bot/base.py | 65 +++++++++++++++-- bot/config.py | 6 ++ bot/docs_client.py | 175 +++++++++++++++++++++++++++++++++++++++++++++ bot/knowledge.py | 70 ++++++++++++++++++ main.py | 56 ++++++++++++++- pyproject.toml | 1 + 7 files changed, 375 insertions(+), 5 deletions(-) create mode 100644 bot/docs_client.py diff --git a/.env.example b/.env.example index 94b6dce..a223daf 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,10 @@ ALLOWED_CHANNELS= # Daily metrics report (optional). If STATUS_CHANNEL_ID is empty the report is disabled. STATUS_CHANNEL_ID= METRICS_SEND_HOUR=9 + +# Docs remote sync +DOCS_BASE_URL=https://nan.builders +DOCS_REFRESH_INTERVAL=900 +DOCS_USE_REMOTE=local +DOCS_CACHE_DIR=vector_db/docs_cache +DOCS_HTTP_TIMEOUT=10 diff --git a/bot/base.py b/bot/base.py index 96032f5..81f560e 100644 --- a/bot/base.py +++ b/bot/base.py @@ -13,7 +13,8 @@ from discord.ext import commands from bot.config import logger, settings -from bot.knowledge import SimpleVectorStore +from bot.docs_client import DocsClient +from bot.knowledge import SimpleVectorStore, load_documentation_from_remote from bot.llm import LLMClient from bot.metrics import send_metrics_report, send_user_metrics_report @@ -75,6 +76,9 @@ def __init__(self) -> None: self._health_port = 9101 self._health_server: HTTPServer | None = None self._health_thread: Thread | None = None + self._docs_last_sync: str | None = None + self._docs_last_sync_ok: bool = False + self._docs_refresh_task: asyncio.Task[None] | None = None def _start_health_server(self) -> None: """Start a lightweight HTTP health check server in a background thread.""" @@ -86,6 +90,8 @@ def do_GET(self) -> None: "status": "healthy" if self.bot._ready else "starting", "initialized": self.bot._initialized, "store_chunks": len(self.bot.store.chunks) if self.bot.store else 0, + "docs_last_sync": self.bot._docs_last_sync, + "docs_last_sync_ok": self.bot._docs_last_sync_ok, } body = json.dumps(health_data).encode() self.send_response(200) @@ -150,6 +156,54 @@ async def on_ready(self) -> None: ) await self.start_daily_metrics() + if self._docs_refresh_task is None or self._docs_refresh_task.done(): + self._docs_refresh_task = asyncio.create_task(self._schedule_docs_refresh()) + else: + logger.info("Docs refresh scheduler already running") + + async def _refresh_docs_once(self) -> None: + from datetime import datetime, UTC + + if self.store is None: + return + + try: + async with DocsClient() as client: + result = await load_documentation_from_remote(self.store, client) + + if result.new_chunks: + embedded = await self.llm.embed_chunks(self.store) + self.store.save() + logger.info("Refresh: embedded %d new chunks", embedded) + elif result.stale_removed: + self.store.save() + + self._docs_last_sync_ok = True + except Exception as e: + logger.error("Docs refresh failed: %s", type(e).__name__) + self._docs_last_sync_ok = False + finally: + self._docs_last_sync = datetime.now(UTC).isoformat() + + async def _schedule_docs_refresh(self) -> None: + if settings.docs_use_remote == "local": + logger.info("DOCS_USE_REMOTE=local, skipping remote docs refresh") + return + + interval = max(60, settings.docs_refresh_interval) + logger.info( + "Docs refresh scheduler started (mode=%s, interval=%ds)", + settings.docs_use_remote, + interval, + ) + + try: + while True: + await self._refresh_docs_once() + await asyncio.sleep(interval) + except asyncio.CancelledError: + logger.info("Docs refresh scheduler cancelled") + async def _schedule_daily_metrics(self) -> None: """Schedule daily metrics to run at the configured hour.""" if settings.status_channel_id_value is None or settings.litellm_admin_key is None: @@ -297,13 +351,16 @@ async def health(self, ctx: commands.Context) -> None: @commands.command(name="docs", description="List available documentation files") async def docs(self, ctx: commands.Context) -> None: - from bot.config import DOCS_DIR + if not self.store: + await ctx.send("Knowledge base not initialized.") + return - docs = list(DOCS_DIR.glob("**/*.md")) + docs = sorted(self.store.get_tracked_sources()) if not docs: await ctx.send("No documentation files loaded yet.") return - doc_list = "\n".join(f"- {d.name}" for d in docs) + + doc_list = "\n".join(f"- {doc}" for doc in docs) embed = discord.Embed(title="Documentation", description=doc_list, color=discord.Color.blue()) await ctx.send(embed=embed) diff --git a/bot/config.py b/bot/config.py index 026ab6f..1d745b5 100644 --- a/bot/config.py +++ b/bot/config.py @@ -23,6 +23,12 @@ class Settings(BaseSettings): status_channel_id: str = "" metrics_send_hour: int = 9 + docs_base_url: str = "https://nan.builders" + docs_refresh_interval: int = 900 + docs_use_remote: str = "local" + docs_cache_dir: str = "vector_db/docs_cache" + docs_http_timeout: int = 10 + @property def allowed_channel_ids(self) -> set[int]: if not self.allowed_channels: diff --git a/bot/docs_client.py b/bot/docs_client.py new file mode 100644 index 0000000..a3554fa --- /dev/null +++ b/bot/docs_client.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urljoin + +import httpx + +from bot.config import logger, settings + + +_FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL) +_SAFE_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$") + + +@dataclass +class ManifestEntry: + slug: str + title: str + description: str + order: int + content_hash: str + content_url: str + + +@dataclass +class Manifest: + version: str + entries: list[ManifestEntry] + + +@dataclass +class DocBody: + slug: str + raw: str + body: str + content_hash: str + + +def _strip_frontmatter(raw: str) -> str: + match = _FRONTMATTER_RE.match(raw) + if not match: + return raw + return raw[match.end():] + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +class DocsClient: + def __init__( + self, + base_url: str | None = None, + cache_dir: Path | None = None, + timeout: float | None = None, + ) -> None: + self._base_url = (base_url or settings.docs_base_url).rstrip("/") + self._cache_dir = Path(cache_dir or settings.docs_cache_dir) + self._cache_dir.mkdir(parents=True, exist_ok=True) + self._timeout = timeout if timeout is not None else settings.docs_http_timeout + self._client: httpx.AsyncClient | None = None + + @property + def manifest_url(self) -> str: + return f"{self._base_url}/api/docs/manifest.json" + + def resolve_content_url(self, content_url: str) -> str: + return urljoin(f"{self._base_url}/", content_url.lstrip("/")) + + async def __aenter__(self) -> "DocsClient": + self._client = httpx.AsyncClient( + timeout=self._timeout, + headers={"User-Agent": "nan-discord-bot/0.1 (+docs-sync)"}, + follow_redirects=False, + ) + return self + + async def __aexit__(self, *args: object) -> None: + if self._client: + await self._client.aclose() + self._client = None + + async def fetch_manifest(self) -> Manifest: + assert self._client is not None, "DocsClient not entered" + resp = await self._client.get(self.manifest_url) + resp.raise_for_status() + data = resp.json() + + entries = [ + ManifestEntry( + slug=e["slug"], + title=e["title"], + description=e["description"], + order=int(e["order"]), + content_hash=e["contentHash"], + content_url=e["contentUrl"], + ) + for e in data["entries"] + if _SAFE_SLUG_RE.match(e["slug"]) + ] + manifest = Manifest(version=data["version"], entries=entries) + + tmp = self._cache_dir / "manifest.json.tmp" + dst = self._cache_dir / "manifest.json" + tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + tmp.replace(dst) + return manifest + + async def fetch_body(self, entry: ManifestEntry) -> DocBody: + assert self._client is not None, "DocsClient not entered" + if not _SAFE_SLUG_RE.match(entry.slug): + raise ValueError(f"Unsafe slug: {entry.slug!r}") + + resp = await self._client.get(self.resolve_content_url(entry.content_url)) + resp.raise_for_status() + raw = resp.text + body = _strip_frontmatter(raw) + computed = f"sha256:{_sha256(body)}" + + if computed != entry.content_hash: + logger.warning( + "Body hash mismatch for %s: manifest=%s computed=%s", + entry.slug, entry.content_hash, computed, + ) + + tmp = self._cache_dir / f"{entry.slug}.md.tmp" + dst = self._cache_dir / f"{entry.slug}.md" + tmp.write_text(raw, encoding="utf-8") + tmp.replace(dst) + + return DocBody(slug=entry.slug, raw=raw, body=body, content_hash=computed) + + def load_cached_manifest(self) -> Manifest | None: + p = self._cache_dir / "manifest.json" + if not p.exists(): + return None + try: + data = json.loads(p.read_text(encoding="utf-8")) + return Manifest( + version=data["version"], + entries=[ + ManifestEntry( + slug=e["slug"], + title=e["title"], + description=e["description"], + order=int(e["order"]), + content_hash=e["contentHash"], + content_url=e["contentUrl"], + ) + for e in data["entries"] + if _SAFE_SLUG_RE.match(e["slug"]) + ], + ) + except (json.JSONDecodeError, KeyError, ValueError) as e: + logger.warning("Failed to load cached manifest: %s", type(e).__name__) + return None + + def load_cached_body(self, slug: str) -> DocBody | None: + if not _SAFE_SLUG_RE.match(slug): + return None + p = self._cache_dir / f"{slug}.md" + if not p.exists(): + return None + raw = p.read_text(encoding="utf-8") + body = _strip_frontmatter(raw) + return DocBody( + slug=slug, + raw=raw, + body=body, + content_hash=f"sha256:{_sha256(body)}", + ) diff --git a/bot/knowledge.py b/bot/knowledge.py index 30f30da..e99dee8 100644 --- a/bot/knowledge.py +++ b/bot/knowledge.py @@ -285,3 +285,73 @@ async def load_documentation(store: SimpleVectorStore, docs_dir: Path) -> LoadRe logger.info("All %d docs unchanged, no re-indexing needed", len(current_sources)) return LoadResult(new_chunks=new_chunks, stale_removed=len(stale_sources)) + + +async def load_documentation_from_remote( + store: SimpleVectorStore, + client: "DocsClient", + fallback_docs_dir: Path | None = None, +) -> LoadResult: + from bot.docs_client import DocsClient # noqa: F401 + + try: + manifest = await client.fetch_manifest() + source_of_truth = "remote" + except Exception as e: + logger.error("Failed to fetch manifest: %s", type(e).__name__) + cached = client.load_cached_manifest() + if cached is not None: + manifest = cached + source_of_truth = "cache" + elif fallback_docs_dir is not None: + logger.warning("No remote manifest and no cache; falling back to local docs") + return await load_documentation(store, fallback_docs_dir) + else: + logger.error("No remote manifest and no cache available") + return LoadResult(new_chunks=0, stale_removed=0) + + current_sources: set[str] = set() + new_chunks = 0 + + for entry in manifest.entries: + source = f"{entry.slug}.md" + current_sources.add(source) + + remote_hash = entry.content_hash.removeprefix("sha256:") + stored_hash = store.get_doc_hash(source) + + if stored_hash == remote_hash: + logger.info("Unchanged (%s), skipping: %s", source_of_truth, source) + continue + + try: + doc_body = await client.fetch_body(entry) + except Exception as e: + logger.warning("Failed to fetch %s from remote (%s), trying cache", source, type(e).__name__) + cached_body = client.load_cached_body(entry.slug) + if cached_body is None: + logger.error("No cached body for %s; skipping this iteration", source) + continue + doc_body = cached_body + + logger.info("Changed (or new), re-indexing from %s: %s", source_of_truth, source) + store.remove_source(source) + chunks = chunk_text(doc_body.body, source=source) + for chunk in chunks: + store.add(chunk) + new_chunks += 1 + store.set_doc_hash(source, doc_body.content_hash.removeprefix("sha256:")) + + stale_sources = store.get_tracked_sources() - current_sources + for source in stale_sources: + logger.info("Source removed from manifest, cleaning up: %s", source) + store.remove_source(source) + + if new_chunks: + logger.info("Re-indexed %d chunks from changed remote files", new_chunks) + elif stale_sources: + logger.info("Removed %d stale sources", len(stale_sources)) + else: + logger.info("All %d remote docs unchanged, no re-indexing needed", len(current_sources)) + + return LoadResult(new_chunks=new_chunks, stale_removed=len(stale_sources)) diff --git a/main.py b/main.py index 76c6cfa..b63a958 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ """Entry point for the nan.discord.bot.""" import asyncio +import hashlib import signal from pathlib import Path @@ -12,8 +13,61 @@ async def init_knowledge_base(store: SimpleVectorStore) -> None: """Load docs and create embeddings. Non-fatal on failure.""" + from bot.docs_client import DocsClient + from bot.knowledge import load_documentation_from_remote + llm = LLMClient() - result = await load_documentation(store, DEFAULT_DOCS_DIR) + mode = settings.docs_use_remote + + if mode == "remote": + async with DocsClient() as client: + result = await load_documentation_from_remote( + store, + client, + fallback_docs_dir=DEFAULT_DOCS_DIR, + ) + elif mode == "shadow": + result = await load_documentation(store, DEFAULT_DOCS_DIR) + + try: + async with DocsClient() as client: + manifest = await client.fetch_manifest() + + local_hashes: dict[str, str] = {} + for md_file in sorted(DEFAULT_DOCS_DIR.glob("*.md")): + local_hashes[md_file.stem] = hashlib.sha256( + md_file.read_text(encoding="utf-8").encode("utf-8") + ).hexdigest() + + remote_hashes = { + entry.slug: entry.content_hash.removeprefix("sha256:") + for entry in manifest.entries + } + + only_local = sorted(set(local_hashes) - set(remote_hashes)) + only_remote = sorted(set(remote_hashes) - set(local_hashes)) + changed = sorted( + slug + for slug in (set(local_hashes) & set(remote_hashes)) + if local_hashes[slug] != remote_hashes[slug] + ) + + logger.info( + "Shadow diff: local_only=%s remote_only=%s changed=%s", + only_local or "-", + only_remote or "-", + changed or "-", + ) + + for entry in manifest.entries: + try: + await client.fetch_body(entry) + except Exception as e: + logger.warning("Shadow fetch failed for %s: %s", entry.slug, type(e).__name__) + except Exception as e: + logger.warning("Shadow mode remote comparison failed: %s", type(e).__name__) + else: + result = await load_documentation(store, DEFAULT_DOCS_DIR) try: if result.new_chunks: diff --git a/pyproject.toml b/pyproject.toml index eca49c5..b262413 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ requires-python = ">=3.11" dependencies = [ "discord.py>=2.3.2", "feedparser>=6.0.0", + "httpx>=0.27.0", "openai>=1.30.0", "pydantic-settings>=2.2.0", ] From 3ba5dbd26597b71316446ff2cbce722aeb7d8038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C3=BAl=20G=C3=B3mez=20Jim=C3=A9nez?= Date: Tue, 26 May 2026 19:25:08 +0200 Subject: [PATCH 2/5] feat(docs): canonical text pipeline, conditional sync, hardened settings Address PR #2 review blockers. The bot now hashes/chunks/caches a single canonical text in local, remote and cache paths, sends conditional GETs, and short-circuits cleanly when the upstream manifest version is unchanged. - knowledge: canonicalize_doc_text() is the single source of truth for the text that enters the chunker. load_documentation() applies it with strip_frontmatter=True; load_documentation_from_remote() with strip_frontmatter=False on the happy path. The hash recorded in doc_hashes is now the hash of canonical text, so local and remote agree byte-for-byte on the same logical content. Expect a one-shot reindex on first deploy as old hashes (raw file with frontmatter) rotate. - SimpleVectorStore: new meta table with get_meta/set_meta, sqlite connection now uses check_same_thread=False. - load_documentation_from_remote: persists manifest.version in meta and short-circuits when the remote returns the same version, avoiding per-entry walks when nothing changed upstream. Skipped when source_of_truth==cache so we still notice drift while remote is unreachable. - docs_client: _EtagStore persists per-resource ETags atomically; fetch_manifest/fetch_body send If-None-Match and reuse cache on 304, refetching unconditionally if the cache went missing. AsyncHTTPTransport(retries=3) for connect-level retries, plus a bounded backoff on 5xx (0.5/1/2s). Body cache stores the canonical text, never the raw frontmatter; load_cached_body canonicalises on read so legacy caches keep working. - main shadow mode: compares canonical hashes on both sides (local strip=True, remote strip=False) and fetches each body once so the diff is real signal, not frontmatter noise. - config: docs_use_remote is Literal["local","remote","shadow"] so typos fail at startup instead of falling silently to local. - base: _refresh_docs_once wrapped in an asyncio.Lock so overlapping refreshes can't interleave around the embed/save cycle. - tests: conftest sets the required Settings env vars; new pytest suites cover _strip_frontmatter, _SAFE_SLUG_RE, ETag/304 round trips, hash-mismatch warnings, legacy cache normalisation, the canonicalize_doc_text contract, load_documentation idempotency, and the full load_documentation_from_remote matrix (unchanged skip, changed reindex, removed cleanup, remote->cache and remote->local fallbacks, manifest.version short-circuit, cache source not short-circuiting). pytest-httpx>=0.30.0 added to the dev extra. Co-Authored-By: Claude Opus 4.7 --- bot/base.py | 36 +-- bot/config.py | 3 +- bot/docs_client.py | 163 ++++++++++++- bot/knowledge.py | 55 ++++- main.py | 32 ++- pyproject.toml | 1 + tests/conftest.py | 7 + tests/test_docs_client.py | 228 +++++++++++++++++++ tests/test_knowledge.py | 73 ++++++ tests/test_load_documentation_from_remote.py | 199 ++++++++++++++++ 10 files changed, 758 insertions(+), 39 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_docs_client.py create mode 100644 tests/test_knowledge.py create mode 100644 tests/test_load_documentation_from_remote.py diff --git a/bot/base.py b/bot/base.py index 81f560e..a315b42 100644 --- a/bot/base.py +++ b/bot/base.py @@ -79,6 +79,7 @@ def __init__(self) -> None: self._docs_last_sync: str | None = None self._docs_last_sync_ok: bool = False self._docs_refresh_task: asyncio.Task[None] | None = None + self._docs_refresh_lock = asyncio.Lock() def _start_health_server(self) -> None: """Start a lightweight HTTP health check server in a background thread.""" @@ -167,23 +168,24 @@ async def _refresh_docs_once(self) -> None: if self.store is None: return - try: - async with DocsClient() as client: - result = await load_documentation_from_remote(self.store, client) - - if result.new_chunks: - embedded = await self.llm.embed_chunks(self.store) - self.store.save() - logger.info("Refresh: embedded %d new chunks", embedded) - elif result.stale_removed: - self.store.save() - - self._docs_last_sync_ok = True - except Exception as e: - logger.error("Docs refresh failed: %s", type(e).__name__) - self._docs_last_sync_ok = False - finally: - self._docs_last_sync = datetime.now(UTC).isoformat() + async with self._docs_refresh_lock: + try: + async with DocsClient() as client: + result = await load_documentation_from_remote(self.store, client) + + if result.new_chunks: + embedded = await self.llm.embed_chunks(self.store) + self.store.save() + logger.info("Refresh: embedded %d new chunks", embedded) + elif result.stale_removed: + self.store.save() + + self._docs_last_sync_ok = True + except Exception as e: + logger.error("Docs refresh failed: %s", type(e).__name__) + self._docs_last_sync_ok = False + finally: + self._docs_last_sync = datetime.now(UTC).isoformat() async def _schedule_docs_refresh(self) -> None: if settings.docs_use_remote == "local": diff --git a/bot/config.py b/bot/config.py index 1d745b5..22199ba 100644 --- a/bot/config.py +++ b/bot/config.py @@ -1,5 +1,6 @@ import logging from pathlib import Path +from typing import Literal from pydantic_settings import BaseSettings, SettingsConfigDict @@ -25,7 +26,7 @@ class Settings(BaseSettings): docs_base_url: str = "https://nan.builders" docs_refresh_interval: int = 900 - docs_use_remote: str = "local" + docs_use_remote: Literal["local", "remote", "shadow"] = "local" docs_cache_dir: str = "vector_db/docs_cache" docs_http_timeout: int = 10 diff --git a/bot/docs_client.py b/bot/docs_client.py index a3554fa..eaaeb50 100644 --- a/bot/docs_client.py +++ b/bot/docs_client.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import hashlib import json import re @@ -10,10 +11,12 @@ import httpx from bot.config import logger, settings +from bot.knowledge import canonicalize_doc_text _FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL) _SAFE_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$") +_BACKOFF_SECONDS = (0.5, 1.0, 2.0) @dataclass @@ -51,6 +54,73 @@ def _sha256(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() +class _EtagStore: + """Persists per-resource ETags in a JSON file alongside the docs cache. + + Layout:: + + { + "manifest": "\"sha256:...\"", + "bodies": {"": "\"sha256:...\""} + } + + Tolerant to a missing or corrupt file (treats it as empty). + """ + + def __init__(self, path: Path) -> None: + self._path = path + self._data: dict[str, object] = {"manifest": None, "bodies": {}} + self._load() + + def _load(self) -> None: + if not self._path.exists(): + return + try: + raw = json.loads(self._path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + logger.warning("Failed to load etags.json (%s); starting empty", type(e).__name__) + return + if not isinstance(raw, dict): + return + manifest = raw.get("manifest") + bodies = raw.get("bodies") or {} + self._data["manifest"] = manifest if isinstance(manifest, str) else None + self._data["bodies"] = { + k: v for k, v in bodies.items() if isinstance(k, str) and isinstance(v, str) + } + + def _save(self) -> None: + tmp = self._path.with_suffix(self._path.suffix + ".tmp") + tmp.write_text(json.dumps(self._data, ensure_ascii=False, indent=2), encoding="utf-8") + tmp.replace(self._path) + + def get_manifest(self) -> str | None: + v = self._data.get("manifest") + return v if isinstance(v, str) else None + + def set_manifest(self, etag: str | None) -> None: + self._data["manifest"] = etag + self._save() + + def get_body(self, slug: str) -> str | None: + bodies = self._data.get("bodies") + if not isinstance(bodies, dict): + return None + v = bodies.get(slug) + return v if isinstance(v, str) else None + + def set_body(self, slug: str, etag: str | None) -> None: + bodies = self._data.setdefault("bodies", {}) + if not isinstance(bodies, dict): + bodies = {} + self._data["bodies"] = bodies + if etag is None: + bodies.pop(slug, None) + else: + bodies[slug] = etag + self._save() + + class DocsClient: def __init__( self, @@ -63,6 +133,7 @@ def __init__( self._cache_dir.mkdir(parents=True, exist_ok=True) self._timeout = timeout if timeout is not None else settings.docs_http_timeout self._client: httpx.AsyncClient | None = None + self._etags = _EtagStore(self._cache_dir / "etags.json") @property def manifest_url(self) -> str: @@ -72,10 +143,12 @@ def resolve_content_url(self, content_url: str) -> str: return urljoin(f"{self._base_url}/", content_url.lstrip("/")) async def __aenter__(self) -> "DocsClient": + transport = httpx.AsyncHTTPTransport(retries=3) self._client = httpx.AsyncClient( timeout=self._timeout, headers={"User-Agent": "nan-discord-bot/0.1 (+docs-sync)"}, follow_redirects=False, + transport=transport, ) return self @@ -84,9 +157,59 @@ async def __aexit__(self, *args: object) -> None: await self._client.aclose() self._client = None + async def _get_with_backoff( + self, + url: str, + *, + headers: dict[str, str] | None = None, + ) -> httpx.Response: + """GET with bounded retries on 5xx and transient connection errors. + + Returns the last response if every attempt returned a 5xx; raises the + last httpx.HTTPError when every attempt raised. Non-5xx responses are + returned immediately. + """ + assert self._client is not None, "DocsClient not entered" + last_exc: BaseException | None = None + last_resp: httpx.Response | None = None + attempts = len(_BACKOFF_SECONDS) + for i in range(attempts): + try: + resp = await self._client.get(url, headers=headers) + except httpx.HTTPError as e: + last_exc = e + if i < attempts - 1: + await asyncio.sleep(_BACKOFF_SECONDS[i]) + continue + raise + last_resp = resp + if resp.status_code < 500: + return resp + if i < attempts - 1: + await asyncio.sleep(_BACKOFF_SECONDS[i]) + continue + assert last_resp is not None or last_exc is not None + if last_resp is not None: + return last_resp + raise last_exc # type: ignore[misc] + async def fetch_manifest(self) -> Manifest: assert self._client is not None, "DocsClient not entered" - resp = await self._client.get(self.manifest_url) + + headers: dict[str, str] = {} + prev_etag = self._etags.get_manifest() + if prev_etag: + headers["If-None-Match"] = prev_etag + + resp = await self._get_with_backoff(self.manifest_url, headers=headers) + + if resp.status_code == 304: + cached = self.load_cached_manifest() + if cached is not None: + return cached + logger.warning("Manifest 304 but local cache missing; refetching unconditionally") + resp = await self._get_with_backoff(self.manifest_url) + resp.raise_for_status() data = resp.json() @@ -108,6 +231,14 @@ async def fetch_manifest(self) -> Manifest: dst = self._cache_dir / "manifest.json" tmp.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") tmp.replace(dst) + + new_etag = resp.headers.get("etag") + if new_etag: + self._etags.set_manifest(new_etag) + else: + # The server forgot to send one — drop our stale token so we stop + # claiming we have a validator we can't actually prove. + self._etags.set_manifest(None) return manifest async def fetch_body(self, entry: ManifestEntry) -> DocBody: @@ -115,10 +246,26 @@ async def fetch_body(self, entry: ManifestEntry) -> DocBody: if not _SAFE_SLUG_RE.match(entry.slug): raise ValueError(f"Unsafe slug: {entry.slug!r}") - resp = await self._client.get(self.resolve_content_url(entry.content_url)) + headers: dict[str, str] = {} + prev_etag = self._etags.get_body(entry.slug) + if prev_etag: + headers["If-None-Match"] = prev_etag + + url = self.resolve_content_url(entry.content_url) + resp = await self._get_with_backoff(url, headers=headers) + + if resp.status_code == 304: + cached = self.load_cached_body(entry.slug) + if cached is not None: + return cached + logger.warning("Body 304 for %s but cache missing; refetching", entry.slug) + resp = await self._get_with_backoff(url) + resp.raise_for_status() raw = resp.text - body = _strip_frontmatter(raw) + # strip_frontmatter=True is intentional for forward-compat: the new + # endpoint never sends frontmatter, but old caches / mixed deploys may. + body = canonicalize_doc_text(raw, strip_frontmatter=True) computed = f"sha256:{_sha256(body)}" if computed != entry.content_hash: @@ -129,9 +276,15 @@ async def fetch_body(self, entry: ManifestEntry) -> DocBody: tmp = self._cache_dir / f"{entry.slug}.md.tmp" dst = self._cache_dir / f"{entry.slug}.md" - tmp.write_text(raw, encoding="utf-8") + tmp.write_text(body, encoding="utf-8") tmp.replace(dst) + new_etag = resp.headers.get("etag") + if new_etag: + self._etags.set_body(entry.slug, new_etag) + else: + self._etags.set_body(entry.slug, None) + return DocBody(slug=entry.slug, raw=raw, body=body, content_hash=computed) def load_cached_manifest(self) -> Manifest | None: @@ -166,7 +319,7 @@ def load_cached_body(self, slug: str) -> DocBody | None: if not p.exists(): return None raw = p.read_text(encoding="utf-8") - body = _strip_frontmatter(raw) + body = canonicalize_doc_text(raw, strip_frontmatter=True) return DocBody( slug=slug, raw=raw, diff --git a/bot/knowledge.py b/bot/knowledge.py index e99dee8..859b453 100644 --- a/bot/knowledge.py +++ b/bot/knowledge.py @@ -16,6 +16,25 @@ from bot.config import logger +_FRONTMATTER_RE = re.compile(r"^---\s*\n.*?\n---\s*\n", re.DOTALL) + + +def canonicalize_doc_text(raw: str, *, strip_frontmatter: bool) -> str: + """Single source of truth for the text that enters the chunker/hasher. + + Normalises line endings to LF, optionally strips a leading YAML frontmatter + block, collapses 3+ consecutive newlines to 2, and trims surrounding + whitespace. Used in three places that must agree byte-for-byte: + local docs (strip_frontmatter=True), remote bodies (strip_frontmatter=False + on the happy path; True when reading legacy caches). + """ + text = raw.replace("\r\n", "\n").replace("\r", "\n") + if strip_frontmatter: + text = _FRONTMATTER_RE.sub("", text, count=1) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + @dataclass class DocumentChunk: """A chunk of text with its embedding.""" @@ -45,7 +64,7 @@ def __init__(self, db_dir: Path) -> None: self._db_dir = db_dir self._db_dir.mkdir(parents=True, exist_ok=True) self._db_path = db_dir / "vectors.db" - self._conn = sqlite3.connect(str(self._db_path)) + self._conn = sqlite3.connect(str(self._db_path), check_same_thread=False) self._conn.row_factory = sqlite3.Row self._chunks: list[DocumentChunk] = [] self._init_schema() @@ -69,6 +88,10 @@ def _init_schema(self) -> None: source TEXT PRIMARY KEY, content_hash TEXT NOT NULL ); + CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); """) def _load_all(self) -> None: @@ -120,6 +143,18 @@ def get_tracked_sources(self) -> set[str]: cursor = self._conn.execute("SELECT source FROM doc_hashes") return {row["source"] for row in cursor} + def get_meta(self, key: str) -> str | None: + cursor = self._conn.execute("SELECT value FROM meta WHERE key = ?", (key,)) + row = cursor.fetchone() + return row["value"] if row else None + + def set_meta(self, key: str, value: str) -> None: + self._conn.execute( + "INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)", + (key, value), + ) + self._conn.commit() + def save(self) -> None: """Persist chunks to SQLite (upsert).""" self._conn.executemany( @@ -255,7 +290,8 @@ async def load_documentation(store: SimpleVectorStore, docs_dir: Path) -> LoadRe source = md_file.name current_sources.add(source) - text = md_file.read_text(encoding="utf-8") + raw = md_file.read_text(encoding="utf-8") + text = canonicalize_doc_text(raw, strip_frontmatter=True) content_hash = hashlib.sha256(text.encode("utf-8")).hexdigest() stored_hash = store.get_doc_hash(source) @@ -310,6 +346,14 @@ async def load_documentation_from_remote( logger.error("No remote manifest and no cache available") return LoadResult(new_chunks=0, stale_removed=0) + # Short-circuit when nothing changed upstream. Only meaningful when the + # manifest came from the remote (cache hits already imply we may have + # missed an update we couldn't fetch). + last_version = store.get_meta("docs_manifest_version") + if source_of_truth == "remote" and last_version == manifest.version: + logger.info("Manifest version unchanged (%s), skipping remote diff", manifest.version) + return LoadResult(new_chunks=0, stale_removed=0) + current_sources: set[str] = set() new_chunks = 0 @@ -336,11 +380,12 @@ async def load_documentation_from_remote( logger.info("Changed (or new), re-indexing from %s: %s", source_of_truth, source) store.remove_source(source) - chunks = chunk_text(doc_body.body, source=source) + text = canonicalize_doc_text(doc_body.body, strip_frontmatter=False) + chunks = chunk_text(text, source=source) for chunk in chunks: store.add(chunk) new_chunks += 1 - store.set_doc_hash(source, doc_body.content_hash.removeprefix("sha256:")) + store.set_doc_hash(source, hashlib.sha256(text.encode("utf-8")).hexdigest()) stale_sources = store.get_tracked_sources() - current_sources for source in stale_sources: @@ -354,4 +399,6 @@ async def load_documentation_from_remote( else: logger.info("All %d remote docs unchanged, no re-indexing needed", len(current_sources)) + store.set_meta("docs_manifest_version", manifest.version) + return LoadResult(new_chunks=new_chunks, stale_removed=len(stale_sources)) diff --git a/main.py b/main.py index b63a958..1e2176f 100644 --- a/main.py +++ b/main.py @@ -7,7 +7,7 @@ from bot.base import NanBot from bot.config import DEFAULT_DOCS_DIR, logger, settings -from bot.knowledge import SimpleVectorStore, load_documentation +from bot.knowledge import SimpleVectorStore, canonicalize_doc_text, load_documentation from bot.llm import LLMClient @@ -33,16 +33,30 @@ async def init_knowledge_base(store: SimpleVectorStore) -> None: async with DocsClient() as client: manifest = await client.fetch_manifest() + # Compare on the same canonical text the chunker would see, so + # the diff is signal (real content divergence) not noise from + # frontmatter or line-ending differences. local_hashes: dict[str, str] = {} for md_file in sorted(DEFAULT_DOCS_DIR.glob("*.md")): + local_text = canonicalize_doc_text( + md_file.read_text(encoding="utf-8"), + strip_frontmatter=True, + ) local_hashes[md_file.stem] = hashlib.sha256( - md_file.read_text(encoding="utf-8").encode("utf-8") + local_text.encode("utf-8") ).hexdigest() - remote_hashes = { - entry.slug: entry.content_hash.removeprefix("sha256:") - for entry in manifest.entries - } + remote_hashes: dict[str, str] = {} + for entry in manifest.entries: + try: + doc_body = await client.fetch_body(entry) + except Exception as e: + logger.warning("Shadow fetch failed for %s: %s", entry.slug, type(e).__name__) + continue + remote_text = canonicalize_doc_text(doc_body.body, strip_frontmatter=False) + remote_hashes[entry.slug] = hashlib.sha256( + remote_text.encode("utf-8") + ).hexdigest() only_local = sorted(set(local_hashes) - set(remote_hashes)) only_remote = sorted(set(remote_hashes) - set(local_hashes)) @@ -58,12 +72,6 @@ async def init_knowledge_base(store: SimpleVectorStore) -> None: only_remote or "-", changed or "-", ) - - for entry in manifest.entries: - try: - await client.fetch_body(entry) - except Exception as e: - logger.warning("Shadow fetch failed for %s: %s", entry.slug, type(e).__name__) except Exception as e: logger.warning("Shadow mode remote comparison failed: %s", type(e).__name__) else: diff --git a/pyproject.toml b/pyproject.toml index b262413..75e96aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ dev = [ "pytest>=8.0.0", "pytest-asyncio>=0.23.0", + "pytest-httpx>=0.30.0", "ruff>=0.4.0", ] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..dbec781 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,7 @@ +import os + +# Required Settings fields must be defined before bot.config is imported. +os.environ.setdefault("DISCORD_TOKEN", "test-token") +os.environ.setdefault("DISCORD_GUILD_ID", "1") +os.environ.setdefault("LITELLM_API_KEY", "test-litellm-key") +os.environ.setdefault("DOCS_USE_REMOTE", "local") diff --git a/tests/test_docs_client.py b/tests/test_docs_client.py new file mode 100644 index 0000000..f43a71e --- /dev/null +++ b/tests/test_docs_client.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from bot.docs_client import ( + DocsClient, + Manifest, + ManifestEntry, + _SAFE_SLUG_RE, + _strip_frontmatter, +) + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _make_manifest_json(version: str, entries: list[dict]) -> str: + return json.dumps({"version": version, "entries": entries}, ensure_ascii=False) + + +def _entry(slug: str, content_hash: str, *, order: int = 1) -> dict: + return { + "slug": slug, + "title": slug.title(), + "description": f"{slug} desc", + "order": order, + "contentHash": content_hash, + "contentUrl": f"/api/docs/{slug}.md", + } + + +class TestStripFrontmatter: + def test_with_frontmatter(self) -> None: + raw = "---\ntitle: foo\n---\nbody" + assert _strip_frontmatter(raw) == "body" + + def test_without_frontmatter(self) -> None: + raw = "no frontmatter here\nbody" + assert _strip_frontmatter(raw) == raw + + def test_malformed(self) -> None: + raw = "---\nincomplete\nbody" + assert _strip_frontmatter(raw) == raw + + +class TestSafeSlugRe: + @pytest.mark.parametrize( + "slug,expected", + [ + ("api", True), + ("getting-started", True), + ("a", True), + ("../etc/passwd", False), + ("Foo", False), + ("", False), + ("a" * 65, False), + ("-bad", False), + ("api/", False), + ("a" * 64, True), + ], + ) + def test_match(self, slug: str, expected: bool) -> None: + assert bool(_SAFE_SLUG_RE.match(slug)) is expected + + +@pytest.mark.asyncio +async def test_fetch_manifest_valid(httpx_mock, tmp_path: Path) -> None: + body = "intro body" + entry = _entry("intro", f"sha256:{_sha256(body)}") + httpx_mock.add_response( + url="https://example.test/api/docs/manifest.json", + method="GET", + text=_make_manifest_json("sha256:" + "a" * 64, [entry]), + headers={"ETag": '"sha256:' + "a" * 64 + '"'}, + ) + + async with DocsClient(base_url="https://example.test", cache_dir=tmp_path) as client: + manifest = await client.fetch_manifest() + + assert manifest.version == "sha256:" + "a" * 64 + assert [e.slug for e in manifest.entries] == ["intro"] + assert (tmp_path / "manifest.json").exists() + assert (tmp_path / "etags.json").exists() + etag_data = json.loads((tmp_path / "etags.json").read_text()) + assert etag_data["manifest"] == '"sha256:' + "a" * 64 + '"' + + +@pytest.mark.asyncio +async def test_fetch_manifest_filters_unsafe_slug(httpx_mock, tmp_path: Path) -> None: + safe = _entry("intro", f"sha256:{_sha256('a')}") + unsafe = _entry("../etc", f"sha256:{_sha256('b')}") + httpx_mock.add_response( + url="https://example.test/api/docs/manifest.json", + method="GET", + text=_make_manifest_json("sha256:" + "b" * 64, [safe, unsafe]), + ) + + async with DocsClient(base_url="https://example.test", cache_dir=tmp_path) as client: + manifest = await client.fetch_manifest() + + assert [e.slug for e in manifest.entries] == ["intro"] + + +@pytest.mark.asyncio +async def test_fetch_manifest_sends_if_none_match(httpx_mock, tmp_path: Path) -> None: + """Second call must reuse the persisted ETag.""" + text = _make_manifest_json("sha256:" + "c" * 64, [_entry("intro", f"sha256:{_sha256('x')}")]) + etag = '"sha256:' + "c" * 64 + '"' + httpx_mock.add_response( + url="https://example.test/api/docs/manifest.json", + method="GET", + text=text, + headers={"ETag": etag}, + ) + + async with DocsClient(base_url="https://example.test", cache_dir=tmp_path) as client: + await client.fetch_manifest() + + # Second client uses persisted ETag from disk. + httpx_mock.add_response( + url="https://example.test/api/docs/manifest.json", + method="GET", + status_code=304, + match_headers={"If-None-Match": etag}, + ) + async with DocsClient(base_url="https://example.test", cache_dir=tmp_path) as client: + manifest = await client.fetch_manifest() + assert [e.slug for e in manifest.entries] == ["intro"] + + +@pytest.mark.asyncio +async def test_fetch_body_hash_mismatch_warns_but_returns( + httpx_mock, tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + body_text = "real body" + declared = "sha256:" + "0" * 64 # wrong on purpose + entry = ManifestEntry( + slug="intro", + title="Intro", + description="d", + order=1, + content_hash=declared, + content_url="/api/docs/intro.md", + ) + httpx_mock.add_response( + url="https://example.test/api/docs/intro.md", + method="GET", + text=body_text, + headers={"ETag": '"sha256:wrong"'}, + ) + + async with DocsClient(base_url="https://example.test", cache_dir=tmp_path) as client: + with caplog.at_level("WARNING"): + doc = await client.fetch_body(entry) + + assert doc.body == "real body" + assert any("hash mismatch" in r.getMessage() for r in caplog.records) + + +@pytest.mark.asyncio +async def test_fetch_body_writes_canonical_to_cache(httpx_mock, tmp_path: Path) -> None: + body_text = "---\ntitle: t\n---\nhello\r\n\r\nworld" + canonical = "hello\n\nworld" + entry = ManifestEntry( + slug="intro", + title="Intro", + description="d", + order=1, + content_hash=f"sha256:{_sha256(canonical)}", + content_url="/api/docs/intro.md", + ) + httpx_mock.add_response( + url="https://example.test/api/docs/intro.md", + method="GET", + text=body_text, + ) + + async with DocsClient(base_url="https://example.test", cache_dir=tmp_path) as client: + await client.fetch_body(entry) + + cached = (tmp_path / "intro.md").read_text(encoding="utf-8") + assert cached == canonical + + +@pytest.mark.asyncio +async def test_fetch_body_304_uses_cache(httpx_mock, tmp_path: Path) -> None: + body_canonical = "cached body" + (tmp_path / "intro.md").write_text(body_canonical, encoding="utf-8") + # Pre-seed an ETag so the next request sends If-None-Match + (tmp_path / "etags.json").write_text( + json.dumps({"manifest": None, "bodies": {"intro": '"sha256:abc"'}}), + encoding="utf-8", + ) + entry = ManifestEntry( + slug="intro", + title="Intro", + description="d", + order=1, + content_hash=f"sha256:{_sha256(body_canonical)}", + content_url="/api/docs/intro.md", + ) + httpx_mock.add_response( + url="https://example.test/api/docs/intro.md", + method="GET", + status_code=304, + match_headers={"If-None-Match": '"sha256:abc"'}, + ) + + async with DocsClient(base_url="https://example.test", cache_dir=tmp_path) as client: + doc = await client.fetch_body(entry) + + assert doc.body == body_canonical + + +@pytest.mark.asyncio +async def test_load_cached_body_normalises_legacy_frontmatter(tmp_path: Path) -> None: + legacy = "---\ntitle: t\n---\nhello\r\n\r\nworld\n" + (tmp_path / "intro.md").write_text(legacy, encoding="utf-8") + + client = DocsClient(base_url="https://example.test", cache_dir=tmp_path) + doc = client.load_cached_body("intro") + assert doc is not None + assert doc.body == "hello\n\nworld" diff --git a/tests/test_knowledge.py b/tests/test_knowledge.py new file mode 100644 index 0000000..651c7a9 --- /dev/null +++ b/tests/test_knowledge.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from bot.knowledge import SimpleVectorStore, canonicalize_doc_text, load_documentation + + +class TestCanonicalizeDocText: + def test_strip_frontmatter(self) -> None: + raw = "---\ntitle: foo\nauthor: bar\n---\nhello world" + assert canonicalize_doc_text(raw, strip_frontmatter=True) == "hello world" + + def test_strip_frontmatter_false_preserves_body(self) -> None: + raw = "---\ntitle: foo\n---\nhello" + # Without the strip flag, the leading dashes stay verbatim. + assert canonicalize_doc_text(raw, strip_frontmatter=False) == raw + + def test_normalises_crlf(self) -> None: + raw = "hello\r\nworld\r\n" + assert canonicalize_doc_text(raw, strip_frontmatter=False) == "hello\nworld" + + def test_collapses_blank_lines(self) -> None: + raw = "a\n\n\n\nb" + assert canonicalize_doc_text(raw, strip_frontmatter=False) == "a\n\nb" + + def test_trims_final(self) -> None: + raw = "\n\n hello\n\n" + assert canonicalize_doc_text(raw, strip_frontmatter=False) == "hello" + + +@pytest.mark.asyncio +async def test_load_documentation_ignores_frontmatter_and_crlf_differences(tmp_path: Path) -> None: + docs = tmp_path / "docs" + docs.mkdir() + db = tmp_path / "db" + + # Two equivalent payloads modulo frontmatter and line endings. + a = "---\ntitle: t\n---\nhello\n\nworld" + (docs / "a.md").write_text(a, encoding="utf-8") + store = SimpleVectorStore(db) + result = await load_documentation(store, docs) + assert result.new_chunks > 0 + store.save() + first_hash = store.get_doc_hash("a.md") + store.close() + + # Same logical content, different frontmatter + CRLF — should not re-index. + b = "---\ntitle: other\n---\nhello\r\n\r\nworld\r\n" + (docs / "a.md").write_text(b, encoding="utf-8") + store2 = SimpleVectorStore(db) + result2 = await load_documentation(store2, docs) + assert result2.new_chunks == 0 + assert store2.get_doc_hash("a.md") == first_hash + store2.close() + + +@pytest.mark.asyncio +async def test_load_documentation_second_run_no_changes_is_noop(tmp_path: Path) -> None: + docs = tmp_path / "docs" + docs.mkdir() + (docs / "x.md").write_text("# Hello\n\nbody", encoding="utf-8") + store = SimpleVectorStore(tmp_path / "db") + + first = await load_documentation(store, docs) + assert first.new_chunks > 0 + store.save() + + second = await load_documentation(store, docs) + assert second.new_chunks == 0 + assert second.stale_removed == 0 + store.close() diff --git a/tests/test_load_documentation_from_remote.py b/tests/test_load_documentation_from_remote.py new file mode 100644 index 0000000..5826ae5 --- /dev/null +++ b/tests/test_load_documentation_from_remote.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, field +from pathlib import Path + +import pytest + +from bot.docs_client import DocBody, Manifest, ManifestEntry +from bot.knowledge import SimpleVectorStore, load_documentation_from_remote + + +def _sha256(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _entry(slug: str, body: str, *, order: int = 1) -> ManifestEntry: + return ManifestEntry( + slug=slug, + title=slug.title(), + description=f"{slug} desc", + order=order, + content_hash=f"sha256:{_sha256(body)}", + content_url=f"/api/docs/{slug}.md", + ) + + +@dataclass +class FakeClient: + """Stub satisfying the surface of DocsClient used by load_documentation_from_remote.""" + + manifest: Manifest | None + bodies: dict[str, str] = field(default_factory=dict) + fail_manifest: bool = False + cached_manifest: Manifest | None = None + fail_bodies: set[str] = field(default_factory=set) + cached_bodies: dict[str, str] = field(default_factory=dict) + + async def fetch_manifest(self) -> Manifest: + if self.fail_manifest: + raise RuntimeError("boom") + assert self.manifest is not None + return self.manifest + + async def fetch_body(self, entry: ManifestEntry) -> DocBody: + if entry.slug in self.fail_bodies: + raise RuntimeError("body boom") + body = self.bodies[entry.slug] + return DocBody( + slug=entry.slug, + raw=body, + body=body, + content_hash=f"sha256:{_sha256(body)}", + ) + + def load_cached_manifest(self) -> Manifest | None: + return self.cached_manifest + + def load_cached_body(self, slug: str) -> DocBody | None: + if slug not in self.cached_bodies: + return None + body = self.cached_bodies[slug] + return DocBody( + slug=slug, + raw=body, + body=body, + content_hash=f"sha256:{_sha256(body)}", + ) + + +@pytest.mark.asyncio +async def test_unchanged_skip(tmp_path: Path) -> None: + body = "hello" + entry = _entry("intro", body) + manifest = Manifest(version="v1", entries=[entry]) + store = SimpleVectorStore(tmp_path / "db") + client = FakeClient(manifest=manifest, bodies={"intro": body}) + + first = await load_documentation_from_remote(store, client) + assert first.new_chunks > 0 + store.save() + + # Reload with a fresh store on the same DB — version short-circuit must + # treat this as unchanged on the second remote sync. + store2 = SimpleVectorStore(tmp_path / "db") + second = await load_documentation_from_remote(store2, client) + assert second.new_chunks == 0 + assert second.stale_removed == 0 + + +@pytest.mark.asyncio +async def test_changed_reindex(tmp_path: Path) -> None: + body1 = "hello v1" + body2 = "hello v2" + store = SimpleVectorStore(tmp_path / "db") + + manifest1 = Manifest(version="v1", entries=[_entry("intro", body1)]) + client1 = FakeClient(manifest=manifest1, bodies={"intro": body1}) + await load_documentation_from_remote(store, client1) + store.save() + first_hash = store.get_doc_hash("intro.md") + + manifest2 = Manifest(version="v2", entries=[_entry("intro", body2)]) + client2 = FakeClient(manifest=manifest2, bodies={"intro": body2}) + result = await load_documentation_from_remote(store, client2) + assert result.new_chunks > 0 + assert store.get_doc_hash("intro.md") != first_hash + + +@pytest.mark.asyncio +async def test_removed_cleanup(tmp_path: Path) -> None: + store = SimpleVectorStore(tmp_path / "db") + body_a = "a body" + body_b = "b body" + manifest1 = Manifest( + version="v1", + entries=[_entry("a", body_a), _entry("b", body_b)], + ) + client1 = FakeClient(manifest=manifest1, bodies={"a": body_a, "b": body_b}) + await load_documentation_from_remote(store, client1) + store.save() + assert {"a.md", "b.md"}.issubset(store.get_tracked_sources()) + + manifest2 = Manifest(version="v2", entries=[_entry("a", body_a)]) + client2 = FakeClient(manifest=manifest2, bodies={"a": body_a}) + result = await load_documentation_from_remote(store, client2) + assert result.stale_removed == 1 + assert "b.md" not in store.get_tracked_sources() + + +@pytest.mark.asyncio +async def test_fallback_remote_to_cache(tmp_path: Path) -> None: + body = "from cache" + cached_manifest = Manifest(version="v-cache", entries=[_entry("intro", body)]) + store = SimpleVectorStore(tmp_path / "db") + client = FakeClient( + manifest=None, + bodies={"intro": body}, + fail_manifest=True, + cached_manifest=cached_manifest, + ) + + result = await load_documentation_from_remote(store, client) + assert result.new_chunks > 0 + # source_of_truth=="cache" means the short-circuit must NOT be triggered. + # But after a successful cache-based sync we still persist the version, + # so a subsequent cache sync with the same version would re-walk entries. + + +@pytest.mark.asyncio +async def test_fallback_remote_to_local(tmp_path: Path) -> None: + local_docs = tmp_path / "local" + local_docs.mkdir() + (local_docs / "intro.md").write_text("local content", encoding="utf-8") + + store = SimpleVectorStore(tmp_path / "db") + client = FakeClient(manifest=None, fail_manifest=True, cached_manifest=None) + + result = await load_documentation_from_remote(store, client, fallback_docs_dir=local_docs) + assert result.new_chunks > 0 + assert "intro.md" in store.get_tracked_sources() + + +@pytest.mark.asyncio +async def test_manifest_version_unchanged_short_circuits(tmp_path: Path) -> None: + body = "hello" + manifest = Manifest(version="v1", entries=[_entry("intro", body)]) + store = SimpleVectorStore(tmp_path / "db") + client = FakeClient(manifest=manifest, bodies={"intro": body}) + + first = await load_documentation_from_remote(store, client) + assert first.new_chunks > 0 + + # Mutate bodies dict so any fetch_body call would change the chunks. + client.bodies["intro"] = "different" + result = await load_documentation_from_remote(store, client) + # Short-circuit must hit before per-entry walk. + assert result.new_chunks == 0 + assert result.stale_removed == 0 + + +@pytest.mark.asyncio +async def test_cache_source_does_not_short_circuit(tmp_path: Path) -> None: + body = "hello" + cached_manifest = Manifest(version="v1", entries=[_entry("intro", body)]) + store = SimpleVectorStore(tmp_path / "db") + # Pretend a prior remote sync already recorded version "v1". + store.set_meta("docs_manifest_version", "v1") + + client = FakeClient( + manifest=None, + bodies={"intro": body}, + fail_manifest=True, + cached_manifest=cached_manifest, + ) + result = await load_documentation_from_remote(store, client) + # We came from cache, so the short-circuit must not fire; the walk + # actually inspects the entry and indexes it (no stored hash yet). + assert result.new_chunks > 0 From e8dbe10311a4542be8d54342937866dcaf33b4a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C3=BAl=20G=C3=B3mez=20Jim=C3=A9nez?= Date: Tue, 26 May 2026 19:35:13 +0200 Subject: [PATCH 3/5] fix(ci): ruff cleanup + TYPE_CHECKING import for DocsClient - Sort imports, drop quoted type annotation, drop unused Manifest import - Import DocsClient under TYPE_CHECKING so the annotation in load_documentation_from_remote resolves without a runtime cycle Co-Authored-By: Claude Opus 4.7 --- bot/base.py | 2 +- bot/docs_client.py | 3 +-- bot/knowledge.py | 4 +++- tests/test_docs_client.py | 3 +-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/base.py b/bot/base.py index a315b42..04e1723 100644 --- a/bot/base.py +++ b/bot/base.py @@ -163,7 +163,7 @@ async def on_ready(self) -> None: logger.info("Docs refresh scheduler already running") async def _refresh_docs_once(self) -> None: - from datetime import datetime, UTC + from datetime import UTC, datetime if self.store is None: return diff --git a/bot/docs_client.py b/bot/docs_client.py index eaaeb50..270f08b 100644 --- a/bot/docs_client.py +++ b/bot/docs_client.py @@ -13,7 +13,6 @@ from bot.config import logger, settings from bot.knowledge import canonicalize_doc_text - _FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL) _SAFE_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,63}$") _BACKOFF_SECONDS = (0.5, 1.0, 2.0) @@ -142,7 +141,7 @@ def manifest_url(self) -> str: def resolve_content_url(self, content_url: str) -> str: return urljoin(f"{self._base_url}/", content_url.lstrip("/")) - async def __aenter__(self) -> "DocsClient": + async def __aenter__(self) -> DocsClient: transport = httpx.AsyncHTTPTransport(retries=3) self._client = httpx.AsyncClient( timeout=self._timeout, diff --git a/bot/knowledge.py b/bot/knowledge.py index 859b453..7ba8e57 100644 --- a/bot/knowledge.py +++ b/bot/knowledge.py @@ -11,10 +11,12 @@ import sqlite3 from dataclasses import dataclass from pathlib import Path -from typing import Self +from typing import TYPE_CHECKING, Self from bot.config import logger +if TYPE_CHECKING: + from bot.docs_client import DocsClient _FRONTMATTER_RE = re.compile(r"^---\s*\n.*?\n---\s*\n", re.DOTALL) diff --git a/tests/test_docs_client.py b/tests/test_docs_client.py index f43a71e..939aac0 100644 --- a/tests/test_docs_client.py +++ b/tests/test_docs_client.py @@ -7,10 +7,9 @@ import pytest from bot.docs_client import ( + _SAFE_SLUG_RE, DocsClient, - Manifest, ManifestEntry, - _SAFE_SLUG_RE, _strip_frontmatter, ) From 4e0962146fc6ff34a9f3ac863ccad56970ba9852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C3=BAl=20G=C3=B3mez=20Jim=C3=A9nez?= Date: Tue, 26 May 2026 19:39:41 +0200 Subject: [PATCH 4/5] style: apply ruff format Align with main's ruff format pass (commit 7a56238). Co-Authored-By: Claude Opus 4.7 --- bot/docs_client.py | 10 +++++----- main.py | 8 ++------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/bot/docs_client.py b/bot/docs_client.py index 270f08b..b561e58 100644 --- a/bot/docs_client.py +++ b/bot/docs_client.py @@ -46,7 +46,7 @@ def _strip_frontmatter(raw: str) -> str: match = _FRONTMATTER_RE.match(raw) if not match: return raw - return raw[match.end():] + return raw[match.end() :] def _sha256(text: str) -> str: @@ -84,9 +84,7 @@ def _load(self) -> None: manifest = raw.get("manifest") bodies = raw.get("bodies") or {} self._data["manifest"] = manifest if isinstance(manifest, str) else None - self._data["bodies"] = { - k: v for k, v in bodies.items() if isinstance(k, str) and isinstance(v, str) - } + self._data["bodies"] = {k: v for k, v in bodies.items() if isinstance(k, str) and isinstance(v, str)} def _save(self) -> None: tmp = self._path.with_suffix(self._path.suffix + ".tmp") @@ -270,7 +268,9 @@ async def fetch_body(self, entry: ManifestEntry) -> DocBody: if computed != entry.content_hash: logger.warning( "Body hash mismatch for %s: manifest=%s computed=%s", - entry.slug, entry.content_hash, computed, + entry.slug, + entry.content_hash, + computed, ) tmp = self._cache_dir / f"{entry.slug}.md.tmp" diff --git a/main.py b/main.py index 1e2176f..26c93c7 100644 --- a/main.py +++ b/main.py @@ -42,9 +42,7 @@ async def init_knowledge_base(store: SimpleVectorStore) -> None: md_file.read_text(encoding="utf-8"), strip_frontmatter=True, ) - local_hashes[md_file.stem] = hashlib.sha256( - local_text.encode("utf-8") - ).hexdigest() + local_hashes[md_file.stem] = hashlib.sha256(local_text.encode("utf-8")).hexdigest() remote_hashes: dict[str, str] = {} for entry in manifest.entries: @@ -54,9 +52,7 @@ async def init_knowledge_base(store: SimpleVectorStore) -> None: logger.warning("Shadow fetch failed for %s: %s", entry.slug, type(e).__name__) continue remote_text = canonicalize_doc_text(doc_body.body, strip_frontmatter=False) - remote_hashes[entry.slug] = hashlib.sha256( - remote_text.encode("utf-8") - ).hexdigest() + remote_hashes[entry.slug] = hashlib.sha256(remote_text.encode("utf-8")).hexdigest() only_local = sorted(set(local_hashes) - set(remote_hashes)) only_remote = sorted(set(remote_hashes) - set(local_hashes)) From 7189e2fa77f6d664376a20c64c27d505c95e7258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sa=C3=BAl=20G=C3=B3mez=20Jim=C3=A9nez?= Date: Tue, 26 May 2026 23:29:40 +0200 Subject: [PATCH 5/5] docs(readme): document remote docs sync + env vars - Add DOCS_USE_REMOTE / DOCS_BASE_URL / DOCS_REFRESH_INTERVAL / DOCS_CACHE_DIR / DOCS_HTTP_TIMEOUT to the env vars table. - Rewrite "Knowledge base" to describe the three modes (local, remote, shadow), the manifest.version short-circuit, and the If-None-Match/304 cache layout under DOCS_CACHE_DIR. - Replace the "no test suite" line with a pointer to pytest + pytest-httpx and CONTRIBUTING.md's testing policy. Co-Authored-By: Claude Opus 4.7 --- README.md | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2cad132..f8c228a 100644 --- a/README.md +++ b/README.md @@ -121,21 +121,28 @@ Auto-response is triggered when the bot is **mentioned** inside a channel listed | `EMBEDDING_DIM` | no | `4096` | Expected embedding dimensionality. Informational; not enforced at write time. | | `TOP_K` | no | `5` | Number of chunks returned by the vector search used to build the RAG context. | | `ALLOWED_CHANNELS` | no | `""` (all channels) | Comma-separated Discord channel IDs the bot will respond in. Empty means every channel is allowed. | -| `STATUS_CHANNEL_ID` | no | `""` (disables daily report) | Channel ID where the daily metrics report is posted. Required for the scheduler to run. | -| `METRICS_SEND_HOUR` | no | `9` | UTC hour (0–23) at which the daily metrics report is posted. | +| `STATUS_CHANNEL_ID` | no | `""` (disables daily report) | Channel ID where the daily metrics report is posted. Required for the scheduler to run. | +| `METRICS_SEND_HOUR` | no | `9` | UTC hour (0–23) at which the daily metrics report is posted. | +| `DOCS_USE_REMOTE` | no | `local` | Source for docs: `local` (`bot/docs/knowledge/`), `remote` (web docs API), or `shadow` (local + warn on remote drift). | +| `DOCS_BASE_URL` | no | `https://nan.builders` | Base URL of the web that serves `/api/docs/manifest.json` and `/api/docs/{slug}.md`. | +| `DOCS_REFRESH_INTERVAL` | no | `900` | Seconds between docs syncs when `DOCS_USE_REMOTE` is not `local`. Aligned with the web `Cache-Control`. | +| `DOCS_CACHE_DIR` | no | `vector_db/docs_cache` | Directory for the cached `manifest.json`, `etags.json`, and per-slug body files used for conditional GETs. | +| `DOCS_HTTP_TIMEOUT` | no | `10` | Per-request HTTP timeout (seconds) for the docs client. | ## Knowledge base -Markdown files in `bot/docs/knowledge/` are loaded at startup by `SimpleVectorStore`, chunked on paragraph boundaries (target ~2000 chars per chunk with overlap), embedded via the LiteLLM embeddings endpoint, and persisted to `vector_db/vectors.db`. A `doc_hashes` table stores a SHA-256 of each source file so unchanged files are skipped on subsequent boots; files that disappear from disk have their chunks evicted from the database. +When `DOCS_USE_REMOTE=local` (the default), markdown files in `bot/docs/knowledge/` are loaded at startup by `SimpleVectorStore`, chunked on paragraph boundaries (target ~2000 chars per chunk with overlap), embedded via the LiteLLM embeddings endpoint, and persisted to `vector_db/vectors.db`. A `doc_hashes` table stores a SHA-256 of each canonical source so unchanged docs are skipped on subsequent boots; sources that disappear have their chunks evicted from the database. -To update the corpus, edit or add `.md` files under `bot/docs/knowledge/` and restart the bot. Only files whose content hash changed will trigger new embedding API calls. +To update the corpus in `local` mode, edit or add `.md` files under `bot/docs/knowledge/` and restart the bot. Only sources whose content hash changed will trigger new embedding API calls. + +When `DOCS_USE_REMOTE=remote`, the corpus is pulled from `/api/docs/manifest.json` instead and refreshed every `DOCS_REFRESH_INTERVAL` seconds (post-ready, in the background). The client honours `If-None-Match`/304 against `DOCS_CACHE_DIR` and short-circuits the entire sync when the manifest `version` (a hash over `[(slug, contentHash)]`) matches the value persisted in the SQLite `meta` table. `shadow` mode runs the local indexer but also fetches the remote bodies and logs any divergence between local and remote canonical hashes — useful while migrating to `remote`. ## Development - Lint: `ruff check .` - Format: `ruff format .` - Ruff is configured in `pyproject.toml` (`line-length = 120`, `target-version = "py311"`, rules `E, F, I, N, W, UP`). -- There is currently no test suite. The `dev` extra installs `pytest` and `pytest-asyncio`, and `pyproject.toml` already configures `asyncio_mode = "auto"` for when tests are added. +- Tests: `pytest` (full suite). The `dev` extra installs `pytest`, `pytest-asyncio`, and `pytest-httpx`. Tests live under `tests/` and mirror the `bot/` package layout. Mock `httpx` responses with `pytest-httpx`; do not hit live services. See `CONTRIBUTING.md` for the full testing policy. ## Deployment