feat(docs): sync knowledge base from remote docs API

.env.example

@@ -23,3 +23,10 @@ ALLOWED_CHANNELS=
 # Daily metrics report (optional). If STATUS_CHANNEL_ID is empty the report is disabled.
 STATUS_CHANNEL_ID=
 METRICS_SEND_HOUR=9
+
+# Docs remote sync
+DOCS_BASE_URL=https://nan.builders
+DOCS_REFRESH_INTERVAL=900
+DOCS_USE_REMOTE=local
+DOCS_CACHE_DIR=vector_db/docs_cache
+DOCS_HTTP_TIMEOUT=10

README.md

@@ -121,21 +121,28 @@ Auto-response is triggered when the bot is **mentioned** inside a channel listed
 | `EMBEDDING_DIM`      | no       | `4096`                           | Expected embedding dimensionality. Informational; not enforced at write time.                            |
 | `TOP_K`              | no       | `5`                              | Number of chunks returned by the vector search used to build the RAG context.                            |
 | `ALLOWED_CHANNELS`   | no       | `""` (all channels)              | Comma-separated Discord channel IDs the bot will respond in. Empty means every channel is allowed.       |
-| `STATUS_CHANNEL_ID`  | no       | `""` (disables daily report)     | Channel ID where the daily metrics report is posted. Required for the scheduler to run.                  |
-| `METRICS_SEND_HOUR`  | no       | `9`                              | UTC hour (0–23) at which the daily metrics report is posted.                                             |
+| `STATUS_CHANNEL_ID`     | no       | `""` (disables daily report)     | Channel ID where the daily metrics report is posted. Required for the scheduler to run.                  |
+| `METRICS_SEND_HOUR`     | no       | `9`                              | UTC hour (0–23) at which the daily metrics report is posted.                                             |
+| `DOCS_USE_REMOTE`       | no       | `local`                          | Source for docs: `local` (`bot/docs/knowledge/`), `remote` (web docs API), or `shadow` (local + warn on remote drift). |
+| `DOCS_BASE_URL`         | no       | `https://nan.builders`           | Base URL of the web that serves `/api/docs/manifest.json` and `/api/docs/{slug}.md`.                     |
+| `DOCS_REFRESH_INTERVAL` | no       | `900`                            | Seconds between docs syncs when `DOCS_USE_REMOTE` is not `local`. Aligned with the web `Cache-Control`.   |
+| `DOCS_CACHE_DIR`        | no       | `vector_db/docs_cache`           | Directory for the cached `manifest.json`, `etags.json`, and per-slug body files used for conditional GETs. |
+| `DOCS_HTTP_TIMEOUT`     | no       | `10`                             | Per-request HTTP timeout (seconds) for the docs client.                                                  |
 
 ## Knowledge base
 
-Markdown files in `bot/docs/knowledge/` are loaded at startup by `SimpleVectorStore`, chunked on paragraph boundaries (target ~2000 chars per chunk with overlap), embedded via the LiteLLM embeddings endpoint, and persisted to `vector_db/vectors.db`. A `doc_hashes` table stores a SHA-256 of each source file so unchanged files are skipped on subsequent boots; files that disappear from disk have their chunks evicted from the database.
+When `DOCS_USE_REMOTE=local` (the default), markdown files in `bot/docs/knowledge/` are loaded at startup by `SimpleVectorStore`, chunked on paragraph boundaries (target ~2000 chars per chunk with overlap), embedded via the LiteLLM embeddings endpoint, and persisted to `vector_db/vectors.db`. A `doc_hashes` table stores a SHA-256 of each canonical source so unchanged docs are skipped on subsequent boots; sources that disappear have their chunks evicted from the database.
 
-To update the corpus, edit or add `.md` files under `bot/docs/knowledge/` and restart the bot. Only files whose content hash changed will trigger new embedding API calls.
+To update the corpus in `local` mode, edit or add `.md` files under `bot/docs/knowledge/` and restart the bot. Only sources whose content hash changed will trigger new embedding API calls.
+
+When `DOCS_USE_REMOTE=remote`, the corpus is pulled from `<DOCS_BASE_URL>/api/docs/manifest.json` instead and refreshed every `DOCS_REFRESH_INTERVAL` seconds (post-ready, in the background). The client honours `If-None-Match`/304 against `DOCS_CACHE_DIR` and short-circuits the entire sync when the manifest `version` (a hash over `[(slug, contentHash)]`) matches the value persisted in the SQLite `meta` table. `shadow` mode runs the local indexer but also fetches the remote bodies and logs any divergence between local and remote canonical hashes — useful while migrating to `remote`.
 
 ## Development
 
 - Lint: `ruff check .`
 - Format: `ruff format .`
 - Ruff is configured in `pyproject.toml` (`line-length = 120`, `target-version = "py311"`, rules `E, F, I, N, W, UP`).
-- There is currently no test suite. The `dev` extra installs `pytest` and `pytest-asyncio`, and `pyproject.toml` already configures `asyncio_mode = "auto"` for when tests are added.
+- Tests: `pytest` (full suite). The `dev` extra installs `pytest`, `pytest-asyncio`, and `pytest-httpx`. Tests live under `tests/` and mirror the `bot/` package layout. Mock `httpx` responses with `pytest-httpx`; do not hit live services. See `CONTRIBUTING.md` for the full testing policy.
 
 ## Deployment
 

bot/base.py

@@ -13,7 +13,8 @@
 from discord.ext import commands
 
 from bot.config import logger, settings
-from bot.knowledge import SimpleVectorStore
+from bot.docs_client import DocsClient
+from bot.knowledge import SimpleVectorStore, load_documentation_from_remote
 from bot.llm import LLMClient
 from bot.metrics import send_metrics_report, send_user_metrics_report
 
@@ -75,6 +76,10 @@ def __init__(self) -> None:
         self._health_port = 9101
         self._health_server: HTTPServer | None = None
         self._health_thread: Thread | None = None
+        self._docs_last_sync: str | None = None
+        self._docs_last_sync_ok: bool = False
+        self._docs_refresh_task: asyncio.Task[None] | None = None
+        self._docs_refresh_lock = asyncio.Lock()
 
     def _start_health_server(self) -> None:
         """Start a lightweight HTTP health check server in a background thread."""
@@ -86,6 +91,8 @@ def do_GET(self) -> None:
                         "status": "healthy" if self.bot._ready else "starting",
                         "initialized": self.bot._initialized,
                         "store_chunks": len(self.bot.store.chunks) if self.bot.store else 0,
+                        "docs_last_sync": self.bot._docs_last_sync,
+                        "docs_last_sync_ok": self.bot._docs_last_sync_ok,
                     }
                     body = json.dumps(health_data).encode()
                     self.send_response(200)
@@ -150,6 +157,55 @@ async def on_ready(self) -> None:
         )
         await self.start_daily_metrics()
 
+        if self._docs_refresh_task is None or self._docs_refresh_task.done():
+            self._docs_refresh_task = asyncio.create_task(self._schedule_docs_refresh())
+        else:
+            logger.info("Docs refresh scheduler already running")
+
+    async def _refresh_docs_once(self) -> None:
+        from datetime import UTC, datetime
+
+        if self.store is None:
+            return
+
+        async with self._docs_refresh_lock:
+            try:
+                async with DocsClient() as client:
+                    result = await load_documentation_from_remote(self.store, client)
+
+                if result.new_chunks:
+                    embedded = await self.llm.embed_chunks(self.store)
+                    self.store.save()
+                    logger.info("Refresh: embedded %d new chunks", embedded)
+                elif result.stale_removed:
+                    self.store.save()
+
+                self._docs_last_sync_ok = True
+            except Exception as e:
+                logger.error("Docs refresh failed: %s", type(e).__name__)
+                self._docs_last_sync_ok = False
+            finally:
+                self._docs_last_sync = datetime.now(UTC).isoformat()
+
+    async def _schedule_docs_refresh(self) -> None:
+        if settings.docs_use_remote == "local":
+            logger.info("DOCS_USE_REMOTE=local, skipping remote docs refresh")
+            return
+
+        interval = max(60, settings.docs_refresh_interval)
+        logger.info(
+            "Docs refresh scheduler started (mode=%s, interval=%ds)",
+            settings.docs_use_remote,
+            interval,
+        )
+
+        try:
+            while True:
+                await self._refresh_docs_once()
+                await asyncio.sleep(interval)
+        except asyncio.CancelledError:
+            logger.info("Docs refresh scheduler cancelled")
+
     async def _schedule_daily_metrics(self) -> None:
         """Schedule daily metrics to run at the configured hour."""
         if settings.status_channel_id_value is None or settings.litellm_admin_key is None:
@@ -297,13 +353,16 @@ async def health(self, ctx: commands.Context) -> None:
 
     @commands.command(name="docs", description="List available documentation files")
     async def docs(self, ctx: commands.Context) -> None:
-        from bot.config import DOCS_DIR
+        if not self.store:
+            await ctx.send("Knowledge base not initialized.")
+            return
 
-        docs = list(DOCS_DIR.glob("**/*.md"))
+        docs = sorted(self.store.get_tracked_sources())
         if not docs:
             await ctx.send("No documentation files loaded yet.")
             return
-        doc_list = "\n".join(f"- {d.name}" for d in docs)
+
+        doc_list = "\n".join(f"- {doc}" for doc in docs)
         embed = discord.Embed(title="Documentation", description=doc_list, color=discord.Color.blue())
         await ctx.send(embed=embed)
 

bot/config.py

@@ -1,5 +1,6 @@
 import logging
 from pathlib import Path
+from typing import Literal
 
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -23,6 +24,12 @@ class Settings(BaseSettings):
     status_channel_id: str = ""
     metrics_send_hour: int = 9
 
+    docs_base_url: str = "https://nan.builders"
+    docs_refresh_interval: int = 900
+    docs_use_remote: Literal["local", "remote", "shadow"] = "local"
+    docs_cache_dir: str = "vector_db/docs_cache"
+    docs_http_timeout: int = 10
+
     @property
     def allowed_channel_ids(self) -> set[int]:
         if not self.allowed_channels: