Refresh docs and screenshots; fix search snippets and citation wrapping

.env.example

@@ -1,35 +1,36 @@
-# Required
-BRAVE_API_KEY=your_key_here
+# Copy to .env and fill in. Only the search key and the Claude key are needed to start.
 
-# Claude backend — choose one: anthropic | vertex
-CLAUDE_BACKEND=anthropic
+# ── Web search ──────────────────────────────────────────────────────────────
+SEARCH_PROVIDER=brave           # brave | searxng
+BRAVE_API_KEY=your_key_here     # needed when SEARCH_PROVIDER=brave
+SEARXNG_URL=                    # e.g. http://localhost:8888 when SEARCH_PROVIDER=searxng
 
-# Anthropic API (used when CLAUDE_BACKEND=anthropic)
+# ── Claude ──────────────────────────────────────────────────────────────────
+# Writes search queries, names topics, finds claims and answers questions.
+CLAUDE_BACKEND=anthropic        # anthropic | vertex
+
+# Anthropic API (CLAUDE_BACKEND=anthropic)
 ANTHROPIC_API_KEY=your_key_here
-CLAUDE_MODEL=claude-sonnet-5-5  # override default model
+CLAUDE_MODEL=claude-sonnet-5-5
 
-# Vertex AI (used when CLAUDE_BACKEND=vertex)
+# Vertex AI (CLAUDE_BACKEND=vertex)
 # Prerequisites: uv sync --extra vertex  &&  gcloud auth application-default login
 GOOGLE_CLOUD_PROJECT=your_gcp_project_id
 VERTEX_REGION=global            # global, a multi-region (us / eu), or a specific region
-VERTEX_CLAUDE_MODEL=claude-sonnet-5-5  # override Vertex model
+VERTEX_CLAUDE_MODEL=claude-sonnet-5-5
 
-# Embeddings — choose one mode
+# ── Embeddings ──────────────────────────────────────────────────────────────
 EMBEDDING_MODE=fastembed        # fastembed (in-process) | ollama | openai
 FASTEMBED_CACHE=./data/models   # where fastembed keeps its model (~520 MB)
 OLLAMA_BASE_URL=http://localhost:11434  # only needed if EMBEDDING_MODE=ollama
 OPENAI_API_KEY=your_key_here    # only needed if EMBEDDING_MODE=openai
 
-# Storage paths
+# ── Storage ─────────────────────────────────────────────────────────────────
 DB_PATH=./data/discourse.db
 LANCE_PATH=./data/vectors
 
-# Optional tuning
-MAX_ARTICLES=120                # cap on total articles to crawl
-UMAP_N_NEIGHBORS=15
-KMEANS_K=auto                   # auto | integer (e.g. 7)
-RAG_TOP_K=15
-CHAT_EFFORT=medium             # low | medium | high: reasoning before chat answers
-RECOVER_ACADEMIC=true         # look up open-access copies of pages academic publishers block
-SEARCH_PROVIDER=brave         # brave | searxng
-SEARXNG_URL=                  # e.g. http://localhost:8888 when SEARCH_PROVIDER=searxng
+# ── Optional tuning ─────────────────────────────────────────────────────────
+MAX_ARTICLES=120                # most pages read per map
+RECOVER_ACADEMIC=true           # look up open-access copies of pages academic publishers block
+RAG_TOP_K=15                    # excerpts searched per question in Ask
+CHAT_EFFORT=medium              # low | medium | high: reasoning before Ask answers

Planning.md

@@ -1,6 +1,8 @@
 # Discourse Lens — Full Project Documentation
 > Research Intelligence Tool | v1.0
 
+> **This is the original v1 plan (April 2026), kept for history.** The app has since been rebuilt (October 2026): many maps instead of one corpus, a topic model in place of k-means, a new SvelteKit 3 frontend, a Claims view, and a different API (`/api/analyses/...`). For how it works now, see [README.md](README.md); for how to check it, see [validation.md](validation.md).
+
 ---
 
 ## Table of Contents

backend/config.py

@@ -47,8 +47,6 @@ class Settings(BaseSettings):
     lance_path: str = "./data/vectors"
 
     # Tuning
-    umap_n_neighbors: int = 15
-    kmeans_k: str = "auto"  # "auto" | integer string
     rag_top_k: int = 15
     # Thinking effort for chat answers: low answers sooner, medium reasons more.
     chat_effort: Literal["low", "medium", "high"] = "medium"

backend/ingestion/search.py

@@ -9,7 +9,9 @@
 
 from __future__ import annotations
 
+import html
 import logging
+import re
 import time
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -23,6 +25,13 @@
 
 RESULTS_PER_QUERY = 20
 
+_TAG = re.compile(r"<[^>]+>")
+
+
+def plain(text: str | None) -> str:
+    """Titles and snippets arrive as HTML (<strong> around matches, &#x27;)."""
+    return " ".join(html.unescape(_TAG.sub("", text or "")).split())
+
 
 @dataclass
 class SearchResult:
@@ -78,7 +87,7 @@ def brave_search(query: str, count: int = RESULTS_PER_QUERY) -> list[SearchResul
         headers={"Accept": "application/json", "X-Subscription-Token": settings.brave_api_key},
     )
     return [
-        SearchResult(r["url"], r.get("title") or "", r.get("description") or "", query)
+        SearchResult(r["url"], plain(r.get("title")), plain(r.get("description")), query)
         for r in resp.json().get("web", {}).get("results", [])
         if "url" in r
     ][:count]
@@ -93,7 +102,7 @@ def searxng_search(query: str, count: int = RESULTS_PER_QUERY) -> list[SearchRes
         headers={"Accept": "application/json"},
     )
     return [
-        SearchResult(r["url"], r.get("title") or "", r.get("content") or "", query)
+        SearchResult(r["url"], plain(r.get("title")), plain(r.get("content")), query)
         for r in resp.json().get("results", [])
         if "url" in r
     ][:count]

frontend/src/lib/citations.spec.ts

@@ -25,7 +25,15 @@ describe('linkCitations', () => {
 	it('leaves markdown links untouched', () => {
 		const text = 'See [the report](https://x.test) [2].';
 		expect(linkCitations(text, sources)).toMatch(
-			/^See \[the report\]\(https:\/\/x\.test\) <button/
+			/^See \[the report\]\(https:\/\/x\.test\) <span class="cite-group"><button/
 		);
 	});
+
+	it('keeps the punctuation after a citation on its line', () => {
+		const html = linkCitations('Lifespan is capped [1][2]. Others disagree [3]', sources);
+		expect(html).toMatch(
+			/<span class="cite-group">(<button[^>]*>\d<\/button>){2}\.<\/span> Others/
+		);
+		expect(html).toMatch(/<button[^>]*>3<\/button><\/span>$/);
+	});
 });

frontend/src/lib/citations.ts

@@ -5,7 +5,9 @@
  * Pure string functions so they're unit-tested without a DOM.
  */
 
-const CITATION = /\[(\d+(?:\s*[,–-]\s*\d+)*)\]/g;
+const MARKER = /\[(\d+(?:\s*[,–-]\s*\d+)*)\]/g;
+// A run of markers ([1][3]) plus any punctuation right after it: kept on one line.
+const RUN = /(?:\[\d+(?:\s*[,–-]\s*\d+)*\])+[.,;:!?)]*/g;
 
 export interface CitableSource {
 	n: number;
@@ -24,18 +26,27 @@ function numbersIn(group: string): number[] {
 	});
 }
 
-/** Replace citation markers with buttons; markers naming no known source stay as text. */
+/**
+ * Replace citation markers with buttons; markers naming no known source stay
+ * as text. Buttons and the punctuation after them share a no-wrap span, so a
+ * line never starts with the full stop that ends a cited sentence.
+ */
 export function linkCitations(markdown: string, sources: CitableSource[]): string {
 	const byN = new Map(sources.map((s) => [s.n, s]));
-	return markdown.replace(CITATION, (match, group: string) => {
-		const ns = numbersIn(group);
-		if (!ns.length || !ns.every((n) => byN.has(n))) return match;
-		return ns
-			.map((n) => {
-				const s = byN.get(n)!;
-				const label = escapeAttr(`${s.title} (${s.domain})`);
-				return `<button type="button" class="cite" data-cite="${n}" title="${label}" aria-label="Source ${n}: ${label}">${n}</button>`;
-			})
-			.join('');
+	return markdown.replace(RUN, (run) => {
+		let linked = false;
+		const html = run.replace(MARKER, (match, group: string) => {
+			const ns = numbersIn(group);
+			if (!ns.length || !ns.every((n) => byN.has(n))) return match;
+			linked = true;
+			return ns
+				.map((n) => {
+					const s = byN.get(n)!;
+					const label = escapeAttr(`${s.title} (${s.domain})`);
+					return `<button type="button" class="cite" data-cite="${n}" title="${label}" aria-label="Source ${n}: ${label}">${n}</button>`;
+				})
+				.join('');
+		});
+		return linked ? `<span class="cite-group">${html}</span>` : run;
 	});
 }

frontend/src/lib/components/Answer.svelte

@@ -134,6 +134,9 @@
 		line-height: 1;
 		cursor: pointer;
 	}
+	.answer :global(.cite-group) {
+		white-space: nowrap;
+	}
 	.answer :global(.cite:hover) {
 		background: var(--accent);
 		color: var(--accent-ink);

frontend/src/routes/+page.svelte

@@ -88,7 +88,8 @@
 			<h1 class="text-[2.6rem] leading-[1.05] display sm:text-[3.6rem]">Map a conversation.</h1>
 			<p class="mt-5 max-w-[34rem] text-[1.075rem] text-ink-2">
 				Enter a topic. Discourse Lens reads around a hundred articles about it and charts the camps,
-				the outliers and how the conversation has moved. A map takes 3 to 8 minutes.
+				where they agree and disagree, and how the conversation has moved. A map takes 3 to 8
+				minutes.
 			</p>
 
 			<form class="mt-9 flex flex-col gap-3 sm:flex-row" onsubmit={start}>

sanity-check.md

@@ -1,18 +1,20 @@
 # Discourse Lens — Personal Sanity Check
-Last updated: v1.0
+Last updated: October 2026, after the modernization (v0.2)
 
 ---
 
 ## What We're Building (Plain English)
 
-You type a topic. The tool searches the web, reads ~100 articles, and builds a semantic map of the whole discourse. You get four views:
+You type a topic. The tool searches the web, reads ~100 articles, and builds a semantic map of the whole discourse. You get six views:
 
-- **Cluster map** — what camps / schools of thought exist
-- **Temporal drift** — how the topic has evolved over time
-- **Outlier finder** — the genuinely novel or fringe takes
-- **Chat interface** — ask questions, get answers with cited sources
+- **Map**: what camps / schools of thought exist, as named topics
+- **Timeline**: how coverage has moved over time
+- **Outliers**: the genuinely novel or fringe takes
+- **Key terms**: what each camp talks about that the others don't
+- **Claims**: where the sources agree, qualify or dispute each other
+- **Ask**: questions answered only from the articles, with clickable citations
 
-Everything runs locally. No accounts, no cloud infra beyond API calls.
+Everything runs locally. No accounts, no cloud infra beyond the search and Claude requests.
 
 > **The insight:** most research tools help you *retrieve*. This one helps you *orient*. It answers "what is the shape of this conversation?" not just "what does this article say?"
 
@@ -22,67 +24,71 @@ Everything runs locally. No accounts, no cloud infra beyond API calls.
 
 | Thing | What it is | How to start it |
 |-------|-----------|-----------------|
-| FastAPI (Python) | Backend — all the heavy lifting | `uvicorn backend.api.main:app --reload` |
-| SvelteKit (Node) | Frontend UI | `cd frontend && npm run dev` |
-| SQLite | Stores articles + chunks as text | Auto-created at `data/discourse.db` |
-| LanceDB | Stores embedding vectors | Auto-created at `data/vectors/` |
-| Brave Search API | Finds articles to crawl | Get free key → brave.com/search/api |
-| Anthropic API | Powers the chat synthesis | Your existing key |
-| Ollama / nomic-embed-text | Turns text into vectors locally | `ollama pull nomic-embed-text` |
+| FastAPI (Python) | Backend, and it serves the built app too | `uv run uvicorn backend.api.main:app --port 8001` |
+| SvelteKit (Node) | Frontend UI, built into `frontend/build` | `cd frontend && npm run build` (or `npm run dev` on :5173 while editing) |
+| SQLite | Maps, articles, chunks, claims, conversations | Auto-created at `data/discourse.db` |
+| LanceDB | Embedding vectors, one table per map | Auto-created at `data/vectors/` |
+| fastembed / nomic-embed-text | Turns text into vectors inside the app | Downloads itself once (~520 MB) to `data/models/` |
+| Brave Search API (or SearXNG) | Finds articles to read | Free key at brave.com/search/api |
+| Anthropic API | Queries, topic names, claims, answers | Your existing key |
 
 ---
 
-## The Six Stages
+## How It Was Built
 
-| Stage | Name | What gets built | Done when... |
-|-------|------|----------------|--------------|
-| 0 | Scaffold | Folder structure, env vars, health endpoint | `curl /health` returns 200 |
-| 1 | Ingestion | Crawls ~100 articles from topic string | 70+ rows in `articles` table |
-| 2 | Processing | Chunks + deduplicates article text | 300+ rows in `chunks` table |
-| 3 | Embedding | Embeds chunks, runs UMAP, labels clusters | Scatter plot shows distinct clusters |
-| 4 | Visualization | All 4 views working in browser | Cross-view filtering works |
-| 5 | Synthesis | RAG chat answers questions with citations | Chat cites 3+ articles |
-| 6 | Polish | Progress bars, error handling, export | You'd show this to someone |
+The original six stages (scaffold → ingestion → processing → embedding → visualization → synthesis → polish) were finished in April 2026. In October 2026 it was overhauled in six steps (merged as PR #2):
 
-**Current stage:** `[ fill this in ]`
+| Step | What changed |
+|------|--------------|
+| 1 | Quality bug fixes: dedup, embedding prefixes, RAG ranking, outliers |
+| 2 | Backend restructure: uv, many maps instead of one corpus, cancellable runs with a live feed |
+| 3 | Frontend rebuilt: SvelteKit 3, Svelte 5, TypeScript, Tailwind 4 |
+| 4 | Topic model: BERTopic's recipe without PyTorch, named topics with summaries |
+| 5 | Ask: streaming answers, hybrid search, stable citations, saved conversations |
+| 6 | Academic page recovery, source review, SearXNG, fastembed, Claims view, e2e tests + CI |
+
+**Current state:** all done and merged to `main`.
 
 ---
 
 ## Decisions That Are Locked
 
-Don't let Claude Code or yourself relitigate these. They're in the spec for good reasons.
+Don't let Claude Code or yourself relitigate these. They're here for good reasons.
 
 | Decision | Why it's locked |
 |----------|----------------|
-| UMAP runs server-side in Python | Browser UMAP is slow and inconsistent for 200+ points |
-| Two-level chunking (1500 parent / 300 child) | Small chunks for retrieval precision, large context for LLM |
+| UMAP and clustering run server-side in Python | Browser UMAP is slow and inconsistent |
+| Cluster articles, not chunks | Chunk-level clustering just gave one topic per article |
+| No PyTorch | BERTopic's recipe works without it; keeps installs small and CPU-only |
+| Two-level chunking (1500 parent / 300 child) | Small chunks for retrieval precision, large context for the answer |
 | LanceDB for vectors, SQLite for everything else | Zero infra, no Postgres, no Redis — keep it simple |
-| Frontend never calls Anthropic directly | Security + easier to add auth later |
-| Embedding mode is configurable via `EMBEDDING_MODE` | Free local (Ollama) or fast API (OpenAI) |
-| No user accounts in v1 | Single active session. State lives in DB files. |
+| Frontend never calls Anthropic directly | Keys stay on the server |
+| Claims run only when asked | They cost Claude requests on every map |
+| No user accounts | Local tool; state lives in the `data/` folder |
 
 ---
 
 ## Things That Will Probably Go Wrong
 
 | Problem | Likely cause | Fix |
 |---------|-------------|-----|
-| Crawl gets < 50 articles | Paywalls, bot detection, dead links | Increase query count; add retry logic; use archive.org fallback |
-| UMAP clusters look like a blob | Not enough articles or topic too broad | Try 150+ articles; narrow the topic string |
-| Embeddings are slow | Running local model on CPU | Switch to `EMBEDDING_MODE=openai` temporarily |
-| Publish dates all null | Sites don't use standard meta tags | Check URL slug extraction fallback in `metadata.py` |
-| RAG answers are vague | Chunks too small or context window too short | Increase parent chunk size to 2000 tokens; raise `RAG_TOP_K` to 20 |
-| D3 scatter is slow at 200+ nodes | Too many SVG DOM nodes | Switch cluster map to canvas renderer |
+| Crawl keeps < 40 articles | Paywalls, bot detection, dead links | Academic pages are already recovered via OpenAlex / Europe PMC; try a broader topic or turn on source review and add pages |
+| Map looks like one blob | Topic too broad or too few articles | Narrow the topic; articles that fit nowhere go to Unclustered on purpose |
+| Embeddings are slow | Running on CPU | It's ~3 s per 10 chunks; `EMBEDDING_MODE=openai` is faster for new maps |
+| Publish dates all null | Sites don't use standard meta tags | Check the URL fallback in `backend/processing/metadata.py` |
+| Answers are vague | Not enough context | Raise `RAG_TOP_K`, or `CHAT_EFFORT=high` |
+| Ask errors about a different model | Map made with another `EMBEDDING_MODE` | Set the mode the message names |
+| A test passes on Windows but fails in CI | UMAP floating point differs by platform | Assert on the meaning, not exact labels |
 
 ---
 
 ## How to Work with Claude Code
 
-- Start each session: `"I am working on Stage N of Discourse Lens. Read PLANNING.md first."`
-- Keep sessions focused on **one stage at a time** — context bleed causes drift
-- If Claude Code suggests a different tech choice, ask it to justify against the spec before agreeing
-- Always run the stage verification command before moving to the next stage
-- If something breaks, share the error message **and** the relevant section of `PLANNING.md` together
+- Start with: `"Read README.md and validation.md first."` They describe the current app; `Planning.md` is the original v1 plan.
+- Keep sessions focused on **one change at a time**; context bleed causes drift
+- If Claude Code suggests a different tech choice, ask it to justify it against the locked decisions above
+- Run the checks in `validation.md` before calling something done
+- If something breaks, share the error message **and** what you were doing
 
 ---
 
@@ -92,21 +98,22 @@ Don't let Claude Code or yourself relitigate these. They're in the spec for good
 |-----|----------------|-----------|--------|
 | `BRAVE_API_KEY` | brave.com/search/api | Yes — 2000 queries/month | `[ ]` |
 | `ANTHROPIC_API_KEY` | console.anthropic.com | No — pay per token | `[ ]` |
-| `OPENAI_API_KEY` | platform.openai.com | No — optional anyway | `[ ]` |
+| `OPENAI_API_KEY` | platform.openai.com | No — optional, only for `EMBEDDING_MODE=openai` | `[ ]` |
+
+Rough costs: a map is a few cents of Claude; Claims is about half a cent per article; an Ask answer is a cent or two.
 
 ---
 
 ## Session Checklist
 
 **Before each session:**
-- [ ] Backend running? (`uvicorn backend.api.main:app --reload`)
-- [ ] Frontend running? (`cd frontend && npm run dev`)
-- [ ] Which stage am I on?
-- [ ] Did I verify the previous stage's output?
+- [ ] `git pull` on `main`
+- [ ] Backend running? (`uv run uvicorn backend.api.main:app --port 8001`)
+- [ ] Frontend rebuilt if it changed? (`cd frontend && npm run build`)
 
 **After each session:**
-- [ ] Run the stage verification command
-- [ ] `git commit -m "Stage N complete: [what works]"`
+- [ ] `uv run pytest` and `cd frontend && npm run check && npm run lint`
+- [ ] Commit on a branch, open a PR, let CI run
 - [ ] Note anything surprising — it will matter later
 
 ---
@@ -118,8 +125,8 @@ Start with these — they have clear discourse structure and good web coverage.
 | Topic string | Why it's good |
 |---|---|
 | `mechanistic interpretability in large language models` | Rich, active, 2022–present, clear camps |
-| `longevity science and aging reversal` | Lots of camps, clear consensus vs. fringe |
-| `urban heat islands and city design` | Good temporal drift, recent policy debate |
-| `jevons paradox and energy efficiency` | Older topic, good for testing temporal view |
+| `longevity science and aging reversal` | Real disagreements, great for Claims |
+| `urban heat islands and city design` | Good timeline, recent policy debate |
+| `jevons paradox and energy efficiency` | Older topic, long timeline |
 
-**Avoid:** Topics too broad (`AI`, `climate change`) or too narrow (< 20 articles exist). The sweet spot is a specific sub-topic with 50–200 good articles.
+**Avoid:** Topics too broad (`AI`, `climate change`) or too narrow (< 20 articles exist). The sweet spot is a specific sub-topic with 50–200 good articles.