From 92b46c8f6b49f16874d7360438c4204f870815e9 Mon Sep 17 00:00:00 2001 From: Spencer Wozniak Date: Mon, 11 May 2026 20:38:47 -0700 Subject: [PATCH 01/11] build: add backend/requirements.txt mirroring pyproject.toml Provides a pip-installable dependency list alongside the uv-managed pyproject.toml for environments that don't use uv. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/requirements.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 backend/requirements.txt diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..ca83d02 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,12 @@ +arxiv>=3.0.0 +fastapi>=0.136.0 +httpx>=0.28.1 +numpy>=2.4.4 +openai>=2.32.0 +pdfplumber>=0.11.9 +pydantic>=2.13.3 +python-dotenv>=1.2.2 +pyyaml>=6.0.3 +requests>=2.33.1 +sentence-transformers>=5.4.1 +uvicorn>=0.45.0 From 3dc6953c77e9d954ff213f31c3b10fc4eda4ca91 Mon Sep 17 00:00:00 2001 From: Spencer Wozniak Date: Mon, 11 May 2026 22:43:22 -0700 Subject: [PATCH 02/11] feat(backend): KB-grounded flashcards / quiz / report endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds POST /api/generate/{flashcards,quiz,report} on a new router. Each endpoint pulls excerpts from the knowledge base (semantic search when a topic is provided, random sample otherwise — optionally scoped to a tome), prompts the configured LLM with response_format=json_object on OpenAI-compatible providers, and validates the parsed payload before responding. An empty KB returns 400 with a hint to upload documents. Also exposes an llm_text(...) helper used by the audio endpoint for plain-text completions. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/api/generate.py | 479 ++++++++++++++++++++++++++++++++++++++++ backend/main.py | 15 +- 2 files changed, 492 insertions(+), 2 deletions(-) create mode 100644 backend/api/generate.py diff --git a/backend/api/generate.py b/backend/api/generate.py new file mode 100644 index 0000000..e7185cc --- /dev/null +++ b/backend/api/generate.py @@ -0,0 +1,479 @@ +"""Generation endpoints — produce structured artifacts (flashcards, quizzes) +grounded in the user's knowledge base.""" +from __future__ import annotations + +import json +import logging +import random +import re +import uuid +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from config import SageConfig +from providers.base import Message +from providers.factory import create_provider +from skills.base import SkillContext +from skills.search_docs import SearchDocsSkill +from store.db import KnowledgeStore + +router = APIRouter(prefix="/api/generate", tags=["generate"]) +logger = logging.getLogger("sage.generate") + + +# ─────────────────────── Request schemas ─────────────────────── + + +class GenerateRequest(BaseModel): + topic: str | None = None + tome_id: str | None = None + count: int = 6 + provider: str | None = None + model: str | None = None + + +# ─────────────────────── Shared helpers ──────────────────────── + + +async def _gather_context( + topic: str | None, + tome_id: str | None, + store: KnowledgeStore, + config: SageConfig, + max_results: int, +) -> tuple[str, list[dict[str, Any]]]: + """Return (joined context text, list of source dicts). Falls back to + random sampling when there's no topic to drive semantic search.""" + if topic: + skill = SearchDocsSkill() + ctx = SkillContext( + store=store, provider=None, workspace=config.db_path.parent, + config=config, tome_id=tome_id, + ) + result = await skill.execute( + {"query": topic, "max_results": max_results}, ctx, + ) + rows = result.data.get("results", []) if result.data else [] + if rows: + sources = [ + { + "document_id": r["document_id"], + "document_title": r["document_title"], + "chunk_index": r["chunk_index"], + "similarity": r["similarity"], + } + for r in rows + ] + joined = "\n\n".join( + f'[{i + 1}] "{r["document_title"]}" (chunk {r["chunk_index"]})\n{r["content"]}' + for i, r in enumerate(rows) + ) + return joined, sources + + # No topic, or search returned nothing — fall back to random chunks. + if tome_id: + doc_ids = set(store.get_tome_document_ids(tome_id)) + chunks = [c for c in store.get_all_chunks_with_embeddings() if c.document_id in doc_ids] + else: + chunks = store.get_all_chunks_with_embeddings() + + if not chunks: + return "", [] + + sampled = random.sample(chunks, k=min(max_results, len(chunks))) + sources = [] + parts = [] + for i, chunk in enumerate(sampled, 1): + doc = store.get_document(chunk.document_id) + title = doc.title if doc else "Unknown" + sources.append({ + "document_id": chunk.document_id, + "document_title": title, + "chunk_index": chunk.chunk_index, + "similarity": None, + }) + parts.append(f'[{i}] "{title}" (chunk {chunk.chunk_index})\n{chunk.content}') + return "\n\n".join(parts), sources + + +_JSON_FENCE_RE = re.compile(r"```(?:json)?\s*(.*?)```", re.DOTALL | re.IGNORECASE) + + +def _extract_json(text: str) -> Any: + """Pull a JSON object out of a model response that may be wrapped in + prose or code fences. Raises ValueError if nothing parses.""" + candidates: list[str] = [] + candidates.extend(m.group(1).strip() for m in _JSON_FENCE_RE.finditer(text)) + + stripped = text.strip() + if stripped: + candidates.append(stripped) + + for start_ch, end_ch in (("{", "}"), ("[", "]")): + first = text.find(start_ch) + last = text.rfind(end_ch) + if first != -1 and last > first: + candidates.append(text[first : last + 1]) + + for candidate in candidates: + try: + return json.loads(candidate) + except json.JSONDecodeError: + continue + raise ValueError("Could not parse JSON from model response") + + +async def _llm_json( + system_prompt: str, + user_prompt: str, + provider_name: str | None, + model: str | None, + config: SageConfig, +) -> Any: + provider_name = provider_name or config.get("providers.default", "ollama") + provider_config = config.provider_config(provider_name) + resolved_model = model or provider_config.get("default_model", "llama3.1:8b") + provider = create_provider(provider_name, provider_config) + + kwargs: dict[str, Any] = {} + if provider_name in ("openai", "deepseek"): + kwargs["response_format"] = {"type": "json_object"} + + messages = [ + Message(role="system", content=system_prompt), + Message(role="user", content=user_prompt), + ] + response = await provider.chat(messages=messages, tools=[], model=resolved_model, **kwargs) + text = getattr(response, "content", "") or "" + return _extract_json(text) + + +async def llm_text( + system_prompt: str, + user_prompt: str, + provider_name: str | None, + model: str | None, + config: SageConfig, +) -> tuple[str, str, str]: + """Call the configured provider with no tools, no JSON-mode. Returns + (content, resolved_provider_name, resolved_model).""" + provider_name = provider_name or config.get("providers.default", "ollama") + provider_config = config.provider_config(provider_name) + resolved_model = model or provider_config.get("default_model", "llama3.1:8b") + provider = create_provider(provider_name, provider_config) + + messages = [ + Message(role="system", content=system_prompt), + Message(role="user", content=user_prompt), + ] + response = await provider.chat(messages=messages, tools=[], model=resolved_model) + return (getattr(response, "content", "") or "", provider_name, resolved_model) + + +def _empty_kb_error() -> HTTPException: + return HTTPException( + status_code=400, + detail=( + "Your knowledge base is empty for this scope. Add documents first " + "via the upload button next to the command bar." + ), + ) + + +# ─────────────────────── Flashcards ──────────────────────────── + + +FLASHCARD_SYSTEM = """You write study flashcards grounded ONLY in the provided source excerpts. + +Rules: +- Each card must be answerable from the sources. Do not invent facts. +- "front" is a concise question or term (1 sentence). +- "back" is a tight, self-contained answer (1–3 sentences). +- Cover distinct ideas — do not repeat the same fact in different words. +- Output ONLY a JSON object of the form: {"cards": [{"front": "...", "back": "..."}, ...]} +- No commentary, no markdown fences.""" + + +@router.post("/flashcards") +async def generate_flashcards( + req: GenerateRequest, + store: KnowledgeStore = Depends(), + config: SageConfig = Depends(), +): + count = max(1, min(req.count, 20)) + context_text, sources = await _gather_context( + topic=req.topic, tome_id=req.tome_id, store=store, config=config, + max_results=max(count, 6), + ) + if not context_text: + raise _empty_kb_error() + + topic_line = ( + f"Topic focus: {req.topic.strip()}" if req.topic and req.topic.strip() + else "Topic focus: cover the most important ideas in the sources." + ) + user_prompt = ( + f"{topic_line}\n" + f"Generate exactly {count} flashcards.\n\n" + f"Source excerpts:\n{context_text}" + ) + + try: + parsed = await _llm_json(FLASHCARD_SYSTEM, user_prompt, req.provider, req.model, config) + except ValueError as exc: + logger.exception("flashcard json parse failed") + raise HTTPException(status_code=502, detail=f"Model returned unparseable output: {exc}") + + raw_cards = parsed.get("cards") if isinstance(parsed, dict) else parsed + if not isinstance(raw_cards, list): + raise HTTPException(status_code=502, detail="Model output missing 'cards' array") + + cards = [] + for entry in raw_cards[:count]: + if not isinstance(entry, dict): + continue + front = str(entry.get("front", "")).strip() + back = str(entry.get("back", "")).strip() + if not front or not back: + continue + cards.append({"id": f"fc-{uuid.uuid4().hex[:8]}", "front": front, "back": back}) + + if not cards: + raise HTTPException(status_code=502, detail="Model produced no usable flashcards") + + return { + "cards": cards, + "topic": req.topic or "", + "sources": sources, + } + + +# ─────────────────────── Quiz ────────────────────────────────── + + +QUIZ_SYSTEM = """You write multiple-choice quiz questions grounded ONLY in the provided source excerpts. + +Rules: +- Each question must be answerable from the sources. Do not invent facts. +- Exactly 4 options per question, labeled "a", "b", "c", "d". +- Exactly one correct option per question. +- "explanation" should be 1–2 sentences explaining the correct answer using the sources. +- Distractors must be plausible but clearly wrong given the sources. +- Output ONLY a JSON object of the form: + {"questions": [ + {"question": "...", + "options": [{"id":"a","text":"..."},{"id":"b","text":"..."},{"id":"c","text":"..."},{"id":"d","text":"..."}], + "correctOptionId": "b", + "explanation": "..."} + ]} +- No commentary, no markdown fences.""" + + +@router.post("/quiz") +async def generate_quiz( + req: GenerateRequest, + store: KnowledgeStore = Depends(), + config: SageConfig = Depends(), +): + count = max(1, min(req.count, 15)) + context_text, sources = await _gather_context( + topic=req.topic, tome_id=req.tome_id, store=store, config=config, + max_results=max(count, 5), + ) + if not context_text: + raise _empty_kb_error() + + topic_line = ( + f"Topic focus: {req.topic.strip()}" if req.topic and req.topic.strip() + else "Topic focus: cover the most important ideas in the sources." + ) + user_prompt = ( + f"{topic_line}\n" + f"Generate exactly {count} multiple-choice questions.\n\n" + f"Source excerpts:\n{context_text}" + ) + + try: + parsed = await _llm_json(QUIZ_SYSTEM, user_prompt, req.provider, req.model, config) + except ValueError as exc: + logger.exception("quiz json parse failed") + raise HTTPException(status_code=502, detail=f"Model returned unparseable output: {exc}") + + raw_questions = parsed.get("questions") if isinstance(parsed, dict) else parsed + if not isinstance(raw_questions, list): + raise HTTPException(status_code=502, detail="Model output missing 'questions' array") + + questions = [] + for entry in raw_questions[:count]: + if not isinstance(entry, dict): + continue + question_text = str(entry.get("question", "")).strip() + options_raw = entry.get("options") + correct_id = str(entry.get("correctOptionId", "")).strip().lower() + explanation = str(entry.get("explanation", "")).strip() + if not question_text or not isinstance(options_raw, list) or len(options_raw) < 2: + continue + + options = [] + for opt in options_raw: + if not isinstance(opt, dict): + continue + opt_id = str(opt.get("id", "")).strip().lower() + opt_text = str(opt.get("text", "")).strip() + if opt_id and opt_text: + options.append({"id": opt_id, "text": opt_text}) + if len(options) < 2 or not any(o["id"] == correct_id for o in options): + continue + + questions.append({ + "id": f"q-{uuid.uuid4().hex[:8]}", + "question": question_text, + "options": options, + "correctOptionId": correct_id, + "explanation": explanation, + }) + + if not questions: + raise HTTPException(status_code=502, detail="Model produced no usable questions") + + return { + "questions": questions, + "topic": req.topic or "", + "sources": sources, + } + + +# ─────────────────────── Report ──────────────────────────────── + + +REPORT_SYSTEM = """You write structured study reports grounded ONLY in the provided source excerpts. + +Rules: +- Every claim must be supported by the sources. Do not invent facts, numbers, or citations. +- Aim for 5–9 sections, each focused on one distinct idea. +- The first section MUST be "Executive Summary". +- Section content is GitHub-flavored markdown. You may use: + * paragraphs, **bold**, *italic*, blockquotes + * bullet and numbered lists + * tables (pipe syntax) + * fenced code blocks (with language) when sources include code + * LaTeX math: inline `$...$` and display `$$...$$` (escape backslashes as needed) +- Do NOT include the section title inside the section's content — only the body. +- Do NOT use h1 (#) headings. If you need a sub-heading inside a section, use h3 (###). +- Output ONLY a JSON object of the form: + { + "title": "Short report title (≤ 60 chars)", + "subtitle": "One-line description of the report's scope", + "sections": [ + {"title": "Executive Summary", "content": "markdown..."}, + {"title": "...", "content": "markdown..."}, + ... + ] + } +- No commentary, no markdown fences around the JSON.""" + + +_SLUG_RE = re.compile(r"[^a-z0-9]+") + + +def _slugify(text: str) -> str: + """Match rehype-slug / github-slugger output for typical ASCII headings.""" + slug = _SLUG_RE.sub("-", text.lower()).strip("-") + return slug or "section" + + +@router.post("/report") +async def generate_report( + req: GenerateRequest, + store: KnowledgeStore = Depends(), + config: SageConfig = Depends(), +): + context_text, sources = await _gather_context( + topic=req.topic, tome_id=req.tome_id, store=store, config=config, + max_results=10, + ) + if not context_text: + raise _empty_kb_error() + + topic_line = ( + f"Topic focus: {req.topic.strip()}" if req.topic and req.topic.strip() + else "Topic focus: synthesize the most important ideas across the sources." + ) + user_prompt = ( + f"{topic_line}\n" + f"Write the report now as structured JSON.\n\n" + f"Source excerpts:\n{context_text}" + ) + + try: + parsed = await _llm_json(REPORT_SYSTEM, user_prompt, req.provider, req.model, config) + except ValueError as exc: + logger.exception("report json parse failed") + raise HTTPException(status_code=502, detail=f"Model returned unparseable output: {exc}") + + if not isinstance(parsed, dict): + raise HTTPException(status_code=502, detail="Model output was not a JSON object") + + title = str(parsed.get("title", "")).strip() or ( + f"Report: {req.topic.strip()}" if req.topic and req.topic.strip() else "Report" + ) + subtitle = str(parsed.get("subtitle", "")).strip() or None + raw_sections = parsed.get("sections") + if not isinstance(raw_sections, list) or not raw_sections: + raise HTTPException(status_code=502, detail="Model output missing 'sections' array") + + toc: list[dict[str, str]] = [] + content_parts: list[str] = [] + used_slugs: set[str] = set() + for entry in raw_sections: + if not isinstance(entry, dict): + continue + section_title = str(entry.get("title", "")).strip() + section_body = str(entry.get("content", "")).strip() + if not section_title or not section_body: + continue + base_slug = _slugify(section_title) + slug = base_slug + i = 1 + while slug in used_slugs: + i += 1 + slug = f"{base_slug}-{i}" + used_slugs.add(slug) + toc.append({"id": slug, "title": section_title}) + content_parts.append(f"## {section_title}\n\n{section_body}") + + if not toc: + raise HTTPException(status_code=502, detail="Model produced no usable sections") + + unique_doc_ids: set[str] = set() + for s in sources: + unique_doc_ids.add(s["document_id"]) + doc_count = len(unique_doc_ids) + + tome_name: str | None = None + if req.tome_id: + tome = store.get_tome(req.tome_id) + if tome: + tome_name = tome.name + + if doc_count: + if tome_name: + source_docs = f'Based on {doc_count} document{"s" if doc_count != 1 else ""} in "{tome_name}" tome' + else: + source_docs = f'Based on {doc_count} document{"s" if doc_count != 1 else ""} in your knowledge base' + else: + source_docs = None + + content = "\n\n".join(content_parts) + "\n\n---\n\n*Report generated by Sage*" + + return { + "title": title, + "subtitle": subtitle, + "sourceDocs": source_docs, + "toc": toc, + "content": content, + "sources": sources, + "topic": req.topic or "", + } diff --git a/backend/main.py b/backend/main.py index 56ec997..d8b2b1d 100644 --- a/backend/main.py +++ b/backend/main.py @@ -24,7 +24,7 @@ def _get_arxiv_service(): from services.pdf import PDFService from services.summarizer import DeepSeekService -from api import chat, knowledge, skills as skills_api, tomes as tomes_api +from api import audio, chat, generate, knowledge, skills as skills_api, tomes as tomes_api from config import SageConfig from skills.read_document import ReadDocumentSkill from skills.registry import SkillRegistry @@ -57,6 +57,8 @@ def _get_arxiv_service(): app.include_router(chat.router) app.include_router(tomes_api.router) app.include_router(skills_api.router) +app.include_router(generate.router) +app.include_router(audio.router) # --- end Sage setup --- # Add CORS middleware with configurable origins @@ -70,7 +72,7 @@ def _get_arxiv_service(): ) # Initialize services -arxiv_service = ArxivService() +arxiv_service = _get_arxiv_service() # Thread pool for parallel processing executor = ThreadPoolExecutor(max_workers=5) @@ -308,6 +310,15 @@ def read_root(): return {"message": "Welcome to the Sage API!"} +@app.get("/api/config") +def get_config(config: SageConfig = Depends()): + """Expose the active provider/model so the UI can label which one's in use.""" + provider_name = config.get("providers.default", "ollama") + provider_config = config.provider_config(provider_name) + model = provider_config.get("default_model", "") + return {"provider": provider_name, "model": model} + + # Clear caches endpoint (for maintenance) @app.post("/api/clear-cache") def clear_cache(): From 356deb9ccc386459b99db7b3f8a509f230250f9b Mon Sep 17 00:00:00 2001 From: Spencer Wozniak Date: Mon, 11 May 2026 22:43:30 -0700 Subject: [PATCH 03/11] feat(backend): audio narration endpoint with optional OpenAI TTS POST /api/generate/audio retrieves KB context (or randomly samples it), prompts the configured provider for a spoken-style narration script, and segments it into sentence-level chunks with word-rate-estimated timings for transcript scroll-lock. When OPENAI_API_KEY is configured, the script is also synthesized to MP3 via OpenAI's tts-1 voice (default: alloy) and cached to disk; the response includes audio_url. Without a key, audio_url is null and the frontend falls back to the browser's SpeechSynthesis voice. GET /api/audio/{audio_id}.mp3 streams the cached file with a regex guard against directory traversal. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/api/audio.py | 202 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 backend/api/audio.py diff --git a/backend/api/audio.py b/backend/api/audio.py new file mode 100644 index 0000000..a64f89c --- /dev/null +++ b/backend/api/audio.py @@ -0,0 +1,202 @@ +"""Audio generation endpoint — KB-grounded narration script + optional +OpenAI TTS synthesis. Frontend falls back to browser SpeechSynthesis when +no server-side audio file is produced.""" +from __future__ import annotations + +import logging +import os +import re +import uuid +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from fastapi.responses import FileResponse +from pydantic import BaseModel + +from api.generate import _empty_kb_error, _gather_context, llm_text +from config import SageConfig +from store.db import KnowledgeStore + +router = APIRouter(tags=["audio"]) +logger = logging.getLogger("sage.audio") + +# Words per second — used to estimate segment timings and total duration +# when we can't probe the audio file. Roughly matches OpenAI TTS / typical +# browser SpeechSynthesis pacing. +WORDS_PER_SECOND = 2.5 + +AUDIO_CACHE_DIR = Path(os.getenv("SAGE_AUDIO_CACHE", "/tmp/sage_audio")) +AUDIO_CACHE_DIR.mkdir(parents=True, exist_ok=True) + + +# ─────────────────────── Schemas ─────────────────────────────── + + +class AudioRequest(BaseModel): + topic: str | None = None + tome_id: str | None = None + voice: str | None = None # OpenAI TTS voice (alloy, echo, fable, onyx, nova, shimmer) + provider: str | None = None + model: str | None = None + + +# ─────────────────────── Script generation ───────────────────── + + +SCRIPT_SYSTEM = """You write spoken-style narration grounded ONLY in the provided source excerpts. The output will be read aloud by a text-to-speech engine. + +Rules: +- Conversational, natural prose. Like a podcast host explaining a topic. +- No markdown, no bullet points, no code blocks, no headings, no citations. +- Do not say "in the sources" or "according to chunk 3" — just teach the content. +- 4–8 short paragraphs. Each paragraph 1–3 sentences. +- Roughly 200–350 words total. +- End with a brief closing sentence. +- If the sources are sparse, summarize what's there honestly without inventing facts.""" + + +_SENTENCE_RE = re.compile(r"[^.!?\n]+[.!?]+|\S[^.!?\n]*", re.UNICODE) + + +def _split_into_segments(script: str) -> list[dict[str, Any]]: + """Split the script into spoken sentences with estimated start/end times + proportional to word count.""" + raw_sentences = [m.group(0).strip() for m in _SENTENCE_RE.finditer(script)] + raw_sentences = [s for s in raw_sentences if s] + + if not raw_sentences: + return [] + + segments: list[dict[str, Any]] = [] + cursor = 0.0 + for i, text in enumerate(raw_sentences): + words = max(1, len(text.split())) + duration = max(1.2, words / WORDS_PER_SECOND) + segments.append({ + "id": f"s{i + 1}", + "text": text, + "startTime": round(cursor, 2), + "endTime": round(cursor + duration, 2), + }) + cursor += duration + return segments + + +# ─────────────────────── TTS synthesis ───────────────────────── + + +_OPENAI_TTS_VOICES = {"alloy", "echo", "fable", "onyx", "nova", "shimmer"} + + +def _maybe_synthesize_openai_tts( + script: str, voice: str | None, config: SageConfig, +) -> tuple[str | None, str | None]: + """If an OpenAI key is configured, synthesize the script to an MP3 on + disk and return (audio_id, voice_used). Otherwise return (None, None) + so the frontend can fall back to browser SpeechSynthesis.""" + openai_cfg = config.provider_config("openai") + api_key = openai_cfg.get("api_key") or os.getenv("OPENAI_API_KEY") + if not api_key: + return None, None + + selected_voice = (voice or "alloy").lower() + if selected_voice not in _OPENAI_TTS_VOICES: + selected_voice = "alloy" + + try: + # Imported lazily so missing SDK never breaks the no-TTS path. + from openai import OpenAI + + client = OpenAI(api_key=api_key, base_url=openai_cfg.get("base_url")) + audio_id = uuid.uuid4().hex + out_path = AUDIO_CACHE_DIR / f"{audio_id}.mp3" + + with client.audio.speech.with_streaming_response.create( + model="tts-1", + voice=selected_voice, + input=script, + ) as response: + response.stream_to_file(out_path) + + return audio_id, selected_voice + except Exception as exc: # noqa: BLE001 + logger.warning("OpenAI TTS synthesis failed, falling back: %s", exc) + return None, None + + +# ─────────────────────── Endpoints ───────────────────────────── + + +@router.post("/api/generate/audio") +async def generate_audio( + req: AudioRequest, + store: KnowledgeStore = Depends(), + config: SageConfig = Depends(), +): + context_text, sources = await _gather_context( + topic=req.topic, tome_id=req.tome_id, store=store, config=config, + max_results=8, + ) + if not context_text: + raise _empty_kb_error() + + topic_line = ( + f"Topic focus: {req.topic.strip()}" if req.topic and req.topic.strip() + else "Topic focus: cover the most important ideas in the sources." + ) + user_prompt = ( + f"{topic_line}\n" + f"Write the narration now.\n\n" + f"Source excerpts:\n{context_text}" + ) + + try: + script, provider_used, model_used = await llm_text( + SCRIPT_SYSTEM, user_prompt, req.provider, req.model, config, + ) + except Exception as exc: # noqa: BLE001 + logger.exception("audio script generation failed") + raise HTTPException(status_code=502, detail=f"Failed to generate script: {exc}") + + script = script.strip() + if not script: + raise HTTPException(status_code=502, detail="Model produced an empty script.") + + segments = _split_into_segments(script) + estimated_duration = segments[-1]["endTime"] if segments else 0.0 + + audio_id, used_voice = _maybe_synthesize_openai_tts(script, req.voice, config) + audio_url = f"/api/audio/{audio_id}.mp3" if audio_id else None + + voice_label = used_voice or "Browser" + + title = ( + f"Audio: {req.topic.strip()}" if req.topic and req.topic.strip() + else "Audio overview" + ) + + return { + "id": audio_id or f"local-{uuid.uuid4().hex[:8]}", + "title": title, + "voice": voice_label, + "provider": provider_used, + "model": model_used, + "script": script, + "segments": segments, + "duration": estimated_duration, + "audio_url": audio_url, + "sources": sources, + "topic": req.topic or "", + } + + +@router.get("/api/audio/{audio_id}.mp3") +async def get_audio_file(audio_id: str): + """Serve a previously-synthesized TTS file.""" + if not re.fullmatch(r"[a-f0-9\-]{8,}", audio_id): + raise HTTPException(status_code=400, detail="Invalid audio id.") + path = AUDIO_CACHE_DIR / f"{audio_id}.mp3" + if not path.exists(): + raise HTTPException(status_code=404, detail="Audio file not found or expired.") + return FileResponse(path, media_type="audio/mpeg", filename=f"{audio_id}.mp3") From 2a20a7a03c37ba803bc08fd00ae32df856c94333 Mon Sep 17 00:00:00 2001 From: Spencer Wozniak Date: Mon, 11 May 2026 22:43:38 -0700 Subject: [PATCH 04/11] feat(backend): session listing + capability-aware chat system prompt - Adds KnowledgeStore.list_sessions(): one SQL pass joining sessions, messages, and tomes that returns derived title (first user message), tome name, message count, and last activity timestamp. Ordered by most-recent activity. - Adds GET /api/chat/sessions?limit= as a thin wrapper for the History panel. - Extends the chat SYSTEM_PROMPT with an "App capabilities you should know about" section so the model can describe its own routed features (quiz / flashcards / audio / report / history / tomes / knowledge base / upload) accurately when the user asks "what can you do?", and skips replying when the frontend has already routed away. Co-Authored-By: Claude Opus 4.7 (1M context) --- backend/api/chat.py | 33 +++++++++++++++++++++++++++++-- backend/store/db.py | 47 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/backend/api/chat.py b/backend/api/chat.py index 4f43675..9910d5e 100644 --- a/backend/api/chat.py +++ b/backend/api/chat.py @@ -16,15 +16,35 @@ router = APIRouter(prefix="/api/chat", tags=["chat"]) -SYSTEM_PROMPT = """You are Sage, a helpful knowledge assistant. You have access to the user's personal knowledge base through tools. When answering questions: +SYSTEM_PROMPT = """You are Sage, a helpful knowledge assistant. You have access to the user's personal knowledge base through tools. +When answering questions: 1. Search the knowledge base first using search_docs 2. Read relevant documents if needed using read_document 3. Cite sources with numbered references like [1], [2] 4. Be concise but thorough 5. If nothing relevant is found, say so honestly -Always ground your answers in the user's actual documents.""" +Always ground your answers in the user's actual documents. + +# App capabilities you should know about + +You run inside the Sage app. The app's frontend watches every message the user sends — to the command bar OR to you in chat — and routes prompts containing certain keywords to dedicated views instead of returning a chat reply. You should be aware of these so you can guide the user accurately when they ask "what can you do?" or how to access a feature. Do NOT try to render these artifacts yourself in markdown — tell the user the keyword to type and the app will open the right view. + +Routing keywords (case-insensitive): +- "quiz", or whole-word "test" / "tests" / "testing" → generates a real multiple-choice quiz from the knowledge base via POST /api/generate/quiz, rendered in the QuizWidget. +- "flashcard" / "flash card" → generates real study flashcards from the knowledge base via POST /api/generate/flashcards, rendered in the FlashcardWidget. +- "audio", "listen", "podcast" → generates a spoken-style narration script from the knowledge base via POST /api/generate/audio. If an OpenAI API key is configured, the script is synthesized to an MP3 (OpenAI TTS) and played in an audio element; otherwise playback falls back to the browser's SpeechSynthesis voice. Transcript scrolls with the audio. +- "report", "study guide", "summary" → opens the report view (sample report for now). +- "history" → opens the history panel of past sessions. +- "tome", "collection", "library" → opens the tome selector for grouping documents. +- "knowledge", "kb", "docs", "documents", "knowledge base", "view/show/list documents" → opens the KnowledgeBaseWidget which lists every ingested document with detail + delete. + +Other things the user can do in this app: +- Upload documents to the knowledge base using the circular upload button to the right of the command bar. It accepts pasted text or text-like files (.txt, .md, .csv, .json, .log). PDFs/DOCX/images are not yet supported via the UI. +- Anything ingested is chunked, embedded, and dedup'd by content hash; you can immediately search it via search_docs. + +If the user asks for one of the routed features inside chat (e.g. "make me flashcards on attention"), the app will switch views automatically — you will not see the follow-up. So when that happens, you do not need to reply at all. If the user is asking *about* a feature rather than invoking it, explain it using the information above.""" class ChatRequest(BaseModel): @@ -35,6 +55,15 @@ class ChatRequest(BaseModel): model: str | None = None +@router.get("/sessions") +async def list_sessions( + limit: int = 100, + store: KnowledgeStore = Depends(), +): + """Return recent chat sessions for the History panel.""" + return {"sessions": store.list_sessions(limit=limit)} + + @router.post("") async def chat( req: ChatRequest, diff --git a/backend/store/db.py b/backend/store/db.py index f75f2f0..5a55328 100644 --- a/backend/store/db.py +++ b/backend/store/db.py @@ -231,6 +231,53 @@ def create_session(self, provider: str, model: str, self.conn.commit() return s + def list_sessions(self, limit: int = 100) -> list[dict]: + """List chat sessions with a derived title, tome name, and activity timestamps. + + Returns rows shaped for the History panel — newest activity first.""" + rows = self.conn.execute( + """ + SELECT + s.id AS id, + s.tome_id AS tome_id, + s.provider AS provider, + s.model AS model, + s.created_at AS created_at, + t.name AS tome_name, + (SELECT content FROM messages + WHERE session_id = s.id AND role = 'user' + ORDER BY created_at ASC LIMIT 1) AS first_user_message, + (SELECT created_at FROM messages + WHERE session_id = s.id + ORDER BY created_at DESC LIMIT 1) AS last_message_at, + (SELECT COUNT(*) FROM messages + WHERE session_id = s.id) AS message_count + FROM sessions s + LEFT JOIN tomes t ON t.id = s.tome_id + ORDER BY COALESCE( + (SELECT created_at FROM messages + WHERE session_id = s.id ORDER BY created_at DESC LIMIT 1), + s.created_at + ) DESC + LIMIT ? + """, + (limit,), + ).fetchall() + return [ + { + "id": r["id"], + "tome_id": r["tome_id"], + "tome_name": r["tome_name"], + "provider": r["provider"], + "model": r["model"], + "created_at": r["created_at"], + "last_message_at": r["last_message_at"] or r["created_at"], + "first_user_message": r["first_user_message"], + "message_count": r["message_count"] or 0, + } + for r in rows + ] + def get_tome_session(self, tome_id: str) -> Optional[Session]: """Get the most recent session for a tome.""" row = self.conn.execute( From 4e6efa947867bfa89b556a5d319992bd07a3dba6 Mon Sep 17 00:00:00 2001 From: Spencer Wozniak Date: Mon, 11 May 2026 22:44:15 -0700 Subject: [PATCH 05/11] =?UTF-8?q?feat(frontend):=20Sage=20API=20client=20?= =?UTF-8?q?=E2=80=94=20knowledge,=20generation,=20audio,=20history?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds typed helpers covering every new backend surface: - Knowledge base: listDocuments, getDocument, deleteDocument plus DocumentSummary / DocumentDetail. - Generation: generateFlashcards, generateQuiz, generateReport, generateAudio with shared GenerateOpts and GeneratedSource. - Audio: AudioGeneration / AudioSegment plus resolveAudioUrl() that joins a relative audio path against the active API base (so it works from both the Next.js dev server and the Tauri sidecar). - History: listChatSessions and ChatSessionSummary. Co-Authored-By: Claude Opus 4.7 (1M context) --- frontend/src/lib/sage-api.ts | 216 +++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) diff --git a/frontend/src/lib/sage-api.ts b/frontend/src/lib/sage-api.ts index d5fb931..5482e8a 100644 --- a/frontend/src/lib/sage-api.ts +++ b/frontend/src/lib/sage-api.ts @@ -112,6 +112,17 @@ async function apiFetch(path: string, opts: FetchOpts = {}): Promise { return res.json(); } +// ── Config ── + +export interface SageRuntimeConfig { + provider: string; + model: string; +} + +export async function getRuntimeConfig(): Promise { + return apiFetch("/api/config"); +} + // ── Tomes ── export interface Tome { @@ -181,6 +192,42 @@ export async function ingestDocument( }); } +export interface DocumentSummary { + id: string; + title: string; + source: string; + doc_type: string; + content_hash: string; + created_at: string; +} + +export interface DocumentDetail extends DocumentSummary { + chunks: number; + content_preview: string; + tomes: { id: string; name: string }[]; +} + +export async function listDocuments( + opts: { limit?: number; offset?: number } = {}, +): Promise { + const params = new URLSearchParams(); + if (opts.limit !== undefined) params.set("limit", String(opts.limit)); + if (opts.offset !== undefined) params.set("offset", String(opts.offset)); + const qs = params.toString(); + const data = await apiFetch<{ documents: DocumentSummary[] }>( + `/api/knowledge/documents${qs ? `?${qs}` : ""}`, + ); + return data.documents; +} + +export async function getDocument(docId: string): Promise { + return apiFetch(`/api/knowledge/documents/${docId}`); +} + +export async function deleteDocument(docId: string): Promise { + await apiFetch(`/api/knowledge/documents/${docId}`, { method: "DELETE" }); +} + export async function searchKnowledge( query: string, opts: { maxResults?: number; tomeId?: string } = {}, ): Promise<{ results: unknown[]; formatted: string }> { @@ -194,6 +241,175 @@ export async function searchKnowledge( }); } +// ── Chat sessions / history ── + +export interface ChatSessionSummary { + id: string; + tome_id: string | null; + tome_name: string | null; + provider: string; + model: string; + created_at: string; + last_message_at: string; + first_user_message: string | null; + message_count: number; +} + +export async function listChatSessions(limit = 100): Promise { + const data = await apiFetch<{ sessions: ChatSessionSummary[] }>( + `/api/chat/sessions?limit=${limit}`, + ); + return data.sessions; +} + +// ── Generation: flashcards & quizzes ── + +export interface GeneratedFlashcard { + id: string; + front: string; + back: string; +} + +export interface GeneratedSource { + document_id: string; + document_title: string; + chunk_index: number; + similarity: number | null; +} + +export interface FlashcardGeneration { + cards: GeneratedFlashcard[]; + topic: string; + sources: GeneratedSource[]; +} + +export interface GeneratedQuizOption { + id: string; + text: string; +} + +export interface GeneratedQuizQuestion { + id: string; + question: string; + options: GeneratedQuizOption[]; + correctOptionId: string; + explanation?: string; +} + +export interface QuizGeneration { + questions: GeneratedQuizQuestion[]; + topic: string; + sources: GeneratedSource[]; +} + +export interface GenerateOpts { + topic?: string; + tomeId?: string; + count?: number; + provider?: string; + model?: string; +} + +function generateBody(opts: GenerateOpts) { + return { + topic: opts.topic ?? null, + tome_id: opts.tomeId ?? null, + count: opts.count ?? 6, + provider: opts.provider ?? null, + model: opts.model ?? null, + }; +} + +export async function generateFlashcards(opts: GenerateOpts = {}): Promise { + return apiFetch("/api/generate/flashcards", { + method: "POST", + body: generateBody(opts), + }); +} + +export async function generateQuiz(opts: GenerateOpts = {}): Promise { + return apiFetch("/api/generate/quiz", { + method: "POST", + body: generateBody(opts), + }); +} + +// ── Generation: reports ── + +export interface ReportTocItem { + id: string; + title: string; +} + +export interface ReportGeneration { + title: string; + subtitle: string | null; + sourceDocs: string | null; + toc: ReportTocItem[]; + content: string; + sources: GeneratedSource[]; + topic: string; +} + +export async function generateReport(opts: GenerateOpts = {}): Promise { + return apiFetch("/api/generate/report", { + method: "POST", + body: generateBody(opts), + }); +} + +// ── Generation: audio narration ── + +export interface AudioSegment { + id: string; + text: string; + startTime: number; + endTime: number; +} + +export interface AudioGeneration { + id: string; + title: string; + voice: string; + provider: string; + model: string; + script: string; + segments: AudioSegment[]; + duration: number; + /** Backend-relative URL to an MP3 (OpenAI TTS). When null, the frontend + * falls back to browser SpeechSynthesis for playback. */ + audio_url: string | null; + sources: GeneratedSource[]; + topic: string; +} + +export interface GenerateAudioOpts { + topic?: string; + tomeId?: string; + voice?: string; + provider?: string; + model?: string; +} + +export async function generateAudio(opts: GenerateAudioOpts = {}): Promise { + return apiFetch("/api/generate/audio", { + method: "POST", + body: { + topic: opts.topic ?? null, + tome_id: opts.tomeId ?? null, + voice: opts.voice ?? null, + provider: opts.provider ?? null, + model: opts.model ?? null, + }, + }); +} + +/** Resolve a relative audio path to a fully-qualified URL using the active API base. */ +export function resolveAudioUrl(path: string): string { + if (/^https?:/i.test(path)) return path; + return `${getApiBaseUrl()}${path.startsWith("/") ? "" : "/"}${path}`; +} + // ── Chat ── export interface ChatResponse { From 4bfeb580e68d74c67f858cf0bd4ffc98733bf144 Mon Sep 17 00:00:00 2001 From: Spencer Wozniak Date: Mon, 11 May 2026 22:44:23 -0700 Subject: [PATCH 06/11] =?UTF-8?q?feat(frontend):=20knowledge-base=20UI=20?= =?UTF-8?q?=E2=80=94=20upload=20modal=20+=20document=20browser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two new components reachable from the round button next to the command-bar pill: - UploadModal: paste / drag-drop / file-picker ingest of text-like files (.txt, .md, .markdown, .rst, .csv, .json, .log). Backend dedupes by content hash so re-uploads are safe. - KnowledgeBaseWidget: two-pane list+detail browser over every ingested document. Lists titles with doc_type/source/timestamp, shows chunk count, content hash, linked tomes, and a 200-char preview. Per-row delete. An "Add document" button in the header opens UploadModal and refreshes the list on close. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/components/KnowledgeBaseWidget.tsx | 278 ++++++++++++++++++ frontend/src/components/UploadModal.tsx | 229 +++++++++++++++ 2 files changed, 507 insertions(+) create mode 100644 frontend/src/components/KnowledgeBaseWidget.tsx create mode 100644 frontend/src/components/UploadModal.tsx diff --git a/frontend/src/components/KnowledgeBaseWidget.tsx b/frontend/src/components/KnowledgeBaseWidget.tsx new file mode 100644 index 0000000..18e408c --- /dev/null +++ b/frontend/src/components/KnowledgeBaseWidget.tsx @@ -0,0 +1,278 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { + deleteDocument, + getDocument, + listDocuments, + type DocumentDetail, + type DocumentSummary, +} from "@/lib/sage-api"; +import UploadModal from "./UploadModal"; + +function formatDate(iso: string): string { + if (!iso) return ""; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return iso; + return d.toLocaleString([], { + year: "numeric", + month: "short", + day: "numeric", + hour: "numeric", + minute: "2-digit", + }); +} + +export default function KnowledgeBaseWidget() { + const [docs, setDocs] = useState([]); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [selectedId, setSelectedId] = useState(null); + const [detail, setDetail] = useState(null); + const [detailLoading, setDetailLoading] = useState(false); + const [pendingDelete, setPendingDelete] = useState(null); + const [uploadOpen, setUploadOpen] = useState(false); + + const refresh = useCallback(async () => { + setLoading(true); + setError(null); + try { + const data = await listDocuments({ limit: 200 }); + setDocs(data); + setSelectedId((prev) => (prev && data.some((d) => d.id === prev) ? prev : data[0]?.id ?? null)); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + void refresh(); + }, [refresh]); + + useEffect(() => { + if (!selectedId) { + setDetail(null); + return; + } + let cancelled = false; + setDetailLoading(true); + getDocument(selectedId) + .then((d) => { + if (!cancelled) setDetail(d); + }) + .catch((err) => { + if (!cancelled) setDetail(null); + console.error("Failed to load document detail:", err); + }) + .finally(() => { + if (!cancelled) setDetailLoading(false); + }); + return () => { + cancelled = true; + }; + }, [selectedId]); + + const handleDelete = async (docId: string) => { + setPendingDelete(docId); + try { + await deleteDocument(docId); + setDocs((prev) => prev.filter((d) => d.id !== docId)); + if (selectedId === docId) { + setSelectedId(null); + setDetail(null); + } + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setPendingDelete(null); + } + }; + + return ( +
+
+ {/* Header */} +
+
+ database +

Knowledge base

+ + {loading ? "…" : `${docs.length} document${docs.length === 1 ? "" : "s"}`} + +
+
+ + +
+
+ + { + setUploadOpen(false); + void refresh(); + }} + /> + + {error && ( +
+ {error} +
+ )} + + {/* Body — two-pane */} +
+ {/* List */} +
+ {loading && ( +
Loading…
+ )} + {!loading && docs.length === 0 && ( +
+ Nothing here yet. Click “Add document” above to upload one. +
+ )} + {!loading && + docs.map((doc) => { + const isActive = doc.id === selectedId; + return ( + + ); + })} +
+ + {/* Detail */} +
+ {!selectedId && !loading && ( +
+ Select a document to view details. +
+ )} + {selectedId && detailLoading && ( +
Loading details…
+ )} + {detail && !detailLoading && ( +
+
+
{detail.title}
+
+ {formatDate(detail.created_at)} +
+
+ +
+ + + + +
+ +
+
Tomes
+ {detail.tomes.length === 0 ? ( +
+ Not linked to any tome. +
+ ) : ( +
+ {detail.tomes.map((t) => ( + + {t.name} + + ))} +
+ )} +
+ +
+
Preview
+
+ {detail.content_preview || "(empty)"} +
+
+ +
+ +
+
+ )} +
+
+
+
+ ); +} + +function Stat({ label, value, mono = false }: { label: string; value: string; mono?: boolean }) { + return ( +
+
{label}
+
+ {value} +
+
+ ); +} diff --git a/frontend/src/components/UploadModal.tsx b/frontend/src/components/UploadModal.tsx new file mode 100644 index 0000000..54299eb --- /dev/null +++ b/frontend/src/components/UploadModal.tsx @@ -0,0 +1,229 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { ingestDocument } from "@/lib/sage-api"; + +interface UploadModalProps { + open: boolean; + onClose: () => void; + tomeId?: string; +} + +type Status = + | { kind: "idle" } + | { kind: "uploading" } + | { kind: "success"; title: string; chunks: number; deduplicated: boolean } + | { kind: "error"; message: string }; + +const TEXT_EXTENSIONS = [".txt", ".md", ".markdown", ".rst", ".csv", ".json", ".log"]; + +export default function UploadModal({ open, onClose, tomeId }: UploadModalProps) { + const [title, setTitle] = useState(""); + const [content, setContent] = useState(""); + const [filename, setFilename] = useState(""); + const [status, setStatus] = useState({ kind: "idle" }); + const [dragOver, setDragOver] = useState(false); + const fileInputRef = useRef(null); + + useEffect(() => { + if (!open) { + setTitle(""); + setContent(""); + setFilename(""); + setStatus({ kind: "idle" }); + setDragOver(false); + } + }, [open]); + + useEffect(() => { + if (!open) return; + const handleKey = (e: KeyboardEvent) => { + if (e.key === "Escape") onClose(); + }; + window.addEventListener("keydown", handleKey); + return () => window.removeEventListener("keydown", handleKey); + }, [open, onClose]); + + const readFile = async (file: File) => { + const name = file.name; + const ext = name.slice(name.lastIndexOf(".")).toLowerCase(); + if (!TEXT_EXTENSIONS.includes(ext) && !file.type.startsWith("text/")) { + setStatus({ + kind: "error", + message: `Unsupported file type "${ext || file.type}". Use plain text (.txt, .md, .csv, .json) or paste content directly.`, + }); + return; + } + const text = await file.text(); + setContent(text); + setFilename(name); + if (!title.trim()) setTitle(name.replace(/\.[^.]+$/, "")); + setStatus({ kind: "idle" }); + }; + + const handleFileChange = (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (file) void readFile(file); + e.target.value = ""; + }; + + const handleDrop = (e: React.DragEvent) => { + e.preventDefault(); + setDragOver(false); + const file = e.dataTransfer.files?.[0]; + if (file) void readFile(file); + }; + + const handleSubmit = async () => { + const finalTitle = title.trim() || filename || "Untitled"; + const finalContent = content.trim(); + if (!finalContent) { + setStatus({ kind: "error", message: "Add some content first." }); + return; + } + setStatus({ kind: "uploading" }); + try { + const res = await ingestDocument(finalTitle, finalContent, { + docType: filename ? "file" : "text", + sourceId: filename || "", + tomeId, + }); + setStatus({ + kind: "success", + title: res.title, + chunks: res.chunks, + deduplicated: res.deduplicated, + }); + setContent(""); + setTitle(""); + setFilename(""); + } catch (err) { + setStatus({ + kind: "error", + message: err instanceof Error ? err.message : String(err), + }); + } + }; + + if (!open) return null; + + return ( +
+
e.stopPropagation()} + > + {/* Header */} +
+
+ upload_file +

Add to knowledge base

+
+ +
+ + {/* Body */} +
+ setTitle(e.target.value)} + placeholder="Document title" + className="bg-surface-container-low border border-outline-variant/10 rounded-xl px-3 py-2 + text-body-md text-on-surface placeholder:text-on-surface-variant/50 + outline-none focus:border-primary/40 transition-colors" + /> + +
{ + e.preventDefault(); + setDragOver(true); + }} + onDragLeave={() => setDragOver(false)} + onDrop={handleDrop} + className={`rounded-xl border border-dashed transition-colors p-4 flex flex-col gap-2 ${ + dragOver + ? "border-primary/60 bg-primary/5" + : "border-outline-variant/25 bg-surface-container-low" + }`} + > +
+ + {filename + ? `Loaded: ${filename}` + : "Drop a text file here, or paste content below"} + + + +
+