diff --git a/apps/api/src/llm/router.ts b/apps/api/src/llm/router.ts index ba27eb9..7794bbb 100644 --- a/apps/api/src/llm/router.ts +++ b/apps/api/src/llm/router.ts @@ -38,6 +38,24 @@ export class LLMRouter { return (response.content[0] as { text: string }).text; } + async *stream( + messages: Array<{ role: 'user' | 'assistant'; content: string }>, + contextBlock: string + ): AsyncIterable { + const system = `${SHORT_RESPONSE_CONTRACT}\n\n## Prior context\n\n${contextBlock}`; + const stream = this.claude.messages.stream({ + model: 'claude-sonnet-4-6', + max_tokens: 1024, + system, + messages, + }); + for await (const chunk of stream) { + if (chunk.type === 'content_block_delta' && chunk.delta.type === 'text_delta') { + yield chunk.delta.text; + } + } + } + async summarise(content: string): Promise { const result = await this.flashModel.generateContent( `Write a 2-3 sentence session summary of this conversation. Be specific about decisions made and next steps.\n\n${content}` diff --git a/apps/api/src/routes/chat.test.ts b/apps/api/src/routes/chat.test.ts index 805e97c..de7315d 100644 --- a/apps/api/src/routes/chat.test.ts +++ b/apps/api/src/routes/chat.test.ts @@ -8,8 +8,13 @@ const mockCache = { all: vi.fn().mockReturnValue([]), } as unknown as IndexCache; +async function* mockStream() { + yield 'assistant '; + yield 'reply'; +} + const mockLlm = { - chat: vi.fn().mockResolvedValue('assistant reply'), + stream: vi.fn().mockImplementation(mockStream), summarise: vi.fn().mockResolvedValue('session summary'), generateFrontmatter: vi.fn().mockResolvedValue({ tags: ['a'], summary: 'a summary' }), } as unknown as LLMRouter; @@ -52,21 +57,30 @@ describe('POST /api/chat', () => { expect(res.status).toBe(400); }); - it('returns reply and context on valid request', async () => { + it('streams reply tokens and context event', async () => { const res = await authedPost(app, sessionCookie, '/api/chat', { messages: validMessages }); expect(res.status).toBe(200); - const data = (await res.json()) as { reply: string; context: unknown[] }; - expect(data.reply).toBe('assistant reply'); - expect(Array.isArray(data.context)).toBe(true); + expect(res.headers.get('content-type')).toContain('text/event-stream'); + const text = await res.text(); + expect(text).toContain('event: context'); + expect(text).toContain('data: assistant '); + expect(text).toContain('data: reply'); + expect(text).toContain('data: [DONE]'); }); - it('propagates LLM error as 500 via global error handler', async () => { - vi.mocked(mockLlm.chat).mockRejectedValueOnce(new Error('LLM timeout')); + it('closes stream gracefully on LLM error', async () => { + const failingIterable: AsyncIterable = { + [Symbol.asyncIterator]: () => ({ + next: () => Promise.reject(new Error('LLM timeout')), + return: () => Promise.resolve({ value: undefined, done: true as const }), + }), + }; + vi.mocked(mockLlm.stream).mockReturnValueOnce(failingIterable); const res = await authedPost(app, sessionCookie, '/api/chat', { messages: validMessages }); - expect(res.status).toBe(500); - const data = (await res.json()) as { error: string }; - expect(data.error).toBe('Internal server error'); - expect(data.error).not.toContain('LLM timeout'); + // SSE always returns 200 once headers are sent; stream closes on error without [DONE] + expect(res.status).toBe(200); + const text = await res.text(); + expect(text).not.toContain('data: [DONE]'); }); }); diff --git a/apps/api/src/routes/chat.ts b/apps/api/src/routes/chat.ts index e5bb363..b0cc549 100644 --- a/apps/api/src/routes/chat.ts +++ b/apps/api/src/routes/chat.ts @@ -34,8 +34,20 @@ export function chatRoutes(llm: LLMRouter, cache: IndexCache) { const { messages, query, relatedToId } = c.req.valid('json'); const contextEntries = assembleContext(cache.all(), query, relatedToId); const contextBlock = formatContextBlock(contextEntries); - const reply = await llm.chat(messages, contextBlock); - return c.json({ reply, context: contextEntries.map((e) => ({ id: e.id, title: e.title })) }); + const { streamSSE } = await import('hono/streaming'); + return streamSSE(c, async (stream) => { + // First event: context metadata + await stream.writeSSE({ + event: 'context', + data: JSON.stringify(contextEntries.map((e) => ({ id: e.id, title: e.title }))), + }); + // Stream tokens + for await (const chunk of llm.stream(messages, contextBlock)) { + await stream.writeSSE({ data: chunk }); + } + // Done sentinel + await stream.writeSSE({ data: '[DONE]' }); + }); }); app.post('/api/chat/summarise', requireAuth(), zValidator('json', summariseSchema), async (c) => { diff --git a/apps/api/src/routes/content.ts b/apps/api/src/routes/content.ts index 11baba6..fb418b5 100644 --- a/apps/api/src/routes/content.ts +++ b/apps/api/src/routes/content.ts @@ -7,7 +7,8 @@ import type { IndexCache } from '../store/index-cache.js'; import { requireAuth } from './auth.js'; const patchSchema = z.object({ - session_summary: z.string().min(1).max(2000), + session_summary: z.string().min(1).max(2000).optional(), + body: z.string().max(50000).optional(), }); export function contentRoutes(cache: IndexCache, github: GitHubClient) { @@ -63,14 +64,17 @@ export function contentRoutes(cache: IndexCache, github: GitHubClient) { const id = c.req.param('id'); const entry = cache.findById(id); if (!entry) return c.json({ error: 'not found' }, 404); - const { session_summary } = c.req.valid('json'); + const input = c.req.valid('json'); const { content, encoding, sha } = await github.getFile(entry.path, entry.branch ?? 'main'); const decoded = encoding === 'base64' ? Buffer.from(content, 'base64').toString('utf-8') : content; - const { data, content: body } = matter(decoded); - const updated = matter.stringify(body, { ...data, session_summary }); + const { data, content: existingBody } = matter(decoded); + const updated = matter.stringify(input.body ?? existingBody, { + ...data, + ...(input.session_summary ? { session_summary: input.session_summary } : {}), + }); await github.upsertFile( entry.path, diff --git a/apps/web/src/lib/api.ts b/apps/web/src/lib/api.ts index 67236f7..a1092b7 100644 --- a/apps/web/src/lib/api.ts +++ b/apps/web/src/lib/api.ts @@ -58,7 +58,7 @@ export const api = { headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input), }), - patch: (id: string, body: { session_summary: string }) => + patch: (id: string, body: { session_summary?: string; body?: string }) => req<{ ok: boolean }>(`/api/content/${id}`, { method: 'PATCH', headers: { 'Content-Type': 'application/json' }, @@ -68,7 +68,7 @@ export const api = { park: (id: string) => req(`/api/content/${id}/park`, { method: 'POST' }), dismiss: (id: string) => req(`/api/content/${id}/dismiss`, { method: 'POST' }), discardPR: (pr: number) => req(`/api/inflight/${pr}/discard`, { method: 'POST' }), - promote: (id: string, title: string, summary: string) => + promote: (id: string, title: string, summary: string, sessionSummary?: string) => req<{ id: string; pr: number; branch: string }>('/api/capture', { method: 'POST', headers: { 'Content-Type': 'application/json' }, @@ -78,7 +78,17 @@ export const api = { tags: [], summary, promoted_from: [id], - body: `## Goal\n${summary || title}\n\n## Steps\n- (to be refined)\n\n## Success criteria\n- (to be refined)`, + body: [ + '## Goal', + summary || title, + '', + ...(sessionSummary ? ['## Context from idea session', sessionSummary, ''] : []), + '## Steps', + '- (to be refined)', + '', + '## Success criteria', + '- (to be refined)', + ].join('\n'), }), }), chat: (messages: unknown[], query?: string, relatedToId?: string) => diff --git a/apps/web/src/routes/chat/[id]/+page.svelte b/apps/web/src/routes/chat/[id]/+page.svelte index 2c19c01..d569dd0 100644 --- a/apps/web/src/routes/chat/[id]/+page.svelte +++ b/apps/web/src/routes/chat/[id]/+page.svelte @@ -14,6 +14,8 @@ let parking = false; let promoting = false; let contextTitles: string[] = []; let error = ''; +let saving = false; +let savedAt: Date | null = null; const idParam = $page.params.id; if (!idParam) throw new Error('Missing route param: id'); @@ -28,6 +30,19 @@ onMount(async () => { } }); +async function save() { + if (messages.length === 0) return; + saving = true; + try { + const conversation = messages.map((m) => `${m.role}: ${m.content}`).join('\n'); + const { summary } = await api.summarise(conversation); + await api.patch(id, { session_summary: summary }); + savedAt = new Date(); + } finally { + saving = false; + } +} + async function send() { if (!input.trim()) return; const userMsg = { role: 'user' as const, content: input }; @@ -35,12 +50,51 @@ async function send() { input = ''; loading = true; error = ''; + const BASE = import.meta.env.PUBLIC_API_URL ?? ''; + const assistantMsg = { role: 'assistant' as const, content: '' }; + messages = [...messages, assistantMsg]; try { - const res = await api.chat(messages, item?.title, id); - messages = [...messages, { role: 'assistant', content: res.reply }]; - contextTitles = res.context.map((c) => c.title); + const res = await fetch(`${BASE}/api/chat`, { + method: 'POST', + credentials: 'include', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + messages: messages.slice(0, -1), + query: item?.title, + relatedToId: id, + }), + }); + if (!res.ok) throw new Error(`API error: ${res.status}`); + if (!res.body) throw new Error('No response body'); + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const parts = buffer.split('\n\n'); + buffer = parts.pop() ?? ''; + for (const part of parts) { + const eventLine = part.split('\n').find((l) => l.startsWith('event:')); + const dataLine = part.split('\n').find((l) => l.startsWith('data:')); + if (!dataLine) continue; + const data = dataLine.slice(5).trim(); + if (eventLine?.includes('context')) { + try { + contextTitles = JSON.parse(data).map((e: { title: string }) => e.title); + } catch { + /* ignore */ + } + } else if (data !== '[DONE]') { + assistantMsg.content += data; + messages = [...messages]; + } + } + } } catch { error = 'Failed to send message. Please try again.'; + messages = messages.slice(0, -1); // remove empty assistant message } finally { loading = false; } @@ -50,6 +104,7 @@ async function park() { parking = true; error = ''; try { + await save(); await api.park(id); goto('/'); } catch { @@ -63,7 +118,7 @@ async function promote() { promoting = true; error = ''; try { - const { id: planId } = await api.promote(id, item.title, item.summary); + const { id: planId } = await api.promote(id, item.title, item.summary, item.session_summary); goto(`/chat/${planId}`); } catch { error = 'Failed to promote. Please try again.'; @@ -97,6 +152,7 @@ async function finish() { {/if} + {#if item.pr} {/if} @@ -115,11 +171,8 @@ async function finish() { align-self:{msg.role==='user'?'flex-end':'flex-start'}; max-width:85%;background:{msg.role==='user'?'#1a3a5c':'#1a1a1a'}; padding:10px 14px;border-radius:12px;font-size:14px;white-space:pre-wrap - ">{msg.content} + ">{msg.content || (loading && msg.role === 'assistant' ? '…' : msg.content)} {/each} - {#if loading} -
...
- {/if} {#if error}

{error}

{/if}