Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
96 changes: 91 additions & 5 deletions kits/sheets/app/src/components/ChatPanel.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@
import { useEffect, useRef, useState } from 'react';
import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
import { ArrowUp, PanelRight, Sparkles } from 'lucide-react';
import { ArrowUp, Mic, PanelRight, Paperclip, Sparkles, X } from 'lucide-react';
import { ChatMessages, ChatMessagesSkeleton, Composer, withReasoning, withResult, withStep, withText } from 'reifyui';
import { sessionTurns, turnsToMessages } from 'reifyui/harness';
import { isPending } from '../lib/sh';
import { runCopilotTurn } from '../lib/copilot';
import { bytesLabel, fileToInput } from '../lib/attach';
import { createDictation } from '../lib/dictate';

const RETRY_MS = 20000;

Expand Down Expand Up @@ -40,9 +42,44 @@ export function ChatColumn({ sheetId, seed, title, agentBusy, onSheetMaybeChange
const [busy, setBusy] = useState(false);
const [connecting, setConnecting] = useState(false);
const [histLoading, setHistLoading] = useState(true);
const [staged, setStaged] = useState([]); // files picked, not yet sent
const [attachErr, setAttachErr] = useState('');
const [listening, setListening] = useState(false);
const pendingRef = useRef(null);
const bodyRef = useRef(null);
const seededRef = useRef(false);
const fileRef = useRef(null);
// Voice input is the browser's own recogniser, and it is null where the browser has none — the
// button is then not rendered at all, rather than rendered disabled. Built once: createDictation
// returns null on an unsupported build, so a ref-with-null-sentinel would rebuild every render.
const [dictation] = useState(() => createDictation());

useEffect(() => () => dictation?.stop(), [dictation]);

async function pickFiles(list) {
setAttachErr('');
const next = [];
for (const f of list) {
try {
// eslint-disable-next-line no-await-in-loop
next.push(await fileToInput(f));
} catch (e) {
setAttachErr(e.message);
}
}
if (next.length) setStaged((cur) => [...cur, ...next]);
}

function toggleDictation() {
if (!dictation) return;
if (listening) { dictation.stop(); return; }
setListening(true);
dictation.start({
onText: (text) => setDraft((cur) => (cur ? `${cur.replace(/\s+$/, '')} ${text}` : text)),
onEnd: () => setListening(false),
onError: () => setListening(false),
});
}

useEffect(() => {
bodyRef.current?.scrollTo({ top: bodyRef.current.scrollHeight });
Expand All @@ -63,7 +100,7 @@ export function ChatColumn({ sheetId, seed, title, agentBusy, onSheetMaybeChange
? m.slice(0, -1) : m));
}

async function deliver(text) {
async function deliver(text, files = []) {
setBusy(true);
setMessages((m) => [...m, { role: 'assistant', blocks: [], status: 'running' }]);
try {
Expand All @@ -80,7 +117,7 @@ export function ChatColumn({ sheetId, seed, title, agentBusy, onSheetMaybeChange
blocks: a.blocks.length ? a.blocks : withText([], 'Something went wrong. Please try again.'),
status: 'failed',
})),
});
}, files.map((f) => f.block));
if (out.connecting) {
dropRunningAsst();
pendingRef.current = text;
Expand All @@ -103,9 +140,16 @@ export function ChatColumn({ sheetId, seed, title, agentBusy, onSheetMaybeChange
function send(text) {
const t = text.trim();
if (!t || busy || histLoading) return;
setMessages((m) => [...m, { role: 'user', text: t }]);
if (listening) { dictation?.stop(); setListening(false); }
const files = staged;
// The names ride on the message so the bubble shows what was sent with it. They are not
// replayed from history — the transcript stores the person's text, not the files — so this
// is what was attached in this session, not a claim about every past turn.
setMessages((m) => [...m, { role: 'user', text: t, files: files.map((f) => f.name) }]);
setDraft('');
deliver(t);
setStaged([]);
setAttachErr('');
deliver(t, files);
}

// Mount: strip ?seed FIRST, then replay history; the seed fires only when there is none.
Expand Down Expand Up @@ -213,6 +257,13 @@ export function ChatColumn({ sheetId, seed, title, agentBusy, onSheetMaybeChange
) : (
<ChatMessages
messages={messages}
userExtras={(m) => (m.files?.length ? (
<span className="msg-files">
{m.files.map((n) => (
<span key={n} className="msg-file"><Paperclip size={11} />{n}</span>
))}
</span>
) : null)}
renderMarkdown={(t) => <ReactMarkdown remarkPlugins={[remarkGfm]}>{t}</ReactMarkdown>}
workingLabel="Working..."
toolLabels={{ workingLabel: 'Working...' }}
Expand All @@ -235,6 +286,41 @@ export function ChatColumn({ sheetId, seed, title, agentBusy, onSheetMaybeChange
rows={2}
autoGrow={false}
classNames={{ root: 'cmp', input: '', row: 'cmp-row' }}
attachments={(staged.length > 0 || attachErr) && (
<div className="cmp-files">
{staged.map((f, i) => (
<span key={`${f.name}-${i}`} className="cmp-file">
<Paperclip size={11} />
<span className="cmp-file-n" title={f.name}>{f.name}</span>
<span className="cmp-file-s">{bytesLabel(f.size)}</span>
<button type="button" aria-label={`Remove ${f.name}`}
onClick={() => setStaged((c) => c.filter((_, j) => j !== i))}>
<X size={11} />
</button>
</span>
))}
{attachErr && <span className="cmp-file-err">{attachErr}</span>}
</div>
)}
accessoriesLeft={(
<>
<input ref={fileRef} type="file" hidden multiple
onChange={(e) => { pickFiles([...e.target.files]); e.target.value = ''; }} />
<button type="button" className="cmp-icon" aria-label="Attach a file"
title="Attach a file — the agent gets it in its working directory"
disabled={busy} onClick={() => fileRef.current?.click()}>
<Paperclip size={16} />
</button>
</>
)}
accessoriesRight={dictation ? (
<button type="button" className={'cmp-icon' + (listening ? ' on' : '')}
aria-label={listening ? 'Stop dictating' : 'Dictate'} aria-pressed={listening}
title={listening ? 'Stop dictating' : 'Dictate'}
disabled={busy} onClick={toggleDictation}>
<Mic size={15} />
</button>
) : null}
renderSend={() => (
<button type="button" className="cmp-send" onClick={() => send(draft)}
disabled={busy || histLoading || !draft.trim()} aria-label="Send message">
Expand Down
58 changes: 58 additions & 0 deletions kits/sheets/app/src/lib/attach.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Files, on their way into a turn.
//
// One mechanism for both places a file gets attached — the person's own attachment in the
// copilot, and an upstream agent column's artifact handed to the next column. Both become an
// `input_file` block on the turn, which the gateway writes into the working directory before the
// agent starts.
//
// Deliberately NOT a workspace PUT: that needs a session which a brand-new sheet does not have
// yet, it is refused 409 while a turn is running, and it cannot carry bytes that are not text.

/** Per-file cap the API enforces. Refuse a bigger file by name rather than truncating it. */
export const FILE_MAX = 25 * 1024 * 1024;

const MIME = {
md: 'text/markdown', txt: 'text/plain', json: 'application/json', csv: 'text/csv',
tsv: 'text/tab-separated-values', html: 'text/html', py: 'text/x-python', js: 'text/javascript',
png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', svg: 'image/svg+xml',
pdf: 'application/pdf', docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
};

export const mimeOf = (name, fallback = 'application/octet-stream') =>
MIME[String(name).split('.').pop().toLowerCase()] || fallback;

/** ArrayBuffer -> data: URL, chunked because a single fromCharCode over a large file overflows
* the argument stack. */
export function bufferToDataUrl(buf, filename, type) {
let bin = '';
const bytes = new Uint8Array(buf);
for (let i = 0; i < bytes.length; i += 0x8000) {
bin += String.fromCharCode.apply(null, bytes.subarray(i, i + 0x8000));
}
return `data:${type || mimeOf(filename)};base64,${btoa(bin)}`;
}

/** A picked File -> the block a turn takes. Throws with the file's own name if it is too large. */
export async function fileToInput(file) {
if (file.size > FILE_MAX) {
throw new Error(`${file.name} is ${Math.round(file.size / 1048576)} MB — the limit is 25 MB.`);
}
const buf = await file.arrayBuffer();
return {
name: file.name,
size: file.size,
block: {
type: 'input_file',
filename: file.name,
file_data: bufferToDataUrl(buf, file.name, file.type),
},
};
}

export function bytesLabel(n) {
if (n == null) return '';
if (n < 1024) return `${n} B`;
if (n < 1048576) return `${Math.round(n / 1024)} KB`;
return `${(n / 1048576).toFixed(1)} MB`;
}
21 changes: 4 additions & 17 deletions kits/sheets/app/src/lib/cell.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
// for the same job. Live text, where it matters, comes from the turn replay in the cell drawer.
import { cancelResponse, containerFileUrl, createResponse, getResponse, patchSession } from 'reifyui/harness';
import { VALUE_MAX, interpolate } from './model.js';
import { FILE_MAX, bufferToDataUrl } from './attach.js';

const POLL_MS = 2000;
/** How long a terminal-but-empty response is given to produce its own content.
Expand All @@ -22,32 +23,18 @@ const POLL_MS = 2000;
* honest rule is to wait for it, bounded. Costs nothing on the normal path — the loop exits the
* instant content appears. */
const SETTLE_MS = 15000;
/** Refuse an attachment larger than the API accepts rather than truncating a file silently. */
const FILE_MAX = 25 * 1024 * 1024;

const now = () => Math.floor(Date.now() / 1000);

const MIME = {
md: 'text/markdown', txt: 'text/plain', json: 'application/json', csv: 'text/csv',
tsv: 'text/tab-separated-values', html: 'text/html', py: 'text/x-python', js: 'text/javascript',
png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', svg: 'image/svg+xml',
pdf: 'application/pdf',
};
const mimeOf = (name) => MIME[String(name).split('.').pop().toLowerCase()] || 'application/octet-stream';

async function fileAsDataUrl(artifact) {
async function artifactAsDataUrl(artifact) {
const res = await fetch(containerFileUrl(artifact.container_id, artifact.file_id), { cache: 'no-store' });
if (!res.ok) throw new Error(`Could not read ${artifact.filename}.`);
const buf = await res.arrayBuffer();
if (buf.byteLength > FILE_MAX) {
throw new Error(`${artifact.filename} is too large to attach (${Math.round(buf.byteLength / 1048576)} MB).`);
}
let bin = '';
const bytes = new Uint8Array(buf);
for (let i = 0; i < bytes.length; i += 0x8000) {
bin += String.fromCharCode.apply(null, bytes.subarray(i, i + 0x8000));
}
return `data:${mimeOf(artifact.filename)};base64,${btoa(bin)}`;
return bufferToDataUrl(buf, artifact.filename);
}

/** A directory per source column, so an agent reading three attachments can tell them apart. */
Expand Down Expand Up @@ -111,7 +98,7 @@ export function makeCellDispatcher({ sheetId, runId, sheetTitle, columns, onCell
for (const u of upstream || []) {
for (const a of u.cell?.artifacts || []) {
// eslint-disable-next-line no-await-in-loop
const file_data = await fileAsDataUrl(a);
const file_data = await artifactAsDataUrl(a);
content.push({ type: 'input_file', filename: `in/${slug(u.column.name)}/${a.filename}`, file_data });
}
}
Expand Down
8 changes: 6 additions & 2 deletions kits/sheets/app/src/lib/copilot.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { materialize } from './model';
* user text for the transcript, so the chat keeps showing the sentence they typed instead of a
* wall of starting JSON. Scaffolding should be invisible.
*/
export async function runCopilotTurn(sheetId, message, handlers) {
export async function runCopilotTurn(sheetId, message, handlers, attachments = []) {
const pending = isPending(sheetId);
let instructions = '';
if (pending) {
Expand All @@ -36,9 +36,13 @@ export async function runCopilotTurn(sheetId, message, handlers) {
}
}

const input = attachments.length
? [{ role: 'user', content: [...attachments, { type: 'input_text', text: message }] }]
: message;

return harnessStreamTurn({
sessionId: pending ? '' : sheetId,
input: message,
input,
instructions,
handlers,
});
Expand Down
56 changes: 56 additions & 0 deletions kits/sheets/app/src/lib/dictate.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Voice input, using the recogniser the browser already has.
//
// There is no speech service in this deployment and no reason to add one: Chrome, Edge and Safari
// ship SpeechRecognition, and it is exactly the feature — speak, get text in the box. Where the
// API is absent, createDictation() returns null and the caller renders no button at all. A
// disabled microphone labelled "coming soon" is a control that does nothing, which is worse than
// its absence: it says the product has a capability it does not have.

const Impl = typeof window !== 'undefined'
? (window.SpeechRecognition || window.webkitSpeechRecognition)
: null;

export function createDictation({ lang } = {}) {
if (!Impl) return null;
let rec = null;

return {
/** onText fires once per finished phrase, so the draft grows as you speak rather than being
* rewritten from scratch on every interim guess. */
start({ onText, onEnd, onError }) {
this.stop();
rec = new Impl();
rec.lang = lang || navigator.language || 'en-US';
rec.continuous = true;
rec.interimResults = false;
rec.onresult = (e) => {
for (let i = e.resultIndex; i < e.results.length; i += 1) {
if (!e.results[i].isFinal) continue;
const text = String(e.results[i][0]?.transcript || '').trim();
if (text) onText?.(text);
}
};
// 'no-speech' and 'aborted' are the two ways a quiet or cancelled session ends. Neither is
// a failure worth telling anyone about; both just stop.
rec.onerror = (e) => {
rec = null;
if (e.error === 'no-speech' || e.error === 'aborted') onEnd?.();
else onError?.(e.error || 'Dictation stopped.');
};
rec.onend = () => { rec = null; onEnd?.(); };
try {
rec.start();
} catch (e) {
rec = null;
onError?.(e?.message || 'Dictation could not start.');
}
},

stop() {
if (!rec) return;
const r = rec;
rec = null;
try { r.stop(); } catch { /* already stopped */ }
},
};
}
33 changes: 33 additions & 0 deletions kits/sheets/app/src/styles/app.css
Original file line number Diff line number Diff line change
Expand Up @@ -735,3 +735,36 @@
.sl-save-chip { display: none; }
.tpl-card { flex-basis: 78vw; }
}

/* ── Composer: attachments and dictation ───────────────────────────────── */
/* Listening is a state the person has to be able to see, because they cannot hear it. */
.cmp-icon.on {
color: var(--brand); border-color: var(--brand); background: var(--brand-50);
animation: cmp-mic 1.4s ease-in-out infinite;
}
@keyframes cmp-mic { 50% { box-shadow: 0 0 0 4px rgba(5, 150, 105, .14); } }
@media (prefers-reduced-motion: reduce) { .cmp-icon.on { animation: none; } }

.cmp-files { display: flex; flex-wrap: wrap; gap: 6px; padding: 0 2px 8px; }
.cmp-file {
display: inline-flex; align-items: center; gap: 5px; max-width: 100%;
padding: 3px 4px 3px 8px; border-radius: 999px; font-size: 11.5px;
background: var(--brand-50); color: var(--brand-ink); border: 1px solid var(--brand-soft);
}
.cmp-file-n { min-width: 0; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; max-width: 170px; }
.cmp-file-s { color: var(--mute); font-variant-numeric: tabular-nums; }
.cmp-file button {
display: inline-flex; border: none; background: transparent; cursor: pointer;
color: var(--mute); padding: 2px; border-radius: 999px; line-height: 0;
}
.cmp-file button:hover { background: rgba(0, 0, 0, .06); color: var(--ink); }
.cmp-file-err { font-size: 11.5px; color: #B91C1C; align-self: center; }

/* The names of the files sent with a message, inside its own bubble. */
.msg-files { display: flex; flex-wrap: wrap; gap: 5px; margin-bottom: 6px; }
.msg-file {
display: inline-flex; align-items: center; gap: 4px; max-width: 100%;
padding: 2px 7px; border-radius: 999px; font-size: 11px;
background: var(--surface); border: 1px solid var(--line); color: var(--ink-2);
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
}
Loading
Loading