diff --git a/AGENTS.md b/AGENTS.md index 3abba97..aa19fd7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,6 +55,7 @@ OpenDex is an **Electron** desktop app (electron-vite + React + Tailwind v4). It ## Skills + permission gate (Phase 5a) - **Skills are self-contained folders** under `src/skills//` (clock, weather, web-search, open, computer), each spanning the process boundary: `meta.ts` (dependency-free id/label/sensitivity + tool-name constants, shared), `skill.ts` (the executable `Skill` — zod/node/ai, MAIN only), `view.tsx` (optional renderer UI: tool labels + result cards, auto-registered). The two halves link only by the tool `name` string declared in `meta.ts`. A skill bundles tools `{ name, description, inputSchema (zod), execute }`; `sensitive: true` routes every call through the permission gate. `src/skills/registry.ts` `buildToolSet({config, requestPermission})` assembles enabled skills' tools (explicit `BUILTIN_SKILLS` list — kept explicit so `pnpm smoke:chat` works under tsx), wrapping sensitive tools' `execute` to `await requestPermission(...)` first. The renderer side (`@skills` alias): `metas.ts` + `tool-views.ts` auto-discover `meta.ts`/`view.tsx` via `import.meta.glob`. **To add a skill, drop a folder — see CONTRIBUTING.md.** +- **Jev fast path (open/computer routing):** before the LLM loop, `tryJevDesktopFastPath` (`src/main/agent/jev/`) calls TypeSafe Jev via the Gateway `/v4/ai/evaluation-model` HTTP route (keeps us on AI SDK 6). Clear `open_app`/`open_url`/`open_path` intents with a parseable target skip the language model and run the Open skill directly (still permission-gated). High-confidence but unparseable open intents constrain the tool set to Open; `computer` intents add a system-prompt bias. Gated by `config.skills.jevFastPath` (default on) and `AI_GATEWAY_API_KEY`. Smoke: `pnpm smoke:jev` / `pnpm smoke:jev --live`. - **Permission gate** (`agent/permissions.ts`): per-request `makePermissionRequester(sender)` checks the standing decision in `config.skills.permissions[id]` (`always`→run, `never`→deny, else prompt via the injected `PermissionUi` → dedicated popup window → `permission:respond`). `allow_once|always|deny|never`; `always`/`never` persist via `updateConfig`. The `chatStart` handler builds tools per-turn with a sender-bound requester. (The prompt UI now lives in its own always-on-top popup window, not the main window — see "Window model".) - Config: `skills: { enabled, permissions }`. A skill may set `optIn: true` (off until the user enables it — see computer-use); `isSkillEnabled(skill, config)` is the single source of truth (`optIn` → on only when `enabled[id] === true`, otherwise on unless `=== false`). Renderer: `components/permission-prompt.tsx` overlay + `lib/use-permission.ts` (queues prompts), rendered as global chrome in `App.tsx`; Settings "Skills & tools" section (enable toggle + Ask/Always/Never) driven by `@skills/metas` (`SKILL_METAS`, auto-discovered from each skill's `meta.ts` — dependency-free, so the renderer never pulls the electron-backed `skill.ts`). - **Session grant:** `makePermissionRequester(sender)` holds a per-requester `sessionAllow` set, so an "Allow once" approval covers the rest of *that command's* multi-step tool loop (a fresh requester is built per `chatStart`, so it never silently persists across commands). Needed so computer-use doesn't re-prompt on every click. diff --git a/package.json b/package.json index 0116119..5a379a2 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "typecheck": "tsc --noEmit", "smoke:chat": "tsx scripts/smoke-chat.ts", "smoke:realtime": "tsx scripts/smoke-realtime.ts", + "smoke:jev": "tsx scripts/smoke-jev.ts", "dist": "electron-vite build && electron-builder", "release": "electron-vite build && electron-builder --publish always", "cut:patch": "pnpm version patch -m \"Release v%s\" && git push --follow-tags", diff --git a/scripts/smoke-jev.ts b/scripts/smoke-jev.ts new file mode 100644 index 0000000..1ab8834 --- /dev/null +++ b/scripts/smoke-jev.ts @@ -0,0 +1,73 @@ +// Smoke-test the Jev desktop router without Electron. +// Usage: +// pnpm smoke:jev # local extractors only +// pnpm smoke:jev --live # also hit Gateway evaluate (needs AI_GATEWAY_API_KEY) +import { config as loadEnv } from "dotenv"; +import { + extractOpenTarget, + routeDesktopIntent, +} from "../src/main/agent/jev/desktop-route"; + +loadEnv(); + +const cases: Array<{ + text: string; + route: "open_app" | "open_url" | "open_path"; + expect: string | null; +}> = [ + { text: "open Safari", route: "open_app", expect: "Safari" }, + { text: "please launch Slack", route: "open_app", expect: "Slack" }, + { text: "can you open the Notes app", route: "open_app", expect: "Notes" }, + { text: "open Google Chrome", route: "open_app", expect: "Google Chrome" }, + { text: "open https://example.com/foo", route: "open_url", expect: "https://example.com/foo" }, + { text: "go to github.com", route: "open_url", expect: "https://github.com" }, + { text: "open ~/Documents", route: "open_path", expect: "~/Documents" }, + // Conjunctions / UI instructions → reject (fall through to LLM) + { text: "open Chrome and click the first tab", route: "open_app", expect: null }, + { text: "open Safari and then go to GitHub", route: "open_app", expect: null }, +]; + +async function main() { + let failed = 0; + console.log("[smoke:jev] extractor checks"); + for (const c of cases) { + const got = extractOpenTarget(c.text, c.route); + const value = + got?.kind === "app" ? got.name : got?.kind === "url" ? got.url : got?.kind === "path" ? got.path : null; + const ok = value === c.expect; + console.log(` ${ok ? "ok" : "FAIL"} "${c.text}" → ${JSON.stringify(value)}`); + if (!ok) failed++; + } + + if (process.argv.includes("--live")) { + if (!process.env.AI_GATEWAY_API_KEY) { + console.error("[smoke:jev] --live needs AI_GATEWAY_API_KEY"); + process.exit(1); + } + const samples = [ + "open Safari", + "click the blue Submit button", + "what's the weather in SF", + "go to github.com", + ]; + console.log("[smoke:jev] live Jev routes"); + for (const text of samples) { + const t0 = Date.now(); + const routed = await routeDesktopIntent(text); + console.log( + ` "${text}" → ${routed ? `${routed.route} p=${routed.probability.toFixed(2)} (${routed.latencyMs}ms)` : "null"} (wall ${Date.now() - t0}ms)`, + ); + } + } + + if (failed) { + console.error(`[smoke:jev] FAIL: ${failed} extractor case(s)`); + process.exit(1); + } + console.log("[smoke:jev] ok"); +} + +main().catch((err) => { + console.error("[smoke:jev] error", err); + process.exit(1); +}); diff --git a/src/main/agent/jev/desktop-route.ts b/src/main/agent/jev/desktop-route.ts new file mode 100644 index 0000000..c843cef --- /dev/null +++ b/src/main/agent/jev/desktop-route.ts @@ -0,0 +1,151 @@ +/** + * Turn-start desktop routing via Jev. + * + * Opening apps through the full LLM+tool loop is the slow path users hit most + * often ("open Safari", "launch Slack"). Jev classifies the utterance into a + * typed route in one cheap parallel evaluate call; clear open_* routes can skip + * the language model entirely. + * + * Computer-use still needs a vision LLM for clicks/typing — Jev only decides + * whether that's the right bucket (and biases the system prompt when it is). + */ + +import { evaluateWithJev } from "./evaluate"; + +export const DESKTOP_ROUTES = [ + "open_app", + "open_url", + "open_path", + "computer", + "other", +] as const; + +export type DesktopRoute = (typeof DESKTOP_ROUTES)[number]; + +export interface DesktopRouteResult { + route: DesktopRoute; + /** Probability of the selected choice (0..1), when the provider returned a distribution. */ + probability: number; + latencyMs: number; +} + +const ROUTE_CRITERIA: Record = { + open_app: + "Launch an installed desktop application by name (Safari, Chrome, Slack, Notes, Spotify, Terminal, Finder, …). No clicking UI elements — just start the app.", + open_url: + "Open a website or http(s)/mailto URL in the default browser. Includes 'go to example.com' / 'open github.com'.", + open_path: + "Open a file or folder on disk by path (e.g. ~/Documents, /tmp/report.pdf).", + computer: + "Control the already-visible desktop with mouse/keyboard: click buttons, type into fields, drag, scroll, take screenshots, or multi-step UI work inside apps. Not a simple 'open/launch X' request.", + other: + "Questions, search, weather, time, conversation, or anything that is not opening something or controlling the screen.", +}; + +/** Minimum selected-option probability before we trust the route. */ +export const ROUTE_CONFIDENCE = 0.8; + +export async function routeDesktopIntent( + utterance: string, + signal?: AbortSignal, +): Promise { + const text = utterance.trim(); + if (!text || !process.env.AI_GATEWAY_API_KEY) return null; + + const started = Date.now(); + try { + const result = await evaluateWithJev({ + signal, + state: { utterance: text }, + questions: { + route: { + type: "choice", + instructions: + "You are routing a single voice command for a desktop voice assistant. Pick the single best handler.", + criteria: ROUTE_CRITERIA, + }, + }, + }); + + const answer = result.answers.route; + if (answer.type !== "choice") return null; + const route = answer.choice as DesktopRoute; + if (!DESKTOP_ROUTES.includes(route)) return null; + // Native Jev returns a full distribution; if a provider omits it, trust the + // chosen option rather than treating missing mass as p=0 (which would skip + // every confident route under ROUTE_CONFIDENCE). + const probability = answer.probabilities?.[answer.choice] ?? 1; + return { route, probability, latencyMs: Date.now() - started }; + } catch (err) { + console.warn("[opendex jev] route failed, falling back to LLM", err); + return null; + } +} + +export type OpenTarget = + | { kind: "app"; name: string } + | { kind: "url"; url: string } + | { kind: "path"; path: string }; + +/** Pull an open target out of a short utterance once Jev has classified the route. + * Jev itself has no free-form string output — extraction is local. Returns null + * when the target is too ambiguous to trust (caller falls back to the LLM). */ +export function extractOpenTarget( + utterance: string, + route: Extract, +): OpenTarget | null { + const text = utterance.trim(); + if (!text) return null; + + if (route === "open_url") { + const mailto = text.match(/\b(mailto:[^\s]+)/i)?.[1]; + if (mailto) return { kind: "url", url: mailto }; + + const explicit = text.match(/\b(https?:\/\/[^\s]+)/i)?.[1]; + if (explicit) return { kind: "url", url: stripTrailingPunct(explicit) }; + + // "open github.com" / "go to example.org/foo" + const domain = text.match( + /\b(?:open|launch|go to|visit|browse(?:\s+to)?)\s+((?:[a-z0-9-]+\.)+[a-z]{2,}(?:\/[^\s]*)?)/i, + )?.[1]; + if (domain) return { kind: "url", url: `https://${stripTrailingPunct(domain)}` }; + + return null; + } + + if (route === "open_path") { + const path = text.match( + /\b(?:open|show|reveal)\s+((?:~|\/)[^\s]+)/i, + )?.[1]; + if (path) return { kind: "path", path: stripTrailingPunct(path) }; + return null; + } + + // open_app + const app = text.match( + /\b(?:please\s+)?(?:can you\s+|could you\s+)?(?:open|launch|start|run|fire up|bring up|pull up)\s+(?:the\s+)?(.+?)(?:\s+app(?:lication)?)?(?:\s+for me)?(?:\s+please)?[.!?]*$/i, + )?.[1]; + if (!app) return null; + const name = cleanAppName(app); + return name ? { kind: "app", name } : null; +} + +function stripTrailingPunct(s: string): string { + return s.replace(/[.,!?;:]+$/g, ""); +} + +function cleanAppName(raw: string): string | null { + let name = stripTrailingPunct(raw.trim()); + // Drop leading articles left over from "open the Notes app". + name = name.replace(/^(?:the|my|a|an)\s+/i, ""); + // Reject leftovers that look like a full sentence / UI instruction. + if (!name || name.length > 48) return null; + if (/\b(and|then|click|type|into|on the|window)\b/i.test(name)) return null; + return name; +} + +export function isOpenRoute( + route: DesktopRoute, +): route is "open_app" | "open_url" | "open_path" { + return route === "open_app" || route === "open_url" || route === "open_path"; +} diff --git a/src/main/agent/jev/evaluate.ts b/src/main/agent/jev/evaluate.ts new file mode 100644 index 0000000..2947015 --- /dev/null +++ b/src/main/agent/jev/evaluate.ts @@ -0,0 +1,93 @@ +/** + * Thin Gateway client for evaluation models (Jev). Evaluation is not exposed on + * the OpenAI-compatible Gateway endpoints and needs AI SDK 7's + * `experimental_evaluate` — we call the Gateway `/v4/ai/evaluation-model` + * route directly so the rest of OpenDex can stay on AI SDK 6 (realtime canary). + * + * Requires `AI_GATEWAY_API_KEY` in the environment. Callers that want a silent + * no-op when the key is missing should check first (see `routeDesktopIntent`). + */ + +const GATEWAY_EVAL_URL = "https://ai-gateway.vercel.sh/v4/ai/evaluation-model"; +const JEV_MODEL_ID = "typesafe-ai/jev"; + +export type EvalQuestion = + | { + type: "boolean"; + instructions: string; + criteria?: { true?: string; false?: string }; + } + | { + type: "choice"; + instructions: string; + criteria: Record; + } + | { + type: "score"; + instructions: string; + criteria: string[]; + }; + +export type EvalAnswer = + | { type: "boolean"; probability: number } + | { + type: "choice"; + choice: string; + probabilities?: Record; + } + | { + type: "score"; + score: number; + probabilities?: Record; + }; + +export interface EvaluateResult> { + answers: { [K in keyof Q]: EvalAnswer }; + usage?: { inputTokens?: number; outputTokens?: number }; + providerMetadata?: Record>; +} + +export async function evaluateWithJev>(opts: { + state: unknown; + questions: Q; + signal?: AbortSignal; +}): Promise> { + const apiKey = process.env.AI_GATEWAY_API_KEY; + if (!apiKey) throw new Error("no AI Gateway API key is set"); + + const res = await fetch(GATEWAY_EVAL_URL, { + method: "POST", + signal: opts.signal, + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + "ai-evaluation-model-specification-version": "4", + "ai-model-id": JEV_MODEL_ID, + }, + body: JSON.stringify({ + state: opts.state, + questions: opts.questions, + providerOptions: { + gateway: { zeroDataRetention: true }, + }, + }), + }); + + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error(`Jev evaluate failed (${res.status}): ${body.slice(0, 240) || res.statusText}`); + } + + const json = (await res.json()) as { + answers?: Record; + usage?: { inputTokens?: number; outputTokens?: number }; + providerMetadata?: Record>; + }; + if (!json.answers) throw new Error("Jev evaluate returned no answers"); + + return { + answers: json.answers as EvaluateResult["answers"], + usage: json.usage, + providerMetadata: json.providerMetadata, + }; +} diff --git a/src/main/agent/jev/fast-open.ts b/src/main/agent/jev/fast-open.ts new file mode 100644 index 0000000..ee6f5a5 --- /dev/null +++ b/src/main/agent/jev/fast-open.ts @@ -0,0 +1,223 @@ +/** + * Fast path for clear "open app / URL / path" commands. + * + * When Jev routes with high confidence and we can extract a target locally, + * execute the Open skill tool directly (still permission-gated) and speak a + * short confirmation — skipping the language-model tool loop entirely. + */ + +import { randomUUID } from "node:crypto"; +import type { ModelMessage } from "ai"; +import type { PermissionRequester } from "../../../skills/types"; +import { openSkill } from "../../../skills/open/skill"; +import { TOOLS } from "../../../skills/open/meta"; +import { + extractOpenTarget, + isOpenRoute, + ROUTE_CONFIDENCE, + routeDesktopIntent, + type DesktopRouteResult, + type OpenTarget, +} from "./desktop-route"; + +export interface FastOpenHandlers { + onDelta: (text: string) => void; + onToolCall?: (call: { toolCallId: string; toolName: string; input: unknown }) => void; + onToolResult?: (result: { + toolCallId: string; + toolName: string; + output: unknown; + }) => void; +} + +export type FastOpenOutcome = + | { + kind: "handled"; + messages: ModelMessage[]; + route: DesktopRouteResult; + target: OpenTarget; + } + | { + kind: "hint"; + route: DesktopRouteResult; + /** System-prompt addendum for the LLM fallthrough. */ + systemAddendum: string; + /** When set, narrow the tool set to these skill ids. */ + skillFilter?: string[]; + } + | { kind: "skip" }; + +/** + * Attempt the Jev-accelerated open path. Returns `handled` when the open tool + * ran; `hint` when we should still call the LLM but with a route bias; `skip` + * when Jev is unavailable / unsure / irrelevant. + */ +export async function tryJevDesktopFastPath(opts: { + utterance: string; + openEnabled: boolean; + computerEnabled: boolean; + requestPermission: PermissionRequester; + signal?: AbortSignal; + handlers: FastOpenHandlers; +}): Promise { + const routed = await routeDesktopIntent(opts.utterance, opts.signal); + if (!routed || routed.probability < ROUTE_CONFIDENCE) return { kind: "skip" }; + if (opts.signal?.aborted) return { kind: "skip" }; + + if (isOpenRoute(routed.route)) { + if (!opts.openEnabled) return { kind: "skip" }; + + const target = extractOpenTarget(opts.utterance, routed.route); + if (target) { + const handled = await executeOpenTarget({ + target, + requestPermission: opts.requestPermission, + handlers: opts.handlers, + signal: opts.signal, + }); + if (handled) { + return { + kind: "handled", + messages: handled, + route: routed, + target, + }; + } + // Permission denied or tool error already spoken — still "handled" so we + // don't also burn an LLM turn. executeOpenTarget returns messages either way. + } + + // Confident it's an open request but we couldn't parse the target — keep + // the LLM, but only expose Open tools so it doesn't wander into computer-use. + return { + kind: "hint", + route: routed, + skillFilter: ["open"], + systemAddendum: + "This utterance is an open/launch request (Jev route=" + + `${routed.route}, p=${routed.probability.toFixed(2)}). ` + + "Use openApp, openUrl, or openPath only — do not drive the mouse or take screenshots.", + }; + } + + if (routed.route === "computer" && opts.computerEnabled) { + return { + kind: "hint", + route: routed, + systemAddendum: + "This utterance is a desktop-control request (Jev route=computer, " + + `p=${routed.probability.toFixed(2)}). Prefer computer tools. ` + + "Do not call openApp/openUrl unless the user explicitly asked to launch something.", + }; + } + + return { kind: "skip" }; +} + +async function executeOpenTarget(opts: { + target: OpenTarget; + requestPermission: PermissionRequester; + handlers: FastOpenHandlers; + signal?: AbortSignal; +}): Promise { + if (opts.signal?.aborted) return null; + + const { toolName, input, spokenOk, spokenFail } = describeTarget(opts.target); + const tool = openSkill.tools.find((t) => t.name === toolName); + if (!tool) return null; + + const detail = tool.summarize ? tool.summarize(input) : toolName; + const allowed = await opts.requestPermission(openSkill.id, openSkill.label, detail); + if (opts.signal?.aborted) return null; + + const toolCallId = randomUUID(); + opts.handlers.onToolCall?.({ toolCallId, toolName, input }); + + let output: unknown; + if (!allowed) { + output = { error: "Permission denied by the user." }; + } else { + try { + output = await tool.execute(input as never); + } catch (err) { + output = { error: err instanceof Error ? err.message : String(err) }; + } + } + + opts.handlers.onToolResult?.({ toolCallId, toolName, output }); + + const failed = + !!output && + typeof output === "object" && + "error" in (output as object) && + Boolean((output as { error?: unknown }).error); + + const spoken = !allowed + ? "I need permission to do that." + : failed + ? spokenFail(String((output as { error: unknown }).error)) + : spokenOk; + + opts.handlers.onDelta(spoken); + + // Match the AI SDK tool-loop message shape so later turns remember the action. + const messages: ModelMessage[] = [ + { + role: "assistant", + content: [ + { + type: "tool-call", + toolCallId, + toolName, + input, + }, + ], + }, + { + role: "tool", + content: [ + { + type: "tool-result", + toolCallId, + toolName, + output: failed + ? { type: "error-text", value: String((output as { error: unknown }).error) } + : { type: "json", value: output as never }, + }, + ], + }, + { role: "assistant", content: spoken }, + ]; + return messages; +} + +function describeTarget(target: OpenTarget): { + toolName: string; + input: Record; + spokenOk: string; + spokenFail: (err: string) => string; +} { + switch (target.kind) { + case "app": + return { + toolName: TOOLS.openApp, + input: { name: target.name }, + spokenOk: `Opening ${target.name}.`, + spokenFail: (err) => `I couldn't open ${target.name}: ${err}`, + }; + case "url": + return { + toolName: TOOLS.openUrl, + input: { url: target.url }, + spokenOk: `Opening ${target.url}.`, + spokenFail: (err) => `I couldn't open that link: ${err}`, + }; + case "path": + return { + toolName: TOOLS.openPath, + input: { path: target.path }, + spokenOk: `Opening ${target.path}.`, + spokenFail: (err) => `I couldn't open that path: ${err}`, + }; + } +} diff --git a/src/main/config/schema.ts b/src/main/config/schema.ts index b699f0a..dc54d7d 100644 --- a/src/main/config/schema.ts +++ b/src/main/config/schema.ts @@ -105,6 +105,12 @@ export interface OpenDexConfig { enabled: Record; /** Standing permission decision per skill: ask each time / always / never. */ permissions: Record; + /** + * Use TypeSafe Jev (via AI Gateway) to classify open/computer intents + * before the language-model loop. Clear "open Safari" commands skip the + * LLM entirely. Needs `AI_GATEWAY_API_KEY`. + */ + jevFastPath: boolean; }; computer: { /** Animate cursor moves (watchable) vs teleport instantly (fastest). */ @@ -177,6 +183,8 @@ export const DEFAULT_CONFIG: OpenDexConfig = { // `computer` is opt-in (off until the user enables it in Settings). enabled: { open: true, computer: false }, permissions: { open: "ask", computer: "ask" }, + // On by default — no-ops when the Gateway key is missing. + jevFastPath: true, }, computer: { animateCursor: true }, // Anonymous usage analytics, on by default (opt-out in onboarding/Settings). diff --git a/src/main/index.ts b/src/main/index.ts index 8d2f4bf..68d57ae 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -23,6 +23,7 @@ import { } from "./ipc/channels"; import { streamChat } from "./agent/chat"; import { resolveModel, checkAppleAvailability } from "./agent/llm/resolve-model"; +import { tryJevDesktopFastPath } from "./agent/jev/fast-open"; import { buildRealtimeInstructions, buildSystemPrompt } from "./agent/system-prompt"; import { endRealtimeSession, @@ -31,7 +32,9 @@ import { } from "./agent/realtime/session-host"; import { buildRealtimeToolDefs, directRealtimeSkills } from "./agent/realtime/realtime-tools"; import { getRealtimeModelMeta } from "./config/realtime-models"; -import { buildToolSet, skillSystemPrompts } from "../skills/registry"; +import { buildToolSet, isSkillEnabled, skillSystemPrompts } from "../skills/registry"; +import { meta as openMeta } from "../skills/open/meta"; +import { meta as computerMeta } from "../skills/computer/meta"; import { makePermissionRequester, pendingPermissions, @@ -521,6 +524,24 @@ function stripImageOutput(output: unknown): unknown { return output; } +/** Latest user utterance as plain text (for Jev routing). */ +function lastUserText(messages: ChatStartPayload["messages"]): string | null { + for (let i = messages.length - 1; i >= 0; i--) { + const m = messages[i]; + if (m.role !== "user") continue; + if (typeof m.content === "string") return m.content.trim() || null; + if (Array.isArray(m.content)) { + const text = m.content + .filter((p): p is { type: "text"; text: string } => p.type === "text" && typeof p.text === "string") + .map((p) => p.text) + .join(" ") + .trim(); + return text || null; + } + } + return null; +} + function registerIpc() { const inFlight = new Map(); @@ -532,16 +553,89 @@ function registerIpc() { const config = getConfig(); const briefing = mode === "briefing"; track("command_run", { mode: briefing ? "briefing" : "command" }); - const system = buildSystemPrompt({ - config, - briefing, - skillPrompts: briefing ? [] : skillSystemPrompts(config), - }); - const tools = buildToolSet({ - config, - requestPermission: makePermissionRequester(sender), - }); + const requestPermission = makePermissionRequester(sender); + + const emitDelta = (delta: string) => { + if (!ac.signal.aborted && !sender.isDestroyed()) { + sender.send(IPC.chatDelta(requestId), delta); + } + }; + const emitToolCall = (call: { toolCallId: string; toolName: string; input: unknown }) => { + track("tool_used", { tool_name: call.toolName }); + if (!ac.signal.aborted && !sender.isDestroyed()) { + sender.send(IPC.chatTool(requestId), call); + } + }; + const emitToolResult = (result: { + toolCallId: string; + toolName: string; + output: unknown; + }) => { + if (!ac.signal.aborted && !sender.isDestroyed()) { + sender.send(IPC.chatToolResult(requestId), { + ...result, + output: stripImageOutput(result.output), + }); + } + }; + try { + // Jev fast path: classify open/computer intents before the LLM loop. + // Clear "open Safari" commands skip the language model entirely. + let systemAddendum = ""; + let skillFilter: string[] | undefined; + if (!briefing && config.skills.jevFastPath) { + const utterance = lastUserText(messages); + if (utterance) { + const fast = await tryJevDesktopFastPath({ + utterance, + openEnabled: isSkillEnabled(openMeta, config), + computerEnabled: isSkillEnabled(computerMeta, config), + requestPermission, + signal: ac.signal, + handlers: { + onDelta: emitDelta, + onToolCall: emitToolCall, + onToolResult: emitToolResult, + }, + }); + if (fast.kind === "handled") { + track("jev_fast_path", { + route: fast.route.route, + latency_ms: fast.route.latencyMs, + handled: true, + }); + if (!sender.isDestroyed()) { + sender.send(IPC.chatDone(requestId), fast.messages); + } + return; + } + if (fast.kind === "hint") { + track("jev_fast_path", { + route: fast.route.route, + latency_ms: fast.route.latencyMs, + handled: false, + }); + systemAddendum = fast.systemAddendum; + skillFilter = fast.skillFilter; + } + } + } + + const system = + buildSystemPrompt({ + config, + briefing, + skillPrompts: briefing ? [] : skillSystemPrompts(config), + }) + (systemAddendum ? `\n\n${systemAddendum}` : ""); + const tools = buildToolSet({ + config, + requestPermission, + include: skillFilter + ? (skill) => skillFilter!.includes(skill.id) + : undefined, + }); + // Resolve the configured provider to a model (may throw for an unset key, // an unavailable Apple model, or the not-yet-built subscription). The // catch below turns it into a spoken apology. @@ -553,28 +647,9 @@ function registerIpc() { tools, briefing, signal: ac.signal, - onDelta: (delta) => { - if (!ac.signal.aborted && !sender.isDestroyed()) { - sender.send(IPC.chatDelta(requestId), delta); - } - }, - onToolCall: (call) => { - // Tool name only — never the input args. - track("tool_used", { tool_name: call.toolName }); - if (!ac.signal.aborted && !sender.isDestroyed()) { - sender.send(IPC.chatTool(requestId), call); - } - }, - onToolResult: (result) => { - if (!ac.signal.aborted && !sender.isDestroyed()) { - sender.send(IPC.chatToolResult(requestId), { - ...result, - // Computer-use returns full screenshots; don't ship megabytes of - // base64 to the activity UI (which never renders them as cards). - output: stripImageOutput(result.output), - }); - } - }, + onDelta: emitDelta, + onToolCall: emitToolCall, + onToolResult: emitToolResult, }); if (!sender.isDestroyed()) { sender.send(IPC.chatDone(requestId), responseMessages); diff --git a/src/renderer/src/components/settings/sections.tsx b/src/renderer/src/components/settings/sections.tsx index f7c6392..d5f65a4 100644 --- a/src/renderer/src/components/settings/sections.tsx +++ b/src/renderer/src/components/settings/sections.tsx @@ -327,9 +327,34 @@ function HotkeyField({ } function SkillsSection({ data, setConfig }: SectionProps) { - const { config } = data; + const { config, secrets } = data; return ( <> + + + setConfig({ + skills: { + ...config.skills, + jevFastPath: v === "on", + }, + }) + } + /> + {SKILL_METAS.map((skill) => { const enabled = skill.optIn ? config.skills.enabled[skill.id] === true diff --git a/src/renderer/src/components/ui/fields.tsx b/src/renderer/src/components/ui/fields.tsx index f78ff39..5f35ea0 100644 --- a/src/renderer/src/components/ui/fields.tsx +++ b/src/renderer/src/components/ui/fields.tsx @@ -182,23 +182,30 @@ export function SegmentedControl({ value, options, onChange, + disabled, }: { value: T; options: { value: T; label: string }[]; onChange: (v: T) => void; + disabled?: boolean; }) { return ( -
+
{options.map((o) => ( diff --git a/src/skills/registry.ts b/src/skills/registry.ts index 466dd51..8367280 100644 --- a/src/skills/registry.ts +++ b/src/skills/registry.ts @@ -49,7 +49,7 @@ export function skillSystemPrompts(config: OpenDexConfig): string[] { * Assemble the tool set for a chat turn: every enabled skill's tools. Sensitive * skills' tools are wrapped so each call passes through the permission gate * first. `include` narrows the set further (realtime sessions pass only their - * direct, non-image skills). + * direct, non-image skills; the Jev fast-path can pin the set to `open`). */ export function buildToolSet({ config,