diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index eb05f084a..7ecf29988 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -1769,7 +1769,7 @@ describe("moveClip — the tool the prompt used to promise", () => { // ── D-CONSENT ─────────────────────────────────────────────────────────────── // -// Settings offers "Project edits — when off, the agent must ask before changing +// Settings offers "Agent edits — when off, the agent must ask before changing // the timeline". `chat-service.ts` computed `editsAllowed` and then did // `void editsAllowed;`. The flag reached nothing: not the tools, not the prompt. describe("allowAgentEdits=false refuses every write", () => { diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index e32527548..c63a56d4c 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -1200,6 +1200,9 @@ function clipRoster(document: AxcutDocument): string { return `The timeline is: ${clips.map((c) => `${c.id}${c.reason ? ` (${c.reason})` : ""}`).join(", ")}.`; } +/** The in-app agent's edit switch, as AI settings label it (`providerSettings.projectEditsLabel`). */ +export const AGENT_EDITS_SETTING = "'Agent edits' in AI settings"; + /** * The refusal the model gets for every write while `allowAgentEdits` is off. * @@ -1215,14 +1218,21 @@ function clipRoster(document: AxcutDocument): string { * * The `{error}` envelope is the shape the workbench's wire oracle already * recognises as a failed tool result — a new one would read as a success. + * + * `setting` names the switch as the user sees it: the in-app agent's by default, + * the MCP server's when an MCP client is refused (`openscreen-mcp-server.ts`). */ -function consentRequired(name: string, args: unknown): AgentToolExecution { +export function consentRequired( + name: string, + args: unknown, + setting: string = AGENT_EDITS_SETTING, +): AgentToolExecution { return { ok: false, resultJson: JSON.stringify({ error: - "Project edits are turned off for this project: the user asked to be consulted " + - "before the timeline changes. Nothing was modified.", + "Edits are turned off: the user asked to be consulted before the timeline changes. " + + "Nothing was modified.", code: "consent_required", tool: name, requestedArgs: args, @@ -1230,7 +1240,7 @@ function consentRequired(name: string, args: unknown): AgentToolExecution { "Describe the exact edit you would make — the tool, the times, the ids — and ask the " + "user to confirm it. Do NOT retry this call and do NOT reach for another write tool: " + "every one of them is refused while the setting is off. Never say an edit was applied. " + - "If they want you to go ahead, they can re-enable 'Project edits' in Settings → AI.", + `If they want you to go ahead, they can turn on ${setting}.`, }), }; } diff --git a/electron/ai-edition/chat-service.toolloop.test.ts b/electron/ai-edition/chat-service.toolloop.test.ts index 3982c901e..d140cf468 100644 --- a/electron/ai-edition/chat-service.toolloop.test.ts +++ b/electron/ai-edition/chat-service.toolloop.test.ts @@ -326,7 +326,7 @@ describe("runChat tool loop", () => { // ── D-CONSENT ─────────────────────────────────────────────────────────────── // -// `allowAgentEdits` is offered in Settings as "Project edits — when off, the +// `allowAgentEdits` is offered in Settings as "Agent edits — when off, the // agent must ask before changing the timeline". `runChat` computed // `const editsAllowed = config.allowAgentEdits !== false;` and then, twenty // lines later, `void editsAllowed;`. That was every use of it: it reached diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index 104c64abc..28390d4bf 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -429,6 +429,22 @@ describe("the prompt when the user has turned project edits off", () => { // selection rules the model needs to describe the edit it is proposing. expect(buildSystemPrompt({ editsAllowed: false }).startsWith(SYSTEM_PROMPT)).toBe(true); }); + + it("names the switch in force: the agent's by default, the caller's when it has its own", () => { + expect(buildSystemPrompt({ editsAllowed: false })).toContain("'Agent edits' in AI settings"); + const mcp = buildSystemPrompt({ editsAllowed: false, editsSetting: "'MCP edits' over there" }); + expect(mcp).toContain("'MCP edits' over there"); + expect(mcp).not.toContain("Agent edits"); + }); +}); + +describe("the prompt for a caller that can export", () => { + it("no longer gives exporting as the example of a missing tool", () => { + expect(SYSTEM_PROMPT).toMatch(/deleting an asset\/recording, exporting\)/); + const prompt = buildSystemPrompt({ editsAllowed: true, canExport: true }); + expect(prompt).not.toMatch(/exporting/); + expect(prompt).toMatch(/deleting an asset\/recording\)/); + }); }); describe("the tools when the user has turned project edits off", () => { diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index f23c19125..345721a60 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -27,6 +27,7 @@ import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; import { ZOOM_DEPTH_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-scale"; import { ZOOM_TRANSITION_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-transitions"; import { + AGENT_EDITS_SETTING, type AgentToolExecution, addAnnotationArgs, addAudioArgs, @@ -83,7 +84,8 @@ export interface OpenScreenAgentSink { // inline read-tool schemas below stay local. /** - * The block appended when the user has turned "Project edits" off. + * The block appended when the user has turned edits off: "Agent edits" for the + * in-app agent, "MCP edits" for an MCP client; `setting` names the one in force. * * ponytail: the executor's refusal is the wall (see `consentRequired` in * agent-tools.ts) but a wall alone does not fix the measured defect. The @@ -93,14 +95,15 @@ export interface OpenScreenAgentSink { * rightly: the model tried. Only the prompt can stop the emission. Two layers, * neither sufficient alone. */ -const CONSENT_PROMPT_BLOCK = [ - "", - "PROJECT EDITS ARE CURRENTLY DISABLED by the user, who asked to be consulted before the timeline changes.", - "- Read freely: getCurrentDocument and getTranscript work as usual.", - "- Do NOT call any tool that writes (removeWords, addTrim, setTrim, setClipRange, moveClip, replaceTimeline, add*/set* effects, remove*). Every one of them will be refused, so calling them wastes the turn and tells the user nothing.", - "- Instead: say precisely what you would change — which tool, which times, which ids — and ask the user to confirm. Be specific enough that they can say yes to it.", - "- Never state or imply that an edit was applied. If the user confirms and you are still refused, tell them the 'Project edits' setting in Settings → AI has to be re-enabled first.", -].join("\n"); +const consentPromptBlock = (setting: string) => + [ + "", + "PROJECT EDITS ARE CURRENTLY DISABLED by the user, who asked to be consulted before the timeline changes.", + "- Read freely: getCurrentDocument and getTranscript work as usual.", + "- Do NOT call any tool that writes (removeWords, addTrim, setTrim, setClipRange, moveClip, replaceTimeline, add*/set* effects, remove*). Every one of them will be refused, so calling them wastes the turn and tells the user nothing.", + "- Instead: say precisely what you would change — which tool, which times, which ids — and ask the user to confirm. Be specific enough that they can say yes to it.", + `- Never state or imply that an edit was applied. If the user confirms and you are still refused, tell them to turn on ${setting} first.`, + ].join("\n"); // ponytail: exported so a test can assert that what the model receives is this // string and NOTHING else — the deepagents regression was invisible precisely @@ -134,14 +137,27 @@ const BASE_SYSTEM_PROMPT = [ "Cursor telemetry: while the screen was captured, OpenScreen recorded where the pointer went. assets[].hasCursorTelemetry in getCurrentDocument says which assets carry it, and getCursorTrack returns the recorded track for one of them — positions over time, and the pointer shape at each moment. What it means is yours to read.", "Blindness is not evidence. When a tool reports it could not read something (reason 'unavailable'), that is a limit of your runtime, not a fact about the project. Only an explicit negative — reason 'no-sidecar', or a false flag — supports telling the user the data is not there.", "", - "Honesty rules: if a request has NO matching tool (e.g. deleting an asset/recording, exporting), say so plainly — do not substitute a different edit and report it as the requested one. After your edits, if you are at all unsure the document ended up as intended, call getCurrentDocument and reconcile what you claim with the real state; each tool result already tells you exactly what it did, so never report a change the results don't support.", ].join("\n"); -/** The system prompt for a turn, which depends on ONE thing: whether the user - * currently allows the agent to write. Exported so the consent wording can be - * tested without a provider. */ -export function buildSystemPrompt(options: { editsAllowed: boolean }): string { - return options.editsAllowed ? BASE_SYSTEM_PROMPT : BASE_SYSTEM_PROMPT + CONSENT_PROMPT_BLOCK; +// Exporting is the example of a missing tool only where there is none: `openscreen mcp` has exportVideo. +const honestyRules = (canExport: boolean) => + `Honesty rules: if a request has NO matching tool (e.g. deleting an asset/recording${canExport ? "" : ", exporting"}), say so plainly — do not substitute a different edit and report it as the requested one. After your edits, if you are at all unsure the document ended up as intended, call getCurrentDocument and reconcile what you claim with the real state; each tool result already tells you exactly what it did, so never report a change the results don't support.`; + +/** The system prompt for a turn: whether the user currently allows writes, and through + * which setting. The MCP server reuses it as its instructions, naming its own setting + * and, over stdio, the export tool it serves. Exported so the wording can be tested + * without a provider. */ +export function buildSystemPrompt(options: { + editsAllowed: boolean; + /** Where the user turns writes back on. Default: the in-app agent's switch. */ + editsSetting?: string; + /** The caller also has a tool that exports the video. */ + canExport?: boolean; +}): string { + const prompt = BASE_SYSTEM_PROMPT + honestyRules(options.canExport === true); + return options.editsAllowed + ? prompt + : prompt + consentPromptBlock(options.editsSetting ?? AGENT_EDITS_SETTING); } /** The prompt of a normal (edits-allowed) turn — the overwhelmingly common diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index ca1bc88b3..328b4f69b 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -5029,10 +5029,11 @@ export function registerIpcHandlers( // dependencies live, but NOT started here: the headless CLI shares this // function and must never bind the port a running app is listening on. // `main.ts` starts it. Its writes have their own switch, off by default and - // read on every call — separate from the in-app agent's "Project edits", so - // turning the server on grants a client read access and nothing more. + // read on every call: "MCP edits", separate from the in-app agent's "Agent edits", + // so turning the server on grants a client read access and nothing more. const mcpSettings = new McpSettingsStore(app.getPath("userData"), safeStorage); const mcpToolDeps: McpToolDeps = { + transport: "http", host: new EditorDocumentHost(ipcMain), projects: aiEditionDocuments, editsAllowed: () => mcpSettings.getSettings().allowEdits, diff --git a/electron/mcp/mcp-controller.test.ts b/electron/mcp/mcp-controller.test.ts index fd8b7ec97..766ccf3db 100644 --- a/electron/mcp/mcp-controller.test.ts +++ b/electron/mcp/mcp-controller.test.ts @@ -16,6 +16,7 @@ const fakeCrypto: McpTokenCrypto = { }; const deps: McpToolDeps = { + transport: "http", host: { snapshot: async () => null, apply: async () => "no-editor" }, projects: { listProjects: async () => [], diff --git a/electron/mcp/mcp-settings-store.ts b/electron/mcp/mcp-settings-store.ts index 4d8aaaa47..4f90768b6 100644 --- a/electron/mcp/mcp-settings-store.ts +++ b/electron/mcp/mcp-settings-store.ts @@ -17,8 +17,8 @@ export interface McpSettings { enabled: boolean; port: number; /** - * Whether MCP clients may run the tools that change the project. Off by - * default and separate from the in-app agent's "Project edits": turning the + * Whether MCP clients may run the tools that change the project ("MCP edits"). Off by + * default and separate from the in-app agent's "Agent edits": turning the * server on gives a client read access, and writing is a second decision. */ allowEdits: boolean; diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.ts index a08774a3d..ef69bfd07 100644 --- a/electron/mcp/media-tools.test.ts +++ b/electron/mcp/media-tools.test.ts @@ -117,6 +117,7 @@ beforeEach(async () => { openVideoFails = false; const toolDeps: McpToolDeps = { + transport: "stdio", host: { snapshot: async () => null, apply: async () => "no-editor" }, projects: documents, editsAllowed: () => true, @@ -175,6 +176,49 @@ async function recordTake(title = "Demo") { }; } +describe("what `openscreen mcp` tells its client", () => { + it("composes the media block with the checkpoint rule, and nothing true only in the app", () => { + const instructions = client.getInstructions() ?? ""; + expect(instructions.startsWith(MEDIA_PREAMBLE)).toBe(true); + expect(instructions).toContain("Before a series of edits, call createCheckpoint"); + expect(instructions).toContain("lost when it exits"); + expect(instructions).not.toMatch( + /project open in the editor|Project edits|Agent edits|Nothing here records|undo step|exporting\)/, + ); + }); + + it("describes projectId and restoreCheckpoint as they work here", async () => { + const { tools } = await client.listTools(); + const byName = new Map(tools.map((tool) => [tool.name, tool])); + const projectId = byName.get("addTrim")?.inputSchema.properties?.projectId as { + description?: string; + }; + expect(projectId.description).toContain("Pass it on every call"); + const restore = byName.get("restoreCheckpoint")?.description ?? ""; + expect(restore).toContain("pointer track"); + expect(restore).not.toContain("undo step"); + }); + + it("answers in its own terms: no editor to fall back on, no Ctrl/Cmd+Z, its own lifetime", async () => { + const noProject = await call("getCurrentDocument"); + expect(textOf(noProject)).toContain("this server has no editor"); + expect(textOf(noProject)).not.toContain("No project is open"); + + const other = await documents.createProject("Other"); + const unguarded = await call("setAspectRatio", { + projectId: other.project.id, + aspectRatio: "1:1", + }); + expect(textOf(unguarded)).toContain("straight to the project's file"); + + const unknown = await call("restoreCheckpoint", { + projectId: other.project.id, + checkpointId: "cp_nope", + }); + expect(textOf(unknown)).toContain("lost when it exits"); + }); +}); + describe("frameTimes", () => { it("centres `count` frames in equal slices, and clamps asked-for times inside the video", () => { expect(frameTimes(12, null, 4)).toEqual([1.5, 4.5, 7.5, 10.5]); @@ -337,6 +381,93 @@ describe("setPointerTrack", () => { expect(click?.cx).toBe(0.6); }); + /** The sidecar beside the take, byte for byte; `null` when there is none. */ + const sidecarFile = (video: string) => + fs.readFile(`${video}.cursor.json`, "utf8").catch(() => null); + const clickAt = (take: { recordingStartedAt: number }, x: number) => [ + { at: take.recordingStartedAt + 5_000, x, y: 0.5, click: true }, + ]; + + it("refuses a project open in the app's editor before writing anything", async () => { + const take = await recordTake(); + const before = await sidecarFile(take.videoPath); + documents.markOpenInEditor(take.projectId); + const refused = await call("setPointerTrack", { + projectId: take.projectId, + events: clickAt(take, 0.2), + }); + expect(refused.isError).toBe(true); + expect(textOf(refused)).toContain("open in the OpenScreen editor"); + expect(await sidecarFile(take.videoPath)).toBe(before); + expect((await documents.getProject(take.projectId)).zoomRanges).toHaveLength(take.autoZooms); + }); + + it("refuses a project with no checkpoint before writing anything", async () => { + const take = await recordTake(); + const before = await sidecarFile(take.videoPath); + // Twenty newer checkpoints push out the one stopRecording made. + const other = await documents.createProject("Other"); + for (let i = 0; i < 20; i++) { + await call("createCheckpoint", { projectId: other.project.id }); + } + const refused = await call("setPointerTrack", { + projectId: take.projectId, + events: clickAt(take, 0.2), + }); + expect(refused.isError).toBe(true); + expect(textOf(refused)).toContain("createCheckpoint"); + expect(await sidecarFile(take.videoPath)).toBe(before); + }); + + it("is undone by restoreCheckpoint, the take's own pointer track included", async () => { + const take = await recordTake(); + const before = await sidecarFile(take.videoPath); + const set = await call("setPointerTrack", { + projectId: take.projectId, + events: clickAt(take, 0.2), + }); + expect(set.isError).toBeFalsy(); + expect(await sidecarFile(take.videoPath)).not.toBe(before); + + const restored = await call("restoreCheckpoint", { + projectId: take.projectId, + checkpointId: take.checkpointId, + }); + expect(restored.isError).toBeFalsy(); + expect(await sidecarFile(take.videoPath)).toBe(before); + expect((await documents.getProject(take.projectId)).zoomRanges).toHaveLength(take.autoZooms); + }); + + it("takes away a track the take never had, and each checkpoint gets back its own", async () => { + await fs.rm(`${videoPath}.cursor.json`); + const take = await recordTake(); + await call("setPointerTrack", { projectId: take.projectId, events: clickAt(take, 0.2) }); + const first = await sidecarFile(take.videoPath); + const second = JSON.parse( + textOf(await call("createCheckpoint", { projectId: take.projectId })), + ).checkpointId; + await call("setPointerTrack", { projectId: take.projectId, events: clickAt(take, 0.8) }); + + await call("restoreCheckpoint", { projectId: take.projectId, checkpointId: take.checkpointId }); + expect(await sidecarFile(take.videoPath)).toBeNull(); + await call("restoreCheckpoint", { projectId: take.projectId, checkpointId: second }); + expect(await sidecarFile(take.videoPath)).toBe(first); + }); + + it("says the edit is partial when the project saved but the track could not be written", async () => { + await fs.rm(`${videoPath}.cursor.json`); + const take = await recordTake(); + // A directory where the sidecar goes: the save succeeds, the sidecar write cannot. + await fs.mkdir(`${take.videoPath}.cursor.json`); + const set = await call("setPointerTrack", { + projectId: take.projectId, + events: clickAt(take, 0.2), + }); + expect(set.isError).toBe(true); + expect(textOf(set)).toContain("The project was saved, but the pointer track"); + expect(textOf(set)).not.toContain("Tool failed"); + }); + it("refuses a project whose take this server did not record", async () => { const other = await documents.createProject("Imported"); const refused = await call("setPointerTrack", { diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts index 4c7fc2bb0..1e332e902 100644 --- a/electron/mcp/media-tools.ts +++ b/electron/mcp/media-tools.ts @@ -6,7 +6,6 @@ // No `electron` import: the runners, the decoder and the documents are handed in, // so the tools run under plain vitest with stand-ins. -import fs from "node:fs/promises"; import path from "node:path"; import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; @@ -28,6 +27,7 @@ import { runDocumentTool } from "../ai-edition/deep-agent/service"; import type { DocumentService } from "../ai-edition/document-service"; import type { RunnerHandlers } from "../cli/runnerHost"; import type { OpenedVideo } from "./hidden-video"; +import { type McpToolRunner, type RunnerEdit, textResult as text } from "./openscreen-mcp-server"; export interface MediaToolDeps { runners: { @@ -42,19 +42,13 @@ export interface MediaToolDeps { }; documents: Pick< DocumentService, - | "createProject" - | "addAsset" - | "saveProject" - | "filePathFor" - | "getProject" - | "getProjectForUpdate" - | "saveProjectIfUnchanged" + "createProject" | "addAsset" | "saveProject" | "filePathFor" | "getProject" >; openVideo(videoPath: string): Promise; /** The pointer samples recorded with a video; empty when there are none. */ readCursor(videoPath: string): Promise; - /** The document tools' runner: a fresh project's checkpoint goes through it. */ - runTool(name: string, args: unknown): Promise; + /** The document tools' runner: a fresh project's checkpoint and every edit go through it. */ + runTool: McpToolRunner; now?(): number; } @@ -137,18 +131,15 @@ export function pointerSamples( return samples; } -function text(body: string, isError = false): CallToolResult { - return { content: [{ type: "text", text: body }], ...(isError ? { isError: true } : {}) }; -} - const READ_ONLY = { readOnlyHint: true, destructiveHint: false, openWorldHint: false } as const; +/** The media tools' capability block, placed before the projects-and-checkpoints rules + * (`mcpInstructions`) on the server that registers them. */ export const MEDIA_PREAMBLE = [ "OpenScreen records the screen, turns each take into a project you edit with the document tools, and exports it as a video.", "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames with the projectId to check the export before you hand it over: it takes the same ruler seconds as the document tools, so a trim or a speed region never shifts what you look at.", "Drive the app with the real mouse when you can: OpenScreen records the pointer, draws a clean cursor and zooms on the clicks as the editor does: clicks close in time and on screen share one zoom, and the first seconds and the end stay on the whole screen. If you drive it from a script instead (Playwright, CDP), the screen pointer never moves: log each action as the script runs it, then hand the log to setPointerTrack after stopRecording.", - "These projects are in the OpenScreen app's project list, so the user can open one in the editor. Do not edit a project while the user has it open there: the editor would keep its own copy.", - "", + "These projects are in the OpenScreen app's project list, so the user can open one in its editor.", ].join("\n"); export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void { @@ -159,8 +150,6 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void startedAt: number; stopping: boolean; } | null = null; - // One pointer track at a time: each reads the project, rewrites the sidecar and saves. - let pointerQueue: Promise = Promise.resolve(); // An export loads its project into the slot a finishing take reads its files from. let exporting = false; /** When each take of this session started, by its video: setPointerTrack's clock. */ @@ -388,7 +377,7 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void "setPointerTrack", { description: [ - "For a take driven by a script (Playwright, CDP, a browser extension) rather than the real mouse: the screen pointer never moved, so the video has no cursor and no zoom. Give the pointer's path here and OpenScreen draws it, gliding between your actions, and zooms on its clicks as for a real mouse. It replaces the pointer recorded with the take, then plans zooms on the new clicks as stopRecording does, clear of the zooms already there; autoZoomsAdded says how many. Only for a take recorded by this server.", + "For a take driven by a script (Playwright, CDP, a browser extension) rather than the real mouse: the screen pointer never moved, so the video has no cursor and no zoom. Give the pointer's path here and OpenScreen draws it, gliding between your actions, and zooms on its clicks as for a real mouse. It replaces the pointer recorded with the take, then plans zooms on the new clicks as stopRecording does, clear of the zooms already there; autoZoomsAdded says how many. Only for a take recorded by this server. Like any edit it needs a checkpoint of the project (stopRecording made one), and restoreCheckpoint undoes it, the pointer track included.", "events, one per action: `at` is Date.now() when your script did it (log it as it runs; the take's recordingStartedAt is on the same clock). `x` and `y` are fractions (0 to 1) of the recorded window. For a point (clientX, clientY) of a page shown in that window, from inside the page: border = (outerWidth - innerWidth) / 2; x = (border + clientX) / outerWidth; y = (outerHeight - innerHeight - border + clientY) / outerHeight.", ].join("\n"), inputSchema: z.object({ @@ -406,51 +395,50 @@ export function registerMediaTools(server: McpServer, deps: MediaToolDeps): void }), annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false }, }, - (args) => { - const next = pointerQueue.then(() => setPointerTrack(args)); - pointerQueue = next.catch(() => undefined); - return next; - }, + // Through the document tools' runner, like any edit: the same queue, the same refusals + // (a project open in the app's editor, one with no checkpoint) before anything is + // written, and the sidecar kept by the checkpoints so restoreCheckpoint puts it back. + (args) => + deps.runTool("setPointerTrack", { projectId: args.projectId }, (document) => + pointerTrackEdit(document, args.events), + ), ); - async function setPointerTrack(args: { - projectId: string; - events: PointerEvent[]; - }): Promise { - const { document, version } = await deps.documents.getProjectForUpdate(args.projectId); + /** The new sidecar, and the zooms planned on its clicks: computed here, written by the runner. */ + async function pointerTrackEdit( + document: AxcutDocument, + events: PointerEvent[], + ): ReturnType { const asset = document.assets.find((a) => a.id === document.project.primaryAssetId); const startedAt = asset ? startedAtByVideo.get(asset.originalPath) : undefined; if (!asset || startedAt === undefined || !asset.durationSec) { - return text( - "This project's take was not recorded by this server, so its start time is unknown. setPointerTrack only works on a take from stopRecording.", - true, - ); + return { + ok: false, + resultJson: + "This project's take was not recorded by this server, so its start time is unknown. setPointerTrack only works on a take from stopRecording.", + }; } - const sidecar: CursorRecordingData = { - version: 2, - provider: "native", - samples: pointerSamples(args.events, startedAt, asset.durationSec * 1000), - assets: [], - }; - await fs.writeFile(`${asset.originalPath}.cursor.json`, JSON.stringify(sidecar), "utf8"); + const samples = pointerSamples(events, startedAt, asset.durationSec * 1000); + const sidecar: CursorRecordingData = { version: 2, provider: "native", samples, assets: [] }; + // Planned from the new track before it is on disk, as readCursor would read it back. const suggestions = await collectAutoZoomSuggestionsForDocument( document, - deps.readCursor, + async () => + samples.map(({ timeMs, cx, cy, visible, interactionType }) => ({ + timeMs, + cx, + cy, + visible, + interactionType, + })), (candidate) => candidate.id === asset.id, ); - const saved = await deps.documents.saveProjectIfUnchanged( - appendAutoZoomSuggestions(document, suggestions), - version, - ); - if (!saved) { - return text( - "The pointer track was written, but the zooms were NOT added: the project changed while this call ran. Call setPointerTrack again.", - true, - ); - } - return text( - JSON.stringify({ samples: sidecar.samples.length, autoZoomsAdded: suggestions.length }), - ); + return { + ok: true, + resultJson: JSON.stringify({ samples: samples.length, autoZoomsAdded: suggestions.length }), + document: appendAutoZoomSuggestions(document, suggestions), + sidecars: [{ path: `${asset.originalPath}.cursor.json`, content: JSON.stringify(sidecar) }], + }; } server.registerTool( diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.ts index 485a95826..942b8eac4 100644 --- a/electron/mcp/openscreen-mcp-server.test.ts +++ b/electron/mcp/openscreen-mcp-server.test.ts @@ -23,6 +23,7 @@ import { MCP_ENDPOINT_PATH, type McpApplyResult, type McpDocumentHost, + mcpInstructions, type RunningMcpServer, startMcpHttpServer, } from "./openscreen-mcp-server"; @@ -145,6 +146,7 @@ async function connect( port: 0, token: TOKEN, deps: { + transport: "http", host: editor, projects, editsAllowed: () => options.editsAllowed ?? true, @@ -212,6 +214,40 @@ describe("the MCP tool surface", () => { const mcp = await connect(new FakeEditor()); expect(mcp.getInstructions()).toContain("Time-bases (do not mix them up)"); }); + + it("tells the client what is true here: an editor, its undo, no media, and its own switch", async () => { + const mcp = await connect(new FakeEditor(), { editsAllowed: false }); + const instructions = mcp.getInstructions() ?? ""; + expect(instructions).toContain( + "Without a projectId they act on the project open in the editor", + ); + expect(instructions).toContain("Nothing here records, exports or imports media."); + expect(instructions).toContain("OpenScreen quits or its MCP server restarts"); + expect(instructions).toContain("turn on 'MCP edits' under MCP server in AI settings"); + expect(instructions).not.toContain("Agent edits"); + const { tools } = await mcp.listTools(); + const restore = tools.find((t) => t.name === "restoreCheckpoint"); + expect(restore?.description).toContain("one undo step"); + expect(restore?.description).not.toContain("pointer track"); + }); +}); + +describe("the instructions, composed per transport", () => { + const media = "MEDIA BLOCK"; + it("puts the media block first, then the same checkpoint rule on both transports", () => { + const stdio = mcpInstructions({ transport: "stdio", editsAllowed: true, media }); + const http = mcpInstructions({ transport: "http", editsAllowed: true }); + expect(stdio.startsWith(media)).toBe(true); + for (const instructions of [stdio, http]) { + expect(instructions).toContain("Before a series of edits, call createCheckpoint"); + expect(instructions).toContain("Time-bases (do not mix them up)"); + } + // Only the in-app server has an editor, its undo stack, and no export. + expect(stdio).not.toMatch(/project open in the editor|undo step|Nothing here records/); + expect(stdio).not.toContain("exporting)"); + expect(http).toContain("exporting)"); + expect(stdio).toContain("lost when it exits"); + }); }); describe("calling a tool", () => { @@ -303,6 +339,9 @@ describe("calling a tool", () => { arguments: { assetId: "asset_1", startSec: 5, endSec: 6 }, }); expect(result.isError).toBe(true); + // The MCP server's switch, not the in-app agent's. + expect(resultText(result)).toContain("'MCP edits' under MCP server in AI settings"); + expect(resultText(result)).not.toContain("Agent edits"); expect(editor.applied).toHaveLength(0); }); @@ -367,10 +406,11 @@ describe("checkpoints", () => { const checkpointId = await checkpoint(mcp); const result = await mcp.callTool({ name: "restoreCheckpoint", arguments: { checkpointId } }); expect(result.isError).toBe(true); + expect(resultText(result)).toContain("'MCP edits'"); expect(editor.applied).toHaveLength(0); }); - it("refuses an unknown checkpoint", async () => { + it("refuses an unknown checkpoint, saying when they are lost", async () => { const editor = new FakeEditor(); const mcp = await connect(editor); const result = await mcp.callTool({ @@ -378,6 +418,7 @@ describe("checkpoints", () => { arguments: { checkpointId: "cp_nope" }, }); expect(result.isError).toBe(true); + expect(resultText(result)).toContain("its port or token changed"); expect(editor.applied).toHaveLength(0); }); @@ -726,7 +767,13 @@ describe("the HTTP guard", () => { running = await startMcpHttpServer({ port: 0, token: TOKEN, - deps: { host: new FakeEditor(), projects, editsAllowed: () => true, version: "0.0.0" }, + deps: { + transport: "http", + host: new FakeEditor(), + projects, + editsAllowed: () => true, + version: "0.0.0", + }, }); return fetch(`http://127.0.0.1:${running.port}${path}`, { method: "POST", diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.ts index a353b6301..f9cef6cce 100644 --- a/electron/mcp/openscreen-mcp-server.ts +++ b/electron/mcp/openscreen-mcp-server.ts @@ -25,11 +25,20 @@ // only way back, so an edit to one is refused until it has a checkpoint. The // in-app agent has no need for them: its whole turn is already a single apply. // +// Two transports serve these tools: the in-app server over HTTP (below) and +// `openscreen mcp` over stdio (`stdio-server.ts`, which adds the media tools). The +// runner and the tools are the same; the texts a client reads are composed per +// transport (`mcpInstructions`, the `*_MESSAGES` tables), so each says what is true +// on the server that serves it. A media tool that edits a project +// (setPointerTrack) hands its change to the runner too, so it meets the same +// refusals and checkpoints, sidecar included. +// // The HTTP layer is local-only: bound to 127.0.0.1, a bearer token on every // request, and a Host/Origin check so a web page cannot reach it by DNS // rebinding. import { randomUUID, timingSafeEqual } from "node:crypto"; +import fs from "node:fs/promises"; import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; import type { AddressInfo } from "node:net"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; @@ -37,7 +46,11 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/ import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { z } from "zod"; import { type AxcutDocument, documentSchema } from "../../src/lib/ai-edition/schema"; -import { type AgentToolExecution, isMutatingTool } from "../ai-edition/agent-tools"; +import { + type AgentToolExecution, + consentRequired, + isMutatingTool, +} from "../ai-edition/agent-tools"; import { buildSystemPrompt, type CursorTelemetryReader, @@ -99,10 +112,19 @@ export interface McpProjectStore { type ProjectRead = Awaited>; +/** + * Which server this is. The tools are the same; what differs is what a call can reach: + * the in-app server (`http`) has the editor, its undo stack and a switch for edits; + * `openscreen mcp` (`stdio`) is a process of its own, with no editor and no switch. + * Every text the client reads says what is true on the one it is connected to. + */ +export type McpTransport = "http" | "stdio"; + export interface McpToolDeps { + transport: McpTransport; host: McpDocumentHost; projects: McpProjectStore; - /** The "Project edits" setting — the same one the in-app agent obeys. */ + /** The MCP server's own edit switch ("MCP edits"), separate from the in-app agent's. */ editsAllowed(): boolean; cursor?: CursorTelemetryReader; version: string; @@ -110,33 +132,51 @@ export interface McpToolDeps { export const LIST_PROJECTS_TOOL = "listProjects"; +/** The in-app server's edit switch, as AI settings label it (`mcpServer.editsLabel`). */ +const MCP_EDITS_SETTING = "'MCP edits' under MCP server in AI settings"; + const LIST_PROJECTS_DESCRIPTION = "List every OpenScreen project: id, title, last update, asset count, and whether it is the one open in the editor. Pass a project's id as `projectId` to any other tool to read or edit that project."; -const PROJECT_ID_ARG = z - .string() - .optional() - .describe( - "The project to act on, from listProjects. Omit it for the project open in the editor.", - ); +const PROJECT_ID_DESCRIPTIONS: Record = { + http: "The project to act on, from listProjects. Omit it for the project open in the editor.", + stdio: + "The project to act on, from listProjects. Pass it on every call: this server has no editor, so there is no open project to fall back on.", +}; -const NO_PROJECT_MESSAGE = - "No project is open in the OpenScreen editor. Call listProjects and pass a projectId, or ask the user to open one in OpenScreen."; +const NO_PROJECT_MESSAGES: Record = { + http: "No project is open in the OpenScreen editor. Call listProjects and pass a projectId, or ask the user to open one in OpenScreen.", + stdio: + "Pass a projectId: this server has no editor, so there is no open project to act on. Call listProjects for the ids.", +}; // The editor would overwrite a file edit with its own copy on its next save, so a // project it opened mid-call is left to it; the retry then goes through the editor. const PROJECT_CHANGED_MESSAGE = "The edit was NOT applied: the project was opened in the editor or saved elsewhere while this call ran. Call getCurrentDocument with the same projectId to re-read it, then retry."; -const CHECKPOINT_REQUIRED_MESSAGE = - "The edit was NOT applied: this project is not open in the editor, so the user cannot undo an edit to it with Ctrl/Cmd+Z. Call createCheckpoint with this projectId first, so restoreCheckpoint can take your edits back in one step, then retry."; +const CHECKPOINT_REQUIRED_MESSAGES: Record = { + http: "The edit was NOT applied: this project is not open in the editor, so the user cannot undo an edit to it with Ctrl/Cmd+Z. Call createCheckpoint with this projectId first, so restoreCheckpoint can take your edits back in one step, then retry.", + stdio: + "The edit was NOT applied: this server saves each edit straight to the project's file, where Ctrl/Cmd+Z cannot reach it. Call createCheckpoint with this projectId first, so restoreCheckpoint can take your edits back in one step, then retry.", +}; + +// ponytail: kept in memory, last 20 only; persist them per project if agents need +// them across sessions. +const MAX_CHECKPOINTS = 20; + +const CHECKPOINT_LIFETIMES: Record = { + http: `Checkpoints live in OpenScreen's memory: only the ${MAX_CHECKPOINTS} most recent are kept, and all of them are lost when OpenScreen quits or its MCP server restarts (turned off and on, or its port or token changed).`, + stdio: `Checkpoints live in this server's memory: only the ${MAX_CHECKPOINTS} most recent are kept, and all of them are lost when it exits, which it does when the client disconnects.`, +}; const APPLY_FAILURE_MESSAGES: Record, string> = { conflict: "The edit was NOT applied: the project changed in the editor while this call ran. Call getCurrentDocument to re-read it, then retry.", "save-failed": "The edit was NOT applied: OpenScreen could not save the project.", - "no-live-document": NO_PROJECT_MESSAGE, - "no-editor": NO_PROJECT_MESSAGE, + // Only the in-app server has an editor to apply through. + "no-live-document": NO_PROJECT_MESSAGES.http, + "no-editor": NO_PROJECT_MESSAGES.http, // Not "NOT applied": the editor may have saved it and only the answer was lost. timeout: "OpenScreen did not confirm this edit in time. Call getCurrentDocument to see whether it landed before retrying.", @@ -149,37 +189,103 @@ const DESTRUCTIVE_TOOLS: ReadonlySet = new Set([ "removeClip", ]); -const MCP_PREAMBLE = [ - "These tools act on the user's OpenScreen projects. Without a projectId they act on the project open in the editor; call listProjects to find any other project and pass its id as projectId. Every edit is saved straight away. An edit to the open project appears in the editor, where the user can undo it with Ctrl/Cmd+Z; an edit to any other project is saved to its file and is not on the editor's undo stack. Nothing here records, exports or imports media.", - "Before a series of edits, call createCheckpoint with the same projectId. Undo is one step per call, so if the result is not what the user wanted, restoreCheckpoint takes the project back in one step instead of many. A project that is not open in the editor cannot be edited until it has a checkpoint, since restoreCheckpoint is its only undo.", - "", -].join("\n"); +/** What both servers say about projects and checkpoints, each in its own terms. */ +const CORE_PREAMBLES: Record = { + http: [ + "These tools act on the user's OpenScreen projects. Without a projectId they act on the project open in the editor; call listProjects to find any other project and pass its id as projectId. Every edit is saved straight away. An edit to the open project appears in the editor, where the user can undo it with Ctrl/Cmd+Z; an edit to any other project is saved to its file and is not on the editor's undo stack.", + "Before a series of edits, call createCheckpoint with the same projectId. Undo is one step per call, so if the result is not what the user wanted, restoreCheckpoint takes the project back in one step instead of many. A project that is not open in the editor cannot be edited until it has a checkpoint, since restoreCheckpoint is its only undo.", + ], + stdio: [ + "These tools act on the user's OpenScreen projects, read from and saved to their files: call listProjects for their ids, and pass one as projectId on every call. Every edit is saved straight away, and Ctrl/Cmd+Z cannot undo it. An edit to a project the user has open in the OpenScreen app is refused, since its editor would save over it: ask them to close it there.", + "Before a series of edits, call createCheckpoint with the same projectId: restoreCheckpoint then takes the project back in one step, and it is the only way back. A project cannot be edited until it has a checkpoint.", + ], +}; -// ponytail: kept in memory, last 20 only. Lost when OpenScreen quits or the MCP -// server is turned off; persist them per project if agents need them across sessions. -const MAX_CHECKPOINTS = 20; +const NO_MEDIA_NOTE = "Nothing here records, exports or imports media."; -const CHECKPOINT_TOOLS = [ - { - name: "createCheckpoint", - description: - "Save the current state of the project and return its checkpointId. Changes nothing. Call it before a series of edits so restoreCheckpoint can revert all of them in one step.", - inputSchema: z.object({}), - mutating: false, - }, - { - name: "restoreCheckpoint", - description: - "Put the project back exactly as it was when createCheckpoint returned this checkpointId, discarding every edit made since, including the user's. Pass the same projectId as to createCheckpoint. On the project open in the editor it lands as one undo step, so the user can undo the restore itself.", - inputSchema: z.object({ checkpointId: z.string() }), - mutating: true, - }, -] as const; - -function textResult(text: string, isError: boolean): CallToolResult { +/** + * The server's instructions: the capability block of the media tools when they are served, + * the projects-and-checkpoints rules of this transport, then the in-app agent's editing + * guidance, naming this server's edit switch and whether it can export. + */ +export function mcpInstructions(options: { + transport: McpTransport; + editsAllowed: boolean; + /** The media tools' preamble, when this server registers them. */ + media?: string; +}): string { + const { transport, editsAllowed, media } = options; + const preamble = [ + ...(media ? [media] : []), + ...CORE_PREAMBLES[transport], + CHECKPOINT_LIFETIMES[transport], + ...(media ? [] : [NO_MEDIA_NOTE]), + "", + ].join("\n"); + return ( + preamble + + buildSystemPrompt({ editsAllowed, editsSetting: MCP_EDITS_SETTING, canExport: !!media }) + ); +} + +function checkpointTools(transport: McpTransport, media: boolean) { + const restore = [ + "Put the project back exactly as it was when createCheckpoint returned this checkpointId, discarding every edit made since, including the user's. Pass the same projectId as to createCheckpoint.", + ...(media ? ["It also puts back the pointer track setPointerTrack replaced."] : []), + ...(transport === "http" + ? [ + "On the project open in the editor it lands as one undo step, so the user can undo the restore itself.", + ] + : []), + ].join(" "); + return [ + { + name: "createCheckpoint", + description: + "Save the current state of the project and return its checkpointId. Changes nothing. Call it before a series of edits so restoreCheckpoint can revert all of them in one step.", + inputSchema: z.object({}), + mutating: false, + }, + { + name: "restoreCheckpoint", + description: restore, + inputSchema: z.object({ checkpointId: z.string() }), + mutating: true, + }, + ] as const; +} + +/** A tool's answer, as both servers' tools give it. */ +export function textResult(text: string, isError = false): CallToolResult { return { content: [{ type: "text", text }], ...(isError ? { isError: true } : {}) }; } +/** A cursor sidecar an edit rewrites beside its video; `null` deletes it. */ +export interface SidecarWrite { + path: string; + content: string | null; +} + +/** + * An edit a tool computes itself rather than through the document executor (setPointerTrack): + * it gets the project as the runner read it and returns the new document, plus the sidecars + * to write once that document is saved. It must write nothing itself. + */ +export type RunnerEdit = ( + document: AxcutDocument, +) => Promise; + +export type McpToolRunner = ReturnType; + +async function readSidecar(file: string): Promise { + try { + return await fs.readFile(file, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } @@ -203,11 +309,19 @@ function splitProjectId(rawArgs: unknown): { projectId: string | undefined; args */ export function createToolRunner(deps: McpToolDeps) { let queue: Promise = Promise.resolve(); - const checkpoints = new Map(); + /** + * Each checkpoint's document, and what each sidecar an edit rewrote since held before + * the first such write: `null` when there was no file. A sidecar no edit touched since + * is already as it was, so it needs no entry. + */ + const checkpoints = new Map< + string, + { document: AxcutDocument; sidecars: Map } + >(); function createCheckpoint(document: AxcutDocument): AgentToolExecution { const checkpointId = `cp_${randomUUID()}`; - checkpoints.set(checkpointId, document); + checkpoints.set(checkpointId, { document, sidecars: new Map() }); // A Map iterates in insertion order: the first key is the oldest. for (const oldest of checkpoints.keys()) { if (checkpoints.size <= MAX_CHECKPOINTS) break; @@ -217,12 +331,14 @@ export function createToolRunner(deps: McpToolDeps) { } /** Hands the checkpoint back as the call's new document, saved like any edit. */ - function restoreCheckpoint(document: AxcutDocument, args: unknown): AgentToolExecution { + function restoreCheckpoint( + document: AxcutDocument, + args: unknown, + ): AgentToolExecution & { sidecars?: SidecarWrite[] } { if (!deps.editsAllowed()) { return { ok: false, - resultJson: - "Project edits are turned off in OpenScreen, so the checkpoint was NOT restored. Ask the user to re-enable 'Project edits' in Settings → AI, or to undo the edits themselves.", + resultJson: `Edits by MCP clients are turned off in OpenScreen, so the checkpoint was NOT restored. Ask the user to turn on ${MCP_EDITS_SETTING}, or to undo the edits themselves.`, }; } const id = (args as { checkpointId?: unknown } | null)?.checkpointId; @@ -230,30 +346,58 @@ export function createToolRunner(deps: McpToolDeps) { if (!checkpoint) { return { ok: false, - resultJson: - "Unknown checkpointId. Checkpoints last until OpenScreen quits, and only the 20 most recent are kept.", + resultJson: `Unknown checkpointId. ${CHECKPOINT_LIFETIMES[deps.transport]}`, }; } - if (checkpoint.project.id !== document.project.id) { + if (checkpoint.document.project.id !== document.project.id) { return { ok: false, - resultJson: `The edit was NOT applied: this checkpoint belongs to another project than the one this call acts on. Pass projectId "${checkpoint.project.id}".`, + resultJson: `The edit was NOT applied: this checkpoint belongs to another project than the one this call acts on. Pass projectId "${checkpoint.document.project.id}".`, }; } return { ok: true, resultJson: JSON.stringify({ ok: true, restored: id }), - document: checkpoint, + document: checkpoint.document, + sidecars: [...checkpoint.sidecars].map(([path, content]) => ({ path, content })), }; } function hasCheckpoint(projectId: string): boolean { for (const checkpoint of checkpoints.values()) { - if (checkpoint.project.id === projectId) return true; + if (checkpoint.document.project.id === projectId) return true; } return false; } + /** + * Writes an edit's sidecars, first keeping what each held for the project's checkpoints. + * Runs once the document is saved, so a failure here is a partial edit, said as one. + */ + async function writeSidecars( + projectId: string, + writes: readonly SidecarWrite[] = [], + ): Promise { + try { + for (const { path, content } of writes) { + const before = await readSidecar(path); + for (const checkpoint of checkpoints.values()) { + if (checkpoint.document.project.id === projectId && !checkpoint.sidecars.has(path)) { + checkpoint.sidecars.set(path, before); + } + } + if (content === null) await fs.rm(path, { force: true }); + else await fs.writeFile(path, content, "utf8"); + } + return null; + } catch (error) { + return textResult( + `The project was saved, but the pointer track beside its video was NOT written: ${errorMessage(error)}. Call this tool again to write it, or restoreCheckpoint to take the project back.`, + true, + ); + } + } + async function listProjects(): Promise { const [projects, snapshot, openElsewhereId] = await Promise.all([ deps.projects.listProjects(), @@ -273,15 +417,18 @@ export function createToolRunner(deps: McpToolDeps) { snapshot: McpDocumentSnapshot, name: string, args: unknown, + edit: RunnerEdit | undefined, ): Promise { const parsed = documentSchema.safeParse(snapshot.document); if (!parsed.success) { return textResult("The project open in the editor could not be read.", true); } - const execution = await execute(parsed.data, name, args); + const execution = await execute(parsed.data, name, args, edit); if (execution.document) { const applied = await deps.host.apply(execution.document, snapshot.revision); if (applied !== "applied") return textResult(APPLY_FAILURE_MESSAGES[applied], true); + const partial = await writeSidecars(execution.document.project.id, execution.sidecars); + if (partial) return partial; } return textResult(execution.resultJson, !execution.ok); } @@ -292,8 +439,9 @@ export function createToolRunner(deps: McpToolDeps) { read: ProjectRead, name: string, args: unknown, + edit: RunnerEdit | undefined, ): Promise { - const execution = await execute(read.document, name, args); + const execution = await execute(read.document, name, args, edit); if (execution.document) { // An editor this server cannot reach (the app, beside `openscreen mcp`) holds // it and would save over this edit. Checked again just before the write. @@ -302,7 +450,7 @@ export function createToolRunner(deps: McpToolDeps) { } // A restore IS the way back, so it needs no checkpoint of its own. if (name !== "restoreCheckpoint" && !hasCheckpoint(projectId)) { - return textResult(CHECKPOINT_REQUIRED_MESSAGE, true); + return textResult(CHECKPOINT_REQUIRED_MESSAGES[deps.transport], true); } // Refused if anything read the project since (the editor opening it would // otherwise drop this edit on its next save) or saved it, inside the app or @@ -310,6 +458,9 @@ export function createToolRunner(deps: McpToolDeps) { if (!(await deps.projects.saveProjectIfUnchanged(execution.document, read.version))) { return textResult(PROJECT_CHANGED_MESSAGE, true); } + // After the save, so an edit refused above has written nothing at all. + const partial = await writeSidecars(projectId, execution.sidecars); + if (partial) return partial; } return textResult(execution.resultJson, !execution.ok); } @@ -318,9 +469,15 @@ export function createToolRunner(deps: McpToolDeps) { document: AxcutDocument, name: string, args: unknown, - ): Promise { + edit: RunnerEdit | undefined, + ): Promise { if (name === "createCheckpoint") return createCheckpoint(document); if (name === "restoreCheckpoint") return restoreCheckpoint(document, args); + // This server's own switch, so the refusal names it rather than the in-app agent's. + if ((edit || isMutatingTool(name)) && !deps.editsAllowed()) { + return consentRequired(name, args, MCP_EDITS_SETTING); + } + if (edit) return edit(document); const availableByAssetId = await probeCursorTelemetry(document, deps.cursor); return runDocumentTool(document, name, args, deps.editsAllowed(), { cursor: deps.cursor, @@ -328,12 +485,14 @@ export function createToolRunner(deps: McpToolDeps) { }); } - async function run(name: string, rawArgs: unknown): Promise { + async function run(name: string, rawArgs: unknown, edit?: RunnerEdit): Promise { if (name === LIST_PROJECTS_TOOL) return listProjects(); const { projectId, args } = splitProjectId(rawArgs); if (projectId === undefined) { const snapshot = await deps.host.snapshot(); - return snapshot ? runOnEditor(snapshot, name, args) : textResult(NO_PROJECT_MESSAGE, true); + return snapshot + ? runOnEditor(snapshot, name, args, edit) + : textResult(NO_PROJECT_MESSAGES[deps.transport], true); } // The file first, the editor second. Reads of a project queue behind each other, // so an editor open of it already under way has returned by now, and the editor @@ -345,7 +504,7 @@ export function createToolRunner(deps: McpToolDeps) { }); const snapshot = await deps.host.snapshot(); if (snapshot && openProjectId(snapshot) === projectId) { - return runOnEditor(snapshot, name, args); + return runOnEditor(snapshot, name, args, edit); } if (!read) { return textResult( @@ -353,13 +512,14 @@ export function createToolRunner(deps: McpToolDeps) { true, ); } - return runOnFile(projectId, read, name, args); + return runOnFile(projectId, read, name, args, edit); } - return (name: string, args: unknown): Promise => { + /** Runs one call; `edit`, when given, computes the call's change in place of the executor. */ + return (name: string, args: unknown, edit?: RunnerEdit): Promise => { const next = queue.then( - () => run(name, args), - () => run(name, args), + () => run(name, args, edit), + () => run(name, args, edit), ); queue = next.catch(() => undefined); return next.catch((error) => textResult(`Tool failed: ${errorMessage(error)}`, true)); @@ -368,14 +528,21 @@ export function createToolRunner(deps: McpToolDeps) { export function createOpenScreenMcpServer( deps: McpToolDeps, - runTool: (name: string, args: unknown) => Promise, - /** What the client reads before the editing guidance; the stdio server says what it adds. */ - preamble = MCP_PREAMBLE, + runTool: McpToolRunner, + /** The media tools' preamble, when the caller registers them on this server. */ + media?: string, ): McpServer { const server = new McpServer( { name: MCP_SERVER_NAME, version: deps.version }, - { instructions: preamble + buildSystemPrompt({ editsAllowed: deps.editsAllowed() }) }, + { + instructions: mcpInstructions({ + transport: deps.transport, + editsAllowed: deps.editsAllowed(), + media, + }), + }, ); + const projectIdArg = z.string().optional().describe(PROJECT_ID_DESCRIPTIONS[deps.transport]); server.registerTool( LIST_PROJECTS_TOOL, { @@ -391,7 +558,7 @@ export function createOpenScreenMcpServer( name, { description: TOOL_DESCRIPTIONS[name], - inputSchema: schema.extend({ projectId: PROJECT_ID_ARG }), + inputSchema: schema.extend({ projectId: projectIdArg }), annotations: { readOnlyHint: !mutating, destructiveHint: DESTRUCTIVE_TOOLS.has(name), @@ -401,12 +568,12 @@ export function createOpenScreenMcpServer( (args: unknown) => runTool(name, args), ); } - for (const tool of CHECKPOINT_TOOLS) { + for (const tool of checkpointTools(deps.transport, media !== undefined)) { server.registerTool( tool.name, { description: tool.description, - inputSchema: tool.inputSchema.extend({ projectId: PROJECT_ID_ARG }), + inputSchema: tool.inputSchema.extend({ projectId: projectIdArg }), annotations: { readOnlyHint: !tool.mutating, destructiveHint: tool.mutating, diff --git a/electron/mcp/stdio-server.ts b/electron/mcp/stdio-server.ts index 7b7701926..0393d5f9b 100644 --- a/electron/mcp/stdio-server.ts +++ b/electron/mcp/stdio-server.ts @@ -26,7 +26,11 @@ export async function serveMcpOverStdio(options: { /** The client went away: nothing is left to serve. */ onClose(): void; }): Promise { - const deps: McpToolDeps = { ...options.mcpToolDeps, editsAllowed: () => true }; + const deps: McpToolDeps = { + ...options.mcpToolDeps, + transport: "stdio", + editsAllowed: () => true, + }; const runTool = createToolRunner(deps); const server = createOpenScreenMcpServer(deps, runTool, MEDIA_PREAMBLE); registerMediaTools(server, { diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.md index 6c4b59f07..94362b8f5 100644 --- a/skills/openscreen-demo/SKILL.md +++ b/skills/openscreen-demo/SKILL.md @@ -82,7 +82,8 @@ Use the document tools, always with the `projectId`. Read the project first with - **Cut dead time:** the setup at the start, the stop at the end, any wait. Use `addTrims`, or `addSpeed` to fast-forward a wait instead of cutting it. - **Zoom on the action:** clicks close in time and on screen share one zoom, and the first 2.5 s stay wide. Check them, then add or move zooms with `addZooms` and `setZoom` where a result deserves its own. Hold a zoom at least 1.5 s. - **Callouts:** one `addAnnotation` per beat, short, placed clear of the action. Text sits on the output frame and stays put when the picture zooms. Use its look for hierarchy: a large title with `animation: "typewriter"`, step callouts with `animation: "pop"`. `setAnnotation` moves or restyles one. -- **Undo:** `restoreCheckpoint` with the `checkpointId` takes the project back to the raw take in one step. +- **Undo:** `restoreCheckpoint` with the `checkpointId` takes the project back to the raw take in one step. It also puts back the take's own pointer track, undoing any `setPointerTrack` made since. +- **Open in the app:** while the user has this project open in the OpenScreen editor, every edit is refused, since the editor would save its own copy over yours. Reads still work. Ask them to close the project there, then retry. Keep the rest as it is: the background, padding, shadow and cursor style are chosen to look good. diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index d85724650..011728781 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "مستوى الاستدلال", "apiKeyLabel": "مفتاح API", "apiKeyHintStored": "محفوظ في safeStorage. اتركه فارغًا للاحتفاظ بالإدخال الحالي.", - "projectEditsLabel": "تعديلات المشروع", + "projectEditsLabel": "تعديلات الوكيل", "projectEditsHint": "عند الإيقاف، يجب أن يستأذن الوكيل قبل تغيير المخطط الزمني. يمكن دائمًا التراجع عن التعديلات.", "allowAgentEdits": "السماح للوكيل بتعديل المشروع", "disconnect": "قطع الاتصال", @@ -413,7 +413,7 @@ "description": "اسمح لعملاء MCP الذين تشغّلهم بنفسك، مثل Claude Code أو Codex، باستخدام أدوات التحرير بالذكاء الاصطناعي على أي من مشاريعك، لا على المشروع المفتوح في المحرر فقط. يسجّلون الدخول بحساباتهم الخاصة، ولا يطّلع OpenScreen عليها أبدًا.", "enableLabel": "الخادم", "enable": "تفعيل خادم MCP", - "editsLabel": "تعديلات المشروع", + "editsLabel": "تعديلات MCP", "allowEdits": "السماح لعملاء MCP بتعديل مشاريعك", "editsHint": "معطّل افتراضيًا: يمكن للعملاء قراءة مشاريعك، وتُرفض كل محاولة تعديل. عند التفعيل، يُحفظ كل تعديل فور وصوله، ويمكن التراجع عن تعديل المشروع المفتوح في المحرر من هناك.", "statusRunning": "يعمل", diff --git a/src/i18n/locales/cs/editor.json b/src/i18n/locales/cs/editor.json index 8424b0985..20d7c47e7 100644 --- a/src/i18n/locales/cs/editor.json +++ b/src/i18n/locales/cs/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Úsilí uvažování", "apiKeyLabel": "API klíč", "apiKeyHintStored": "Uloženo v safeStorage. Ponechte prázdné pro zachování stávajícího záznamu.", - "projectEditsLabel": "Úpravy projektu", + "projectEditsLabel": "Úpravy agentem", "projectEditsHint": "Pokud je vypnuto, agent musí požádat před změnou časové osy. Úpravy lze vždy vrátit zpět.", "allowAgentEdits": "Povolit agentovi úpravy projektu", "disconnect": "Odpojit", @@ -413,7 +413,7 @@ "description": "Umožněte MCP klientům, které sami spouštíte, například Claude Code nebo Codex, používat nástroje pro úpravy s AI na kterémkoli z vašich projektů, nejen na tom otevřeném v editoru. Přihlašují se vlastními účty; OpenScreen je nikdy nevidí.", "enableLabel": "Server", "enable": "Zapnout MCP server", - "editsLabel": "Úpravy projektu", + "editsLabel": "Úpravy přes MCP", "allowEdits": "Povolit MCP klientům úpravy vašich projektů", "editsHint": "Ve výchozím stavu vypnuto: klienti mohou vaše projekty číst a každá úprava, o kterou se pokusí, je odmítnuta. Po zapnutí se každá úprava hned uloží; úpravu projektu otevřeného v editoru lze tam vrátit zpět.", "statusRunning": "Běží", diff --git a/src/i18n/locales/de/editor.json b/src/i18n/locales/de/editor.json index 24d35321d..b5859a41d 100644 --- a/src/i18n/locales/de/editor.json +++ b/src/i18n/locales/de/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Denkaufwand", "apiKeyLabel": "API-Schlüssel", "apiKeyHintStored": "Wird in safeStorage gespeichert. Leer lassen, um den vorhandenen Eintrag zu behalten.", - "projectEditsLabel": "Projektänderungen", + "projectEditsLabel": "Agenten-Änderungen", "projectEditsHint": "Wenn aus, muss der Agent fragen, bevor er die Timeline ändert. Änderungen lassen sich immer rückgängig machen.", "allowAgentEdits": "Dem Agenten erlauben, das Projekt zu bearbeiten", "disconnect": "Trennen", @@ -413,7 +413,7 @@ "description": "Erlaube MCP-Clients, die du selbst ausführst, etwa Claude Code oder Codex, die KI-Bearbeitungswerkzeuge für jedes deiner Projekte zu nutzen, nicht nur für das im Editor geöffnete. Sie melden sich mit ihren eigenen Konten an; OpenScreen sieht diese nie.", "enableLabel": "Server", "enable": "MCP-Server aktivieren", - "editsLabel": "Projektänderungen", + "editsLabel": "MCP-Änderungen", "allowEdits": "MCP-Clients erlauben, deine Projekte zu bearbeiten", "editsHint": "Standardmäßig aus: Clients können deine Projekte lesen, und jede Bearbeitung wird abgelehnt. Wenn aktiviert, wird jede Bearbeitung sofort gespeichert; eine Bearbeitung des im Editor geöffneten Projekts lässt sich dort rückgängig machen.", "statusRunning": "Läuft", diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index 44e256d2e..5113242b8 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Reasoning effort", "apiKeyLabel": "API key", "apiKeyHintStored": "Stored securely on this computer. Leave blank to keep the current key.", - "projectEditsLabel": "Project edits", + "projectEditsLabel": "Agent edits", "projectEditsHint": "When off, the agent must ask before changing the timeline. Edits are always undoable.", "allowAgentEdits": "Allow the agent to edit the project", "disconnect": "Disconnect", @@ -413,7 +413,7 @@ "description": "Let MCP clients you run yourself, such as Claude Code or Codex, use the AI editing tools on any of your projects, not only the one open in the editor. They sign in with their own accounts; OpenScreen never sees them.", "enableLabel": "Server", "enable": "Enable MCP server", - "editsLabel": "Project edits", + "editsLabel": "MCP edits", "allowEdits": "Allow MCP clients to edit your projects", "editsHint": "Off by default: clients can read your projects, and every edit they try is refused. When on, each edit is saved as it lands; an edit to the project open in the editor can be undone there.", "statusRunning": "Running", diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index dd3ea0821..05da13ca0 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Esfuerzo de razonamiento", "apiKeyLabel": "Clave API", "apiKeyHintStored": "Guardada en safeStorage. Déjalo en blanco para conservar la entrada existente.", - "projectEditsLabel": "Ediciones del proyecto", + "projectEditsLabel": "Ediciones del agente", "projectEditsHint": "Si está desactivado, el agente debe preguntar antes de cambiar la línea de tiempo. Las ediciones siempre se pueden deshacer.", "allowAgentEdits": "Permitir que el agente edite el proyecto", "disconnect": "Desconectar", @@ -413,7 +413,7 @@ "description": "Permite que los clientes MCP que ejecutas tú mismo, como Claude Code o Codex, usen las herramientas de edición con IA en cualquiera de tus proyectos, no solo en el abierto en el editor. Inician sesión con sus propias cuentas; OpenScreen nunca las ve.", "enableLabel": "Servidor", "enable": "Activar servidor MCP", - "editsLabel": "Ediciones del proyecto", + "editsLabel": "Ediciones por MCP", "allowEdits": "Permitir que los clientes MCP editen tus proyectos", "editsHint": "Desactivado por defecto: los clientes pueden leer tus proyectos y se rechaza cada edición que intenten. Si se activa, cada edición se guarda al llegar; una edición del proyecto abierto en el editor se puede deshacer allí.", "statusRunning": "En ejecución", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 7344c6c40..d242f893f 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Effort de raisonnement", "apiKeyLabel": "Clé API", "apiKeyHintStored": "Stockée dans safeStorage. Laissez vide pour conserver l'entrée existante.", - "projectEditsLabel": "Modifications du projet", + "projectEditsLabel": "Modifications par l'agent", "projectEditsHint": "Si désactivé, l'agent doit demander avant de modifier la timeline. Les modifications sont toujours annulables.", "allowAgentEdits": "Autoriser l'agent à modifier le projet", "disconnect": "Déconnecter", @@ -413,7 +413,7 @@ "description": "Permettez aux clients MCP que vous lancez vous-même, comme Claude Code ou Codex, d’utiliser les outils d’édition IA sur n’importe lequel de vos projets, pas seulement celui ouvert dans l’éditeur. Ils se connectent avec leurs propres comptes ; OpenScreen ne les voit jamais.", "enableLabel": "Serveur", "enable": "Activer le serveur MCP", - "editsLabel": "Modifications du projet", + "editsLabel": "Modifications par MCP", "allowEdits": "Autoriser les clients MCP à modifier vos projets", "editsHint": "Désactivé par défaut : les clients peuvent lire vos projets, et chaque modification tentée est refusée. Une fois activé, chaque modification est enregistrée dès son arrivée ; une modification du projet ouvert dans l’éditeur peut y être annulée.", "statusRunning": "En cours", diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index e648dc9aa..c7a25b8e3 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Sforzo di ragionamento", "apiKeyLabel": "Chiave API", "apiKeyHintStored": "Salvata in safeStorage. Lascia vuoto per mantenere la voce esistente.", - "projectEditsLabel": "Modifiche al progetto", + "projectEditsLabel": "Modifiche dell'agente", "projectEditsHint": "Se disattivato, l'agente deve chiedere prima di cambiare la timeline. Le modifiche sono sempre annullabili.", "allowAgentEdits": "Consenti all'agente di modificare il progetto", "disconnect": "Disconnetti", @@ -413,7 +413,7 @@ "description": "Consenti ai client MCP che esegui tu stesso, come Claude Code o Codex, di usare gli strumenti di modifica con IA su qualsiasi tuo progetto, non solo su quello aperto nell’editor. Accedono con i propri account; OpenScreen non li vede mai.", "enableLabel": "Server", "enable": "Attiva server MCP", - "editsLabel": "Modifiche al progetto", + "editsLabel": "Modifiche via MCP", "allowEdits": "Consenti ai client MCP di modificare i tuoi progetti", "editsHint": "Disattivato per impostazione predefinita: i client possono leggere i tuoi progetti e ogni modifica tentata viene rifiutata. Se attivo, ogni modifica viene salvata appena arriva; una modifica al progetto aperto nell’editor si può annullare lì.", "statusRunning": "In esecuzione", diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index e55e90cec..1ec988af8 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "推論の深さ", "apiKeyLabel": "API キー", "apiKeyHintStored": "safeStorage に保存されています。空欄のままにすると既存の値を保持します。", - "projectEditsLabel": "プロジェクトの編集", + "projectEditsLabel": "エージェントによる編集", "projectEditsHint": "オフの場合、エージェントはタイムラインを変更する前に確認します。編集はいつでも元に戻せます。", "allowAgentEdits": "エージェントによるプロジェクトの編集を許可", "disconnect": "接続を解除", @@ -413,7 +413,7 @@ "description": "Claude Code や Codex など、ご自身で実行する MCP クライアントが、エディターで開いているプロジェクトだけでなく、すべてのプロジェクトに AI 編集ツールを使えるようにします。クライアントは各自のアカウントでサインインし、OpenScreen がそれを見ることはありません。", "enableLabel": "サーバー", "enable": "MCP サーバーを有効にする", - "editsLabel": "プロジェクトの編集", + "editsLabel": "MCP による編集", "allowEdits": "MCP クライアントによるプロジェクトの編集を許可", "editsHint": "既定ではオフです。クライアントはプロジェクトを読み取れますが、編集はすべて拒否されます。オンにすると、各編集は届いた時点で保存されます。エディターで開いているプロジェクトへの編集は、エディターで元に戻せます。", "statusRunning": "実行中", diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index 7865a0e7b..fbd59e7ca 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "추론 강도", "apiKeyLabel": "API 키", "apiKeyHintStored": "safeStorage에 저장되어 있습니다. 기존 항목을 유지하려면 비워 두세요.", - "projectEditsLabel": "프로젝트 편집", + "projectEditsLabel": "에이전트 편집", "projectEditsHint": "끄면 에이전트가 타임라인을 변경하기 전에 확인을 요청합니다. 편집은 항상 실행 취소할 수 있습니다.", "allowAgentEdits": "에이전트의 프로젝트 편집 허용", "disconnect": "연결 해제", @@ -413,7 +413,7 @@ "description": "Claude Code나 Codex처럼 직접 실행하는 MCP 클라이언트가 편집기에 열린 프로젝트뿐 아니라 모든 프로젝트에서 AI 편집 도구를 사용할 수 있게 합니다. 클라이언트는 자체 계정으로 로그인하며, OpenScreen은 이를 절대 보지 않습니다.", "enableLabel": "서버", "enable": "MCP 서버 사용", - "editsLabel": "프로젝트 편집", + "editsLabel": "MCP 편집", "allowEdits": "MCP 클라이언트의 프로젝트 편집 허용", "editsHint": "기본값은 꺼짐입니다. 클라이언트는 프로젝트를 읽을 수 있지만 모든 편집 시도는 거부됩니다. 켜면 각 편집이 도착하는 즉시 저장되며, 편집기에 열린 프로젝트의 편집은 편집기에서 실행 취소할 수 있습니다.", "statusRunning": "실행 중", diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 3724a1aa3..39af8ad00 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Esforço de raciocínio", "apiKeyLabel": "Chave de API", "apiKeyHintStored": "Armazenada no safeStorage. Deixe em branco para manter a entrada existente.", - "projectEditsLabel": "Edições do projeto", + "projectEditsLabel": "Edições do agente", "projectEditsHint": "Quando desativado, o agente precisa perguntar antes de alterar a linha do tempo. As edições sempre podem ser desfeitas.", "allowAgentEdits": "Permitir que o agente edite o projeto", "disconnect": "Desconectar", @@ -413,7 +413,7 @@ "description": "Permita que clientes MCP que você mesmo executa, como Claude Code ou Codex, usem as ferramentas de edição com IA em qualquer um dos seus projetos, não só no aberto no editor. Eles entram com as próprias contas; o OpenScreen nunca as vê.", "enableLabel": "Servidor", "enable": "Ativar servidor MCP", - "editsLabel": "Edições do projeto", + "editsLabel": "Edições via MCP", "allowEdits": "Permitir que clientes MCP editem seus projetos", "editsHint": "Desativado por padrão: os clientes podem ler seus projetos, e toda edição que tentarem é recusada. Quando ativado, cada edição é salva assim que chega; uma edição do projeto aberto no editor pode ser desfeita lá.", "statusRunning": "Em execução", diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index 6fac2c415..85ddc1908 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Глубина рассуждений", "apiKeyLabel": "API-ключ", "apiKeyHintStored": "Хранится в safeStorage. Оставьте пустым, чтобы сохранить текущее значение.", - "projectEditsLabel": "Правки проекта", + "projectEditsLabel": "Правки агента", "projectEditsHint": "Если выключено, агент должен спрашивать перед изменением таймлайна. Правки всегда можно отменить.", "allowAgentEdits": "Разрешить агенту редактировать проект", "disconnect": "Отключить", @@ -413,7 +413,7 @@ "description": "Разрешите MCP-клиентам, которые вы запускаете сами, например Claude Code или Codex, использовать инструменты ИИ-редактирования для любого вашего проекта, а не только для открытого в редакторе. Они входят со своими учётными записями; OpenScreen их никогда не видит.", "enableLabel": "Сервер", "enable": "Включить MCP-сервер", - "editsLabel": "Правки проекта", + "editsLabel": "Правки через MCP", "allowEdits": "Разрешить MCP-клиентам редактировать ваши проекты", "editsHint": "По умолчанию выключено: клиенты могут читать ваши проекты, а любая попытка правки отклоняется. Если включено, каждая правка сохраняется сразу; правку проекта, открытого в редакторе, можно отменить там.", "statusRunning": "Работает", diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index 74e5db229..ac764226e 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Akıl yürütme düzeyi", "apiKeyLabel": "API anahtarı", "apiKeyHintStored": "safeStorage'da saklanıyor. Mevcut kaydı korumak için boş bırakın.", - "projectEditsLabel": "Proje düzenlemeleri", + "projectEditsLabel": "Aracı düzenlemeleri", "projectEditsHint": "Kapalıyken, aracı zaman çizelgesini değiştirmeden önce izin ister. Düzenlemeler her zaman geri alınabilir.", "allowAgentEdits": "Aracının projeyi düzenlemesine izin ver", "disconnect": "Bağlantıyı kes", @@ -413,7 +413,7 @@ "description": "Claude Code veya Codex gibi kendi çalıştırdığınız MCP istemcilerinin, yalnızca düzenleyicide açık olan projede değil, projelerinizin herhangi birinde yapay zekâ düzenleme araçlarını kullanmasına izin verin. Kendi hesaplarıyla oturum açarlar; OpenScreen bunları hiçbir zaman görmez.", "enableLabel": "Sunucu", "enable": "MCP sunucusunu etkinleştir", - "editsLabel": "Proje düzenlemeleri", + "editsLabel": "MCP düzenlemeleri", "allowEdits": "MCP istemcilerinin projelerinizi düzenlemesine izin ver", "editsHint": "Varsayılan olarak kapalı: istemciler projelerinizi okuyabilir ve denedikleri her düzenleme reddedilir. Açıkken her düzenleme geldiği anda kaydedilir; düzenleyicide açık olan projedeki bir düzenleme orada geri alınabilir.", "statusRunning": "Çalışıyor", diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 340886e6f..b99882eee 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "Mức độ suy luận", "apiKeyLabel": "Khóa API", "apiKeyHintStored": "Được lưu trong safeStorage. Để trống để giữ mục hiện có.", - "projectEditsLabel": "Chỉnh sửa dự án", + "projectEditsLabel": "Chỉnh sửa bởi tác nhân", "projectEditsHint": "Khi tắt, tác nhân phải hỏi trước khi thay đổi dòng thời gian. Mọi chỉnh sửa đều có thể hoàn tác.", "allowAgentEdits": "Cho phép tác nhân chỉnh sửa dự án", "disconnect": "Ngắt kết nối", @@ -413,7 +413,7 @@ "description": "Cho phép các ứng dụng khách MCP do bạn tự chạy, như Claude Code hoặc Codex, dùng công cụ chỉnh sửa bằng AI trên bất kỳ dự án nào của bạn, không chỉ dự án đang mở trong trình chỉnh sửa. Chúng đăng nhập bằng tài khoản riêng; OpenScreen không bao giờ thấy các tài khoản đó.", "enableLabel": "Máy chủ", "enable": "Bật máy chủ MCP", - "editsLabel": "Chỉnh sửa dự án", + "editsLabel": "Chỉnh sửa qua MCP", "allowEdits": "Cho phép ứng dụng khách MCP chỉnh sửa các dự án của bạn", "editsHint": "Tắt theo mặc định: ứng dụng khách có thể đọc các dự án của bạn và mọi thao tác chỉnh sửa đều bị từ chối. Khi bật, mỗi chỉnh sửa được lưu ngay khi đến; chỉnh sửa trên dự án đang mở trong trình chỉnh sửa có thể được hoàn tác tại đó.", "statusRunning": "Đang chạy", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 9e7be62fd..b02212816 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "推理强度", "apiKeyLabel": "API 密钥", "apiKeyHintStored": "已存储在 safeStorage 中。留空可保留现有条目。", - "projectEditsLabel": "项目编辑", + "projectEditsLabel": "智能体编辑", "projectEditsHint": "关闭时，智能体在更改时间轴前必须先询问。所有编辑始终可以撤销。", "allowAgentEdits": "允许智能体编辑项目", "disconnect": "断开连接", @@ -413,7 +413,7 @@ "description": "让你自己运行的 MCP 客户端（如 Claude Code 或 Codex）在你的任意项目上使用 AI 编辑工具，而不只是编辑器中打开的项目。它们使用各自的账号登录，OpenScreen 永远不会看到这些账号。", "enableLabel": "服务器", "enable": "启用 MCP 服务器", - "editsLabel": "项目编辑", + "editsLabel": "MCP 编辑", "allowEdits": "允许 MCP 客户端编辑你的项目", "editsHint": "默认关闭：客户端可以读取你的项目，但所有编辑都会被拒绝。开启后，每次编辑都会立即保存；对编辑器中打开的项目所做的编辑可以在编辑器中撤销。", "statusRunning": "运行中", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index e9c482d04..c7e9c3ad1 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -400,7 +400,7 @@ "reasoningEffortLabel": "推理強度", "apiKeyLabel": "API 金鑰", "apiKeyHintStored": "已儲存在 safeStorage 中。留空可保留現有項目。", - "projectEditsLabel": "專案編輯", + "projectEditsLabel": "代理編輯", "projectEditsHint": "關閉時，代理在變更時間軸前必須先詢問。所有編輯都可以復原。", "allowAgentEdits": "允許代理編輯專案", "disconnect": "中斷連線", @@ -413,7 +413,7 @@ "description": "讓你自行執行的 MCP 用戶端（例如 Claude Code 或 Codex）在你的任何專案上使用 AI 編輯工具，而不只是編輯器中開啟的專案。它們使用各自的帳號登入，OpenScreen 永遠不會看到這些帳號。", "enableLabel": "伺服器", "enable": "啟用 MCP 伺服器", - "editsLabel": "專案編輯", + "editsLabel": "MCP 編輯", "allowEdits": "允許 MCP 用戶端編輯你的專案", "editsHint": "預設關閉：用戶端可以讀取你的專案，但所有編輯都會被拒絕。開啟後，每次編輯都會立即儲存；對編輯器中開啟的專案所做的編輯可以在編輯器中復原。", "statusRunning": "執行中", diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md index 5cebdfa35..1a97895e0 100644 --- a/technical-documentation/architecture/mcp-server.md +++ b/technical-documentation/architecture/mcp-server.md @@ -1,42 +1,70 @@ # MCP server -OpenScreen can offer the in-app agent's tools to MCP clients the user runs themselves — Claude Code, Codex, Cursor, anything that speaks MCP over Streamable HTTP — on any of the user's projects, not only the one open in the editor. The client brings its own model and its own sign-in; OpenScreen never sees, stores or relays those credentials. It is off by default and turned on in **Settings → AI → MCP server**. +OpenScreen offers the in-app agent's tools to MCP clients the user runs themselves (Claude Code, Codex, Cursor, anything that speaks MCP), on any of the user's projects, not only the one open in the editor. The client brings its own model and its own sign-in; OpenScreen never sees, stores or relays those credentials. + +The same tools are served over two transports, by two different processes: + +| | In-app server (HTTP) | `openscreen mcp` (stdio) | +|---|---|---| +| Process | The running app (the GUI) | A process of its own, started by the client: the app's executable with `mcp` | +| Setup | **AI settings → MCP server**: turn it on, copy the `claude mcp add` / `codex mcp add` command (it carries the token) | `claude mcp add openscreen -- mcp` ([executables](../../website/docs/cli.md#running-the-cli)) | +| Auth | Bearer token, `127.0.0.1` only, Host/Origin check | None: the client launched the process with the user's rights | +| Edits | Off until the user turns on **MCP edits** | Always allowed | +| Project open in the editor | Edited live through the editor, one undo step per call | Read only: an edit to it is refused ([the marker](#the-open-in-editor-marker)) | +| Other projects | Read from and saved to their file, a checkpoint first | Same | +| Tools | `listProjects`, the agent's tools, the checkpoint tools | The same, plus the media tools: `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` | +| Checkpoints | In memory, lost when the app quits or the server restarts | In memory, lost when the process exits, which it does when the client disconnects | + +Neither server knows about the other. Two processes may therefore write the same project folder; the [open-in-editor marker](#the-open-in-editor-marker) and the byte compare of `saveProjectIfUnchanged` are what keep them from overwriting each other's work. | File | Role | |---|---| -| [`electron/mcp/openscreen-mcp-server.ts`](../../electron/mcp/openscreen-mcp-server.ts) | The MCP server and its local HTTP guard. Registers the tools, routes each call to the editor or a project file, runs it. | +| [`electron/mcp/openscreen-mcp-server.ts`](../../electron/mcp/openscreen-mcp-server.ts) | The tools both transports serve, the tool runner (routing, refusals, checkpoints), the texts per transport (`mcpInstructions`), and the local HTTP guard. | +| [`electron/mcp/stdio-server.ts`](../../electron/mcp/stdio-server.ts) | `openscreen mcp`: the same server and runner on stdin/stdout, with the media tools. | +| [`electron/mcp/media-tools.ts`](../../electron/mcp/media-tools.ts) | The media tools and their capability block (`MEDIA_PREAMBLE`). | | [`electron/mcp/editor-document-host.ts`](../../electron/mcp/editor-document-host.ts) | Main-process side of the live document: asks the editor window for a snapshot and hands edits back. | -| [`electron/mcp/mcp-controller.ts`](../../electron/mcp/mcp-controller.ts) | Lifecycle: start when enabled, restart on a port or token change, status for the settings UI. | +| [`electron/mcp/mcp-controller.ts`](../../electron/mcp/mcp-controller.ts) | In-app server lifecycle: start when enabled, restart on a port or token change, status for the settings UI. | | [`electron/mcp/mcp-settings-store.ts`](../../electron/mcp/mcp-settings-store.ts) | `mcp-server.json` (enabled, port, allowEdits) and `mcp-token.enc` (bearer token, `safeStorage`). | | [`src/lib/ai-edition/store/mcpDocumentHost.ts`](../../src/lib/ai-edition/store/mcpDocumentHost.ts) | Renderer side: answers snapshot / apply requests from the project store. Mounted by `NewEditorShell`. | -| [`src/components/ai-edition/McpServerSettings.tsx`](../../src/components/ai-edition/McpServerSettings.tsx) | The settings section: server toggle, its own Project edits toggle, port, token, copyable `claude mcp add` / `codex mcp add` commands. | +| [`src/components/ai-edition/McpServerSettings.tsx`](../../src/components/ai-edition/McpServerSettings.tsx) | The settings section: server toggle, the **MCP edits** toggle, port, token, copyable `claude mcp add` / `codex mcp add` commands. | ## One tool surface, two agents -Nothing about the tools is reimplemented. The server registers `TOOL_ARG_SCHEMAS` (names and zod schemas), `TOOL_DESCRIPTIONS`, and hands `buildSystemPrompt` to the client as the server `instructions` — all exported from [`deep-agent/service.ts`](../../electron/ai-edition/deep-agent/service.ts), where `buildTools` builds the in-app agent from the same table. Every call goes through `runDocumentTool`, the function the in-app agent's `documentTool` also calls: the cursor-telemetry read the zoom tools need, then `executeAgentTool`. +Nothing about the tools is reimplemented. The server registers `TOOL_ARG_SCHEMAS` (names and zod schemas) and `TOOL_DESCRIPTIONS`, and builds its `instructions` around `buildSystemPrompt`, all exported from [`deep-agent/service.ts`](../../electron/ai-edition/deep-agent/service.ts), where `buildTools` builds the in-app agent from the same table. Every call goes through `runDocumentTool`, the function the in-app agent's `documentTool` also calls: the cursor-telemetry read the zoom tools need, then `executeAgentTool`. So a tool added to the agent appears over MCP with no further work, and the MCP test asserts the listed tools equal `listProjects`, then `OPENSCREEN_TOOL_NAMES`, then the two checkpoint tools below. What the server adds: -- **`listProjects`**, a tool of its own: every project's id, title, `updatedAt`, asset count, and `open` for the one in the editor. +- **`listProjects`**, a tool of its own: every project's id, title, `updatedAt`, asset count, and `open` for the one in an editor (this process's, or the app's seen through the marker). - **An optional `projectId` on every agent tool and both checkpoint tools**, added to its schema with `.extend()` at registration and stripped again before the call reaches the executor, which never sees it. See [Which project a call acts on](#which-project-a-call-acts-on). - **`createCheckpoint` / `restoreCheckpoint`**, see [Checkpoints](#checkpoints). - MCP metadata: `readOnlyHint` for the reads (`!isMutatingTool`), `listProjects` and `createCheckpoint`, and `destructiveHint` for `replaceTimeline`, the three `remove*` tools and `restoreCheckpoint`. -**Writes are a second opt-in.** MCP clients have their own "Project edits" switch, `allowEdits` in `mcp-server.json`, **off by default** and independent of the in-app agent's `allowAgentEdits`. Turning the server on therefore grants read access only. The value is read on every call and passed to the executor as `editsAllowed`, the same gate the in-app agent's switch drives: while it is off every mutating tool is refused with the executor's consent message, and the consent block of the system prompt is in the instructions. It is independent because `allowAgentEdits` defaults to allowed and lives in the provider form, so a user with no provider configured could never have turned MCP writes off. +## What each transport tells its client + +Every text a client reads must be true on the server that serves it, so the texts are composed rather than shared verbatim. `McpToolDeps.transport` (`"http"` or `"stdio"`) selects them: + +- **Instructions** (`mcpInstructions`): the media tools' capability block when they are registered, then the projects-and-checkpoints rules of the transport, then how long checkpoints live there, then `buildSystemPrompt`. The checkpoint rule is in both. Over HTTP the rules speak of the editor and its undo stack and add "Nothing here records, exports or imports media"; over stdio they say a `projectId` is needed on every call, that `Ctrl/Cmd+Z` cannot undo an edit, and that a project open in the app is refused. +- **`buildSystemPrompt`** takes the transport's facts as options: `editsSetting` names the switch its consent block points to (the MCP server's **MCP edits**, not the in-app agent's **Agent edits**), and `canExport` drops "exporting" from its examples of a request with no tool when `exportVideo` is served. +- **Tool descriptions**: `projectId` is "omit it for the open project" over HTTP and "pass it on every call" over stdio; `restoreCheckpoint` mentions the undo step only over HTTP, and the pointer track only where `setPointerTrack` is served. +- **Messages**: no project to fall back on, a checkpoint required, an unknown checkpoint, edits turned off. Each has a version per transport, or exists only on the one that can produce it. + +The in-app chat agent keeps `buildSystemPrompt`'s defaults: its own switch, no export. + +**Writes on the in-app server are a second opt-in.** MCP clients have their own **MCP edits** switch, `allowEdits` in `mcp-server.json`, **off by default** and independent of the in-app agent's **Agent edits** (`allowAgentEdits`). Turning the server on therefore grants read access only. The value is read on every call; while it is off every mutating tool and `setPointerTrack` are refused with the executor's `consent_required` payload naming **MCP edits**, `restoreCheckpoint` with its own refusal text naming it too, and the consent block of the system prompt is in the instructions. It is independent because `allowAgentEdits` defaults to allowed and lives in the provider form, so a user with no provider configured could never have turned MCP writes off. Toggling it does not restart the server. `openscreen mcp` has no such switch: the client started it itself, and there is no other caller to keep out. ## Which project a call acts on -| `projectId` | Path | -|---|---| -| omitted | The project open in the editor, through the editor. With none open the call fails with "No project is open" and points at `listProjects`. | -| the open project's id | The same: through the editor, never its file. The file is read first all the same (see below). | -| any other id | That project's file, through the app's one `DocumentService` (`getProjectForUpdate` → tool → `saveProjectIfUnchanged`). Works with no editor window at all. A write needs a checkpoint of that project first. | +| `projectId` | In-app server | `openscreen mcp` | +|---|---|---| +| omitted | The project open in the editor, through the editor. With none open the call fails and points at `listProjects`. | Fails: there is no editor to fall back on. The message asks for a `projectId`. | +| the id of the project open in this process's editor | The same: through the editor, never its file. The file is read first all the same (see below). | Cannot happen: this process has no editor. | +| any other id | That project's file, through the process's one `DocumentService` (`getProjectForUpdate` → tool → `saveProjectIfUnchanged`). Works with no editor window at all. A write needs a checkpoint of that project first, and is refused while another process's editor holds it. | Same. | The open project is never written to disk from here because the editor holds it in memory and saves it as a whole: a file edit under it would be overwritten by the editor's next save, and the editor would never show it. ## Where the document comes from -### The open project +### The open project (in-app server) The in-app agent is handed a document snapshot per chat turn. An MCP client has no turn, so each call reads the **live** document from the editor window, which owns it (`useProjectStore`, with its `revision`): @@ -48,45 +76,70 @@ A user edit that lands between 1 and 3 moves the revision and the apply is refus Only the webContents that registered on `ai-edition.mcp-host` is asked, and only its replies count. An editor that unmounts or is destroyed stops being asked. A request unanswered for 30 s resolves to "no project" (reads) or `timeout` (writes, reported as "did not confirm", not as a failure, since the save may have landed). +`openscreen mcp` builds the same `EditorDocumentHost`, but no editor window ever exists in that process, so its snapshot is always `null`. + ### Any other project -Read with `DocumentService.getProjectForUpdate` (`getProject`, migrated and relinked like any open, plus a version) and, if the tool changed it, saved with `saveProjectIfUnchanged` — the same instance every other save in the app goes through. Reads and saves of one project share its queue, so a read returns only once every read and save asked for before it is done. +Read with `DocumentService.getProjectForUpdate` (`getProject`, migrated and relinked like any open, plus a version) and, if the tool changed it, saved with `saveProjectIfUnchanged` — the same instance every other save in the process goes through. Reads and saves of one project share its queue, so a read returns only once every read and save asked for before it is done. A call naming a project reads its file **before** asking the editor for a snapshot. An editor open of that project already under way has then returned, and the editor installs what it read before it answers, so the snapshot shows the project and the call goes through the editor. -There is no revision to guard the file with, so the save is refused with "NOT applied … re-read, then retry" when, since the call's read: +Before the save, the runner refuses, in this order and with nothing written: + +1. **Another process's editor holds the project** ([the marker](#the-open-in-editor-marker)). That editor would save over the edit. +2. **The project has no checkpoint** in this server's memory. An edit to a file is on no undo stack, so the checkpoint is its only way back ([Checkpoints](#checkpoints)). +3. **The project changed since the call read it.** There is no revision to guard a file with, so the save is refused with "NOT applied … re-read, then retry" when, since the read: + - **anything read the project.** This is the editor opening it: from that read on it holds a copy without the MCP edit, and its next save would drop the edit without a word. A snapshot cannot see this in time, since the editor installs what it read only after the read returns. `DocumentService` counts every read and save of a project instead. The client's retry, with the same `projectId`, then goes through the editor. The open-file dialog never saves back a project's own file: it loads that project by id (`storedProjectId`), a read like any other. + - **anything saved it, or its file changed.** The file must still be, byte for byte, the one the tool ran against. This also covers the other process, a writer outside the app (a sync tool, a restored copy) and a delete. + +The check runs inside the project's queue, so nothing in the process reads or saves the project between the check and the write. -1. **Anything read the project.** This is the editor opening it: from that read on it holds a copy without the MCP edit, and its next save would drop the edit without a word. A snapshot cannot see this in time, since the editor installs what it read only after the read returns. `DocumentService` counts every read and save of a project instead. The client's retry, with the same `projectId`, then goes through the editor. The open-file dialog never saves back a project's own file: it loads that project by id (`storedProjectId`), a read like any other. -2. **Anything saved it, or its file changed.** The file must still be, byte for byte, the one the tool ran against. This also covers a writer outside the app (a sync tool, a restored copy) and a delete. +### The open-in-editor marker -The check runs inside the project's queue, so nothing in the app reads or saves the project between the check and the write. +`openscreen mcp` cannot ask the app's editor anything, so the app says which project its editor holds: the editor sends the project id on `AI_EDITION_OPEN_PROJECT_CHANNEL`, and `DocumentService.markOpenInEditor` writes `.open-in-editor.json` (`{pid, projectId}`, temp file then rename) in the projects folder. Only the window that set it clears it, on closing the project or on being destroyed. `projectOpenInEditor()` reads it back and ignores a marker whose `pid` is no longer running, so an app that crashed blocks nothing. The runner checks it just before any write to a file; `listProjects` reports the project as `open`. -An edit made this way is saved, but it is not on any undo stack: the editor never held it. Its undo is a checkpoint, so the write is refused until the client has created one for that project (below). +An edit refused this way is answered with `ProjectOpenInEditorError`'s message: close the project in the app, or connect to the in-app server, which edits the open project through the editor. ## Checkpoints -A client chains several edits in one turn, and each is its own undo step, so reverting a turn by hand means one Ctrl+Z per call, interleaved with whatever the user did meanwhile. `createCheckpoint` saves the live document in the main process and returns a `checkpointId`; `restoreCheckpoint` applies that document back through the same revision-guarded apply, so the whole revert is **one undo step** the user can itself undo. The server instructions tell the client to checkpoint before a series of edits. +A client chains several edits in one turn, and each is its own undo step, so reverting a turn by hand means one Ctrl+Z per call, interleaved with whatever the user did meanwhile. `createCheckpoint` saves the document in the server's memory and returns a `checkpointId`; `restoreCheckpoint` hands that document back as the call's new document. On the project open in the in-app server's editor it goes through the same revision-guarded apply, so the whole revert is **one undo step** the user can itself undo. The server instructions tell the client to checkpoint before a series of edits. -- They take `projectId` like every tool, so a project that is not open has them too: the checkpoint is read from its file, and the restore is saved to it with the same guard as any edit. That restore is not one undo step, since nothing holds the project's history. -- **A project that is not open cannot be edited without one.** Ctrl+Z cannot reach an edit to it, so a write there is refused until a checkpoint of that project is in memory, and the client is told to call `createCheckpoint` first. The server instructions say so up front. -- A restore is a write: refused while MCP "Project edits" is off, and refused when the checkpoint belongs to another project than the one the call acts on. That message names the checkpoint's `projectId`: restoring a closed project's checkpoint takes it explicitly. +- They take `projectId` like every tool, so a project that is not open has them too: the checkpoint is read from its file, and the restore is saved to it with the same guards as any edit. That restore is not one undo step, since nothing holds the project's history. +- **A project that is not open cannot be edited without one.** Ctrl+Z cannot reach an edit to it, so a write there is refused until a checkpoint of that project is in memory, and the client is told to call `createCheckpoint` first. Over stdio that is every project. `stopRecording` creates one for each new take and returns its id. +- **They cover the cursor sidecar.** `setPointerTrack` rewrites `