From f034cd1098161309cbd9fe02394af487a7df9801 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:18:27 +0200 Subject: [PATCH 1/3] feat(mcp): rename a project, import and remove its media, create one setProjectInfo, importMedia and removeMedia run in the tool runner like any edit, so the project open in the editor changes there live, and openscreen mcp forwards them to the running app. createProject is a media tool, empty or from a video, and returns a checkpoint as stopRecording does. Deleting a project stays out on purpose. Part of #1180 --- electron/ai-edition/agent-tools.ts | 4 +- electron/ai-edition/deep-agent/service.ts | 2 +- electron/ai-edition/document-service.ts | 137 +++++++------- electron/ipc/handlers.ts | 7 + electron/mcp/app-channel.ts | 3 +- electron/mcp/media-tools.test.ts | 47 ++++- electron/mcp/media-tools.ts | 87 ++++++++- electron/mcp/openscreen-mcp-server.test.ts | 157 ++++++++++++++- electron/mcp/openscreen-mcp-server.ts | 54 +++++- electron/mcp/project-tools.ts | 210 +++++++++++++++++++++ 10 files changed, 624 insertions(+), 84 deletions(-) create mode 100644 electron/mcp/project-tools.ts diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 9e01e253b..4c73bc919 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -1235,7 +1235,7 @@ export function documentSnapshotForModel( timeBaseNote: "clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds: the editor's ruler, where a trim leaves a hole instead of shortening it and a speed region stretches nothing. The exported video's clock skips those holes and plays sped-up spans faster, so it differs from the ruler once there are any.", audioNote: - "audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio'; nothing here can import a file from disk or record one, so if the project has no audio asset, say so rather than inventing an id.", + "audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio': if the project has none, a file has to be imported first (importMedia, where it is offered), so never invent an id.", zoomNote: `renderedScale is what the viewer sees (depth is an ordinal, not a factor: ${ZOOM_DEPTH_LEGEND}). ` + "When a zoom carries customScale it wins over depth and depthIsOverridden is true — " + @@ -2960,7 +2960,7 @@ export function executeAgentTool( `Unknown asset: ${assetId}.` + (available.length ? ` Imported audio in this project: ${available.map((a) => `${a.id} (${a.label})`).join(", ")}.` - : " This project has no imported audio; a file can only be imported or recorded from the editor, not from here."), + : " This project has no imported audio: a file has to be imported into it first (importMedia, where it is offered)."), ); } if (asset.kind !== "audio") { diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 7b05c0444..1465b9ada 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -219,7 +219,7 @@ export const TOOL_DESCRIPTIONS: Record = { "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", setCameraFullscreen: "Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.", - addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`, + addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them. If there is none, a file has to be imported first (importMedia, where it is offered): never guess an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`, setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`, removeTrim: "Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.", diff --git a/electron/ai-edition/document-service.ts b/electron/ai-edition/document-service.ts index 8fea6d027..0d44274af 100644 --- a/electron/ai-edition/document-service.ts +++ b/electron/ai-edition/document-service.ts @@ -149,6 +149,75 @@ function isSupportedAudioPath(filePath: string): boolean { return SUPPORTED_AUDIO_EXTENSIONS.has(ext); } +/** Why `input` cannot be filed as its kind of asset, or `null` when it can. */ +export function unsupportedAssetPath(input: Pick): string | null { + if (!input.path) return "Asset path is required."; + if ((input.kind ?? "video") === "audio") { + return isSupportedAudioPath(input.path) + ? null + : `Unsupported audio extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_AUDIO_EXTENSIONS].join(", ")})`; + } + return isSupportedVideoPath(input.path) + ? null + : `Unsupported video extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_VIDEO_EXTENSIONS].join(", ")})`; +} + +/** The kind a media file is imported as, from its extension: audio only when it cannot be a video. */ +export function assetKindForPath(filePath: string): "video" | "audio" { + return !isSupportedVideoPath(filePath) && isSupportedAudioPath(filePath) ? "audio" : "video"; +} + +/** `doc` with `asset` added to its media, as an import adds it. */ +export function documentWithAsset(doc: AxcutDocument, asset: AxcutAsset): AxcutDocument { + // An audio import is an overlay, never the thing the timeline is built + // around, so it must not claim the empty primaryAssetId slot — otherwise the + // first file dropped into a fresh project (a BGM track) would become its + // primary asset and the editor would try to lay out clips from a file with + // no video. + const claimsPrimary = asset.kind !== "audio" && !doc.project.primaryAssetId; + return { + ...doc, + assets: [...doc.assets, asset], + project: { + ...doc.project, + ...(claimsPrimary ? { primaryAssetId: asset.id } : {}), + updatedAt: new Date().toISOString(), + }, + }; +} + +/** `doc` without the asset, its clips, trims and audio tracks. */ +export function documentWithoutAsset(doc: AxcutDocument, assetId: string): AxcutDocument { + const assets = doc.assets.filter((a) => a.id !== assetId); + // Primary is the thing the timeline is built around, so it must fall to the + // next VIDEO asset — never an audio overlay (issue #350), which can't be + // primary (see addAsset). Falling back to `assets[0]` would hand primary to + // an audio asset when the removed one was the last video. + const primaryAssetId = + doc.project.primaryAssetId === assetId + ? (assets.find((a) => a.kind !== "audio")?.id ?? undefined) + : doc.project.primaryAssetId; + const withoutAssetClips = doc.timeline.clips + .filter((clip) => clip.assetId === assetId) + .reduce((current, clip) => removeClip(current, clip.id), doc); + return { + ...withoutAssetClips, + assets, + // Drop imported audio tracks that referenced the removed asset — they + // would otherwise dangle, pointing at an asset the document no longer has. + audioTracks: withoutAssetClips.audioTracks.filter((t) => t.assetId !== assetId), + timeline: { + ...withoutAssetClips.timeline, + trimRanges: withoutAssetClips.timeline.trimRanges.filter((r) => r.assetId !== assetId), + }, + project: { + ...withoutAssetClips.project, + primaryAssetId, + updatedAt: new Date().toISOString(), + }, + }; +} + function safeProjectId(raw: string): string { // ponytail: project ids are uuid-prefixed strings (e.g. "proj_"). Reject // anything that smells like path traversal before we ever touch the disk. @@ -532,23 +601,8 @@ export class DocumentService { async addAsset(projectId: string, input: AddAssetInput): Promise { const doc = await this.getProject(projectId); - if (!input.path) { - throw new ProjectFileError("Asset path is required.", projectId); - } - const kind = input.kind ?? "video"; - if (kind === "audio") { - if (!isSupportedAudioPath(input.path)) { - throw new ProjectFileError( - `Unsupported audio extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_AUDIO_EXTENSIONS].join(", ")})`, - projectId, - ); - } - } else if (!isSupportedVideoPath(input.path)) { - throw new ProjectFileError( - `Unsupported video extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_VIDEO_EXTENSIONS].join(", ")})`, - projectId, - ); - } + const unsupported = unsupportedAssetPath(input); + if (unsupported) throw new ProjectFileError(unsupported, projectId); const absolutePath = path.isAbsolute(input.path) ? input.path : path.resolve(input.path); // P3.1 — capture the file size at import. Non-fatal: a stat failure // (network drive, permissions) just leaves sizeBytes undefined. @@ -560,28 +614,13 @@ export class DocumentService { } const asset: AxcutAsset = { id: createId("asset"), - kind, + kind: input.kind ?? "video", label: input.label?.trim() || path.basename(absolutePath), originalPath: absolutePath, sizeBytes, cameraTrack: null, }; - // An audio import is an overlay, never the thing the timeline is built - // around, so it must not claim the empty primaryAssetId slot — otherwise the - // first file dropped into a fresh project (a BGM track) would become its - // primary asset and the editor would try to lay out clips from a file with - // no video. - const claimsPrimary = kind !== "audio" && !doc.project.primaryAssetId; - const next: AxcutDocument = { - ...doc, - assets: [...doc.assets, asset], - project: { - ...doc.project, - ...(claimsPrimary ? { primaryAssetId: asset.id } : {}), - updatedAt: new Date().toISOString(), - }, - }; - return this.saveProject(next); + return this.saveProject(documentWithAsset(doc, asset)); } async removeAsset(projectId: string, assetId: string): Promise { @@ -589,35 +628,7 @@ export class DocumentService { if (!doc.assets.some((a) => a.id === assetId)) { throw new ProjectFileError(`Asset ${assetId} not found in project ${projectId}.`, projectId); } - const assets = doc.assets.filter((a) => a.id !== assetId); - // Primary is the thing the timeline is built around, so it must fall to the - // next VIDEO asset — never an audio overlay (issue #350), which can't be - // primary (see addAsset). Falling back to `assets[0]` would hand primary to - // an audio asset when the removed one was the last video. - const primaryAssetId = - doc.project.primaryAssetId === assetId - ? (assets.find((a) => a.kind !== "audio")?.id ?? undefined) - : doc.project.primaryAssetId; - const withoutAssetClips = doc.timeline.clips - .filter((clip) => clip.assetId === assetId) - .reduce((current, clip) => removeClip(current, clip.id), doc); - const next: AxcutDocument = { - ...withoutAssetClips, - assets, - // Drop imported audio tracks that referenced the removed asset — they - // would otherwise dangle, pointing at an asset the document no longer has. - audioTracks: withoutAssetClips.audioTracks.filter((t) => t.assetId !== assetId), - timeline: { - ...withoutAssetClips.timeline, - trimRanges: withoutAssetClips.timeline.trimRanges.filter((r) => r.assetId !== assetId), - }, - project: { - ...withoutAssetClips.project, - primaryAssetId, - updatedAt: new Date().toISOString(), - }, - }; - return this.saveProject(next); + return this.saveProject(documentWithoutAsset(doc, assetId)); } /** diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 2df583509..d40eee6db 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -5059,6 +5059,13 @@ export function registerIpcHandlers( editsAllowed: () => mcpSettings.getSettings().allowEdits, cursor: agentCursorTelemetryReader, version: app.getVersion(), + // importMedia: the editor's own probe, and the grant a picked file gets, so the editor + // can play a file a client imported into the project it has open. + openVideo: openVideoInHiddenWindow, + approveMedia: (filePath) => { + const media = normalizeVideoSourcePath(filePath); + if (media && hasAllowedImportMediaExtension(media)) approveFilePath(media); + }, }; // One runner, so one queue and one checkpoint store, for the in-app server and every // `openscreen mcp` the app answers through its channel (app-channel.ts, started by main.ts). diff --git a/electron/mcp/app-channel.ts b/electron/mcp/app-channel.ts index 5a2a56f72..09d7f6be4 100644 --- a/electron/mcp/app-channel.ts +++ b/electron/mcp/app-channel.ts @@ -45,6 +45,7 @@ import { type McpToolDeps, type McpToolRunner, } from "./openscreen-mcp-server"; +import { PROJECT_EDIT_TOOLS } from "./project-tools"; const DISCOVERY_FILE = "mcp-app.json"; /** Connect, token and MCP handshake: the app answers in milliseconds when it is alive. */ @@ -260,7 +261,7 @@ export async function connectToApp( function isReadOnly(name: string): boolean { if (name === LIST_PROJECTS_TOOL) return true; if (name === "createCheckpoint" || name === "restoreCheckpoint") return false; - return !isMutatingTool(name); + return !isMutatingTool(name) && !PROJECT_EDIT_TOOLS.has(name); } /** diff --git a/electron/mcp/media-tools.test.ts b/electron/mcp/media-tools.test.ts index 3b9c01d76..9ad599c27 100644 --- a/electron/mcp/media-tools.test.ts +++ b/electron/mcp/media-tools.test.ts @@ -309,6 +309,48 @@ describe("recording", () => { }); }); +describe("createProject", () => { + it("creates an empty project the document tools can edit at once", async () => { + const created = await call("createProject", { title: " Blank " }); + expect(created.isError).toBeFalsy(); + const { projectId, checkpointId } = JSON.parse(textOf(created)); + expect(checkpointId).toEqual(expect.any(String)); + const document = await documents.getProject(projectId); + expect(document.project.title).toBe("Blank"); + expect(document.assets).toEqual([]); + // Its checkpoint lets an edit land on the file straight away. + const renamed = await call("setProjectInfo", { projectId, title: "Named" }); + expect(renamed.isError).toBeFalsy(); + expect((await documents.getProject(projectId)).project.title).toBe("Named"); + }); + + it("lays a video out as one clip, titled after the file, with the zooms of its clicks", async () => { + await fs.writeFile(videoPath, "video"); + const created = await call("createProject", { videoPath }); + expect(created.isError).toBeFalsy(); + const result = JSON.parse(textOf(created)); + expect(result).toMatchObject({ durationSec: 12 }); + expect(result.autoZooms).toBeGreaterThan(0); + const document = await documents.getProject(result.projectId); + expect(document.project.title).toBe("take"); + expect(document.timeline.clips).toHaveLength(1); + expect(document.assets[0]).toMatchObject({ originalPath: videoPath, durationSec: 12 }); + }); + + it("creates nothing from a path it cannot use", async () => { + const missing = await call("createProject", { videoPath: path.join(dir, "gone.mp4") }); + expect(missing.isError).toBe(true); + expect(textOf(missing)).toContain("No file"); + const relative = await call("createProject", { videoPath: "take.mp4" }); + expect(textOf(relative)).toContain("must be absolute"); + await fs.writeFile(videoPath, "video"); + openVideoFails = true; + const unreadable = await call("createProject", { videoPath }); + expect(textOf(unreadable)).toContain("media error 4"); + expect(await documents.listProjects()).toEqual([]); + }); +}); + describe("pointerSamples", () => { const startedAt = 10_000; const samples = pointerSamples( @@ -716,7 +758,7 @@ describe("the media tools in the app", () => { allowed = false; const app = await connectApp(); expect(app.getInstructions()).toContain( - "startRecording and exportVideo are refused too while 'MCP edits'", + "startRecording, exportVideo and createProject are refused too while 'MCP edits'", ); const started = await callOn(app, "startRecording", {}); expect(started.isError).toBe(true); @@ -728,6 +770,9 @@ describe("the media tools in the app", () => { }); expect(textOf(exported)).toContain("'MCP edits'"); expect(runners.started).toEqual([]); + const created = await callOn(app, "createProject", {}); + expect(textOf(created)).toContain("'MCP edits'"); + expect(await documents.listProjects()).toEqual([]); // Reading stays open: listing what could be recorded changes nothing. runners.results.sources = { success: true, diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts index 675abff3b..745526707 100644 --- a/electron/mcp/media-tools.ts +++ b/electron/mcp/media-tools.ts @@ -40,6 +40,7 @@ import { type RunnerEdit, textResult as text, } from "./openscreen-mcp-server"; +import { mediaPathProblem } from "./project-tools"; /** What startRecording asks a recorder for. */ export type RecordRequest = Pick< @@ -192,6 +193,7 @@ export function mediaPreamble(transport: McpTransport): string { return [ "OpenScreen records the screen, turns each take into a project you edit with the document tools, and exports it as a video.", "A demo goes: listSources, then startRecording, then drive the app, then stopRecording. It returns the projectId and a checkpointId. Edit with the document tools, always passing that projectId: every edit is saved to the project straight away, and restoreCheckpoint with the checkpointId takes the project back to the raw take in one step. Then exportVideo, and getFrames with the projectId to check the export before you hand it over: it takes the same ruler seconds as the document tools, so a trim or a speed region never shifts what you look at.", + "To edit a video you already have instead, createProject with its videoPath; importMedia adds a video or an audio file to a project.", "Drive the app with the real mouse when you can: OpenScreen records the pointer, draws a clean cursor and zooms on the clicks as the editor does: clicks close in time and on screen share one zoom, and the first seconds and the end stay on the whole screen. If you drive it from a script instead (Playwright, CDP), the screen pointer never moves: log each action as the script runs it, then hand the log to setPointerTrack after stopRecording.", TAKE_NOTES[transport], ].join("\n"); @@ -216,6 +218,11 @@ const STOP_RECORDING_DESCRIPTIONS: Record = { "Stop the recording and turn the take into a new project, laid out as the editor lays out a fresh recording: one clip, plus the zooms planned from the recorded clicks. Not one per click: clicks close in time and on screen share one zoom, a click in the first seconds or the one that stopped the take gets none, and autoZooms says how many there are. Returns the projectId to edit, a checkpointId that restores the raw take, and the take's videoPath.", }; +const CREATE_PROJECT_NOTES: Record = { + http: "The editor keeps the project it has open: the new one is in OpenScreen's project list, for the user to open.", + stdio: "It is in the OpenScreen app's project list, so the user can open it in its editor.", +}; + const EXPORT_PATH_DESCRIPTIONS: Record = { http: "Where to write the video: an absolute path ending in .mp4 or .gif. This server runs inside the app, so it has no working directory of yours to resolve a relative one against.", stdio: @@ -434,6 +441,15 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { /** Each project's last export of this session, and when it was made: what getFrames reads. */ const lastExports = new Map(); + /** A fresh project's checkpoint, so a client can edit it at once and take it back in one step. */ + async function checkpointOf(projectId: string): Promise { + const checkpoint = await deps.runTool("createCheckpoint", { projectId }); + const checkpointText = checkpoint.content[0]?.type === "text" ? checkpoint.content[0].text : ""; + return checkpoint.isError + ? null + : ((JSON.parse(checkpointText) as { checkpointId?: string }).checkpointId ?? null); + } + /** Stops the take, then hands back its project with a checkpoint of it. */ async function stopTake( running: RunningTake, @@ -447,15 +463,10 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { } startedAtByVideo.set(recorded.videoPath, running.startedAt); const { projectId } = recorded; - const checkpoint = await deps.runTool("createCheckpoint", { projectId }); - const checkpointText = checkpoint.content[0]?.type === "text" ? checkpoint.content[0].text : ""; - const checkpointId = checkpoint.isError - ? null - : (JSON.parse(checkpointText) as { checkpointId?: string }).checkpointId; return text( JSON.stringify({ projectId, - checkpointId, + checkpointId: await checkpointOf(projectId), videoPath: recorded.videoPath, durationSec: recorded.durationSec, recordingStartedAt: running.startedAt, @@ -794,6 +805,70 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { } }, ); + + server.registerTool( + "createProject", + { + description: [ + "Create a new project, in the look new projects start with in OpenScreen (the style preset marked for them, else the last edited project's), as the editor's New does. Empty, or from a video file laid out as one clip, as the editor lays out an opened video, with zooms planned from the clicks when the video is an OpenScreen recording with its pointer track; autoZooms says how many.", + `Returns the projectId to edit and a checkpointId that restores the project as it was created. Fill an empty one with importMedia. ${CREATE_PROJECT_NOTES[transport]}`, + ].join("\n"), + inputSchema: z.object({ + title: z + .string() + .optional() + .describe( + "The project's title in OpenScreen. Default: the video's file name, or Untitled Project.", + ), + videoPath: z + .string() + .optional() + .describe( + "Absolute path of a video (.mp4, .mov, .m4v, .webm, .mkv, .avi, .wmv) to start from. Omit it for an empty project.", + ), + }), + annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, + }, + async (args) => { + if (!deps.editsAllowed()) { + return text( + `No project was created: OpenScreen lets MCP clients create projects only while the user allows it. Ask the user to turn on ${MCP_EDITS_SETTING}.`, + true, + ); + } + const title = args.title?.trim(); + if (args.videoPath === undefined) { + const created = await deps.documents.createProject(title ?? ""); + const projectId = created.project.id; + return text(JSON.stringify({ projectId, checkpointId: await checkpointOf(projectId) })); + } + const videoPath = args.videoPath; + const problem = await mediaPathProblem(videoPath, "video"); + if (problem) return text(problem, true); + let project: Awaited>; + try { + project = await createProjectFromVideo( + deps, + videoPath, + title || path.parse(videoPath).name, + ); + } catch (error) { + return text( + `No project was created: ${error instanceof Error ? error.message : String(error)}`, + true, + ); + } + const projectId = project.document.project.id; + return text( + JSON.stringify({ + projectId, + checkpointId: await checkpointOf(projectId), + durationSec: project.durationSec, + autoZooms: project.autoZooms, + }), + ); + }, + ); } return { preamble: mediaPreamble(transport), register }; diff --git a/electron/mcp/openscreen-mcp-server.test.ts b/electron/mcp/openscreen-mcp-server.test.ts index edb226ff5..85e0fbf2d 100644 --- a/electron/mcp/openscreen-mcp-server.test.ts +++ b/electron/mcp/openscreen-mcp-server.test.ts @@ -24,6 +24,7 @@ import { MCP_ENDPOINT_PATH, type McpApplyResult, type McpDocumentHost, + type McpToolDeps, mcpInstructions, type RunningMcpServer, startMcpHttpServer, @@ -141,7 +142,11 @@ afterEach(async () => { async function connect( editor: FakeEditor, - options: { editsAllowed?: boolean; headers?: Record } = {}, + options: { + editsAllowed?: boolean; + headers?: Record; + media?: Pick; + } = {}, ) { const editsAllowed = () => options.editsAllowed ?? true; const deps = { @@ -150,6 +155,7 @@ async function connect( projects, editsAllowed, version: "0.0.0", + ...options.media, }; const unused = () => { throw new Error("not used by these tests"); @@ -192,6 +198,9 @@ describe("the MCP tool surface", () => { expect(tools.map((t) => t.name)).toEqual([ LIST_PROJECTS_TOOL, ...OPENSCREEN_TOOL_NAMES, + "setProjectInfo", + "importMedia", + "removeMedia", "createCheckpoint", "restoreCheckpoint", // The same as `openscreen mcp` serves (#1174). @@ -201,6 +210,7 @@ describe("the MCP tool surface", () => { "setPointerTrack", "exportVideo", "getFrames", + "createProject", ]); for (const tool of tools.slice(1, 1 + OPENSCREEN_TOOL_NAMES.length)) { expect(tool.description).toBe(TOOL_DESCRIPTIONS[tool.name]); @@ -231,6 +241,8 @@ describe("the MCP tool surface", () => { expect(byName.get(LIST_PROJECTS_TOOL)?.readOnlyHint).toBe(true); expect(byName.get("addTrim")?.readOnlyHint).toBe(false); expect(byName.get("removeClip")?.destructiveHint).toBe(true); + expect(byName.get("removeMedia")?.destructiveHint).toBe(true); + expect(byName.get("importMedia")?.readOnlyHint).toBe(false); expect(byName.get("createCheckpoint")?.readOnlyHint).toBe(true); expect(byName.get("restoreCheckpoint")?.destructiveHint).toBe(true); }); @@ -247,7 +259,7 @@ describe("the MCP tool surface", () => { "Without a projectId they act on the project open in the editor", ); expect(instructions).toContain("startRecording hands over to OpenScreen's recording bar"); - expect(instructions).toContain("startRecording and exportVideo are refused too"); + expect(instructions).toContain("startRecording, exportVideo and createProject are refused too"); expect(instructions).toContain("all of them are lost when OpenScreen quits."); expect(instructions).toContain("turn on 'MCP edits' under MCP server in AI settings"); expect(instructions).not.toContain("Agent edits"); @@ -462,6 +474,147 @@ describe("checkpoints", () => { }); }); +describe("the project's name and media", () => { + function probe(durationSec = 12) { + return vi.fn(async () => ({ + durationSec, + width: 1920, + height: 1080, + close: () => undefined, + capture: async () => ({ mimeType: "image/jpeg" as const, data: "" }), + })); + } + + function mediaFile(name: string): string { + const file = path.join(dir, name); + writeFileSync(file, "media"); + return file; + } + + it("renames the open project through the editor, trimmed as the editor commits it", async () => { + const editor = new FakeEditor(); + const mcp = await connect(editor); + const result = await mcp.callTool({ + name: "setProjectInfo", + arguments: { title: " Launch demo " }, + }); + expect(result.isError).toBeFalsy(); + expect(editor.applied).toHaveLength(1); + expect(editor.document?.project.title).toBe("Launch demo"); + + const empty = await mcp.callTool({ name: "setProjectInfo", arguments: { title: " " } }); + expect(empty.isError).toBe(true); + expect(editor.applied).toHaveLength(1); + }); + + it("imports a video into the open project, measured and readable by the editor", async () => { + const editor = new FakeEditor(); + const openVideo = probe(); + const approveMedia = vi.fn(); + const mcp = await connect(editor, { media: { openVideo, approveMedia } }); + const file = mediaFile("broll.mp4"); + const result = await mcp.callTool({ name: "importMedia", arguments: { path: file } }); + expect(result.isError).toBeFalsy(); + const { assetId, kind, durationSec } = JSON.parse(resultText(result)); + expect([kind, durationSec]).toEqual(["video", 12]); + expect(editor.applied).toHaveLength(1); + const asset = editor.document?.assets.find((a) => a.id === assetId); + expect(asset).toMatchObject({ originalPath: file, label: "broll.mp4", durationSec: 12 }); + expect(asset?.video).toMatchObject({ width: 1920, height: 1080 }); + // The project already has its footage: an import never takes its place. + expect(editor.document?.project.primaryAssetId).toBe("asset_1"); + expect(approveMedia).toHaveBeenCalledWith(file); + }); + + it("files an audio-only extension as audio, and still imports a file it cannot measure", async () => { + const editor = new FakeEditor(); + const openVideo = vi.fn(async () => { + throw new Error("media error 4"); + }); + const mcp = await connect(editor, { media: { openVideo } }); + const result = await mcp.callTool({ + name: "importMedia", + arguments: { path: mediaFile("voice.mp3"), label: "Voiceover" }, + }); + expect(result.isError).toBeFalsy(); + expect(JSON.parse(resultText(result))).toMatchObject({ + kind: "audio", + label: "Voiceover", + durationSec: null, + }); + expect(editor.document?.assets.at(-1)?.kind).toBe("audio"); + }); + + it("refuses a relative, missing or unsupported path before probing anything", async () => { + const editor = new FakeEditor(); + const openVideo = probe(); + const mcp = await connect(editor, { media: { openVideo } }); + for (const [filePath, says] of [ + ["broll.mp4", "must be absolute"], + [path.join(dir, "missing.mp4"), "No file"], + [mediaFile("notes.txt"), "Unsupported video extension"], + ]) { + const result = await mcp.callTool({ name: "importMedia", arguments: { path: filePath } }); + expect(result.isError).toBe(true); + expect(resultText(result)).toContain(says); + } + expect(openVideo).not.toHaveBeenCalled(); + expect(editor.applied).toHaveLength(0); + }); + + it("imports nothing while MCP edits is off", async () => { + const editor = new FakeEditor(); + const openVideo = probe(); + const mcp = await connect(editor, { editsAllowed: false, media: { openVideo } }); + const refused = await mcp.callTool({ + name: "importMedia", + arguments: { path: mediaFile("ok.mp4") }, + }); + expect(refused.isError).toBe(true); + expect(resultText(refused)).toContain("'MCP edits'"); + expect(openVideo).not.toHaveBeenCalled(); + expect(editor.applied).toHaveLength(0); + }); + + it("removes media only once the timeline no longer uses it", async () => { + const editor = new FakeEditor(); + const mcp = await connect(editor, { media: { openVideo: probe() } }); + const inUse = await mcp.callTool({ name: "removeMedia", arguments: { assetId: "asset_1" } }); + expect(inUse.isError).toBe(true); + expect(resultText(inUse)).toContain("clip_1 (removeClip)"); + expect(editor.applied).toHaveLength(0); + + const imported = await mcp.callTool({ + name: "importMedia", + arguments: { path: mediaFile("unused.mov") }, + }); + const { assetId } = JSON.parse(resultText(imported)); + const removed = await mcp.callTool({ name: "removeMedia", arguments: { assetId } }); + expect(removed.isError).toBeFalsy(); + expect(editor.document?.assets.map((a) => a.id)).toEqual(["asset_1"]); + }); + + it("edits a project that is not open on its file, once it has a checkpoint", async () => { + await projects.saveProject(fixtureDocument("proj_2")); + const editor = new FakeEditor(); + const mcp = await connect(editor, { media: { openVideo: probe() } }); + const args = { projectId: "proj_2", path: mediaFile("music.wav") }; + const refused = await mcp.callTool({ name: "importMedia", arguments: args }); + expect(resultText(refused)).toContain("createCheckpoint"); + await mcp.callTool({ name: "createCheckpoint", arguments: { projectId: "proj_2" } }); + const result = await mcp.callTool({ name: "importMedia", arguments: args }); + expect(result.isError).toBeFalsy(); + await mcp.callTool({ + name: "setProjectInfo", + arguments: { projectId: "proj_2", title: "Renamed" }, + }); + const saved = await projects.getProject("proj_2"); + expect(saved.assets.map((a) => a.kind)).toEqual(["video", "audio"]); + expect(saved.project.title).toBe("Renamed"); + expect(editor.applied).toHaveLength(0); + }); +}); + describe("projects other than the open one", () => { const addTrimArgs = { assetId: "asset_1", startSec: 5, endSec: 6 }; diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.ts index 0a78778ff..3d8523f51 100644 --- a/electron/mcp/openscreen-mcp-server.ts +++ b/electron/mcp/openscreen-mcp-server.ts @@ -24,6 +24,8 @@ // back to where the turn started. For a project that is not open they are the // only way back, so an edit to one is refused until it has a checkpoint. The // in-app agent has no need for them: its whole turn is already a single apply. +// Nor for the project's own edits (project-tools.ts: its name, its media), which +// reach a file on disk the in-app agent has no way to name. // // Two transports serve these tools: the in-app server over HTTP (below) and // `openscreen mcp` over stdio (`stdio-server.ts`), both with the media tools around @@ -68,6 +70,17 @@ import { type ProjectVersion, } from "../ai-edition/document-service"; import type { MediaTools } from "./media-tools"; +import { + IMPORT_MEDIA_TOOL, + type ImportMediaDeps, + PROJECT_EDIT_TOOLS, + PROJECT_TOOLS, + prepareImportMedia, + REMOVE_MEDIA_TOOL, + removeMedia, + SET_PROJECT_INFO_TOOL, + setProjectInfo, +} from "./project-tools"; export const MCP_SERVER_NAME = "openscreen"; export const MCP_ENDPOINT_PATH = "/mcp"; @@ -124,7 +137,7 @@ type ProjectRead = Awaited>; */ export type McpTransport = "http" | "stdio"; -export interface McpToolDeps { +export interface McpToolDeps extends ImportMediaDeps { transport: McpTransport; host: McpDocumentHost; projects: McpProjectStore; @@ -225,7 +238,9 @@ export function mcpInstructions(options: { ...CORE_PREAMBLES[transport], CHECKPOINT_LIFETIMES[transport], ...(media && !editsAllowed - ? [`startRecording and exportVideo are refused too while ${MCP_EDITS_SETTING} is off.`] + ? [ + `startRecording, exportVideo and createProject are refused too while ${MCP_EDITS_SETTING} is off.`, + ] : []), "", ].join("\n"); @@ -494,10 +509,12 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner { if (name === "createCheckpoint") return createCheckpoint(document); if (name === "restoreCheckpoint") return restoreCheckpoint(document, args, editsAllowed); // This server's own switch, so the refusal names it rather than the in-app agent's. - if ((edit || isMutatingTool(name)) && !editsAllowed) { + if ((edit || isMutatingTool(name) || PROJECT_EDIT_TOOLS.has(name)) && !editsAllowed) { return consentRequired(name, args, MCP_EDITS_SETTING); } if (edit) return edit(document); + if (name === SET_PROJECT_INFO_TOOL) return setProjectInfo(document, args); + if (name === REMOVE_MEDIA_TOOL) return removeMedia(document, args); const availableByAssetId = await probeCursorTelemetry(document, deps.cursor); return runDocumentTool(document, name, args, editsAllowed, { cursor: deps.cursor, @@ -537,11 +554,17 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner { return (name, args, options = {}) => { // The setting is read when the call's turn comes: it may change while it waits. - const runNow = () => - run(name, args, { - edit: options.edit, - editsAllowed: options.editsAllowed ?? deps.editsAllowed(), - }); + const runNow = async () => { + const editsAllowed = options.editsAllowed ?? deps.editsAllowed(); + let edit = options.edit; + // The file is checked and probed before the project is read: see prepareImportMedia. + if (name === IMPORT_MEDIA_TOOL && !edit && editsAllowed) { + const prepared = await prepareImportMedia(splitProjectId(args).args, deps); + if (typeof prepared === "string") return textResult(prepared, true); + edit = prepared; + } + return run(name, args, { edit, editsAllowed }); + }; const next = queue.then(runNow, runNow); queue = next.catch(() => undefined); return next.catch((error) => textResult(`Tool failed: ${errorMessage(error)}`, true)); @@ -590,6 +613,21 @@ export function createOpenScreenMcpServer( (args: unknown) => runTool(name, args), ); } + for (const tool of PROJECT_TOOLS) { + server.registerTool( + tool.name, + { + description: tool.description, + inputSchema: tool.inputSchema.extend({ projectId: projectIdArg }), + annotations: { + readOnlyHint: false, + destructiveHint: tool.destructive, + openWorldHint: false, + }, + }, + (args: unknown) => runTool(tool.name, args), + ); + } for (const tool of checkpointTools(media !== undefined)) { server.registerTool( tool.name, diff --git a/electron/mcp/project-tools.ts b/electron/mcp/project-tools.ts new file mode 100644 index 000000000..bb51dc398 --- /dev/null +++ b/electron/mcp/project-tools.ts @@ -0,0 +1,210 @@ +// The project's own fields and its media, as MCP tools: rename the project, import a +// file into its media, remove one. Run by the tool runner like any document edit +// (openscreen-mcp-server.ts), so the project open in the editor changes there, live +// and one Ctrl/Cmd+Z step, and any other project meets the same checkpoint rule. The +// app's channel serves them too, so `openscreen mcp` hands them to the running app. +// +// Deleting a project is deliberately not here: it cannot be undone, and nothing an +// agent makes needs it. + +import fs from "node:fs/promises"; +import path from "node:path"; +import { z } from "zod"; +import { createId } from "../../src/lib/ai-edition/document/ids"; +import type { AxcutAsset, AxcutDocument } from "../../src/lib/ai-edition/schema"; +import type { AgentToolExecution } from "../ai-edition/agent-tools"; +import { + assetKindForPath, + documentWithAsset, + documentWithoutAsset, + unsupportedAssetPath, +} from "../ai-edition/document-service"; +import type { OpenedVideo } from "./hidden-video"; + +export const SET_PROJECT_INFO_TOOL = "setProjectInfo"; +export const IMPORT_MEDIA_TOOL = "importMedia"; +export const REMOVE_MEDIA_TOOL = "removeMedia"; + +/** The runner's edits to a project that are not the document executor's. */ +export const PROJECT_EDIT_TOOLS: ReadonlySet = new Set([ + SET_PROJECT_INFO_TOOL, + IMPORT_MEDIA_TOOL, + REMOVE_MEDIA_TOOL, +]); + +const setProjectInfoArgs = z.object({ + title: z.string().describe("The project's new name."), +}); + +const importMediaArgs = z.object({ + path: z + .string() + .describe( + "Absolute path of the file. Video: .mp4, .mov, .m4v, .webm, .mkv, .avi, .wmv. Audio: .mp3, .wav, .m4a, .aac, .flac, .ogg, .oga, .opus, .webm.", + ), + kind: z + .enum(["video", "audio"]) + .optional() + .describe( + "What the file is. Default: audio for an audio-only extension, video otherwise, so pass audio for a .webm that holds only sound.", + ), + label: z.string().optional().describe("Its name in the project's media. Default: the file name."), +}); + +const removeMediaArgs = z.object({ + assetId: z.string().describe("The asset to remove, from getCurrentDocument's assets."), +}); + +export const PROJECT_TOOLS = [ + { + name: SET_PROJECT_INFO_TOOL, + description: + "Rename the project: its title in OpenScreen's project list and in the editor's top bar, as renaming it there does. Changes nothing else.", + inputSchema: setProjectInfoArgs, + destructive: false, + }, + { + name: IMPORT_MEDIA_TOOL, + description: + "Add a video or audio file from disk to the project's media, as the editor's Import does. It does not place it on the timeline. An audio file is then played by a track you lay with addAudio. A video becomes the project's footage when it has none yet (a project createProject made without a video): replaceTimeline then lays it out. Returns the new assetId and the file's durationSec, null when OpenScreen could not read it (the editor reads it again when it opens the project).", + inputSchema: importMediaArgs, + destructive: false, + }, + { + name: REMOVE_MEDIA_TOOL, + description: + "Remove a file from the project's media. Refused while the timeline still uses it: remove its clips (removeClip) and its audio tracks (removeModifier) first. The file on disk is left alone.", + inputSchema: removeMediaArgs, + destructive: true, + }, +] as const; + +function failure(message: string): AgentToolExecution { + return { ok: false, resultJson: JSON.stringify({ error: message }) }; +} + +/** Why `filePath` cannot be imported as `kind`, or `null` when it can. */ +export async function mediaPathProblem( + filePath: string, + kind: "video" | "audio", +): Promise { + if (!path.isAbsolute(filePath)) return `The path must be absolute, got "${filePath}".`; + const unsupported = unsupportedAssetPath({ path: filePath, kind }); + if (unsupported) return unsupported; + const stat = await fs.stat(filePath).catch(() => null); + if (!stat?.isFile()) return `No file at "${filePath}".`; + return null; +} + +export function setProjectInfo(document: AxcutDocument, rawArgs: unknown): AgentToolExecution { + const parsed = setProjectInfoArgs.safeParse(rawArgs); + if (!parsed.success) return failure(parsed.error.message); + // Trimmed and never empty, as the editor's name field commits it. + const title = parsed.data.title.trim(); + if (!title) return failure("title must not be empty."); + return { + ok: true, + resultJson: JSON.stringify({ ok: true, title }), + document: { ...document, project: { ...document.project, title } }, + }; +} + +export function removeMedia(document: AxcutDocument, rawArgs: unknown): AgentToolExecution { + const parsed = removeMediaArgs.safeParse(rawArgs); + if (!parsed.success) return failure(parsed.error.message); + const { assetId } = parsed.data; + if (!document.assets.some((asset) => asset.id === assetId)) { + return failure(`Unknown asset: ${assetId}. getCurrentDocument lists the project's assets.`); + } + const clips = document.timeline.clips.filter((clip) => clip.assetId === assetId); + const tracks = document.audioTracks.filter((track) => track.assetId === assetId); + if (clips.length > 0 || tracks.length > 0) { + const uses = [ + ...(clips.length ? [`clips ${clips.map((clip) => clip.id).join(", ")} (removeClip)`] : []), + ...(tracks.length + ? [`audio tracks ${tracks.map((track) => track.id).join(", ")} (removeModifier)`] + : []), + ]; + return failure( + `Asset ${assetId} is still on the timeline, so nothing was removed. Remove its ${uses.join(" and its ")} first.`, + ); + } + const next = documentWithoutAsset(document, assetId); + return { + ok: true, + resultJson: JSON.stringify({ + ok: true, + removed: assetId, + primaryAssetId: next.project.primaryAssetId ?? null, + }), + document: next, + }; +} + +export interface ImportMediaDeps { + /** The duration probe the editor's footage goes through (hidden-video.ts). */ + openVideo?(filePath: string): Promise; + /** Lets the editor read the file, as picking it in the editor would. */ + approveMedia?(filePath: string): void; +} + +/** + * Checks and measures the file, then returns the edit that files it: computed before the + * runner reads the project, so the slow probe never holds a snapshot open. A string is + * why it cannot be imported. + */ +export async function prepareImportMedia( + rawArgs: unknown, + deps: ImportMediaDeps, +): Promise<((document: AxcutDocument) => Promise) | string> { + const parsed = importMediaArgs.safeParse(rawArgs); + if (!parsed.success) return JSON.stringify({ error: parsed.error.message }); + const { path: filePath, label } = parsed.data; + const kind = parsed.data.kind ?? assetKindForPath(filePath); + const problem = await mediaPathProblem(filePath, kind); + if (problem) return JSON.stringify({ error: problem }); + + const sizeBytes = (await fs.stat(filePath)).size; + // Not fatal, as in the editor: a file Chromium cannot decode may still be one the + // compositor plays, and the editor probes it again once it opens the project. + const probed = await deps.openVideo?.(filePath).then( + (video) => { + video.close(); + return video; + }, + () => null, + ); + const durationSec = + probed && Number.isFinite(probed.durationSec) && probed.durationSec > 0 + ? probed.durationSec + : undefined; + deps.approveMedia?.(filePath); + + return async (document) => { + const asset: AxcutAsset = { + id: createId("asset"), + kind, + label: label?.trim() || path.basename(filePath), + originalPath: filePath, + sizeBytes, + cameraTrack: null, + ...(durationSec ? { durationSec } : {}), + ...(kind === "video" && probed && probed.width > 0 + ? { video: { codec: "unknown", width: probed.width, height: probed.height, fps: 0 } } + : {}), + }; + const next = documentWithAsset(document, asset); + return { + ok: true, + resultJson: JSON.stringify({ + ok: true, + assetId: asset.id, + kind, + label: asset.label, + durationSec: durationSec ?? null, + primaryAssetId: next.project.primaryAssetId ?? null, + }), + document: next, + }; + }; +} From dcdc485f08a1e001a89c9464e86e12787c792cd8 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:18:31 +0200 Subject: [PATCH 2/3] docs(mcp): the project and media tools, and why deleting a project is not one Part of #1180 --- skills/openscreen-demo/SKILL.md | 4 +++ .../architecture/mcp-server.md | 30 ++++++++++++++----- website/docs/ai-editing.md | 4 +-- .../current/ai-editing.md | 4 +-- .../current/ai-editing.md | 4 +-- .../current/ai-editing.md | 4 +-- .../current/ai-editing.md | 4 +-- .../current/ai-editing.md | 4 +-- .../current/ai-editing.md | 4 +-- .../current/ai-editing.md | 4 +-- 10 files changed, 42 insertions(+), 24 deletions(-) diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.md index 5baf48627..15e5f556c 100644 --- a/skills/openscreen-demo/SKILL.md +++ b/skills/openscreen-demo/SKILL.md @@ -68,6 +68,8 @@ Write a storyboard and show it to the user in a few lines, then carry on unless - **Real mouse** (computer-use, or any tool that moves the actual pointer): OpenScreen records the pointer, draws a clean cursor and already zooms on the clicks (see *Zoom on the action* below). If your mouse tool cannot click in the app's window (some only read browsers, for one), drive it from a script instead. - **A script** (Playwright, CDP): the screen pointer never moves, so the take has no cursor and no zoom. In the script, log every action as it runs: `at` is `Date.now()`, `x` and `y` are the action's position as fractions of the window, `click` is true for a click. Hand the log to `setPointerTrack` after `stopRecording`; its description gives the formula from a page's coordinates. A script also gives the cleanest take: same timing on every run, no hesitation. +**A video you already have** needs no recording: `createProject` with its absolute `videoPath` lays it out as a project, and returns a `projectId` and a `checkpointId` as `stopRecording` does. Then go on from step 6. + ## 5. Check the take `getFrames` on the `videoPath`, about 6 frames. Is it the right window? Is anything private visible? Did every beat happen? If not, record again: a new take costs less than a fix in editing. @@ -83,6 +85,8 @@ Use the document tools, always with the `projectId`. Read the project first with - **Zoom on the action:** clicks close in time and on screen share one zoom, and the first 2.5 s stay wide. Check them, then add or move zooms with `addZooms` and `setZoom` where a result deserves its own. Hold a zoom at least 1.5 s. - **Callouts:** one `addAnnotation` per beat, short, placed clear of the action. Text sits on the output frame and stays put when the picture zooms. Use its look for hierarchy: a large title with `animation: "typewriter"`, step callouts with `animation: "pop"`. An arrow, `type: "figure"`, points at a control the text names. `setAnnotation` moves or restyles one. - **Hide what is private:** an email address, a token or a name that shows on screen gets `addAnnotation` with `type: "blur"` over the span it is visible. Its x, y, width and height are percent of the recording's picture, not of the frame, so it stays on what it hides when a zoom moves the picture. Check it with `getFrames`. +- **Music or a voiceover:** `importMedia` with the file's absolute path adds it to the project's media, then `addAudio` plays it over a span. `removeMedia` takes back a file the timeline no longer uses. +- **Name:** `setProjectInfo` renames the project, if the `title` given to `stopRecording` no longer fits. - **Undo:** `restoreCheckpoint` with the `checkpointId` takes the project back to the raw take in one step. It also puts back the take's own pointer track, undoing any `setPointerTrack` made since. - **Open in the app:** while OpenScreen is running, your edits go through it, so a project the user has open in its editor changes there live and they can undo each edit. If an edit is refused as "open in the OpenScreen editor", the app did not answer: retry once, then ask the user to close the project there or restart OpenScreen. diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md index a5208678c..a6449808f 100644 --- a/technical-documentation/architecture/mcp-server.md +++ b/technical-documentation/architecture/mcp-server.md @@ -12,7 +12,7 @@ The same tools are served over two transports, by two different processes: | Edits, recording, export | Off until the user turns on **MCP edits** | Always allowed | | Project open in the editor | Edited live through the editor, one undo step per call | The same while the app runs: the call is forwarded to it. With no app answering, refused ([the marker](#the-open-in-editor-marker)) | | Other projects | Read from and saved to their file, a checkpoint first | Same | -| Tools | `listProjects`, the agent's tools, the checkpoint tools, and the media tools: `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` | The same | +| Tools | `listProjects`, the agent's tools, the project tools (`setProjectInfo`, `importMedia`, `removeMedia`), the checkpoint tools, and the media tools: `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames`, `createProject` | The same | | A take | Recorded by the app's recording bar, as its Record button records; the editor opens it ([below](#the-media-tools)) | Recorded by a hidden runner of its own process, and made a project there | | Checkpoints | In the app's memory, shared with forwarded `openscreen mcp` calls, lost when the app quits | In the app's while it runs; otherwise in this process's, lost when it exits, which it does when the client disconnects | @@ -22,6 +22,7 @@ The same tools are served over two transports, by two different processes: |---|---| | [`electron/mcp/openscreen-mcp-server.ts`](../../electron/mcp/openscreen-mcp-server.ts) | The tools both transports serve, the tool runner (routing, refusals, checkpoints), the texts per transport (`mcpInstructions`), and the local HTTP guard. | | [`electron/mcp/stdio-server.ts`](../../electron/mcp/stdio-server.ts) | `openscreen mcp`: the same server, runner and media tools on stdin/stdout. | +| [`electron/mcp/project-tools.ts`](../../electron/mcp/project-tools.ts) | The project tools the runner runs: rename a project, import a file into its media, remove one. | | [`electron/mcp/media-tools.ts`](../../electron/mcp/media-tools.ts) | The media tools both servers serve (`createMediaTools`), their capability block per transport (`mediaPreamble`), and `openscreen mcp`'s recorder. | | [`electron/mcp/app-recorder.ts`](../../electron/mcp/app-recorder.ts) | The in-app server's recorder: a take through the app's recording bar, opened by its editor. | | [`electron/mcp/editor-document-host.ts`](../../electron/mcp/editor-document-host.ts) | Main-process side of the live document: asks the editor window for a snapshot and hands edits back. | @@ -35,25 +36,26 @@ The same tools are served over two transports, by two different processes: Nothing about the tools is reimplemented. The server registers `TOOL_ARG_SCHEMAS` (names and zod schemas) and `TOOL_DESCRIPTIONS`, and builds its `instructions` around `buildSystemPrompt`, all exported from [`deep-agent/service.ts`](../../electron/ai-edition/deep-agent/service.ts), where `buildTools` builds the in-app agent from the same table. Every call goes through `runDocumentTool`, the function the in-app agent's `documentTool` also calls: the cursor-telemetry read the zoom tools need, then `executeAgentTool`. -So a tool added to the agent appears over MCP with no further work, and the MCP test asserts the listed tools equal `listProjects`, then `OPENSCREEN_TOOL_NAMES`, then the two checkpoint tools, then the [media tools](#the-media-tools). What the server adds: +So a tool added to the agent appears over MCP with no further work, and the MCP test asserts the listed tools equal `listProjects`, then `OPENSCREEN_TOOL_NAMES`, then the three project tools, then the two checkpoint tools, then the [media tools](#the-media-tools). What the server adds: - **`listProjects`**, a tool of its own: every project's id, title, `updatedAt`, asset count, and `open` for the one in an editor (this process's, or the app's seen through the marker). - **An optional `projectId` on every agent tool and both checkpoint tools**, added to its schema with `.extend()` at registration and stripped again before the call reaches the executor, which never sees it. See [Which project a call acts on](#which-project-a-call-acts-on). - **`createCheckpoint` / `restoreCheckpoint`**, see [Checkpoints](#checkpoints). -- MCP metadata: `readOnlyHint` for the reads (`!isMutatingTool`), `listProjects` and `createCheckpoint`, and `destructiveHint` for `replaceTimeline`, the three `remove*` tools and `restoreCheckpoint`. +- **The project tools**, see [The project and its media](#the-project-and-its-media). +- MCP metadata: `readOnlyHint` for the reads (`!isMutatingTool`), `listProjects` and `createCheckpoint`, and `destructiveHint` for `replaceTimeline`, the three `remove*` agent tools, `removeMedia` and `restoreCheckpoint`. ## What each transport tells its client Every text a client reads must be true on the server that serves it, so the texts are composed rather than shared verbatim. `McpToolDeps.transport` (`"http"` or `"stdio"`) selects them: -- **Instructions** (`mcpInstructions`): the media tools' capability block (`mediaPreamble`), then the projects-and-checkpoints rules of the transport, then how long checkpoints live there, then `buildSystemPrompt`. The checkpoint rule is in both. Over HTTP the media block says a take goes through the app's recording bar and opens in the editor, and the rules speak of the editor and its undo stack; while **MCP edits** is off, a line says `startRecording` and `exportVideo` are refused too. Over stdio the media block says the takes land in the app's project list, and the rules say that calls go through the app while it runs (the open project live, with `Ctrl/Cmd+Z`), and to the files when it does not. +- **Instructions** (`mcpInstructions`): the media tools' capability block (`mediaPreamble`), then the projects-and-checkpoints rules of the transport, then how long checkpoints live there, then `buildSystemPrompt`. The checkpoint rule is in both. Over HTTP the media block says a take goes through the app's recording bar and opens in the editor, and the rules speak of the editor and its undo stack; while **MCP edits** is off, a line says `startRecording`, `exportVideo` and `createProject` are refused too. Over stdio the media block says the takes land in the app's project list, and the rules say that calls go through the app while it runs (the open project live, with `Ctrl/Cmd+Z`), and to the files when it does not. - **`buildSystemPrompt`** takes the transport's facts as options: `editsSetting` names the switch its consent block points to (the MCP server's **MCP edits**, not the in-app agent's **Agent edits**), and `canExport` drops "exporting" from its examples of a request with no tool, since both serve `exportVideo`. - **Tool descriptions**: `projectId` is "omit it for the open project" over HTTP and "omit it only for the open project, while the app runs" over stdio. `startRecording` and `stopRecording` say how each server makes a take ([below](#the-media-tools)), and the paths of `exportVideo` and `getFrames` must be absolute over HTTP: the app has no working directory of the client's. - **Messages**: no project to fall back on, a checkpoint required, an unknown checkpoint, edits turned off. Each has a version per transport, or exists only on the one that can produce it. The stdio versions are only met with no app answering, and say so. The in-app chat agent keeps `buildSystemPrompt`'s defaults: its own switch, no export. The app's channel for `openscreen mcp` registers none of the media tools, only a `setPointerTrack` of its own ([below](#the-media-tools)), so its instructions have no media block: no agent reads them. -**Writes on the in-app server are a second opt-in.** MCP clients have their own **MCP edits** switch, `allowEdits` in `mcp-server.json`, **off by default** and independent of the in-app agent's **Agent edits** (`allowAgentEdits`). Turning the server on therefore grants read access only. The value is read on every call; while it is off every mutating tool and `setPointerTrack` are refused with the executor's `consent_required` payload naming **MCP edits**, `restoreCheckpoint` with its own refusal text naming it too, `startRecording` and `exportVideo` with theirs, and the consent block of the system prompt is in the instructions. It is independent because `allowAgentEdits` defaults to allowed and lives in the provider form, so a user with no provider configured could never have turned MCP writes off. Toggling it does not restart the server. `openscreen mcp` has no such switch: the client started it itself, and there is no other caller to keep out. Its calls forwarded to the app keep that: the runner takes `editsAllowed` per call. +**Writes on the in-app server are a second opt-in.** MCP clients have their own **MCP edits** switch, `allowEdits` in `mcp-server.json`, **off by default** and independent of the in-app agent's **Agent edits** (`allowAgentEdits`). Turning the server on therefore grants read access only. The value is read on every call; while it is off every mutating tool and `setPointerTrack` are refused with the executor's `consent_required` payload naming **MCP edits**, `restoreCheckpoint` with its own refusal text naming it too, `startRecording`, `exportVideo` and `createProject` with theirs, and the consent block of the system prompt is in the instructions. It is independent because `allowAgentEdits` defaults to allowed and lives in the provider form, so a user with no provider configured could never have turned MCP writes off. Toggling it does not restart the server. `openscreen mcp` has no such switch: the client started it itself, and there is no other caller to keep out. Its calls forwarded to the app keep that: the runner takes `editsAllowed` per call. **Recording and export wait for the same switch, with no setting of their own.** The settings model has two steps: the server on is read access, **MCP edits** on is a client changing things. A take captures the screen and creates a project; an export writes a file wherever the client says, over one already there. Both change things, more than an edit does, so both wait for **MCP edits**, whose label says so: *Allow MCP clients to edit your projects, record and export*. A third switch would split one decision, whether this client may act for the user, in two. `stopRecording` is not gated, so a take started before the switch went off can still end. `listSources` and `getFrames` only read, so the server being on is enough. @@ -116,9 +118,19 @@ A client chains several edits in one turn, and each is its own undo step, so rev - It discards every edit since the checkpoint, the user's included. The tool description says so. - **Lifetime.** Checkpoints live in the runner's memory, the 20 most recent. In the app that is one runner (`createToolRunner`, built in `registerIpcHandlers`), shared by the in-app server and every forwarded `openscreen mcp` call; a restart of the in-app server keeps them, and they are lost when the app quits. An `openscreen mcp` with no app answering keeps its own, lost when the process exits, which it does when the client disconnects. The unknown-checkpoint message says which. The in-app agent does not need them: its whole turn is a single apply. +## The project and its media + +Three tools of the runner's own ([`project-tools.ts`](../../electron/mcp/project-tools.ts)) edit what the agent's tools cannot reach: the project's name and its media. They take `projectId` and go through the runner like any edit, so the project open in the editor changes there, live and one undo step, any other project needs a checkpoint first, and **MCP edits** gates them. The app's channel serves them, so `openscreen mcp` forwards them to the running app like any document call. The in-app agent does not get them: it has no file on disk to name. + +- **`setProjectInfo`** sets `project.title`, trimmed and never empty, as the editor's name field commits it. +- **`importMedia`** files a video or audio file into `assets`, through the same `documentWithAsset` as `DocumentService.addAsset`: an audio file never claims the primary slot, a video claims it only when the project has none. Before the runner reads the project, the path is checked (absolute, a file, an extension `addAsset` accepts for the kind) and measured with the hidden-window probe (`openVideo`, hidden-video.ts), so the slow probe never holds a snapshot open. A probe that fails is not fatal, as in the editor: the asset has no `durationSec` and the editor measures it once it opens the project. In the app, `approveMedia` grants the file to the renderer, as picking it would, so the editor can play what a client imported into the project it has open. The kind defaults to the extension: audio for an audio-only one, video otherwise. The file lands in the media, not on the timeline: `addAudio` plays an audio file. +- **`removeMedia`** drops an asset with `documentWithoutAsset`, `DocumentService.removeAsset`'s own change, but refuses while a clip or an audio track still uses it, naming them: removing footage the timeline plays is a `removeClip`, not a side effect. The file on disk is never touched. + +**Deleting a project is left out on purpose.** It cannot be undone, no checkpoint holds a deleted file, and nothing an agent makes needs it. The user deletes projects in the app. + ## The media tools -`createMediaTools` builds the tools that make a video around the edits, once per server, and `createOpenScreenMcpServer` registers them. Both servers serve the same six, from the same code. Their state (the take running, the export running, each take's start time, each project's last export) lives in what `createMediaTools` returns, so the in-app server, which builds a fresh `McpServer` per request, keeps it across requests and across a restart of its port. They share the runner the document tools use: the app's one runner in the app, the routed one of `stdio-server.ts` over stdio, so their document calls go to the app while it runs. +`createMediaTools` builds the tools that make a video around the edits, once per server, and `createOpenScreenMcpServer` registers them. Both servers serve the same seven, from the same code. Their state (the take running, the export running, each take's start time, each project's last export) lives in what `createMediaTools` returns, so the in-app server, which builds a fresh `McpServer` per request, keeps it across requests and across a restart of its port. They share the runner the document tools use: the app's one runner in the app, the routed one of `stdio-server.ts` over stdio, so their document calls go to the app while it runs. What differs between the two servers is who records, handed in as a `TakeRecorder`: @@ -135,10 +147,11 @@ Then both servers carry on alike: - `setPointerTrack` gives a script-driven take a pointer track and plans zooms on its clicks. `pointerTrackEdit` computes the new sidecar and the zooms from the project the runner read, writes nothing itself, and hands both to the runner as a `RunnerEdit`. The runner then applies the same refusals as any edit (another process's editor, no checkpoint, a project changed mid-call) before writing anything, saves the project, and only then writes the sidecar, keeping its previous content for the checkpoints. Every refusal comes back as the tool's own text result. A sidecar write that fails after the save is reported as a partial edit: the project was saved, the pointer track was not. A `RunnerEdit` is a closure and cannot be forwarded, so the call carries what the edit needs (`events`, and `takes`, each take's start time by video): the app's channel serves `setPointerTrack` with those args and runs `pointerTrackEdit` in its own runner. That is the channel's own registration, not the in-app server's tool: the takes of `openscreen mcp` are known to it alone. - `exportVideo` renders in a hidden `cli-export` runner, the same in both processes: in the app, `listSources` and `exportVideo` run the CLI's runners (`createRunnerHost` with `createRunnerWindow`), whose IPC the app registers on first use. The runner reads the project by id (`CliExportRequest.projectId`, the editor's own `aiEdition.get`), not by path: opening a file by path moves the main process's current project and recording-session slot, which in the app is the hand-off from the recorder to the editor. - `exportVideo` and `getFrames` read projects; they write none. Over HTTP their paths must be absolute. +- `createProject` makes a project with `DocumentService.createProject`, so it starts in the look new projects take (the preset marked for them, else the last edited project's), as the editor's New does. With a `videoPath` it lays the video out with `createProjectFromVideo`, the path a stdio take goes through: one clip, measured, and the zooms on its clicks when a pointer track sits beside it. Either way it returns a checkpoint, as `stopRecording` does, so the client can edit the closed project at once. It runs where the media tools run (in the CLI process for `openscreen mcp`): a new project is one no editor holds. **MCP edits** gates it. ## `openscreen mcp` and the running app -The stdio server (`openscreen mcp`, [`stdio-server.ts`](../../electron/mcp/stdio-server.ts)) is a process apart from the app. **When the app runs, it is the authority** (#1174): `listProjects`, every document tool, both checkpoint tools and `setPointerTrack` are forwarded to it, and the app runs them through the same runner as the in-app server. The project open in the editor is then edited through the editor, live and one undo step per call, and checkpoints live in one place. The other media tools (record, export, `getFrames`, `listSources`) stay in the CLI process. +The stdio server (`openscreen mcp`, [`stdio-server.ts`](../../electron/mcp/stdio-server.ts)) is a process apart from the app. **When the app runs, it is the authority** (#1174): `listProjects`, every document tool, the project tools, both checkpoint tools and `setPointerTrack` are forwarded to it, and the app runs them through the same runner as the in-app server. The project open in the editor is then edited through the editor, live and one undo step per call, and checkpoints live in one place. The other media tools (record, export, `getFrames`, `listSources`, `createProject`) stay in the CLI process. - **Channel.** MCP itself, over a Windows named pipe (a fresh random name per launch) or a Unix socket (`/mcp.sock`, mode 0600). The app serves `createOpenScreenMcpServer` on it, `openscreen mcp` is its client and forwards each `tools/call` unchanged. The handshake lists the app's tools, so a tool an older app lacks runs locally. - **Discovery.** Once listening, the app writes `/mcp-app.json` (`{ pid, endpoint, token }`, mode 0600) and removes it as it quits. `openscreen mcp` shares that userData and reads the file on each call it is not connected for: the app starting or quitting mid-session is picked up on the next call. A file whose pid is dead, or whose endpoint does not answer within 3 s, is a crashed app's and is ignored. @@ -189,4 +202,5 @@ codex mcp add openscreen -- "" mcp - **An in-app take is ready by a quiet period.** The editor lays a take out in several saves and says none of them is the last: the recorder waits for its revision to hold for 1.5 s. A save later than that (a slow auto-zoom pass) lands after the checkpoint, so `restoreCheckpoint` takes it back with the client's edits. - **No Ctrl+Z for edits to closed projects.** Their undo is the checkpoint the write requires, which lives in memory: it is gone once its server goes away or 20 newer ones push it out, and the edits stay. - **An editor that stops answering looks closed.** A snapshot unanswered for 30 s reads as "no project open", so an edit naming the project that editor holds would go to its file, and the editor's next save would drop it. It takes a renderer hung for 30 s. -- **No project management.** Projects can be listed, read and edited, and each take makes one; only a take's `title` names it, none can be renamed or deleted afterwards, and nothing imports media. +- **No project deletion**, on purpose ([above](#the-project-and-its-media)). +- **An imported recording comes without its camera.** The editor's import links the webcam an OpenScreen recording was made with (`find-recording-camera`); `importMedia` and `createProject` do not, so the camera of a recording imported over MCP is not in the project. diff --git a/website/docs/ai-editing.md b/website/docs/ai-editing.md index 04984cfca..8bc9da436 100644 --- a/website/docs/ai-editing.md +++ b/website/docs/ai-editing.md @@ -67,7 +67,7 @@ If you already use an AI coding agent such as Claude Code or Codex, it can drive 2. Copy the **Claude Code** or **Codex** command shown there and run it in a terminal. The Claude Code command carries the access token. Codex reads it from the `OPENSCREEN_MCP_TOKEN` environment variable instead: set that in the shell you start Codex from. 3. Ask your agent for the edit, on the project open in the OpenScreen editor or on any other project in your list. -The tools are the ones the built-in agent uses, plus the ones that record, export and look at a video ([below](#letting-an-agent-make-the-whole-video)). They act on the project open in the editor unless the client names another one: it can list every project and read or edit any of them. MCP clients can only **read** until you also turn on **MCP edits** in the MCP server section: until then every edit, recording and export they try is refused. This switch is separate from the built-in agent's **Agent edits**, and it starts off. Once on, each edit is saved as it lands: +The tools are the ones the built-in agent uses, plus the ones that record, export and look at a video ([below](#letting-an-agent-make-the-whole-video)). They act on the project open in the editor unless the client names another one: it can list every project and read or edit any of them. It can also create a project, empty or from a video file, rename one, and add a video or audio file to a project's media or remove one. Deleting a project is not among the tools: that stays yours to do in the app. MCP clients can only **read** until you also turn on **MCP edits** in the MCP server section: until then every edit, recording, export and new project they try is refused. This switch is separate from the built-in agent's **Agent edits**, and it starts off. Once on, each edit is saved as it lands: - **On the project open in the editor**, an edit is undone with `Ctrl/Cmd + Z`. Clients are told to save a checkpoint before a series of edits, so you can ask them to put the project back as it was in a single step, which `Ctrl/Cmd + Z` can itself undo. - **On any other project**, an edit is saved to its file, where `Ctrl/Cmd + Z` cannot reach it. A client must save a checkpoint of that project before its first edit, and putting the project back from it is the only undo. @@ -76,7 +76,7 @@ Checkpoints are kept in OpenScreen's memory: they are lost when it quits or when ## Letting an agent make the whole video -An agent can make a whole video on its own, from the recording to the export. Besides the editing tools, both ways of adding OpenScreen to it give it `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo` and `getFrames`. +An agent can make a whole video on its own, from the recording to the export. Besides the editing tools, both ways of adding OpenScreen to it give it `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` and `createProject`. **Through the server above**, a take is recorded by OpenScreen itself, as with its Record button: the editor closes, the recording bar counts down and records, and the take opens in the editor as a new project, where you watch each edit land. It needs **MCP edits** on, and the recording bar gets your own settings back once the take ends. diff --git a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md index 1d435b948..9417b4e1d 100644 --- a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md @@ -67,7 +67,7 @@ Wenn du bereits einen KI-Coding-Agenten wie Claude Code oder Codex nutzt, kann e 2. Kopiere den dort angezeigten Befehl für **Claude Code** oder **Codex** und führe ihn in einem Terminal aus. Der Befehl für Claude Code enthält das Zugriffstoken. Codex liest es stattdessen aus der Umgebungsvariable `OPENSCREEN_MCP_TOKEN`: Setze sie in der Shell, aus der du Codex startest. 3. Bitte deinen Agenten um die Änderung, am im OpenScreen-Editor geöffneten Projekt oder an einem beliebigen anderen Projekt deiner Liste. -Die Werkzeuge sind die, die der eingebaute Agent nutzt, dazu die, die ein Video aufnehmen, exportieren und ansehen ([unten](#letting-an-agent-make-the-whole-video)). Sie wirken auf das im Editor geöffnete Projekt, es sei denn, der Client nennt ein anderes: Er kann alle Projekte auflisten und jedes davon lesen oder bearbeiten. MCP-Clients können nur **lesen**, bis du im Abschnitt MCP-Server auch **MCP-Änderungen** einschaltest: Bis dahin wird jede Bearbeitung, Aufnahme oder jeder Export, den sie versuchen, abgelehnt. Dieser Schalter ist von **Agenten-Änderungen**, dem des eingebauten Agenten, getrennt und steht anfangs auf aus. Ist er an, wird jede Änderung gespeichert, sobald sie ankommt: +Die Werkzeuge sind die, die der eingebaute Agent nutzt, dazu die, die ein Video aufnehmen, exportieren und ansehen ([unten](#letting-an-agent-make-the-whole-video)). Sie wirken auf das im Editor geöffnete Projekt, es sei denn, der Client nennt ein anderes: Er kann alle Projekte auflisten und jedes davon lesen oder bearbeiten. Er kann auch ein Projekt anlegen, leer oder aus einer Videodatei, eines umbenennen und den Medien eines Projekts eine Video- oder Audiodatei hinzufügen oder eine daraus entfernen. Ein Projekt löschen gehört nicht zu den Werkzeugen: Das bleibt dir in der App vorbehalten. MCP-Clients können nur **lesen**, bis du im Abschnitt MCP-Server auch **MCP-Änderungen** einschaltest: Bis dahin wird jede Bearbeitung, Aufnahme, jeder Export und jedes neue Projekt, das sie versuchen, abgelehnt. Dieser Schalter ist von **Agenten-Änderungen**, dem des eingebauten Agenten, getrennt und steht anfangs auf aus. Ist er an, wird jede Änderung gespeichert, sobald sie ankommt: - **Am im Editor geöffneten Projekt** wird eine Änderung mit `Ctrl/Cmd + Z` rückgängig gemacht. Clients werden angewiesen, vor einer Reihe von Änderungen einen Wiederherstellungspunkt zu speichern. So kannst du sie bitten, das Projekt in einem einzigen Schritt wiederherzustellen, wie es war, und auch diesen Schritt kann `Ctrl/Cmd + Z` rückgängig machen. - **An jedem anderen Projekt** wird eine Änderung in seiner Datei gespeichert, wo `Ctrl/Cmd + Z` sie nicht erreicht. Ein Client muss vor der ersten Änderung einen Wiederherstellungspunkt dieses Projekts speichern, und dorthin zurückzukehren ist der einzige Weg, sie rückgängig zu machen. @@ -76,7 +76,7 @@ Wiederherstellungspunkte liegen im Speicher von OpenScreen: Sie gehen verloren, ## Einen Agenten das ganze Video erstellen lassen {#letting-an-agent-make-the-whole-video} -Ein Agent kann ein ganzes Video selbstständig erstellen, von der Aufnahme bis zum Export. Neben den Bearbeitungswerkzeugen geben ihm beide Wege, OpenScreen hinzuzufügen, `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo` und `getFrames`. +Ein Agent kann ein ganzes Video selbstständig erstellen, von der Aufnahme bis zum Export. Neben den Bearbeitungswerkzeugen geben ihm beide Wege, OpenScreen hinzuzufügen, `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` und `createProject`. **Über den Server oben** nimmt OpenScreen eine Aufnahme selbst auf, wie mit seinem Aufnahme-Button: Der Editor schließt sich, die Aufnahmeleiste zählt herunter und nimmt auf, und die Aufnahme öffnet sich im Editor als neues Projekt, wo du jede Änderung ankommen siehst. Dafür muss **MCP-Änderungen** an sein, und die Aufnahmeleiste bekommt am Ende der Aufnahme deine eigenen Einstellungen zurück. diff --git a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md index 658fa2a42..d6f3c1c42 100644 --- a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md @@ -67,7 +67,7 @@ Si ya usas un agente de código con IA como Claude Code o Codex, puede manejar l 2. Copia el comando de **Claude Code** o de **Codex** que aparece ahí y ejecútalo en una terminal. El comando de Claude Code lleva el token de acceso. Codex, en cambio, lo lee de la variable de entorno `OPENSCREEN_MCP_TOKEN`: defínela en el shell desde el que inicias Codex. 3. Pídele la edición a tu agente, sobre el proyecto abierto en el editor de OpenScreen o sobre cualquier otro proyecto de tu lista. -Las herramientas son las que usa el agente integrado, además de las que graban, exportan y miran un video ([más abajo](#letting-an-agent-make-the-whole-video)). Actúan sobre el proyecto abierto en el editor, salvo que el cliente indique otro: puede listar todos los proyectos y leer o editar cualquiera de ellos. Los clientes MCP solo pueden **leer** hasta que también actives **Ediciones por MCP** en la sección Servidor MCP: hasta entonces se rechaza cada edición, grabación o exportación que intenten. Este interruptor es independiente de **Ediciones del agente**, el del agente integrado, y empieza desactivado. Una vez activado, cada edición se guarda en cuanto se aplica: +Las herramientas son las que usa el agente integrado, además de las que graban, exportan y miran un video ([más abajo](#letting-an-agent-make-the-whole-video)). Actúan sobre el proyecto abierto en el editor, salvo que el cliente indique otro: puede listar todos los proyectos y leer o editar cualquiera de ellos. También puede crear un proyecto, vacío o a partir de un archivo de video, cambiarle el nombre a uno, y agregar un archivo de video o de audio a los medios de un proyecto o quitar uno. Eliminar un proyecto no está entre las herramientas: eso queda en tus manos, en la aplicación. Los clientes MCP solo pueden **leer** hasta que también actives **Ediciones por MCP** en la sección Servidor MCP: hasta entonces se rechaza cada edición, grabación, exportación o proyecto nuevo que intenten. Este interruptor es independiente de **Ediciones del agente**, el del agente integrado, y empieza desactivado. Una vez activado, cada edición se guarda en cuanto se aplica: - **En el proyecto abierto en el editor**, una edición se deshace con `Ctrl/Cmd + Z`. A los clientes se les indica que guarden un punto de restauración antes de una serie de ediciones, así que puedes pedirles que dejen el proyecto como estaba en un solo paso, que el propio `Ctrl/Cmd + Z` puede deshacer. - **En cualquier otro proyecto**, una edición se guarda en su archivo, fuera del alcance de `Ctrl/Cmd + Z`. Un cliente debe guardar un punto de restauración de ese proyecto antes de su primera edición, y volver a él es la única forma de deshacerla. @@ -76,7 +76,7 @@ Los puntos de restauración se guardan en la memoria de OpenScreen: se pierden c ## Dejar que un agente haga todo el video {#letting-an-agent-make-the-whole-video} -Un agente puede hacer un video completo por sí solo, desde la grabación hasta la exportación. Además de las herramientas de edición, las dos formas de agregar OpenScreen al agente le dan `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo` y `getFrames`. +Un agente puede hacer un video completo por sí solo, desde la grabación hasta la exportación. Además de las herramientas de edición, las dos formas de agregar OpenScreen al agente le dan `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` y `createProject`. **Con el servidor de arriba**, la toma la graba el propio OpenScreen, como con su botón Grabar: el editor se cierra, la barra de grabación hace su cuenta regresiva y graba, y la toma se abre en el editor como un proyecto nuevo, donde ves llegar cada edición. Necesita **Ediciones por MCP** activado, y la barra de grabación recupera tus propios ajustes al terminar la toma. diff --git a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md index ccfbd7398..b64e969e0 100644 --- a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md @@ -67,7 +67,7 @@ Si vous utilisez déjà un agent de code IA comme Claude Code ou Codex, il peut 2. Copiez la commande **Claude Code** ou **Codex** affichée à cet endroit et lancez-la dans un terminal. La commande Claude Code contient le jeton d'accès. Codex, lui, le lit dans la variable d'environnement `OPENSCREEN_MCP_TOKEN` : définissez-la dans le shell depuis lequel vous lancez Codex. 3. Demandez la modification à votre agent, sur le projet ouvert dans l'éditeur d'OpenScreen ou sur n'importe quel autre projet de votre liste. -Les outils sont ceux qu'utilise l'agent intégré, plus ceux qui enregistrent, exportent et regardent une vidéo ([plus bas](#letting-an-agent-make-the-whole-video)). Ils agissent sur le projet ouvert dans l'éditeur, sauf si le client en désigne un autre : il peut lister tous les projets et lire ou modifier n'importe lequel. Les clients MCP ne peuvent que **lire** tant que vous n'avez pas aussi activé **Modifications par MCP** dans la section Serveur MCP : jusque-là, chaque modification, enregistrement ou export qu'ils tentent est refusé. Cet interrupteur est distinct de **Modifications par l'agent**, celui de l'agent intégré, et il est désactivé au départ. Une fois activé, chaque modification est enregistrée dès qu'elle est appliquée : +Les outils sont ceux qu'utilise l'agent intégré, plus ceux qui enregistrent, exportent et regardent une vidéo ([plus bas](#letting-an-agent-make-the-whole-video)). Ils agissent sur le projet ouvert dans l'éditeur, sauf si le client en désigne un autre : il peut lister tous les projets et lire ou modifier n'importe lequel. Il peut aussi créer un projet, vide ou à partir d'un fichier vidéo, en renommer un, et ajouter un fichier vidéo ou audio aux médias d'un projet ou en retirer un. Supprimer un projet ne fait pas partie des outils : cela reste à vous, dans l'application. Les clients MCP ne peuvent que **lire** tant que vous n'avez pas aussi activé **Modifications par MCP** dans la section Serveur MCP : jusque-là, chaque modification, enregistrement, export ou nouveau projet qu'ils tentent est refusé. Cet interrupteur est distinct de **Modifications par l'agent**, celui de l'agent intégré, et il est désactivé au départ. Une fois activé, chaque modification est enregistrée dès qu'elle est appliquée : - **Sur le projet ouvert dans l'éditeur**, une modification s'annule avec `Ctrl/Cmd + Z`. Les clients ont pour consigne d'enregistrer un point de restauration avant une série de modifications : vous pouvez donc leur demander de remettre le projet dans son état d'avant en une seule étape, que `Ctrl/Cmd + Z` peut lui-même annuler. - **Sur tout autre projet**, une modification est enregistrée dans son fichier, hors de portée de `Ctrl/Cmd + Z`. Un client doit enregistrer un point de restauration de ce projet avant sa première modification, et y revenir est la seule façon de l'annuler. @@ -76,7 +76,7 @@ Les points de restauration sont gardés dans la mémoire d'OpenScreen : ils son ## Laisser un agent faire toute la vidéo {#letting-an-agent-make-the-whole-video} -Un agent peut faire une vidéo entière tout seul, de l'enregistrement à l'export. En plus des outils d'édition, les deux façons d'ajouter OpenScreen à l'agent lui donnent `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo` et `getFrames`. +Un agent peut faire une vidéo entière tout seul, de l'enregistrement à l'export. En plus des outils d'édition, les deux façons d'ajouter OpenScreen à l'agent lui donnent `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` et `createProject`. **Par le serveur ci-dessus**, une prise est enregistrée par OpenScreen lui-même, comme avec son bouton Enregistrer : l'éditeur se ferme, la barre d'enregistrement fait son décompte et enregistre, et la prise s'ouvre dans l'éditeur comme nouveau projet, où vous voyez chaque modification arriver. Il faut que **Modifications par MCP** soit activé, et la barre d'enregistrement retrouve vos propres réglages à la fin de la prise. diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md index f885f150b..6a7ee8095 100644 --- a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md @@ -67,7 +67,7 @@ Claude Code や Codex などの AI コーディングエージェントをすで 2. そこに表示される **Claude Code** または **Codex** のコマンドをコピーし、ターミナルで実行します。Claude Code のコマンドにはアクセストークンが含まれています。Codex は代わりに環境変数 `OPENSCREEN_MCP_TOKEN` からトークンを読み取るため、Codex を起動するシェルでこの変数を設定してください。 3. OpenScreen のエディターで開いているプロジェクト、または一覧にあるほかのプロジェクトについて、エージェントに編集を依頼します。 -ツールは組み込みのエージェントが使うものと同じで、さらに動画を録画、エクスポートし、確認するツールがあります（[後述](#letting-an-agent-make-the-whole-video)）。クライアントが別のプロジェクトを指定しない限り、エディターで開いているプロジェクトに作用します。クライアントはすべてのプロジェクトを一覧でき、どれでも読み取りや編集ができます。MCP サーバーのセクションで**MCP による編集**もオンにするまで、MCP クライアントは**読み取る**ことしかできず、編集、録画、エクスポートはすべて拒否されます。このスイッチは組み込みのエージェントの**エージェントによる編集**とは別で、最初はオフです。オンにすると、各編集は反映された時点で保存されます。 +ツールは組み込みのエージェントが使うものと同じで、さらに動画を録画、エクスポートし、確認するツールがあります（[後述](#letting-an-agent-make-the-whole-video)）。クライアントが別のプロジェクトを指定しない限り、エディターで開いているプロジェクトに作用します。クライアントはすべてのプロジェクトを一覧でき、どれでも読み取りや編集ができます。プロジェクトの作成（空のもの、または動画ファイルから）、名前の変更、プロジェクトのメディアへの動画・音声ファイルの追加や削除もできます。プロジェクトの削除はツールにありません。それはアプリであなたが行います。MCP サーバーのセクションで**MCP による編集**もオンにするまで、MCP クライアントは**読み取る**ことしかできず、編集、録画、エクスポート、新しいプロジェクトの作成はすべて拒否されます。このスイッチは組み込みのエージェントの**エージェントによる編集**とは別で、最初はオフです。オンにすると、各編集は反映された時点で保存されます。 - **エディターで開いているプロジェクト**では、編集は `Ctrl/Cmd + Z` で元に戻せます。クライアントには一連の編集の前にチェックポイントを保存するよう指示されているため、プロジェクトを 1 回の操作で元の状態に戻すよう頼むことができ、その操作自体も `Ctrl/Cmd + Z` で取り消せます。 - **それ以外のプロジェクト**では、編集はそのファイルに保存され、`Ctrl/Cmd + Z` では元に戻せません。クライアントは最初の編集の前にそのプロジェクトのチェックポイントを保存する必要があり、そこへ戻すことが唯一の取り消し方法です。 @@ -76,7 +76,7 @@ Claude Code や Codex などの AI コーディングエージェントをすで ## エージェントに動画全体を作らせる {#letting-an-agent-make-the-whole-video} -エージェントは、録画からエクスポートまで、動画全体を自力で作れます。どちらの方法で OpenScreen を追加しても、編集ツールに加えて `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames` を使えます。 +エージェントは、録画からエクスポートまで、動画全体を自力で作れます。どちらの方法で OpenScreen を追加しても、編集ツールに加えて `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames`、`createProject` を使えます。 **上のサーバーを使う場合**、録画は OpenScreen 自身が、録画ボタンを押したときと同じように行います。エディターが閉じ、録画バーがカウントダウンして録画し、録画は新しいプロジェクトとしてエディターで開きます。そこで各編集が反映されるのを確認できます。**MCP による編集**をオンにする必要があり、録画が終わると録画バーの設定はあなた自身のものに戻ります。 diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md index 2eac6c45a..6e8dd8bcd 100644 --- a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md @@ -67,7 +67,7 @@ Se você já usa um agente de código com IA, como o Claude Code ou o Codex, ele 2. Copie o comando do **Claude Code** ou do **Codex** exibido ali e execute-o em um terminal. O comando do Claude Code leva o token de acesso. Já o Codex o lê da variável de ambiente `OPENSCREEN_MCP_TOKEN`: defina-a no shell a partir do qual você inicia o Codex. 3. Peça a edição ao seu agente, no projeto aberto no editor do OpenScreen ou em qualquer outro projeto da sua lista. -As ferramentas são as que o agente integrado usa, mais as que gravam, exportam e olham um vídeo ([mais abaixo](#letting-an-agent-make-the-whole-video)). Elas agem sobre o projeto aberto no editor, a menos que o cliente indique outro: ele pode listar todos os projetos e ler ou editar qualquer um deles. Os clientes MCP só podem **ler** até você também ativar **Edições via MCP** na seção Servidor MCP: até lá, toda edição, gravação ou exportação que tentarem é recusada. Esse interruptor é separado de **Edições do agente**, o do agente integrado, e começa desativado. Depois de ativado, cada edição é salva assim que é aplicada: +As ferramentas são as que o agente integrado usa, mais as que gravam, exportam e olham um vídeo ([mais abaixo](#letting-an-agent-make-the-whole-video)). Elas agem sobre o projeto aberto no editor, a menos que o cliente indique outro: ele pode listar todos os projetos e ler ou editar qualquer um deles. Ele também pode criar um projeto, vazio ou a partir de um arquivo de vídeo, renomear um, e adicionar um arquivo de vídeo ou de áudio às mídias de um projeto ou remover um. Excluir um projeto não está entre as ferramentas: isso fica com você, no aplicativo. Os clientes MCP só podem **ler** até você também ativar **Edições via MCP** na seção Servidor MCP: até lá, toda edição, gravação, exportação ou novo projeto que tentarem é recusado. Esse interruptor é separado de **Edições do agente**, o do agente integrado, e começa desativado. Depois de ativado, cada edição é salva assim que é aplicada: - **No projeto aberto no editor**, uma edição se desfaz com `Ctrl/Cmd + Z`. Os clientes são instruídos a salvar um ponto de restauração antes de uma série de edições, então você pode pedir a eles que deixem o projeto como estava em um único passo, que o próprio `Ctrl/Cmd + Z` pode desfazer. - **Em qualquer outro projeto**, uma edição é salva no arquivo dele, fora do alcance de `Ctrl/Cmd + Z`. Um cliente precisa salvar um ponto de restauração desse projeto antes da primeira edição, e voltar a ele é a única forma de desfazê-la. @@ -76,7 +76,7 @@ Os pontos de restauração ficam na memória do OpenScreen: eles se perdem quand ## Como deixar um agente fazer o vídeo inteiro {#letting-an-agent-make-the-whole-video} -Um agente pode fazer um vídeo inteiro sozinho, da gravação à exportação. Além das ferramentas de edição, as duas formas de adicionar o OpenScreen ao agente lhe dão `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo` e `getFrames`. +Um agente pode fazer um vídeo inteiro sozinho, da gravação à exportação. Além das ferramentas de edição, as duas formas de adicionar o OpenScreen ao agente lhe dão `listSources`, `startRecording`, `stopRecording`, `setPointerTrack`, `exportVideo`, `getFrames` e `createProject`. **Pelo servidor acima**, a gravação é feita pelo próprio OpenScreen, como com o botão Gravar: o editor fecha, a barra de gravação faz a contagem regressiva e grava, e a gravação abre no editor como um novo projeto, onde você vê cada edição chegar. É preciso ativar **Edições via MCP**, e a barra de gravação volta às suas próprias configurações quando a gravação termina. diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md index 97192274a..592784919 100644 --- a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md @@ -67,7 +67,7 @@ ChatGPT 和 GitHub Copilot 登录选项已**在 1.8.0 中移除**。它们的工 2. 复制那里显示的 **Claude Code** 或 **Codex** 命令，并在终端中运行。Claude Code 命令中带有访问令牌。Codex 则改为从 `OPENSCREEN_MCP_TOKEN` 环境变量读取令牌：请在启动 Codex 的终端中设置它。 3. 让你的 AI 代理进行编辑，对象可以是 OpenScreen 编辑器中打开的项目，也可以是你列表中的任何其他项目。 -这些工具就是内置 AI 代理所使用的工具，另外还有录制、导出和查看视频的工具（[见下文](#letting-an-agent-make-the-whole-video)）。除非客户端指定其他项目，它们作用于编辑器中打开的项目：客户端可以列出所有项目，并读取或编辑其中任何一个。在你同时开启 MCP 服务器部分中的**MCP 编辑**之前，MCP 客户端只能**读取**，它们尝试的每项编辑、录制或导出都会被拒绝。这个开关独立于内置 AI 代理的**智能体编辑**开关，默认处于关闭状态。开启后，每项编辑都会在生效时立即保存： +这些工具就是内置 AI 代理所使用的工具，另外还有录制、导出和查看视频的工具（[见下文](#letting-an-agent-make-the-whole-video)）。除非客户端指定其他项目，它们作用于编辑器中打开的项目：客户端可以列出所有项目，并读取或编辑其中任何一个。它还可以创建项目（空项目，或基于一个视频文件），重命名项目，以及向项目的媒体添加或从中移除视频或音频文件。删除项目不在这些工具之列：这仍由你在应用中完成。在你同时开启 MCP 服务器部分中的**MCP 编辑**之前，MCP 客户端只能**读取**，它们尝试的每项编辑、录制、导出或新建项目都会被拒绝。这个开关独立于内置 AI 代理的**智能体编辑**开关，默认处于关闭状态。开启后，每项编辑都会在生效时立即保存： - **对编辑器中打开的项目**，编辑可用 `Ctrl/Cmd + Z` 撤销。客户端会被要求在进行一系列编辑之前保存一个检查点，因此你可以让它们一步把项目恢复原样，而这一步本身也可以用 `Ctrl/Cmd + Z` 撤销。 - **对其他任何项目**，编辑会保存到它的文件中，`Ctrl/Cmd + Z` 无法撤销。客户端必须在第一次编辑之前保存该项目的检查点，恢复到这个检查点是唯一的撤销方式。 @@ -76,7 +76,7 @@ ChatGPT 和 GitHub Copilot 登录选项已**在 1.8.0 中移除**。它们的工 ## 让 AI 代理制作整段视频 {#letting-an-agent-make-the-whole-video} -AI 代理可以从录制到导出独立完成一整段视频。无论以哪种方式添加 OpenScreen，除编辑工具外，AI 代理都能使用 `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo` 和 `getFrames`。 +AI 代理可以从录制到导出独立完成一整段视频。无论以哪种方式添加 OpenScreen，除编辑工具外，AI 代理都能使用 `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames` 和 `createProject`。 **通过上面的服务器**，录制由 OpenScreen 自己完成，就像按下它的录制按钮一样：编辑器关闭，录制栏倒计时后开始录制，录好的内容作为新项目在编辑器中打开，你可以看到每项编辑实时生效。这需要开启**MCP 编辑**，录制结束后，录制栏会恢复你自己的设置。 diff --git a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md index b48e12f5d..c91106b37 100644 --- a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md @@ -67,7 +67,7 @@ ChatGPT 與 GitHub Copilot 的登入選項已**在 1.8.0 中移除**。它們的 2. 複製那裡顯示的 **Claude Code** 或 **Codex** 指令，並在終端機中執行。Claude Code 指令中帶有存取權杖。Codex 則改為從 `OPENSCREEN_MCP_TOKEN` 環境變數讀取權杖：請在啟動 Codex 的終端機中設定它。 3. 請你的代理進行剪輯，對象可以是 OpenScreen 編輯器中開啟的專案，也可以是清單中的任何其他專案。 -這些工具就是內建代理所使用的工具，另外還有錄影、匯出和查看影片的工具（[見下文](#letting-an-agent-make-the-whole-video)）。除非用戶端指定其他專案，否則它們作用於編輯器中開啟的專案：用戶端可以列出所有專案，並讀取或編輯其中任何一個。在你同時開啟 MCP 伺服器區段中的**MCP 編輯**之前，MCP 用戶端只能**讀取**，它們嘗試的每項編輯、錄影或匯出都會被拒絕。這個開關與內建代理的**代理編輯**開關各自獨立，預設為關閉。開啟後，每項編輯都會在生效時立即儲存： +這些工具就是內建代理所使用的工具，另外還有錄影、匯出和查看影片的工具（[見下文](#letting-an-agent-make-the-whole-video)）。除非用戶端指定其他專案，否則它們作用於編輯器中開啟的專案：用戶端可以列出所有專案，並讀取或編輯其中任何一個。它也可以建立專案（空白專案，或從一個影片檔建立）、重新命名專案，以及在專案的媒體中加入或移除影片或音訊檔。刪除專案不在這些工具之中：這仍由你在應用程式中進行。在你同時開啟 MCP 伺服器區段中的**MCP 編輯**之前，MCP 用戶端只能**讀取**，它們嘗試的每項編輯、錄影、匯出或新建專案都會被拒絕。這個開關與內建代理的**代理編輯**開關各自獨立，預設為關閉。開啟後，每項編輯都會在生效時立即儲存： - **在編輯器中開啟的專案上**，編輯可用 `Ctrl/Cmd + Z` 復原。用戶端會被告知在進行一系列編輯之前先儲存檢查點，因此你可以請它們一步將專案還原成原本的樣子，而這一步本身也可以用 `Ctrl/Cmd + Z` 復原。 - **在其他任何專案上**，編輯會儲存到它的檔案中，`Ctrl/Cmd + Z` 無法復原。用戶端必須在第一次編輯之前儲存該專案的檢查點，還原到這個檢查點是唯一的復原方式。 @@ -76,7 +76,7 @@ ChatGPT 與 GitHub Copilot 的登入選項已**在 1.8.0 中移除**。它們的 ## 讓代理製作整支影片 {#letting-an-agent-make-the-whole-video} -代理可以從錄影到匯出自行製作一整支影片。無論以哪種方式加入 OpenScreen，除了剪輯工具之外，代理都能使用 `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo` 與 `getFrames`。 +代理可以從錄影到匯出自行製作一整支影片。無論以哪種方式加入 OpenScreen，除了剪輯工具之外，代理都能使用 `listSources`、`startRecording`、`stopRecording`、`setPointerTrack`、`exportVideo`、`getFrames` 與 `createProject`。 **透過上述伺服器**，錄影由 OpenScreen 自己完成，就像按下它的錄影按鈕一樣：編輯器會關閉，錄影列倒數後開始錄影，錄好的內容會以新專案在編輯器中開啟，你可以看到每項編輯即時生效。這需要開啟**MCP 編輯**，錄影結束後，錄影列會恢復你自己的設定。 From 726fb410eef912cca3a63adaae70a73a7465a3f6 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:28:27 +0200 Subject: [PATCH 3/3] fix(mcp): let the editor read an imported .oga, keep a created project's id when its checkpoint fails, check an existing video for private content Part of #1180 --- electron/ipc/handlers.ts | 1 + electron/mcp/media-tools.ts | 27 ++++++++++++++++++++------- skills/openscreen-demo/SKILL.md | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index d40eee6db..ead867cb1 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -274,6 +274,7 @@ const ALLOWED_IMPORT_AUDIO_EXTENSIONS = new Set([ ".aac", ".flac", ".ogg", + ".oga", ".opus", ]); diff --git a/electron/mcp/media-tools.ts b/electron/mcp/media-tools.ts index 745526707..25148ff88 100644 --- a/electron/mcp/media-tools.ts +++ b/electron/mcp/media-tools.ts @@ -443,11 +443,17 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { /** A fresh project's checkpoint, so a client can edit it at once and take it back in one step. */ async function checkpointOf(projectId: string): Promise { - const checkpoint = await deps.runTool("createCheckpoint", { projectId }); - const checkpointText = checkpoint.content[0]?.type === "text" ? checkpoint.content[0].text : ""; - return checkpoint.isError - ? null - : ((JSON.parse(checkpointText) as { checkpointId?: string }).checkpointId ?? null); + // Never fails the call: the project exists either way, and its id is the answer. + try { + const checkpoint = await deps.runTool("createCheckpoint", { projectId }); + const checkpointText = + checkpoint.content[0]?.type === "text" ? checkpoint.content[0].text : ""; + return checkpoint.isError + ? null + : ((JSON.parse(checkpointText) as { checkpointId?: string }).checkpointId ?? null); + } catch { + return null; + } } /** Stops the take, then hands back its project with a checkpoint of it. */ @@ -838,8 +844,15 @@ export function createMediaTools(deps: MediaToolDeps): MediaTools { } const title = args.title?.trim(); if (args.videoPath === undefined) { - const created = await deps.documents.createProject(title ?? ""); - const projectId = created.project.id; + let projectId: string; + try { + projectId = (await deps.documents.createProject(title ?? "")).project.id; + } catch (error) { + return text( + `No project was created: ${error instanceof Error ? error.message : String(error)}`, + true, + ); + } return text(JSON.stringify({ projectId, checkpointId: await checkpointOf(projectId) })); } const videoPath = args.videoPath; diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.md index 15e5f556c..99f75ebb6 100644 --- a/skills/openscreen-demo/SKILL.md +++ b/skills/openscreen-demo/SKILL.md @@ -68,7 +68,7 @@ Write a storyboard and show it to the user in a few lines, then carry on unless - **Real mouse** (computer-use, or any tool that moves the actual pointer): OpenScreen records the pointer, draws a clean cursor and already zooms on the clicks (see *Zoom on the action* below). If your mouse tool cannot click in the app's window (some only read browsers, for one), drive it from a script instead. - **A script** (Playwright, CDP): the screen pointer never moves, so the take has no cursor and no zoom. In the script, log every action as it runs: `at` is `Date.now()`, `x` and `y` are the action's position as fractions of the window, `click` is true for a click. Hand the log to `setPointerTrack` after `stopRecording`; its description gives the formula from a page's coordinates. A script also gives the cleanest take: same timing on every run, no hesitation. -**A video you already have** needs no recording: `createProject` with its absolute `videoPath` lays it out as a project, and returns a `projectId` and a `checkpointId` as `stopRecording` does. Then go on from step 6. +**A video you already have** needs no recording: `createProject` with its absolute `videoPath` lays it out as a project, and returns a `projectId` and a `checkpointId` as `stopRecording` does. Check it as a take in step 5, its `videoPath` included, then go on from step 6. ## 5. Check the take