wwwyo · Oct 13, 2026
@@ -78,6 +78,7 @@ import {
 	setRecordingsDir,
 } from "../main";
 import { EditorDocumentHost } from "../mcp/editor-document-host";
+import { openVideoInHiddenWindow } from "../mcp/hidden-video";
 import { McpController } from "../mcp/mcp-controller";
 import { McpSettingsStore } from "../mcp/mcp-settings-store";
 import type { McpToolDeps } from "../mcp/openscreen-mcp-server";
@@ -5038,6 +5039,7 @@ export function registerIpcHandlers(
 		editsAllowed: () => mcpSettings.getSettings().allowEdits,
 		cursor: agentCursorTelemetryReader,
 		version: app.getVersion(),
+		openVideo: openVideoInHiddenWindow,
 	};
 	const mcpController = new McpController(mcpSettings, mcpToolDeps);
 

@@ -60,7 +60,7 @@ export interface MediaToolDeps {
 
 export const MAX_FRAMES = 12;
 // Wide enough to read UI text, small enough that a dozen frames stay cheap.
-const FRAME_WIDTH = 1280;
+export const FRAME_WIDTH = 1280;
 // A seek to the exact end shows no frame in Chromium; stop one frame short of it.
 const END_MARGIN_SEC = 1 / 60;
 

@@ -18,6 +18,7 @@ import {
 import { OPENSCREEN_TOOL_NAMES } from "../ai-edition/agent-tools";
 import { TOOL_DESCRIPTIONS } from "../ai-edition/deep-agent/service";
 import { DocumentService } from "../ai-edition/document-service";
+import type { OpenedVideo } from "./hidden-video";
 import {
 	LIST_PROJECTS_TOOL,
 	MCP_ENDPOINT_PATH,
@@ -123,8 +124,21 @@ let running: RunningMcpServer | null = null;
 let client: Client | null = null;
 let dir: string;
 let projects: DocumentService;
+const openVideo = vi.fn(
+	async (_videoPath: string): Promise<OpenedVideo> => ({
+		durationSec: 10,
+		width: 1920,
+		height: 1080,
+		capture: async (timeSec) => ({
+			mimeType: "image/jpeg",
+			data: Buffer.from(`frame@${timeSec}`).toString("base64"),
+		}),
+		close: () => undefined,
+	}),
+);
 
 beforeEach(() => {
+	openVideo.mockClear();
 	dir = mkdtempSync(path.join(os.tmpdir(), "openscreen-mcp-projects-"));
 	projects = new DocumentService(path.join(dir, "projects"), dir);
 });
@@ -149,6 +163,7 @@ async function connect(
 			projects,
 			editsAllowed: () => options.editsAllowed ?? true,
 			version: "0.0.0",
+			openVideo,
 		},
 	});
 	client = new Client({ name: "test", version: "0.0.0" });
@@ -167,22 +182,23 @@ function resultText(result: Awaited<ReturnType<Client["callTool"]>>): string {
 }
 
 describe("the MCP tool surface", () => {
-	it("is listProjects, then exactly the in-app agent's tools with its descriptions, then the checkpoints", async () => {
+	it("lists document tools, checkpoints, and source frames", async () => {
 		const mcp = await connect(new FakeEditor());
 		const { tools } = await mcp.listTools();
 		expect(tools.map((t) => t.name)).toEqual([
 			LIST_PROJECTS_TOOL,
 			...OPENSCREEN_TOOL_NAMES,
 			"createCheckpoint",
 			"restoreCheckpoint",
+			"getFrames",
 		]);
 		for (const tool of tools.slice(1, 1 + OPENSCREEN_TOOL_NAMES.length)) {
 			expect(tool.description).toBe(TOOL_DESCRIPTIONS[tool.name]);
 			expect(tool.inputSchema.type).toBe("object");
 			expect(Object.keys(tool.inputSchema.properties ?? {})).toContain("projectId");
 			expect(tool.inputSchema.required ?? []).not.toContain("projectId");
 		}
-		for (const tool of tools.slice(-2)) {
+		for (const tool of tools.slice(-3, -1)) {
 			expect(Object.keys(tool.inputSchema.properties ?? {})).toContain("projectId");
 		}
 		// The zod schemas survive the trip to JSON Schema with their fields intact.
@@ -206,6 +222,66 @@ describe("the MCP tool surface", () => {
 		expect(byName.get("removeClip")?.destructiveHint).toBe(true);
 		expect(byName.get("createCheckpoint")?.readOnlyHint).toBe(true);
 		expect(byName.get("restoreCheckpoint")?.destructiveHint).toBe(true);
+		expect(byName.get("getFrames")?.readOnlyHint).toBe(true);
+	});
+
+	it("returns only frames from the selected project's video asset", async () => {
+		const mcp = await connect(new FakeEditor());
+		const { tools } = await mcp.listTools();
+		const framesTool = tools.find((tool) => tool.name === "getFrames");
+		expect(Object.keys(framesTool?.inputSchema.properties ?? {})).not.toContain("path");
+		const result = await mcp.callTool({ name: "getFrames", arguments: { atSec: [1, 5] } });
+		expect(result.isError).toBeFalsy();
+		expect(openVideo).toHaveBeenCalledWith("/tmp/rec.mp4");
+		const content = result.content as Array<{ type: string }>;
+		expect(content.filter((part) => part.type === "image")).toHaveLength(2);
+		expect(resultText(result)).toContain("These do not show edits");
+		const missing = await mcp.callTool({ name: "getFrames", arguments: { assetId: "unknown" } });
+		expect(missing.isError).toBe(true);
+		expect(openVideo).toHaveBeenCalledTimes(1);
+	});
+
+	it("reads a named project without enabling edits", async () => {
+		const other = await projects.createProject("Other");
+		const source = path.join(dir, "other.mp4");
+		writeFileSync(source, "video stub");
+		await projects.addAsset(other.project.id, { path: source });
+		const mcp = await connect(new FakeEditor(), { editsAllowed: false });
+		const result = await mcp.callTool({
+			name: "getFrames",
+			arguments: { projectId: other.project.id, atSec: [2] },
+		});
+		expect(result.isError).toBeFalsy();
+		expect(openVideo).toHaveBeenCalledWith(source);
+	});
+
+	it("limits concurrent frame extraction to one decoder window", async () => {
+		let releaseCapture: () => void = () => undefined;
+		const paused = new Promise<void>((resolve) => {
+			releaseCapture = resolve;
+		});
+		openVideo.mockImplementationOnce(async () => ({
+			durationSec: 10,
+			width: 1920,
+			height: 1080,
+			capture: async () => {
+				await paused;
+				return { mimeType: "image/jpeg", data: "aGVsbG8=" };
+			},
+			close: () => undefined,
+		}));
+		const mcp = await connect(new FakeEditor());
+		const first = mcp.callTool({ name: "getFrames", arguments: { atSec: [1] } });
+		try {
+			await vi.waitFor(() => expect(openVideo).toHaveBeenCalledTimes(1));
+			const second = await mcp.callTool({ name: "getFrames", arguments: { atSec: [2] } });
+			expect(second.isError).toBe(true);
+			expect(resultText(second)).toContain("already running");
+			expect(openVideo).toHaveBeenCalledTimes(1);
+		} finally {
+			releaseCapture();
+		}
+		expect((await first).isError).toBeFalsy();
 	});
 
 	it("hands the client the in-app agent's guidance as server instructions", async () => {

@@ -52,6 +52,8 @@ import {
 	type ProjectSummary,
 	type ProjectVersion,
 } from "../ai-edition/document-service";
+import type { OpenedVideo } from "./hidden-video";
+import { createProjectFrameToolRegistrar } from "./project-frame-tools";
 
 export const MCP_SERVER_NAME = "openscreen";
 export const MCP_ENDPOINT_PATH = "/mcp";
@@ -106,6 +108,8 @@ export interface McpToolDeps {
 	editsAllowed(): boolean;
 	cursor?: CursorTelemetryReader;
 	version: string;
+	/** Available in the GUI process for read-only source frames over HTTP. */
+	openVideo?: (videoPath: string) => Promise<OpenedVideo>;
 }
 
 export const LIST_PROJECTS_TOOL = "listProjects";
@@ -150,7 +154,7 @@ const DESTRUCTIVE_TOOLS: ReadonlySet<string> = new Set([
 ]);
 
 const MCP_PREAMBLE = [
-	"These tools act on the user's OpenScreen projects. Without a projectId they act on the project open in the editor; call listProjects to find any other project and pass its id as projectId. Every edit is saved straight away. An edit to the open project appears in the editor, where the user can undo it with Ctrl/Cmd+Z; an edit to any other project is saved to its file and is not on the editor's undo stack. Nothing here records, exports or imports media.",
+	"These tools act on the user's OpenScreen projects. Without a projectId they act on the project open in the editor; call listProjects to find any other project and pass its id as projectId. Every edit is saved straight away. An edit to the open project appears in the editor, where the user can undo it with Ctrl/Cmd+Z; an edit to any other project is saved to its file and is not on the editor's undo stack. getFrames reads source recording frames, not the edited preview or export. Nothing here records, exports or imports media.",
 	"Before a series of edits, call createCheckpoint with the same projectId. Undo is one step per call, so if the result is not what the user wanted, restoreCheckpoint takes the project back in one step instead of many. A project that is not open in the editor cannot be edited until it has a checkpoint, since restoreCheckpoint is its only undo.",
 	"",
 ].join("\n");
@@ -453,6 +457,7 @@ export async function startMcpHttpServer(options: {
 }): Promise<RunningMcpServer> {
 	const { token, deps } = options;
 	const runTool = createToolRunner(deps);
+	const registerFrames = createProjectFrameToolRegistrar(deps);
 	let boundPort = options.port;
 
 	const handle = async (req: IncomingMessage, res: ServerResponse) => {
@@ -470,6 +475,7 @@ export async function startMcpHttpServer(options: {
 		// for a server that keeps no per-session state. Tools read the live editor
 		// on every call, so there is nothing a session would need to remember.
 		const server = createOpenScreenMcpServer(deps, runTool);
+		registerFrames(server);
 		const transport = new StreamableHTTPServerTransport({
 			sessionIdGenerator: undefined,
 			enableJsonResponse: true,

@@ -0,0 +1,93 @@
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
+import { z } from "zod";
+import { documentSchema } from "../../src/lib/ai-edition/schema";
+import { DocumentNotFoundError } from "../ai-edition/document-service";
+import type { OpenedVideo } from "./hidden-video";
+import { FRAME_WIDTH, frameTimes, MAX_FRAMES } from "./media-tools";
+import type { McpToolDeps } from "./openscreen-mcp-server";
+
+function error(message: string): CallToolResult {
+	return { content: [{ type: "text", text: message }], isError: true };
+}
+
+export function createProjectFrameToolRegistrar(deps: McpToolDeps): (server: McpServer) => void {
+	const openVideo = deps.openVideo;
+	if (!openVideo) return () => undefined;
+	let busy = false;
+	return (server) =>
+		server.registerTool(
+			"getFrames",
+			{
+				description:
+					"Read still frames from a video asset in an OpenScreen project. These are frames of the original recording, not the edited preview or export. Omit projectId for the project open in the editor; omit assetId for its primary video. Times are seconds in that source video. At most 12 frames per call.",
+				inputSchema: z.object({
+					projectId: z.string().optional(),
+					assetId: z.string().optional(),
+					atSec: z.array(z.number().nonnegative()).max(MAX_FRAMES).optional(),
+					count: z.number().int().min(1).max(MAX_FRAMES).optional(),
+				}),
+				annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: false },
+			},
+			async ({ projectId, assetId, atSec, count }) => {
+				if (busy) return error("Frame extraction is already running. Try again when it finishes.");
+				busy = true;
+				try {
+					const snapshot = await deps.host.snapshot();
+					let rawDocument: unknown;
+					if (projectId === undefined) {
+						if (!snapshot) return error("No project is open in the OpenScreen editor.");
+						rawDocument = snapshot.document;
+					} else if (
+						(snapshot?.document as { project?: { id?: unknown } } | null)?.project?.id === projectId
+					) {
+						rawDocument = snapshot?.document;
+					} else {
+						try {
+							rawDocument = (await deps.projects.getProjectForUpdate(projectId)).document;
+						} catch (cause) {
+							if (cause instanceof DocumentNotFoundError)
+								return error(`No project has the id "${projectId}".`);
+							throw cause;
+						}
+					}
+					const parsed = documentSchema.safeParse(rawDocument);
+					if (!parsed.success) return error("The project could not be read.");
+					const document = parsed.data;
+					const selectedId = assetId ?? document.project.primaryAssetId;
+					const asset = document.assets.find(
+						(item) => item.id === selectedId && item.kind === "video",
+					);
+					if (!asset) return error("No matching video asset exists in this project.");
+
+					let video: OpenedVideo;
+					try {
+						video = await openVideo(asset.originalPath);
+					} catch {
+						return error("OpenScreen could not decode this project's video asset.");
+					}
+					try {
+						const times = frameTimes(video.durationSec, atSec ?? null, count ?? 6);
+						const content: CallToolResult["content"] = [
+							{
+								type: "text",
+								text: `${times.length} source frame(s) from asset ${asset.id}; ${video.width}x${video.height}, ${video.durationSec.toFixed(2)}s. These do not show edits.`,
+							},
+						];
+						for (const time of times) {
+							const frame = await video.capture(time, FRAME_WIDTH);
+							content.push({ type: "text", text: `${time.toFixed(2)}s in source video` });
+							content.push({ type: "image", data: frame.data, mimeType: frame.mimeType });
+						}
+						return { content };
+					} catch {
+						return error("OpenScreen could not capture frames from this project's video asset.");
+					} finally {
+						video.close();
+					}
+				} finally {
+					busy = false;
+				}
+			},
+		);
+}
@@ -67,7 +67,7 @@ If you already use an AI coding agent such as Claude Code or Codex, it can drive
 2. Copy the **Claude Code** or **Codex** command shown there and run it in a terminal. The Claude Code command carries the access token. Codex reads it from the `OPENSCREEN_MCP_TOKEN` environment variable instead: set that in the shell you start Codex from.
 3. Keep a project open in the OpenScreen editor, then ask your agent for the edit.
 
-The tools are the ones the built-in agent uses, and they act on the project open in the editor. MCP clients can only **read** the project until you also turn on **Project edits** in the MCP server section; this switch is separate from the built-in agent's, and it starts off. Once on, each edit is saved as it lands and undone with `Ctrl/Cmd + Z`. Clients are told to save a checkpoint before a series of edits, so you can ask them to put the project back as it was in a single step, which `Ctrl/Cmd + Z` can itself undo. **Regenerate** the token to disconnect every client set up with the old one.
+The tools are the ones the built-in agent uses, and they act on the project open in the editor. The HTTP server also offers `getFrames` for a project's original video asset. Its images show the source recording, not trims, zooms or other edits; it does not accept an arbitrary file path. MCP clients can only **read** the project until you also turn on **Project edits** in the MCP server section; this switch is separate from the built-in agent's, and it starts off. Once on, each edit is saved as it lands and undone with `Ctrl/Cmd + Z`. Clients are told to save a checkpoint before a series of edits, so you can ask them to put the project back as it was in a single step, which `Ctrl/Cmd + Z` can itself undo. **Regenerate** the token to disconnect every client set up with the old one.
 
 ## Letting an agent make the whole video
 

@@ -67,7 +67,7 @@ Wenn du bereits einen KI-Coding-Agenten wie Claude Code oder Codex nutzt, kann e
 2. Kopiere den dort angezeigten Befehl für **Claude Code** oder **Codex** und führe ihn in einem Terminal aus. Der Befehl für Claude Code enthält das Zugriffstoken. Codex liest es stattdessen aus der Umgebungsvariable `OPENSCREEN_MCP_TOKEN`: Setze sie in der Shell, aus der du Codex startest.
 3. Lass ein Projekt im OpenScreen-Editor geöffnet und bitte dann deinen Agenten um die Änderung.
 
-Die Werkzeuge sind die, die der eingebaute Agent nutzt, und sie wirken auf das im Editor geöffnete Projekt. MCP-Clients können das Projekt nur **lesen**, bis du im Abschnitt MCP-Server auch **Project edits** einschaltest; dieser Schalter ist von dem des eingebauten Agenten getrennt und steht anfangs auf aus. Ist er an, wird jede Änderung gespeichert, sobald sie ankommt, und mit `Ctrl/Cmd + Z` rückgängig gemacht. Clients werden angewiesen, vor einer Reihe von Änderungen einen Wiederherstellungspunkt zu speichern. So kannst du sie bitten, das Projekt in einem einzigen Schritt wiederherzustellen, wie es war, und auch diesen Schritt kann `Ctrl/Cmd + Z` rückgängig machen. Wenn du das Token mit **Regenerate** ersetzt, werden alle Clients getrennt, die mit dem alten eingerichtet wurden.
+Die Werkzeuge sind die, die der eingebaute Agent nutzt, und sie wirken auf das im Editor geöffnete Projekt. Der HTTP-Server bietet außerdem `getFrames` für Einzelbilder aus dem Originalvideo eines Projekts. Die Bilder zeigen die Quellaufnahme ohne Schnitte, Zooms oder andere Bearbeitungen; ein beliebiger Dateipfad ist nicht zulässig. MCP-Clients können das Projekt nur **lesen**, bis du im Abschnitt MCP-Server auch **Project edits** einschaltest; dieser Schalter ist von dem des eingebauten Agenten getrennt und steht anfangs auf aus. Ist er an, wird jede Änderung gespeichert, sobald sie ankommt, und mit `Ctrl/Cmd + Z` rückgängig gemacht. Clients werden angewiesen, vor einer Reihe von Änderungen einen Wiederherstellungspunkt zu speichern. So kannst du sie bitten, das Projekt in einem einzigen Schritt wiederherzustellen, wie es war, und auch diesen Schritt kann `Ctrl/Cmd + Z` rückgängig machen. Wenn du das Token mit **Regenerate** ersetzt, werden alle Clients getrennt, die mit dem alten eingerichtet wurden.
 
 ## Einen Agenten das ganze Video erstellen lassen {#letting-an-agent-make-the-whole-video}
 

@@ -67,7 +67,7 @@ Si ya usas un agente de código con IA como Claude Code o Codex, puede manejar l
 2. Copia el comando de **Claude Code** o de **Codex** que aparece ahí y ejecútalo en una terminal. El comando de Claude Code lleva el token de acceso. Codex, en cambio, lo lee de la variable de entorno `OPENSCREEN_MCP_TOKEN`: defínela en el shell desde el que inicias Codex.
 3. Mantén un proyecto abierto en el editor de OpenScreen y luego pídele la edición a tu agente.
 
-Las herramientas son las que usa el agente integrado, y actúan sobre el proyecto abierto en el editor. Los clientes MCP solo pueden **leer** el proyecto hasta que también actives **Ediciones del proyecto** en la sección Servidor MCP; este interruptor es independiente del del agente integrado, y empieza desactivado. Una vez activado, cada edición se guarda en cuanto se aplica y se deshace con `Ctrl/Cmd + Z`. A los clientes se les indica que guarden un punto de restauración antes de una serie de ediciones, así que puedes pedirles que dejen el proyecto como estaba en un solo paso, que el propio `Ctrl/Cmd + Z` puede deshacer. **Regenerar** el token desconecta a todos los clientes configurados con el anterior.
+Las herramientas son las que usa el agente integrado, y actúan sobre el proyecto abierto en el editor. El servidor HTTP también ofrece `getFrames` para obtener fotogramas del vídeo original de un proyecto. Las imágenes muestran la grabación de origen, sin recortes, zooms ni otras ediciones; no acepta una ruta de archivo arbitraria. Los clientes MCP solo pueden **leer** el proyecto hasta que también actives **Ediciones del proyecto** en la sección Servidor MCP; este interruptor es independiente del del agente integrado, y empieza desactivado. Una vez activado, cada edición se guarda en cuanto se aplica y se deshace con `Ctrl/Cmd + Z`. A los clientes se les indica que guarden un punto de restauración antes de una serie de ediciones, así que puedes pedirles que dejen el proyecto como estaba en un solo paso, que el propio `Ctrl/Cmd + Z` puede deshacer. **Regenerar** el token desconecta a todos los clientes configurados con el anterior.
 
 ## Dejar que un agente haga todo el video {#letting-an-agent-make-the-whole-video}