From 26af1ce26b3fec835267b550c027f7625658e363 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:05:37 +0200 Subject: [PATCH 1/4] feat(mcp): setEditorSettings and a settings block in getCurrentDocument One tool writes the project's look through applySettingsPatch: background, frame, format (Original shapes included), camera layout, cursor, autoFocusAll and audioGainDb, with the pane's own side effects. Bounds come from SETTING_BOUNDS and wallpapers from parseStylePresetWallpaper. getCurrentDocument reads it back in the same shape, with the options on offer and why each hidden setting is inert. setAspectRatio becomes an alias. Part of #1180 --- electron/ai-edition/agent-tools.test.ts | 1 + electron/ai-edition/agent-tools.ts | 33 +- electron/ai-edition/chat-service.ts | 4 + .../ai-edition/deep-agent/service.test.ts | 1 + electron/ai-edition/deep-agent/service.ts | 17 +- .../ai-edition/editor-settings-tool.test.ts | 224 +++++++++ electron/ai-edition/editor-settings-tool.ts | 444 ++++++++++++++++++ electron/ipc/handlers.ts | 2 + electron/mcp/openscreen-mcp-server.ts | 4 + src/lib/ai-edition/store/editorSettings.ts | 2 +- 10 files changed, 719 insertions(+), 13 deletions(-) create mode 100644 electron/ai-edition/editor-settings-tool.test.ts create mode 100644 electron/ai-edition/editor-settings-tool.ts diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 4a9b07a6e6..f9811fa81e 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -203,6 +203,7 @@ describe("the mutating-tool table", () => { "setAudio", "setCameraFullscreen", "setClipRange", + "setEditorSettings", "setSpeed", "setTrim", "setWordText", diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 9e01e253b9..34da2f2cd8 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -48,7 +48,6 @@ import { figureDataSchema, } from "../../src/lib/ai-edition/schema"; import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings"; -import { applySettingsPatch } from "../../src/lib/ai-edition/store/settingsPatch"; import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; import { isGeneratedAssetId } from "../../src/lib/ai-edition/timeline/clip-parts"; import { @@ -79,8 +78,10 @@ import { zoomTransitions, } from "../../src/lib/ai-edition/timeline/zoom-transitions"; import { SETTING_BOUNDS } from "../../src/lib/projectDefaults"; +import type { SegmentationSupport } from "../../src/native/contracts"; import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription"; import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils"; +import { editorSettingsSnapshot, executeSetEditorSettings } from "./editor-settings-tool"; import { measureTextInShippedFonts } from "./text-measure"; export interface AgentToolExecution { @@ -670,7 +671,11 @@ export const setClipRangeArgs = z.object({ }); export const setAspectRatioArgs = z.object({ - aspectRatio: z.enum(["auto", ...ASPECT_RATIO_PRESETS]), + aspectRatio: z + .string() + .describe( + `"auto", one of ${ASPECT_RATIO_PRESETS.join(", ")}, or a token from getCurrentDocument's settingsContext.originalAspectRatios.`, + ), }); export const replaceTimelineArgs = z.object({ @@ -1027,6 +1032,7 @@ export const OPENSCREEN_TOOL_NAMES = [ "removeModifier", "removeClip", "setAspectRatio", + "setEditorSettings", ] as const; /** @@ -1100,6 +1106,7 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "removeModifier", "removeClip", "setAspectRatio", + "setEditorSettings", ]); export function isMutatingTool(name: string): boolean { @@ -1512,6 +1519,9 @@ export interface AgentToolOptions { * Absent means the runtime has no telemetry reader wired at all, which the * executor reports as "could not look", never as "there is none". */ cursorTelemetry?: CursorTelemetryContext; + /** Whether this machine can cut out the camera background, asked only by the two tools that + * report it. Absent where nothing can ask the compositor: reported as not checked. */ + cameraSegmentation?: () => SegmentationSupport; } /** @@ -1724,28 +1734,33 @@ export function executeAgentTool( switch (name) { case "setAspectRatio": { + // The format alone, kept for clients that know it: setEditorSettings does the work. const parsed = setAspectRatioArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); const previous = getEditorSettings(document).aspectRatio; - // The editor's own write: the frame-placed annotations keep their centre, and a text is - // refitted to its words in the new frame, measured in the shipped fonts. - const next = applySettingsPatch( + const execution = executeSetEditorSettings( document, { aspectRatio: parsed.data.aspectRatio }, - measureTextInShippedFonts, + options?.cameraSegmentation?.() ?? null, ); + if (!execution.ok) return execution; return { - ok: true, - document: next, + ...execution, resultJson: JSON.stringify({ aspectRatio: parsed.data.aspectRatio, previous }), summary: `set the format to ${parsed.data.aspectRatio}`, }; } + case "setEditorSettings": + return executeSetEditorSettings(document, args, options?.cameraSegmentation?.() ?? null); + case "getCurrentDocument": { return { ok: true, - resultJson: JSON.stringify(documentSnapshotForModel(document, options?.cursorTelemetry)), + resultJson: JSON.stringify({ + ...documentSnapshotForModel(document, options?.cursorTelemetry), + ...editorSettingsSnapshot(document, options?.cameraSegmentation?.() ?? null), + }), }; } diff --git a/electron/ai-edition/chat-service.ts b/electron/ai-edition/chat-service.ts index b224565bc7..e1e55c94c9 100644 --- a/electron/ai-edition/chat-service.ts +++ b/electron/ai-edition/chat-service.ts @@ -20,6 +20,7 @@ import type { AiEditionChatMessage, AiEditionChatResult, AiEditionToolCallSummary, + SegmentationSupport, } from "../../src/native/contracts"; import { applyCompaction, @@ -319,6 +320,8 @@ export interface ChatRunEnv { /** Reads recorded cursor telemetry for an asset. Built in `electron/ipc/ * handlers.ts`, where the path allow-list lives. */ cursor?: CursorTelemetryReader; + /** Whether this machine can cut out the camera background, asked of the compositor. */ + cameraSegmentation?: () => SegmentationSupport; /** Runtime-only model retry override used by bounded measurement harnesses. */ maxRetries?: number; } @@ -464,6 +467,7 @@ export async function runChat( sink: agentSink, editsAllowed, cursor: env.cursor, + cameraSegmentation: env.cameraSegmentation, }); // The project was deleted while the agent ran: its sessions are gone, so a diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index d14b3e9efc..a718552ffd 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -85,6 +85,7 @@ const ARGS: Record = { removeModifier: { id: "nope" }, removeClip: { clipId: "clip_1" }, setAspectRatio: { aspectRatio: "16:9" }, + setEditorSettings: { padding: 30 }, }; /** ponytail: the fixture starts from a FIXED instant. `createEmptyDocument` reads diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 7b05c0444f..53d122e10d 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -31,6 +31,7 @@ import { ZOOM_DEPTH_LEGEND, } from "../../../src/lib/ai-edition/timeline/zoom-scale"; import { ZOOM_TRANSITION_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-transitions"; +import type { SegmentationSupport } from "../../../src/native/contracts"; import { AGENT_EDITS_SETTING, type AgentToolExecution, @@ -65,6 +66,7 @@ import { setWordTextArgs, setZoomArgs, } from "../agent-tools"; +import { setEditorSettingsArgs } from "../editor-settings-tool"; import { createOpenScreenChatModel, messageContentToText, @@ -132,7 +134,7 @@ const BASE_SYSTEM_PROMPT = [ "- Cutting specific spoken words (fillers, a repeated or flubbed phrase, a false start, a duplicate take) is removeWords: read getTranscriptWords, choose the occurrences from their surrounding speech, then pass that assetId and ONLY those word IDs. Word IDs restart for each asset, so keep the read's assetId. Consecutive words become one cut. Do not cut every occurrence of a word just because its text matches one you were asked to remove.", "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.", `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text, an arrow or a privacy blur on screen: a blur is how an email address, a token or any private detail the recording shows is hidden. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, - "- setAspectRatio picks the shape of the whole video, for where it will be shown; auto follows the recording.", + "- setEditorSettings sets the look of the whole video: the background (wallpaper, its motion and blur), padding, rounded corners, shadow, a device frame, motion blur, depth of field, the shape of the video (aspectRatio: auto follows the recording) and how a recording of another shape fills it, the camera's layout (picture-in-picture, side by side, stacked, its shape, size, position, framing and background cutout), the cursor, and the master volume. getCurrentDocument returns all of them under settings, in the shape this tool takes, and settingsContext.inactive names each one the editor hides in the current state and why: setting one of those changes nothing, so say so instead of reporting the look as changed. setAspectRatio is the same write for the shape alone.", "- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.", "- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.", "- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.", @@ -181,7 +183,7 @@ const CUT_TRANSITIONS_RESULT = `When the result cuts a zoom's move in or out, it export const TOOL_DESCRIPTIONS: Record = { getCurrentDocument: - "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / camera-fullscreen / audio regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", + "Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / camera-fullscreen / audio regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. It also gives the project's look under `settings`, in the shape setEditorSettings takes, with `settingsContext`: the formats, wallpapers and cursor themes on offer, and under `inactive` each setting the editor hides in the current state and why. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.", getTranscript: "Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.", getCursorTrack: @@ -214,7 +216,9 @@ export const TOOL_DESCRIPTIONS: Record = { setAnnotation: "Change an existing annotation by id: its span (virtual-timeline seconds), its type, its centre (x/y, percent of its box), its size (width/height, arrow and blur only), its text and look (text only: fontSize, color, background, bold, animation), or its figureData / blurData, of which only the keys you pass change. Only the fields you pass are changed. A type change converts it as the editor's type switch does: it stays where it is on screen, a text turned into an arrow or a blur takes the default box, and the text is kept to come back to. A text box is refitted to its text around its centre. Arguments that belong to another kind (width on a text, blurData on an arrow) are refused. A new span that would run into another annotation is refused, naming it.", setAspectRatio: - "Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording. Annotations on the frame keep their place and shape.", + "Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording, or one of the clips' own shapes (getCurrentDocument's settingsContext.originalAspectRatios). Annotations on the frame keep their place and shape. The same as setEditorSettings with aspectRatio alone.", + setEditorSettings: + 'Change the project\'s look: pass only the settings to change, named as getCurrentDocument returns them under `settings`. Background: wallpaper (a bundled path from settingsContext.bundledWallpapers, a CSS colour or gradient, or a base64 PNG/JPEG data URL; never a file path or web URL; an embedded image reads back as a marker, leave the key out to keep it), wallpaperMotion (moves a gradient or image only), backgroundBlur 0–1. Frame: aspectRatio ("auto", a preset, or an originalAspectRatios token; a fixed format picked for the first time fills the frame with a window that follows the cursor, which formatFollowCursor turns off when settingsContext.formatFill is "available"), padding 0–100 %, borderRadius 0–64 px, shadowIntensity 0–1, frame (none, window, laptop, phone, monitor) and frameTheme (light, dark), motionBlurAmount 0–1, depthOfField (blurs 3D-tilted zooms only). Camera: webcamLayoutPreset, webcamMirrored, webcamReactiveZoom (shrinks on zoom), webcamMaskShape (square or rectangle, the camera\'s own proportions; an untouched roundness follows it), webcamRoundness 0–1, webcamSizePreset 15–60, webcamAnchor (a corner or an edge middle), webcamCropRegion (the framing: a square share of the picture, width = height ≥ 1/3) and webcamCropPan (where that window sits, 0–1 per axis, 0.5 centred), webcamBackgroundMode (none, transparent, blur, custom: needs settingsContext.cameraSegmentation ready), webcamBlurIntensity 0–1, webcamWallpaper (as wallpaper). Cursor, under `cursor`: show, autoHide, theme (settingsContext.cursorThemes), asArrow (the recorded cursor kinds drawn as the arrow instead), size 1.5–6, smoothing 0–1, motionBlur 0–1, clickBounce 0–2, clickImpact, clickSound (only heard when the take recorded clicks) and clickSoundGainDb −24–0, model3d. Also autoFocusAll (every zoom follows the cursor) and audioGainDb −12–12 dB, the master volume. A value out of range or a key not listed is refused with the reason, and nothing changes. The result lists what was applied, including what the editor changes with it, and `inactive` for any of those the current state leaves without effect.', addCameraFullscreen: "Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.", setCameraFullscreen: @@ -259,6 +263,8 @@ export interface CursorTelemetryReader { export interface ToolRuntime { cursor?: CursorTelemetryReader; availableByAssetId?: Record; + /** Asks the compositor whether this machine can cut out the camera background. */ + cameraSegmentation?: () => SegmentationSupport; } /** @@ -339,6 +345,7 @@ export async function runDocumentTool( return executeAgentTool(document, name, JSON.stringify(args ?? {}), { editsAllowed, cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, + cameraSegmentation: runtime.cameraSegmentation, }); } @@ -423,6 +430,7 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ ["removeModifier", removeModifierArgs], ["removeClip", removeClipArgs], ["setAspectRatio", setAspectRatioArgs], + ["setEditorSettings", setEditorSettingsArgs], ]; /** @@ -460,6 +468,8 @@ export interface InvokeArgs { /** Injected by `chat-service` from the Electron layer. Absent in tests and in * the workbench unless one is supplied on purpose. */ cursor?: CursorTelemetryReader; + /** Injected the same way: whether this machine can cut out the camera background. */ + cameraSegmentation?: () => SegmentationSupport; } /** One cheap probe per asset, run before the tools are built so the very first @@ -534,6 +544,7 @@ export async function invokeOpenScreenAgent(args: InvokeArgs): Promise segmentation : undefined, + }); +} + +function read(document: AxcutDocument, segmentation?: "ready" | "no-runtime") { + return JSON.parse( + executeAgentTool(document, "getCurrentDocument", "", { + cameraSegmentation: segmentation ? () => segmentation : undefined, + }).resultJson, + ); +} + +describe("setEditorSettings", () => { + it("writes the look through the editor's patch, and reads it back in the same shape", () => { + const result = set(fixture(), { + wallpaper: "/wallpapers/wallpaper3.jpg", + padding: 20, + borderRadius: 12, + frame: "laptop", + audioGainDb: -3, + autoFocusAll: true, + cursor: { size: 2, theme: "glass-lens", show: true, asArrow: ["text"] }, + }); + expect(result.ok).toBe(true); + const document = result.document as AxcutDocument; + const settings = getEditorSettings(document); + expect(settings).toMatchObject({ + wallpaper: "/wallpapers/wallpaper3.jpg", + padding: 20, + borderRadius: 12, + frame: "laptop", + audioGainDb: -3, + autoFocusAll: true, + cursorTheme: "glass-lens", + }); + expect(settings.cursor.size).toBe(2); + expect(settings.cursor.asArrow).toEqual(["text"]); + + const snapshot = read(document); + expect(snapshot.settings).toMatchObject({ padding: 20, aspectRatio: "auto" }); + expect(snapshot.settings.cursor).toMatchObject({ size: 2, theme: "glass-lens" }); + // What is read is what the tool takes: the whole block goes back in unchanged. + expect(set(document, snapshot.settings).ok).toBe(true); + }); + + it("refuses a wallpaper that reaches outside the project, a value out of range and an unknown key", () => { + const document = fixture(); + for (const args of [ + { wallpaper: "file:///C:/Users/me/secret.png" }, + { wallpaper: "https://example.com/bg.jpg" }, + { webcamWallpaper: "url(https://example.com/a.png)" }, + { padding: 101 }, + { cursor: { size: 10 } }, + { audioGainDb: 13 }, + { cursorSize: 2 }, + { cursor: { colour: "red" } }, + {}, + ]) { + const result = set(document, args); + expect(result.ok, JSON.stringify(args)).toBe(false); + expect(result.document).toBeUndefined(); + } + }); + + it("offers the presets, auto and the clips' own shapes, and fills a first fixed format", () => { + const document = fixture({ width: 1366, height: 768 }); + expect(read(document).settingsContext.originalAspectRatios).toEqual([ + { token: "683:384", width: 1366, height: 768, clipCount: 1 }, + ]); + expect(set(document, { aspectRatio: "7:3" }).ok).toBe(false); + const original = set(document, { aspectRatio: "683:384" }); + expect(original.ok).toBe(true); + const vertical = set(document, { aspectRatio: "9:16" }); + expect(JSON.parse(vertical.resultJson).applied).toEqual(["aspectRatio", "formatFollowCursor"]); + expect(getEditorSettings(vertical.document as AxcutDocument).formatFollowCursor).toBe(true); + // The alias takes the same shapes. + expect( + executeAgentTool(document, "setAspectRatio", JSON.stringify({ aspectRatio: "683:384" })).ok, + ).toBe(true); + }); + + it("moves an untouched camera roundness with its shape, and stores the framing as size and pan", () => { + const document = fixture({ camera: true }); + const square = set(document, { webcamMaskShape: "rectangle" }).document as AxcutDocument; + expect(getEditorSettings(square).webcamRoundness).toBe(0.4); + const tuned = set(document, { webcamRoundness: 0.1 }).document as AxcutDocument; + expect( + getEditorSettings(set(tuned, { webcamMaskShape: "rectangle" }).document as AxcutDocument) + .webcamRoundness, + ).toBe(0.1); + + const framed = set(document, { + webcamCropRegion: { x: 0.5, y: 0, width: 0.5, height: 0.5 }, + }).document as AxcutDocument; + expect(getEditorSettings(framed).webcamCropPan).toEqual({ x: 1, y: 0 }); + expect(getEditorSettings(framed).webcamCropRegion).toEqual({ + x: 0.5, + y: 0, + width: 0.5, + height: 0.5, + }); + const panned = set(framed, { webcamCropPan: { x: 0.5, y: 0.5 } }).document as AxcutDocument; + expect(getEditorSettings(panned).webcamCropRegion).toEqual({ + x: 0.25, + y: 0.25, + width: 0.5, + height: 0.5, + }); + expect(set(document, { webcamCropRegion: { x: 0, y: 0, width: 0.5, height: 0.6 } }).ok).toBe( + false, + ); + }); + + it("says which of the settings it wrote change nothing in the current state", () => { + const result = JSON.parse(set(fixture(), { frameTheme: "dark", padding: 10 }).resultJson); + expect(Object.keys(result.inactive)).toEqual(["frameTheme"]); + expect(set(fixture(), { webcamMirrored: true }).resultJson).toContain( + "No clip on the timeline has a camera", + ); + }); + + it("is refused while edits are off", () => { + const result = executeAgentTool(fixture(), "setEditorSettings", '{"padding":10}', { + editsAllowed: false, + }); + expect(result.ok).toBe(false); + expect(result.document).toBeUndefined(); + }); +}); + +describe("getCurrentDocument's settings", () => { + it("elides an embedded image and gives the context the editor gates its controls on", () => { + const image = `data:image/png;base64,${"A".repeat(4000)}`; + const document = set(fixture(), { wallpaper: image }).document as AxcutDocument; + const snapshot = read(document); + expect(snapshot.settings.wallpaper).toMatch(/^\[embedded image/); + expect(JSON.stringify(snapshot)).not.toContain("AAAA"); + expect(snapshot.settingsContext).toMatchObject({ + aspectRatios: ["auto", "16:9", "9:16", "1:1", "4:3", "4:5", "16:10", "10:16"], + formatFill: "none", + hasTiltedZoom: false, + cameraSegmentation: null, + }); + expect(snapshot.settingsContext.bundledWallpapers).toContain("/wallpapers/wallpaper11.jpg"); + expect(snapshot.settingsContext.cursorThemes).toContainEqual({ + id: "glass-lens", + name: "Glass Lens", + }); + expect(Object.keys(snapshot.settingsContext.inactive)).toEqual( + expect.arrayContaining(["formatFollowCursor", "depthOfField", "webcamLayoutPreset"]), + ); + }); + + it("follows the camera layout, the cutout capability and the hidden cursor", () => { + const document = fixture({ camera: true }); + const ready = read(document, "ready").settingsContext.inactive; + expect(ready.webcamBackgroundMode).toBeUndefined(); + expect(ready.webcamBlurIntensity).toMatch(/blur/); + expect(ready.webcamMaskShape).toBeUndefined(); + + const missing = read(document, "no-runtime").settingsContext; + expect(missing.cameraSegmentation).toBe("no-runtime"); + expect(missing.inactive.webcamBackgroundMode).toMatch(/cannot cut out/); + + const sideBySide = set(document, { + webcamLayoutPreset: "dual-frame", + cursor: { show: false }, + }).document as AxcutDocument; + const inactive = read(sideBySide, "ready").settingsContext.inactive; + expect(inactive.webcamMaskShape).toMatch(/picture-in-picture/); + expect(inactive.webcamReactiveZoom).toMatch(/picture-in-picture/); + expect(inactive["cursor.model3d"]).toMatch(/hidden/); + }); +}); diff --git a/electron/ai-edition/editor-settings-tool.ts b/electron/ai-edition/editor-settings-tool.ts new file mode 100644 index 0000000000..d0660910c2 --- /dev/null +++ b/electron/ai-edition/editor-settings-tool.ts @@ -0,0 +1,444 @@ +// The project's look for the agent: `setEditorSettings`, which writes it, and the `settings` +// block `getCurrentDocument` reads it back through (#1180, batch A). +// +// Both sides speak `EditorSettingsPatch`, so what the agent reads it can write back as is. The +// write goes through `applySettingsPatch`, the editor's own, with what the editor does around a +// pick: a format picked for the first time fills the frame, an untouched camera roundness follows +// its shape, and the camera framing is stored as the size-and-pan pair the pane writes. +// +// The read also says, per knob, why the editor hides it in the current state, so an agent does +// not set a value that changes nothing and report it as done. + +import { z } from "zod"; +import { + WALLPAPER_MOTIONS, + WEBCAM_BACKGROUND_MODES, +} from "../../src/components/video-editor/types"; +import { + collectNativeFormats, + type FormatFillAvailability, + formatFillAvailability, + isAutoFormatAvailable, +} from "../../src/lib/ai-edition/document/outputFormat"; +import type { AxcutDocument } from "../../src/lib/ai-edition/schema"; +import { + AUDIO_GAIN_DB_LIMIT, + type EditorSettingsPatch, + getEditorSettings, + panOfCropRegion, +} from "../../src/lib/ai-edition/store/editorSettings"; +import { applySettingsPatch } from "../../src/lib/ai-edition/store/settingsPatch"; +import { parseStylePresetWallpaper } from "../../src/lib/ai-edition/stylePresets"; +import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; +import { + resolveWebcamLayoutPreset, + supportsWebcamReactiveZoom, + type WebcamLayoutPreset, +} from "../../src/lib/compositeLayout"; +import { + CURSOR_KIND_IDS, + CURSOR_THEME_IDS, + CURSOR_THEMES, + DEFAULT_CURSOR_THEME_ID, +} from "../../src/lib/cursor/cursorThemes"; +import { + DEFAULT_WEBCAM_ROUNDNESS, + FRAME_THEMES, + RECORDING_FRAMES, + SETTING_BOUNDS, + type SettingBound, + WEBCAM_ANCHORS, +} from "../../src/lib/projectDefaults"; +import { classifyWallpaper, WALLPAPER_PATHS } from "../../src/lib/wallpaper"; +import type { SegmentationSupport } from "../../src/native/contracts"; +import { ASPECT_RATIO_PRESETS, type AspectRatio } from "../../src/utils/aspectRatioUtils"; +import type { AgentToolExecution } from "./agent-tools"; +import { measureTextInShippedFonts } from "./text-measure"; + +const WEBCAM_LAYOUT_PRESETS = [ + "picture-in-picture", + "dual-frame", + "vertical-stack", + "no-webcam", +] as const satisfies readonly WebcamLayoutPreset[]; + +/** The tightest the camera framing goes: a third of the picture (the pane's `MIN_CROP_SIZE`). */ +const MIN_WEBCAM_CROP_SIZE = 1 / 3; + +const bounded = (bound: SettingBound) => + z.number().min(SETTING_BOUNDS[bound][0]).max(SETTING_BOUNDS[bound][1]); + +/** The security boundary: only self-contained values, never a file path or a URL. */ +const wallpaperArg = (key: string) => + z.string().superRefine((value, ctx) => { + try { + parseStylePresetWallpaper(value, key); + } catch (error) { + ctx.addIssue({ + code: "custom", + message: String(error instanceof Error ? error.message : error).replace( + /^Style preset /, + "", + ), + }); + } + }); + +const fraction = z.number().min(0).max(1); + +export const setEditorSettingsArgs = z + .object({ + wallpaper: wallpaperArg("wallpaper").describe( + "A bundled wallpaper path (settingsContext.bundledWallpapers), a CSS colour (#1e1e2e, rgb(…)), a CSS gradient, or a base64 PNG/JPEG data URL. No file paths, no web URLs.", + ), + wallpaperMotion: z.enum(WALLPAPER_MOTIONS), + frame: z.enum(RECORDING_FRAMES), + frameTheme: z.enum(FRAME_THEMES), + aspectRatio: z + .string() + .describe( + '"auto", one of the presets, or a token from settingsContext.originalAspectRatios.', + ), + formatFollowCursor: z.boolean(), + shadowIntensity: bounded("shadowIntensity"), + backgroundBlur: bounded("backgroundBlur"), + motionBlurAmount: bounded("motionBlurAmount"), + depthOfField: z.boolean(), + borderRadius: bounded("borderRadius").describe("Corner radius in px at 1080p."), + padding: bounded("padding").describe("Percent."), + webcamLayoutPreset: z.enum(WEBCAM_LAYOUT_PRESETS), + webcamMaskShape: z.enum(["rectangle", "square"]), + webcamRoundness: bounded("webcamRoundness"), + webcamMirrored: z.boolean(), + webcamReactiveZoom: z.boolean(), + webcamSizePreset: bounded("webcamSizePreset").describe( + "The camera's long side, in percent of the frame's short side.", + ), + webcamAnchor: z.enum(WEBCAM_ANCHORS), + webcamCropRegion: z + .object({ + x: fraction, + y: fraction, + width: z.number().min(MIN_WEBCAM_CROP_SIZE).max(1), + height: z.number().min(MIN_WEBCAM_CROP_SIZE).max(1), + }) + .strict() + .refine((r) => Math.abs(r.width - r.height) < 1e-6, { + message: "width and height must be equal: the framing keeps the camera's proportions.", + }) + .refine((r) => r.x + r.width <= 1 + 1e-6 && r.y + r.height <= 1 + 1e-6, { + message: "The framing must stay inside the picture: x + width and y + height ≤ 1.", + }), + webcamCropPan: z.object({ x: fraction, y: fraction }).strict(), + webcamBackgroundMode: z.enum(WEBCAM_BACKGROUND_MODES), + webcamWallpaper: wallpaperArg("webcamWallpaper"), + webcamBlurIntensity: bounded("webcamBlurIntensity"), + audioGainDb: z.number().min(-AUDIO_GAIN_DB_LIMIT).max(AUDIO_GAIN_DB_LIMIT), + autoFocusAll: z.boolean(), + cursor: z + .object({ + show: z.boolean(), + autoHide: z.boolean(), + theme: z.enum([...CURSOR_THEME_IDS] as [string, ...string[]]), + asArrow: z.array(z.enum(CURSOR_KIND_IDS as [string, ...string[]])), + size: bounded("cursorSize"), + smoothing: bounded("cursorSmoothing"), + motionBlur: bounded("cursorMotionBlur"), + clickBounce: bounded("cursorClickBounce"), + clickImpact: z.boolean(), + clickSound: z.boolean(), + clickSoundGainDb: bounded("cursorClickSoundGainDb"), + model3d: z.boolean(), + }) + .partial() + .strict(), + }) + .partial() + .strict(); + +export type SetEditorSettingsArgs = z.infer; + +/** A data URL is megabytes the model cannot read: it sees what it is, and leaves the key out. */ +function elideDataUrl(value: string): string { + return value.startsWith("data:") + ? `[embedded image, ${Math.round((value.length * 0.75) / 1024)} KB: leave this key out to keep it]` + : value; +} + +/** The look, in the shape `setEditorSettings` takes. */ +function settingsForModel(document: AxcutDocument) { + const s = getEditorSettings(document); + return { + wallpaper: elideDataUrl(s.wallpaper), + wallpaperMotion: s.wallpaperMotion, + frame: s.frame, + frameTheme: s.frameTheme, + aspectRatio: s.aspectRatio, + // `null` is "never chosen", which only a first fixed format reads; absent, not a third value. + ...(s.formatFollowCursor === null ? {} : { formatFollowCursor: s.formatFollowCursor }), + shadowIntensity: s.shadowIntensity, + backgroundBlur: s.backgroundBlur, + motionBlurAmount: s.motionBlurAmount, + depthOfField: s.depthOfField, + borderRadius: s.borderRadius, + padding: s.padding, + webcamLayoutPreset: s.webcamLayoutPreset, + webcamMaskShape: s.webcamMaskShape, + webcamRoundness: s.webcamRoundness, + webcamMirrored: s.webcamMirrored, + webcamReactiveZoom: s.webcamReactiveZoom, + webcamSizePreset: s.webcamSizePreset, + webcamAnchor: s.webcamAnchor, + webcamCropRegion: s.webcamCropRegion, + webcamCropPan: s.webcamCropPan, + webcamBackgroundMode: s.webcamBackgroundMode, + webcamWallpaper: elideDataUrl(s.webcamWallpaper), + webcamBlurIntensity: s.webcamBlurIntensity, + audioGainDb: s.audioGainDb, + autoFocusAll: s.autoFocusAll, + cursor: { + show: s.cursorShow, + autoHide: s.cursorAutoHide, + theme: s.cursorTheme, + asArrow: s.cursor.asArrow, + size: s.cursor.size, + smoothing: s.cursor.smoothing, + motionBlur: s.cursor.motionBlur, + clickBounce: s.cursor.clickBounce, + clickImpact: s.cursor.clickImpact, + clickSound: s.cursor.clickSound, + clickSoundGainDb: s.cursor.clickSoundGainDb, + model3d: s.cursor.model3d, + }, + }; +} + +const FORMAT_FILL_REASONS: Record, string> = { + none: "The format is auto or the recording's own shape, so the recording already fills it.", + mixed: "The clips differ in shape, so no single window rule suits them all.", + layout: "A side-by-side or stacked camera layout already fills its own slot.", + frame: "A device frame is drawn around the whole recording.", +}; + +const WEBCAM_KEYS = [ + "webcamLayoutPreset", + "webcamMaskShape", + "webcamRoundness", + "webcamMirrored", + "webcamReactiveZoom", + "webcamSizePreset", + "webcamAnchor", + "webcamCropRegion", + "webcamCropPan", + "webcamBackgroundMode", + "webcamWallpaper", + "webcamBlurIntensity", +] as const; + +/** + * Each setting the editor hides (or disables) in this state, with why. Its value is stored and + * comes back when the state changes, but it changes nothing on screen until then. Keys are the + * `setEditorSettings` ones, `cursor.*` for the cursor's. + */ +export function inactiveSettings( + document: AxcutDocument, + segmentation: SegmentationSupport | null, +): Record { + const s = getEditorSettings(document); + const out: Record = {}; + if (classifyWallpaper(s.wallpaper).kind === "color") { + out.wallpaperMotion = "Only a gradient or image wallpaper moves; this one is a solid colour."; + } + const fill = formatFillAvailability(document); + if (fill !== "available") out.formatFollowCursor = FORMAT_FILL_REASONS[fill]; + if (s.frame === "none") out.frameTheme = "No frame is drawn (frame is none)."; + if (s.padding === 0 && s.frame === "none") { + out.borderRadius = "At padding 0 with no frame the corners are square."; + } + if (!document.zoomRanges.some((zoom) => zoom.rotationPreset != null)) { + out.depthOfField = "It only blurs a 3D-tilted zoom (rotationPreset), and there is none."; + } + + if (!hasAnyClipWithCamera(document.assets, document.timeline.clips)) { + for (const key of WEBCAM_KEYS) out[key] = "No clip on the timeline has a camera."; + } else { + const preset = resolveWebcamLayoutPreset(s.webcamLayoutPreset, true); + if (preset !== "picture-in-picture") { + const why = `Only the picture-in-picture layout has it; the layout is ${preset}.`; + for (const key of [ + "webcamMaskShape", + "webcamRoundness", + "webcamSizePreset", + "webcamAnchor", + ] as const) { + out[key] = why; + } + } + if (!supportsWebcamReactiveZoom(preset)) { + out.webcamReactiveZoom = `Only a picture-in-picture camera shrinks on zoom; the layout is ${preset}.`; + } + if (segmentation !== null && segmentation !== "ready") { + const why = `This machine cannot cut out the camera background (${segmentation}).`; + out.webcamBackgroundMode = why; + out.webcamBlurIntensity = why; + out.webcamWallpaper = why; + } else { + if (s.webcamBackgroundMode !== "blur") { + out.webcamBlurIntensity = "Only used when webcamBackgroundMode is blur."; + } + if (s.webcamBackgroundMode !== "custom") { + out.webcamWallpaper = "Only used when webcamBackgroundMode is custom."; + } + } + } + + if (!s.cursorShow) { + const why = "The cursor is hidden (cursor.show is false)."; + for (const key of ["autoHide", "model3d", "asArrow", "clickImpact", "clickSound"]) { + out[`cursor.${key}`] = why; + } + out["cursor.clickSoundGainDb"] = why; + } else if (!s.cursor.clickSound) { + out["cursor.clickSoundGainDb"] = "The click sound is off (cursor.clickSound is false)."; + } + return out; +} + +/** The project's look and what the editor would offer for it, for `getCurrentDocument`. */ +export function editorSettingsSnapshot( + document: AxcutDocument, + segmentation: SegmentationSupport | null, +) { + const autoAvailable = isAutoFormatAvailable(document); + return { + settings: settingsForModel(document), + settingsContext: { + note: "settings uses setEditorSettings' own shape. inactive lists each setting the editor hides in this state, and why: a value set there is stored but changes nothing until that state changes. cameraSegmentation null means this server could not check whether the machine can cut out the camera background.", + aspectRatios: [...(autoAvailable ? ["auto"] : []), ...ASPECT_RATIO_PRESETS], + originalAspectRatios: collectNativeFormats(document).map(({ ratio: _ratio, ...f }) => f), + formatFill: formatFillAvailability(document), + hasTiltedZoom: document.zoomRanges.some((zoom) => zoom.rotationPreset != null), + bundledWallpapers: WALLPAPER_PATHS, + cursorThemes: [ + { id: DEFAULT_CURSOR_THEME_ID, name: "Default" }, + ...CURSOR_THEMES.map(({ id, name }) => ({ id, name })), + ], + cameraSegmentation: segmentation, + inactive: inactiveSettings(document, segmentation), + }, + }; +} + +/** `"auto"`, a preset, or one of the timeline's own shapes: what the format picker offers. */ +function aspectRatioError(document: AxcutDocument, value: string): string | null { + const original = collectNativeFormats(document).map((f) => f.token); + const allowed = ["auto", ...ASPECT_RATIO_PRESETS, ...original]; + return allowed.includes(value) + ? null + : `aspectRatio ${JSON.stringify(value)} is not offered for this project. Use one of: ${allowed.join(", ")}.`; +} + +/** The parsed args as the editor would write them, with what its pane does around each pick. */ +function editorPatch(document: AxcutDocument, args: SetEditorSettingsArgs): EditorSettingsPatch { + const current = getEditorSettings(document); + const { webcamCropRegion, webcamCropPan, wallpaper, webcamWallpaper, ...rest } = args; + const patch: EditorSettingsPatch = { + ...(rest as Omit), + aspectRatio: args.aspectRatio as AspectRatio | undefined, + }; + if (wallpaper !== undefined) patch.wallpaper = parseStylePresetWallpaper(wallpaper); + if (webcamWallpaper !== undefined) { + patch.webcamWallpaper = parseStylePresetWallpaper(webcamWallpaper, "webcamWallpaper"); + } + // A format picked while the project never chose how to fill it fills the frame (`fillDefault`). + if ( + args.aspectRatio !== undefined && + args.aspectRatio !== "auto" && + args.formatFollowCursor === undefined && + current.formatFollowCursor === null + ) { + patch.formatFollowCursor = true; + } + // An untouched roundness follows the shape to its own default; a tuned one stays. + if ( + args.webcamMaskShape !== undefined && + args.webcamRoundness === undefined && + current.webcamRoundness === DEFAULT_WEBCAM_ROUNDNESS[current.webcamMaskShape] + ) { + patch.webcamRoundness = DEFAULT_WEBCAM_ROUNDNESS[args.webcamMaskShape]; + } + // The pan is what is stored; the rect is rebuilt from it. A rect given alone says where it + // sits, so its pan is read off it; a pan given alone keeps the current size. + if (webcamCropRegion || webcamCropPan) { + const size = webcamCropRegion?.width ?? current.webcamCropRegion.width; + const pan = + webcamCropPan ?? + (webcamCropRegion ? panOfCropRegion(webcamCropRegion) : current.webcamCropPan); + patch.webcamCropPan = pan; + patch.webcamCropRegion = { + x: pan.x * (1 - size), + y: pan.y * (1 - size), + width: size, + height: size, + }; + } + return patch; +} + +/** Every key a patch sets, `cursor.*` for the cursor's: what the result reports as applied. */ +function patchKeys(patch: EditorSettingsPatch): string[] { + const { cursor, ...rest } = patch; + return [ + ...Object.keys(rest).filter((key) => rest[key as keyof typeof rest] !== undefined), + ...Object.keys(cursor ?? {}).map((key) => `cursor.${key}`), + ]; +} + +export function executeSetEditorSettings( + document: AxcutDocument, + args: unknown, + segmentation: SegmentationSupport | null, +): AgentToolExecution { + const parsed = setEditorSettingsArgs.safeParse(args); + if (!parsed.success) { + return { ok: false, resultJson: JSON.stringify({ error: parsed.error.message }) }; + } + const patch = editorPatch(document, parsed.data); + const keys = patchKeys(patch); + if (keys.length === 0) { + return { + ok: false, + resultJson: JSON.stringify({ + error: "Pass at least one setting; getCurrentDocument lists them under settings.", + }), + }; + } + if (patch.aspectRatio !== undefined) { + const error = aspectRatioError(document, patch.aspectRatio); + if (error) return { ok: false, resultJson: JSON.stringify({ error }) }; + } + // The editor's own write: frame-placed annotations keep their centre through a new format, + // a text refitted to its words in the shipped fonts. + const next = applySettingsPatch(document, patch, measureTextInShippedFonts); + const inactive = inactiveSettings(next, segmentation); + const stillInactive = Object.fromEntries( + keys.filter((key) => key in inactive).map((key) => [key, inactive[key]]), + ); + const settings = settingsForModel(next); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + applied: keys, + settings, + ...(Object.keys(stillInactive).length + ? { + inactive: stillInactive, + inactiveNote: + "Stored, but these change nothing in the current state, for the reason given. Say so rather than reporting the look as changed.", + } + : {}), + }), + summary: `set ${keys.join(", ")}`, + }; +} diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 2df583509a..8cd32244d1 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -5058,6 +5058,7 @@ export function registerIpcHandlers( projects: aiEditionDocuments, editsAllowed: () => mcpSettings.getSettings().allowEdits, cursor: agentCursorTelemetryReader, + cameraSegmentation: () => loudnessService.probeSegmentation(), version: app.getVersion(), }; // One runner, so one queue and one checkpoint store, for the in-app server and every @@ -5243,6 +5244,7 @@ export function registerIpcHandlers( runAiEditionChat: (projectId, sessionId, message, document, sink) => runChat(projectId, sessionId, message, getAiEditionLlmConfig(), document, sink, { cursor: agentCursorTelemetryReader, + cameraSegmentation: () => loudnessService.probeSegmentation(), }), undoAiEditionToolBatch: (_projectId, _sessionId) => ({ success: false, diff --git a/electron/mcp/openscreen-mcp-server.ts b/electron/mcp/openscreen-mcp-server.ts index 0a78778ff7..4e4b3d02b5 100644 --- a/electron/mcp/openscreen-mcp-server.ts +++ b/electron/mcp/openscreen-mcp-server.ts @@ -48,6 +48,7 @@ import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/ import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; import { z } from "zod"; import { type AxcutDocument, documentSchema } from "../../src/lib/ai-edition/schema"; +import type { SegmentationSupport } from "../../src/native/contracts"; import { type AgentToolExecution, consentRequired, @@ -131,6 +132,8 @@ export interface McpToolDeps { /** The MCP server's own edit switch ("MCP edits"), separate from the in-app agent's. */ editsAllowed(): boolean; cursor?: CursorTelemetryReader; + /** Whether this machine can cut out the camera background; absent where nothing can ask. */ + cameraSegmentation?: () => SegmentationSupport; version: string; } @@ -502,6 +505,7 @@ export function createToolRunner(deps: McpToolDeps): McpToolRunner { return runDocumentTool(document, name, args, editsAllowed, { cursor: deps.cursor, availableByAssetId, + cameraSegmentation: deps.cameraSegmentation, }); } diff --git a/src/lib/ai-edition/store/editorSettings.ts b/src/lib/ai-edition/store/editorSettings.ts index 2fc464ad6b..61284e2119 100644 --- a/src/lib/ai-edition/store/editorSettings.ts +++ b/src/lib/ai-edition/store/editorSettings.ts @@ -428,7 +428,7 @@ const MIN_CROP_SIZE = 0.01; * full-frame crop has no room to sit in, so it reports centred — the one case where the * answer is arbitrary, and the one case where nothing depends on it. */ -function panOfCropRegion(crop: CropRegion): CropPan { +export function panOfCropRegion(crop: CropRegion): CropPan { // Guarded at 1 and not at some epsilon below it: the zoom slider is integer percent, so // the smallest window short of full frame is 100/101 and leaves ~0.0099 of room — a // perfectly ordinary divisor that a wider guard would throw away as if it were centred. From 52db6a17d1271ffd95b02f1c47ef2f0d772072bd Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:05:41 +0200 Subject: [PATCH 2/4] docs(mcp): document setEditorSettings and the settings read Part of #1180 --- skills/openscreen-demo/SKILL.md | 2 +- technical-documentation/architecture/mcp-server.md | 9 +++++++++ website/docs/ai-editing.md | 2 +- .../docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- 10 files changed, 18 insertions(+), 9 deletions(-) diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.md index 5baf486272..3ccc2cef33 100644 --- a/skills/openscreen-demo/SKILL.md +++ b/skills/openscreen-demo/SKILL.md @@ -86,7 +86,7 @@ Use the document tools, always with the `projectId`. Read the project first with - **Undo:** `restoreCheckpoint` with the `checkpointId` takes the project back to the raw take in one step. It also puts back the take's own pointer track, undoing any `setPointerTrack` made since. - **Open in the app:** while OpenScreen is running, your edits go through it, so a project the user has open in its editor changes there live and they can undo each edit. If an edit is refused as "open in the OpenScreen editor", the app did not answer: retry once, then ask the user to close the project there or restart OpenScreen. -Keep the rest as it is: the background, padding, shadow and cursor style are chosen to look good. +Keep the rest as it is: the background, padding, shadow and cursor style are chosen to look good. When the user asks for a different look, `setEditorSettings` sets it (background, frame, camera layout, cursor, volume); `getCurrentDocument` shows the current values under `settings`, and `settingsContext.inactive` says which ones would change nothing right now. ## 7. Export and review diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md index a5208678c6..37376e9e2b 100644 --- a/technical-documentation/architecture/mcp-server.md +++ b/technical-documentation/architecture/mcp-server.md @@ -42,6 +42,15 @@ So a tool added to the agent appears over MCP with no further work, and the MCP - **`createCheckpoint` / `restoreCheckpoint`**, see [Checkpoints](#checkpoints). - MCP metadata: `readOnlyHint` for the reads (`!isMutatingTool`), `listProjects` and `createCheckpoint`, and `destructiveHint` for `replaceTimeline`, the three `remove*` tools and `restoreCheckpoint`. +### The project's look + +`setEditorSettings` ([`editor-settings-tool.ts`](../../electron/ai-edition/editor-settings-tool.ts)) writes every setting of the editor's look panes: background, frame, format, camera layout, cursor, `autoFocusAll` and `audioGainDb`. It takes a partial `EditorSettingsPatch` and applies it with `applySettingsPatch`, the editor's own write, plus what the panes do around a pick (a first fixed format fills the frame, an untouched camera roundness follows its shape, the camera framing is stored as size and pan). `setAspectRatio` is the same write for the format alone. + +- **Validation.** Numbers are held to `SETTING_BOUNDS`, enums to the constant lists, and both wallpapers go through `parseStylePresetWallpaper`: a colour, a gradient, a bundled path or an inline PNG/JPEG, never `file://` nor a URL. The schema is strict, so a misnamed key is refused rather than dropped. +- **Read side.** `getCurrentDocument` adds `settings`, in the tool's own shape with data-URL wallpapers elided, and `settingsContext`: the formats on offer (`originalAspectRatios` from `collectNativeFormats`), `formatFill`, `hasTiltedZoom`, the bundled wallpapers, the cursor themes, `cameraSegmentation`, and `inactive`, each setting the editor hides in the current state with why. A write reports the same for the keys it set. +- **Segmentation.** `McpToolDeps.cameraSegmentation` and the in-app agent's `ChatRunEnv` ask the compositor (`probeSegmentation`), wired in `registerIpcHandlers`; `openscreen mcp` inherits it with `mcpToolDeps`. Where nothing is wired it reads `null`, "not checked". +- **Not covered yet:** style presets (`StylePresetService`) are not exposed; `setEditorSettings` reaches every field a preset holds. + ## What each transport tells its client Every text a client reads must be true on the server that serves it, so the texts are composed rather than shared verbatim. `McpToolDeps.transport` (`"http"` or `"stdio"`) selects them: diff --git a/website/docs/ai-editing.md b/website/docs/ai-editing.md index 04984cfca7..65055f5410 100644 --- a/website/docs/ai-editing.md +++ b/website/docs/ai-editing.md @@ -41,7 +41,7 @@ The ChatGPT and GitHub Copilot sign-in options were **removed in 1.8.0**. They w ## Using the agent -Describe the edit in plain language — "cut the dead air in the intro", "zoom in when I open the terminal". The agent works through real, undoable timeline operations, not a re-render: it can add and adjust trims, zooms, speed regions, annotations (text, arrows, and blurs that hide private details on screen), imported audio tracks with their fades, and Full Camera segments, edit clip in/out points, reorder or remove clips, and read the transcript to find what you're referring to. +Describe the edit in plain language — "cut the dead air in the intro", "zoom in when I open the terminal". The agent works through real, undoable timeline operations, not a re-render: it can add and adjust trims, zooms, speed regions, annotations (text, arrows, and blurs that hide private details on screen), imported audio tracks with their fades, and Full Camera segments, edit clip in/out points, reorder or remove clips, and read the transcript to find what you're referring to. It also sets the look of the video: background, padding, rounded corners, shadow, device frame, format, the camera layout, the cursor and the master volume. The panel around it: diff --git a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md index 1d435b948e..b178b82515 100644 --- a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Die Anmeldeoptionen für ChatGPT und GitHub Copilot wurden **in 1.8.0 entfernt** ## Den Agenten nutzen {#using-the-agent} -Beschreibe die Änderung in Alltagssprache, etwa „schneide die Stille im Intro heraus“ oder „zoome hinein, wenn ich das Terminal öffne“. Der Agent arbeitet mit echten, rückgängig machbaren Operationen auf der Zeitleiste, nicht mit einem neuen Rendering: Er kann Schnitte, Zooms, Geschwindigkeitsbereiche, Annotationen (Text, Pfeile und Unschärfen, die private Details auf dem Bildschirm verbergen), importierte Audiospuren samt Ein- und Ausblendung und Full-Camera-Segmente hinzufügen und anpassen, Start- und Endpunkte von Clips ändern, Clips umsortieren oder entfernen und das Transkript lesen, um herauszufinden, was du meinst. +Beschreibe die Änderung in Alltagssprache, etwa „schneide die Stille im Intro heraus“ oder „zoome hinein, wenn ich das Terminal öffne“. Der Agent arbeitet mit echten, rückgängig machbaren Operationen auf der Zeitleiste, nicht mit einem neuen Rendering: Er kann Schnitte, Zooms, Geschwindigkeitsbereiche, Annotationen (Text, Pfeile und Unschärfen, die private Details auf dem Bildschirm verbergen), importierte Audiospuren samt Ein- und Ausblendung und Full-Camera-Segmente hinzufügen und anpassen, Start- und Endpunkte von Clips ändern, Clips umsortieren oder entfernen und das Transkript lesen, um herauszufinden, was du meinst. Außerdem stellt er das Aussehen des Videos ein: Hintergrund, Innenabstand, abgerundete Ecken, Schatten, Geräterahmen, Format, das Kamera-Layout, den Cursor und die Gesamtlautstärke. Der Bereich drumherum: diff --git a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md index 658fa2a428..b12d7fe7d1 100644 --- a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Las opciones de inicio de sesión con ChatGPT y GitHub Copilot se **eliminaron e ## Usar el agente {#using-the-agent} -Describe la edición con tus propias palabras: "corta el tiempo muerto de la introducción", "haz zoom cuando abro la terminal". El agente trabaja con operaciones reales de la línea de tiempo, que se pueden deshacer, y no con un nuevo renderizado: puede agregar y ajustar recortes, zooms, regiones de velocidad, anotaciones (texto, flechas y desenfoques que ocultan datos privados en pantalla), pistas de audio importadas con sus fundidos y segmentos de cámara a pantalla completa, editar los puntos de entrada y salida de los clips, reordenar o quitar clips, y leer la transcripción para encontrar aquello a lo que te refieres. +Describe la edición con tus propias palabras: "corta el tiempo muerto de la introducción", "haz zoom cuando abro la terminal". El agente trabaja con operaciones reales de la línea de tiempo, que se pueden deshacer, y no con un nuevo renderizado: puede agregar y ajustar recortes, zooms, regiones de velocidad, anotaciones (texto, flechas y desenfoques que ocultan datos privados en pantalla), pistas de audio importadas con sus fundidos y segmentos de cámara a pantalla completa, editar los puntos de entrada y salida de los clips, reordenar o quitar clips, y leer la transcripción para encontrar aquello a lo que te refieres. También ajusta el aspecto del vídeo: fondo, margen, esquinas redondeadas, sombra, marco de dispositivo, formato, la disposición de la cámara, el cursor y el volumen general. El panel que lo rodea: diff --git a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md index ccfbd7398a..ba3c134232 100644 --- a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Les options de connexion ChatGPT et GitHub Copilot ont été **supprimées dans ## Utiliser l'agent {#using-the-agent} -Décrivez la modification en langage courant : « coupe les temps morts de l'intro », « zoome quand j'ouvre le terminal ». L'agent travaille avec de vraies opérations de timeline annulables, pas avec un nouveau rendu : il peut ajouter et ajuster des coupes, des zooms, des régions de vitesse, des annotations (texte, flèches et flous qui masquent une information privée à l'écran), des pistes audio importées avec leurs fondus, et des segments Caméra plein écran, modifier les points d'entrée et de sortie des clips, réordonner ou retirer des clips, et lire la transcription pour trouver ce dont vous parlez. +Décrivez la modification en langage courant : « coupe les temps morts de l'intro », « zoome quand j'ouvre le terminal ». L'agent travaille avec de vraies opérations de timeline annulables, pas avec un nouveau rendu : il peut ajouter et ajuster des coupes, des zooms, des régions de vitesse, des annotations (texte, flèches et flous qui masquent une information privée à l'écran), des pistes audio importées avec leurs fondus, et des segments Caméra plein écran, modifier les points d'entrée et de sortie des clips, réordonner ou retirer des clips, et lire la transcription pour trouver ce dont vous parlez. Il règle aussi l'apparence de la vidéo : fond, marge, coins arrondis, ombre, cadre d'appareil, format, disposition de la caméra, curseur et volume général. Le panneau qui l'entoure : diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md index f885f150b5..ae812f06ce 100644 --- a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT と GitHub Copilot のサインインは、**1.8.0 で削除されまし ## エージェントを使う {#using-the-agent} -行いたい編集を、普段の言葉で伝えます（「イントロの無音部分をカットして」「ターミナルを開いたところでズームして」など）。エージェントは再レンダリングではなく、元に戻せる実際のタイムライン操作で作業します。トリム、ズーム、再生速度の範囲、注釈（テキスト、矢印、画面上の個人情報を隠すぼかし）、取り込んだオーディオトラックとそのフェード、フルスクリーンカメラのセグメントの追加と調整、クリップのイン点とアウト点の編集、クリップの並べ替えと削除ができ、文字起こしを読んで、あなたが指している箇所を見つけます。 +行いたい編集を、普段の言葉で伝えます（「イントロの無音部分をカットして」「ターミナルを開いたところでズームして」など）。エージェントは再レンダリングではなく、元に戻せる実際のタイムライン操作で作業します。トリム、ズーム、再生速度の範囲、注釈（テキスト、矢印、画面上の個人情報を隠すぼかし）、取り込んだオーディオトラックとそのフェード、フルスクリーンカメラのセグメントの追加と調整、クリップのイン点とアウト点の編集、クリップの並べ替えと削除ができ、文字起こしを読んで、あなたが指している箇所を見つけます。背景、余白、角の丸み、影、デバイスフレーム、フォーマット、カメラのレイアウト、カーソル、全体の音量といった動画の見た目も設定できます。 エージェントを囲むパネル： diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md index 2eac6c45a8..c082545dfc 100644 --- a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ As opções de login do ChatGPT e do GitHub Copilot foram **removidas na 1.8.0** ## Como usar o agente {#using-the-agent} -Descreva a edição em linguagem natural — "corte o tempo morto da introdução", "dê zoom quando eu abrir o terminal". O agente trabalha com operações reais da linha do tempo, que podem ser desfeitas, e não com uma nova renderização: ele pode adicionar e ajustar recortes, zooms, regiões de velocidade, anotações (texto, setas e desfoques que escondem dados privados na tela), faixas de áudio importadas com seus fades e segmentos de Câmera em Tela Cheia, editar os pontos de entrada/saída dos clipes, reordenar ou remover clipes e ler a transcrição para encontrar o trecho a que você se refere. +Descreva a edição em linguagem natural — "corte o tempo morto da introdução", "dê zoom quando eu abrir o terminal". O agente trabalha com operações reais da linha do tempo, que podem ser desfeitas, e não com uma nova renderização: ele pode adicionar e ajustar recortes, zooms, regiões de velocidade, anotações (texto, setas e desfoques que escondem dados privados na tela), faixas de áudio importadas com seus fades e segmentos de Câmera em Tela Cheia, editar os pontos de entrada/saída dos clipes, reordenar ou remover clipes e ler a transcrição para encontrar o trecho a que você se refere. Ele também ajusta a aparência do vídeo: fundo, margem, cantos arredondados, sombra, moldura de dispositivo, formato, o layout da câmera, o cursor e o volume geral. O painel em volta dele: diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md index 97192274ad..2d5807d891 100644 --- a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT 和 GitHub Copilot 登录选项已**在 1.8.0 中移除**。它们的工 ## 使用 AI 代理 {#using-the-agent} -用日常语言描述你想要的编辑，例如“剪掉开头的空白”“我打开终端时放大画面”。AI 代理通过真实、可撤销的时间轴操作来完成编辑，而不是重新渲染：它可以添加和调整删减区间、缩放、速度区间、标注（文字、箭头，以及遮住屏幕上隐私信息的模糊）、导入的音轨及其淡入淡出和全屏摄像头区间，编辑片段的入点/出点，重新排序或移除片段，还会读取转录文本，找到你所指的内容。 +用日常语言描述你想要的编辑，例如“剪掉开头的空白”“我打开终端时放大画面”。AI 代理通过真实、可撤销的时间轴操作来完成编辑，而不是重新渲染：它可以添加和调整删减区间、缩放、速度区间、标注（文字、箭头，以及遮住屏幕上隐私信息的模糊）、导入的音轨及其淡入淡出和全屏摄像头区间，编辑片段的入点/出点，重新排序或移除片段，还会读取转录文本，找到你所指的内容。它还能设置视频的外观：背景、边距、圆角、阴影、设备边框、画面比例、摄像头布局、光标和总音量。 它周围的面板包括： diff --git a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md index b48e12f5d1..90524eab21 100644 --- a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT 與 GitHub Copilot 的登入選項已**在 1.8.0 中移除**。它們的 ## 使用代理 {#using-the-agent} -用一般的語言描述你要的剪輯，例如「剪掉開頭的空檔」、「我打開終端機時放大畫面」。代理會透過真實、可復原的時間軸操作來完成工作，而不是重新算繪：它可以新增與調整剪輯、縮放、速度區域、標註（文字、箭頭，以及遮住畫面上隱私資訊的模糊）、匯入的音軌及其淡入淡出與全螢幕攝影機片段，編輯片段的入點與出點，重新排序或移除片段，也會讀取逐字稿來找出你指的是哪一段。 +用一般的語言描述你要的剪輯，例如「剪掉開頭的空檔」、「我打開終端機時放大畫面」。代理會透過真實、可復原的時間軸操作來完成工作，而不是重新算繪：它可以新增與調整剪輯、縮放、速度區域、標註（文字、箭頭，以及遮住畫面上隱私資訊的模糊）、匯入的音軌及其淡入淡出與全螢幕攝影機片段，編輯片段的入點與出點，重新排序或移除片段，也會讀取逐字稿來找出你指的是哪一段。它也能設定影片的外觀：背景、邊距、圓角、陰影、裝置外框、畫面比例、攝影機版面、游標與整體音量。 它周圍的面板包括： From 93351db4e5af00aa85124e902e3a36e73b9f7084 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:08:54 +0200 Subject: [PATCH 3/4] fix(mcp): offer auto only while the timeline holds one composition, keep the current format writable Part of #1180 --- .../ai-edition/editor-settings-tool.test.ts | 25 +++++++++++++++++++ electron/ai-edition/editor-settings-tool.ts | 9 ++++--- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/electron/ai-edition/editor-settings-tool.test.ts b/electron/ai-edition/editor-settings-tool.test.ts index f86fa53a50..cb939a6c5e 100644 --- a/electron/ai-edition/editor-settings-tool.test.ts +++ b/electron/ai-edition/editor-settings-tool.test.ts @@ -123,6 +123,31 @@ describe("setEditorSettings", () => { const vertical = set(document, { aspectRatio: "9:16" }); expect(JSON.parse(vertical.resultJson).applied).toEqual(["aspectRatio", "formatFollowCursor"]); expect(getEditorSettings(vertical.document as AxcutDocument).formatFollowCursor).toBe(true); + // Two shapes on the timeline: auto is not offered, as in the picker. + const mixed = documentSchema.parse({ + ...document, + assets: [ + ...document.assets, + { ...document.assets[0], id: "asset_2", video: { width: 1080, height: 1920 } }, + ], + timeline: { + ...document.timeline, + clips: [ + ...document.timeline.clips, + { + ...document.timeline.clips[0], + id: "clip_2", + assetId: "asset_2", + timelineStartSec: 30, + timelineEndSec: 60, + }, + ], + }, + }); + expect( + set(set(mixed, { aspectRatio: "9:16" }).document as AxcutDocument, { aspectRatio: "auto" }) + .ok, + ).toBe(false); // The alias takes the same shapes. expect( executeAgentTool(document, "setAspectRatio", JSON.stringify({ aspectRatio: "683:384" })).ok, diff --git a/electron/ai-edition/editor-settings-tool.ts b/electron/ai-edition/editor-settings-tool.ts index d0660910c2..9bae3c6031 100644 --- a/electron/ai-edition/editor-settings-tool.ts +++ b/electron/ai-edition/editor-settings-tool.ts @@ -329,11 +329,14 @@ export function editorSettingsSnapshot( }; } -/** `"auto"`, a preset, or one of the timeline's own shapes: what the format picker offers. */ +/** What the format picker offers: auto (while the timeline holds one composition), a preset, or + * one of the timeline's own shapes. The current value always passes, so a read written back + * as is never fails on it. */ function aspectRatioError(document: AxcutDocument, value: string): string | null { const original = collectNativeFormats(document).map((f) => f.token); - const allowed = ["auto", ...ASPECT_RATIO_PRESETS, ...original]; - return allowed.includes(value) + const auto = isAutoFormatAvailable(document) ? ["auto"] : []; + const allowed = [...auto, ...ASPECT_RATIO_PRESETS, ...original]; + return allowed.includes(value) || value === getEditorSettings(document).aspectRatio ? null : `aspectRatio ${JSON.stringify(value)} is not offered for this project. Use one of: ${allowed.join(", ")}.`; } From e47b1d43f0f56e9b7590bef326f6eaac5c436e72 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:21:31 +0200 Subject: [PATCH 4/4] fix(mcp): refuse a camera crop and pan that place the framing in two spots Part of #1180 --- electron/ai-edition/editor-settings-tool.test.ts | 13 +++++++++++++ electron/ai-edition/editor-settings-tool.ts | 14 ++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/electron/ai-edition/editor-settings-tool.test.ts b/electron/ai-edition/editor-settings-tool.test.ts index cb939a6c5e..d838e95ce6 100644 --- a/electron/ai-edition/editor-settings-tool.test.ts +++ b/electron/ai-edition/editor-settings-tool.test.ts @@ -184,6 +184,19 @@ describe("setEditorSettings", () => { expect(set(document, { webcamCropRegion: { x: 0, y: 0, width: 0.5, height: 0.6 } }).ok).toBe( false, ); + // A rect and a pan that disagree are refused; at full size any pan agrees. + expect( + set(framed, { + webcamCropRegion: { x: 0, y: 0, width: 0.5, height: 0.5 }, + webcamCropPan: { x: 1, y: 0 }, + }).ok, + ).toBe(false); + expect( + set(framed, { + webcamCropRegion: { x: 0, y: 0, width: 1, height: 1 }, + webcamCropPan: { x: 1, y: 0 }, + }).ok, + ).toBe(true); }); it("says which of the settings it wrote change nothing in the current state", () => { diff --git a/electron/ai-edition/editor-settings-tool.ts b/electron/ai-edition/editor-settings-tool.ts index 9bae3c6031..f233494b09 100644 --- a/electron/ai-edition/editor-settings-tool.ts +++ b/electron/ai-edition/editor-settings-tool.ts @@ -388,6 +388,18 @@ function editorPatch(document: AxcutDocument, args: SetEditorSettingsArgs): Edit return patch; } +/** A rect and a pan sent together that place the window in two different spots. At full size + * there is no room to place it in, so any pan agrees with the rect there. */ +function cropFramingConflict(args: SetEditorSettingsArgs): string | null { + const { webcamCropRegion: region, webcamCropPan: pan } = args; + if (!region || !pan) return null; + const agrees = (offset: number, size: number, share: number) => + Math.abs(offset - share * (1 - size)) < 1e-6; + return agrees(region.x, region.width, pan.x) && agrees(region.y, region.height, pan.y) + ? null + : "webcamCropRegion and webcamCropPan place the framing in two different spots: send one of them, or both as getCurrentDocument returns them."; +} + /** Every key a patch sets, `cursor.*` for the cursor's: what the result reports as applied. */ function patchKeys(patch: EditorSettingsPatch): string[] { const { cursor, ...rest } = patch; @@ -406,6 +418,8 @@ export function executeSetEditorSettings( if (!parsed.success) { return { ok: false, resultJson: JSON.stringify({ error: parsed.error.message }) }; } + const conflict = cropFramingConflict(parsed.data); + if (conflict) return { ok: false, resultJson: JSON.stringify({ error: conflict }) }; const patch = editorPatch(document, parsed.data); const keys = patchKeys(patch); if (keys.length === 0) {