feat(mcp): setEditorSettings and the project's look in getCurrentDocument

electron/ai-edition/agent-tools.test.ts

@@ -203,6 +203,7 @@ describe("the mutating-tool table", () => {
 				"setAudio",
 				"setCameraFullscreen",
 				"setClipRange",
+				"setEditorSettings",
 				"setSpeed",
 				"setTrim",
 				"setWordText",

electron/ai-edition/agent-tools.ts

@@ -48,7 +48,6 @@ import {
 	figureDataSchema,
 } from "../../src/lib/ai-edition/schema";
 import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings";
-import { applySettingsPatch } from "../../src/lib/ai-edition/store/settingsPatch";
 import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera";
 import { isGeneratedAssetId } from "../../src/lib/ai-edition/timeline/clip-parts";
 import {
@@ -79,8 +78,10 @@ import {
 	zoomTransitions,
 } from "../../src/lib/ai-edition/timeline/zoom-transitions";
 import { SETTING_BOUNDS } from "../../src/lib/projectDefaults";
+import type { SegmentationSupport } from "../../src/native/contracts";
 import { annotationFootageRect, zoomScaleLimit } from "../../src/native/sceneDescription";
 import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils";
+import { editorSettingsSnapshot, executeSetEditorSettings } from "./editor-settings-tool";
 import { measureTextInShippedFonts } from "./text-measure";
 
 export interface AgentToolExecution {
@@ -670,7 +671,11 @@ export const setClipRangeArgs = z.object({
 });
 
 export const setAspectRatioArgs = z.object({
-	aspectRatio: z.enum(["auto", ...ASPECT_RATIO_PRESETS]),
+	aspectRatio: z
+		.string()
+		.describe(
+			`"auto", one of ${ASPECT_RATIO_PRESETS.join(", ")}, or a token from getCurrentDocument's settingsContext.originalAspectRatios.`,
+		),
 });
 
 export const replaceTimelineArgs = z.object({
@@ -1027,6 +1032,7 @@ export const OPENSCREEN_TOOL_NAMES = [
 	"removeModifier",
 	"removeClip",
 	"setAspectRatio",
+	"setEditorSettings",
 ] as const;
 
 /**
@@ -1100,6 +1106,7 @@ export const MUTATING_TOOL_NAMES: ReadonlySet<string> = new Set([
 	"removeModifier",
 	"removeClip",
 	"setAspectRatio",
+	"setEditorSettings",
 ]);
 
 export function isMutatingTool(name: string): boolean {
@@ -1512,6 +1519,9 @@ export interface AgentToolOptions {
 	 *  Absent means the runtime has no telemetry reader wired at all, which the
 	 *  executor reports as "could not look", never as "there is none". */
 	cursorTelemetry?: CursorTelemetryContext;
+	/** Whether this machine can cut out the camera background, asked only by the two tools that
+	 *  report it. Absent where nothing can ask the compositor: reported as not checked. */
+	cameraSegmentation?: () => SegmentationSupport;
 }
 
 /**
@@ -1724,28 +1734,33 @@ export function executeAgentTool(
 
 	switch (name) {
 		case "setAspectRatio": {
+			// The format alone, kept for clients that know it: setEditorSettings does the work.
 			const parsed = setAspectRatioArgs.safeParse(args);
 			if (!parsed.success) return failure(parsed.error.message);
 			const previous = getEditorSettings(document).aspectRatio;
-			// The editor's own write: the frame-placed annotations keep their centre, and a text is
-			// refitted to its words in the new frame, measured in the shipped fonts.
-			const next = applySettingsPatch(
+			const execution = executeSetEditorSettings(
 				document,
 				{ aspectRatio: parsed.data.aspectRatio },
-				measureTextInShippedFonts,
+				options?.cameraSegmentation?.() ?? null,
 			);
+			if (!execution.ok) return execution;
 			return {
-				ok: true,
-				document: next,
+				...execution,
 				resultJson: JSON.stringify({ aspectRatio: parsed.data.aspectRatio, previous }),
 				summary: `set the format to ${parsed.data.aspectRatio}`,
 			};
 		}
 
+		case "setEditorSettings":
+			return executeSetEditorSettings(document, args, options?.cameraSegmentation?.() ?? null);
+
 		case "getCurrentDocument": {
 			return {
 				ok: true,
-				resultJson: JSON.stringify(documentSnapshotForModel(document, options?.cursorTelemetry)),
+				resultJson: JSON.stringify({
+					...documentSnapshotForModel(document, options?.cursorTelemetry),
+					...editorSettingsSnapshot(document, options?.cameraSegmentation?.() ?? null),
+				}),
 			};
 		}
 

electron/ai-edition/chat-service.ts

@@ -20,6 +20,7 @@ import type {
 	AiEditionChatMessage,
 	AiEditionChatResult,
 	AiEditionToolCallSummary,
+	SegmentationSupport,
 } from "../../src/native/contracts";
 import {
 	applyCompaction,
@@ -319,6 +320,8 @@ export interface ChatRunEnv {
 	/** Reads recorded cursor telemetry for an asset. Built in `electron/ipc/
 	 *  handlers.ts`, where the path allow-list lives. */
 	cursor?: CursorTelemetryReader;
+	/** Whether this machine can cut out the camera background, asked of the compositor. */
+	cameraSegmentation?: () => SegmentationSupport;
 	/** Runtime-only model retry override used by bounded measurement harnesses. */
 	maxRetries?: number;
 }
@@ -464,6 +467,7 @@ export async function runChat(
 		sink: agentSink,
 		editsAllowed,
 		cursor: env.cursor,
+		cameraSegmentation: env.cameraSegmentation,
 	});
 
 	// The project was deleted while the agent ran: its sessions are gone, so a

electron/ai-edition/deep-agent/service.test.ts

@@ -85,6 +85,7 @@ const ARGS: Record<string, unknown> = {
 	removeModifier: { id: "nope" },
 	removeClip: { clipId: "clip_1" },
 	setAspectRatio: { aspectRatio: "16:9" },
+	setEditorSettings: { padding: 30 },
 };
 
 /** ponytail: the fixture starts from a FIXED instant. `createEmptyDocument` reads

electron/ai-edition/deep-agent/service.ts

@@ -31,6 +31,7 @@ import {
 	ZOOM_DEPTH_LEGEND,
 } from "../../../src/lib/ai-edition/timeline/zoom-scale";
 import { ZOOM_TRANSITION_LEGEND } from "../../../src/lib/ai-edition/timeline/zoom-transitions";
+import type { SegmentationSupport } from "../../../src/native/contracts";
 import {
 	AGENT_EDITS_SETTING,
 	type AgentToolExecution,
@@ -65,6 +66,7 @@ import {
 	setWordTextArgs,
 	setZoomArgs,
 } from "../agent-tools";
+import { setEditorSettingsArgs } from "../editor-settings-tool";
 import {
 	createOpenScreenChatModel,
 	messageContentToText,
@@ -132,7 +134,7 @@ const BASE_SYSTEM_PROMPT = [
 	"- Cutting specific spoken words (fillers, a repeated or flubbed phrase, a false start, a duplicate take) is removeWords: read getTranscriptWords, choose the occurrences from their surrounding speech, then pass that assetId and ONLY those word IDs. Word IDs restart for each asset, so keep the read's assetId. Consecutive words become one cut. Do not cut every occurrence of a word just because its text matches one you were asked to remove.",
 	"- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.",
 	`- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text, an arrow or a privacy blur on screen: a blur is how an email address, a token or any private detail the recording shows is hidden. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`,
-	"- setAspectRatio picks the shape of the whole video, for where it will be shown; auto follows the recording.",
+	"- setEditorSettings sets the look of the whole video: the background (wallpaper, its motion and blur), padding, rounded corners, shadow, a device frame, motion blur, depth of field, the shape of the video (aspectRatio: auto follows the recording) and how a recording of another shape fills it, the camera's layout (picture-in-picture, side by side, stacked, its shape, size, position, framing and background cutout), the cursor, and the master volume. getCurrentDocument returns all of them under settings, in the shape this tool takes, and settingsContext.inactive names each one the editor hides in the current state and why: setting one of those changes nothing, so say so instead of reporting the look as changed. setAspectRatio is the same write for the shape alone.",
 	"- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.",
 	"- addAudio lays an imported voiceover or music file over a span. It plays an asset the project already has (kind 'audio'); importing or recording one is the editor's job, not a tool you have — so when the project has none, say so rather than naming an id that does not exist.",
 	"- moveClip changes the order of placed clips, one call per clip that moves, preserving ids, source ranges, trims and anchored effects. replaceTimeline rebuilds the timeline from kept intervals and sorts them, so it cannot reorder anything.",
@@ -181,7 +183,7 @@ const CUT_TRANSITIONS_RESULT = `When the result cuts a zoom's move in or out, it
 
 export const TOOL_DESCRIPTIONS: Record<string, string> = {
 	getCurrentDocument:
-		"Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / camera-fullscreen / audio regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.",
+		"Read a compact snapshot of the current project: assets (with durations), timeline clips and trim ranges (source-time), and the zoom / speed / annotation / camera-fullscreen / audio regions (virtual, edited-timeline time), each with the settings its writes take: a zoom's level, focus mode and hidden cursor, an annotation's type, box, look and arrow or blur data, an audio track's level and fades. It also gives the project's look under `settings`, in the shape setEditorSettings takes, with `settingsContext`: the formats, wallpapers and cursor themes on offer, and under `inactive` each setting the editor hides in the current state and why. Call this before editing if the snapshot in the system prompt may be stale. The AxcutDocument is the single source of truth — your edits should preserve the user's placed clips and any timeline state they have already set up.",
 	getTranscript:
 		"Read the transcript segments (speech and silence, with start/end seconds and text) for an asset. Omit assetId to read the primary asset's transcript.",
 	getCursorTrack:
@@ -214,7 +216,9 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {
 	setAnnotation:
 		"Change an existing annotation by id: its span (virtual-timeline seconds), its type, its centre (x/y, percent of its box), its size (width/height, arrow and blur only), its text and look (text only: fontSize, color, background, bold, animation), or its figureData / blurData, of which only the keys you pass change. Only the fields you pass are changed. A type change converts it as the editor's type switch does: it stays where it is on screen, a text turned into an arrow or a blur takes the default box, and the text is kept to come back to. A text box is refitted to its text around its centre. Arguments that belong to another kind (width on a text, blurData on an arrow) are refused. A new span that would run into another annotation is refused, naming it.",
 	setAspectRatio:
-		"Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording. Annotations on the frame keep their place and shape.",
+		"Set the shape of the exported video: 16:9 for X, YouTube or a slide, 9:16 for a vertical story or short, 1:1 or 4:5 for a square feed, auto to follow the recording, or one of the clips' own shapes (getCurrentDocument's settingsContext.originalAspectRatios). Annotations on the frame keep their place and shape. The same as setEditorSettings with aspectRatio alone.",
+	setEditorSettings:
+		'Change the project\'s look: pass only the settings to change, named as getCurrentDocument returns them under `settings`. Background: wallpaper (a bundled path from settingsContext.bundledWallpapers, a CSS colour or gradient, or a base64 PNG/JPEG data URL; never a file path or web URL; an embedded image reads back as a marker, leave the key out to keep it), wallpaperMotion (moves a gradient or image only), backgroundBlur 0–1. Frame: aspectRatio ("auto", a preset, or an originalAspectRatios token; a fixed format picked for the first time fills the frame with a window that follows the cursor, which formatFollowCursor turns off when settingsContext.formatFill is "available"), padding 0–100 %, borderRadius 0–64 px, shadowIntensity 0–1, frame (none, window, laptop, phone, monitor) and frameTheme (light, dark), motionBlurAmount 0–1, depthOfField (blurs 3D-tilted zooms only). Camera: webcamLayoutPreset, webcamMirrored, webcamReactiveZoom (shrinks on zoom), webcamMaskShape (square or rectangle, the camera\'s own proportions; an untouched roundness follows it), webcamRoundness 0–1, webcamSizePreset 15–60, webcamAnchor (a corner or an edge middle), webcamCropRegion (the framing: a square share of the picture, width = height ≥ 1/3) and webcamCropPan (where that window sits, 0–1 per axis, 0.5 centred), webcamBackgroundMode (none, transparent, blur, custom: needs settingsContext.cameraSegmentation ready), webcamBlurIntensity 0–1, webcamWallpaper (as wallpaper). Cursor, under `cursor`: show, autoHide, theme (settingsContext.cursorThemes), asArrow (the recorded cursor kinds drawn as the arrow instead), size 1.5–6, smoothing 0–1, motionBlur 0–1, clickBounce 0–2, clickImpact, clickSound (only heard when the take recorded clicks) and clickSoundGainDb −24–0, model3d. Also autoFocusAll (every zoom follows the cursor) and audioGainDb −12–12 dB, the master volume. A value out of range or a key not listed is refused with the reason, and nothing changes. The result lists what was applied, including what the editor changes with it, and `inactive` for any of those the current state leaves without effect.',
 	addCameraFullscreen:
 		"Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.",
 	setCameraFullscreen:
@@ -259,6 +263,8 @@ export interface CursorTelemetryReader {
 export interface ToolRuntime {
 	cursor?: CursorTelemetryReader;
 	availableByAssetId?: Record<string, boolean>;
+	/** Asks the compositor whether this machine can cut out the camera background. */
+	cameraSegmentation?: () => SegmentationSupport;
 }
 
 /**
@@ -339,6 +345,7 @@ export async function runDocumentTool(
 	return executeAgentTool(document, name, JSON.stringify(args ?? {}), {
 		editsAllowed,
 		cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load },
+		cameraSegmentation: runtime.cameraSegmentation,
 	});
 }
 
@@ -423,6 +430,7 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray<readonly [string, z.ZodObject]> = [
 	["removeModifier", removeModifierArgs],
 	["removeClip", removeClipArgs],
 	["setAspectRatio", setAspectRatioArgs],
+	["setEditorSettings", setEditorSettingsArgs],
 ];
 
 /**
@@ -460,6 +468,8 @@ export interface InvokeArgs {
 	/** Injected by `chat-service` from the Electron layer. Absent in tests and in
 	 *  the workbench unless one is supplied on purpose. */
 	cursor?: CursorTelemetryReader;
+	/** Injected the same way: whether this machine can cut out the camera background. */
+	cameraSegmentation?: () => SegmentationSupport;
 }
 
 /** One cheap probe per asset, run before the tools are built so the very first
@@ -534,6 +544,7 @@ export async function invokeOpenScreenAgent(args: InvokeArgs): Promise<InvokeRes
 	const tools = buildTools(holder, sink, editsAllowed, {
 		cursor: args.cursor,
 		availableByAssetId,
+		cameraSegmentation: args.cameraSegmentation,
 	});
 	const agent = createAgent({
 		model: chatModel,