From 2400f05259bd18fa0bac3f36316ff73daf747362 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:20:46 +0200 Subject: [PATCH 1/5] refactor(timeline): share the editor's insert-clip and crop writes as pure mutators insertAssetClip and setClipCrop move the media panel's insert and the Edit clip crop out of useTimeline, so the agent tools reuse them. Part of #1180. --- src/lib/ai-edition/document/timeline.test.ts | 69 ++++++++++++++++++++ src/lib/ai-edition/document/timeline.ts | 54 ++++++++++++++- src/lib/ai-edition/store/useTimeline.ts | 31 ++------- 3 files changed, 126 insertions(+), 28 deletions(-) diff --git a/src/lib/ai-edition/document/timeline.test.ts b/src/lib/ai-edition/document/timeline.test.ts index a33f1c6e7..7f203f32a 100644 --- a/src/lib/ai-edition/document/timeline.test.ts +++ b/src/lib/ai-edition/document/timeline.test.ts @@ -9,6 +9,7 @@ import { audioLanePills } from "./audioTracks"; import { buildTimelineFromIntervals, duplicateClip, + insertAssetClip, invertIntervals, moveClip, normalizeIntervals, @@ -23,6 +24,7 @@ import { resequenceClips, resolvePlaybackSegments, restoreFullTimeline, + setClipCrop, setClipSourceRange, subtractInterval, timelineIntervals, @@ -2260,3 +2262,70 @@ describe("readSpeedRegions", () => { expect(readSpeedRegions({ legacyEditor: null })).toEqual([]); }); }); + +describe("insertAssetClip", () => { + const doc = () => + makeDoc({ + timeline: { + ...makeDoc().timeline, + clips: [makeClip({ id: "clip_a", sourceStartSec: 0, sourceEndSec: 5 })], + }, + }); + const clip = (sourceStartSec: number, sourceEndSec: number) => ({ + id: "clip_new", + assetId: "asset_1", + sourceStartSec, + sourceEndSec, + origin: "user" as const, + reason: "", + }); + + it("places the clip at the index, lays the timeline back to back", () => { + const next = insertAssetClip(doc(), 0, clip(20, 23)); + expect(next.timeline.clips.map((c) => [c.id, c.timelineStartSec, c.timelineEndSec])).toEqual([ + ["clip_new", 0, 3], + ["clip_a", 3, 8], + ]); + }); + + it("clamps the index and folds media that continues its neighbour into it", () => { + const next = insertAssetClip(doc(), 99, clip(5, 9)); + expect(next.timeline.clips).toMatchObject([ + { id: "clip_a", sourceStartSec: 0, sourceEndSec: 9 }, + ]); + }); +}); + +describe("setClipCrop", () => { + const crop = { x: 0, y: 0, width: 0.5, height: 0.5 }; + + it("sets and clears a clip's crop", () => { + const doc = makeDoc({ + timeline: { ...makeDoc().timeline, clips: [makeClip()] }, + }); + const cropped = setClipCrop(doc, "clip_a", crop); + expect(cropped.timeline.clips[0].cropRegion).toEqual(crop); + expect(setClipCrop(cropped, "clip_a", null).timeline.clips[0].cropRegion).toBeUndefined(); + }); + + it("makes one clip of two meeting neighbours once their crops match", () => { + const doc = makeDoc({ + timeline: { + ...makeDoc().timeline, + clips: [ + makeClip({ id: "clip_a", cropRegion: crop }), + makeClip({ + id: "clip_b", + sourceStartSec: 5, + sourceEndSec: 9, + timelineStartSec: 5, + timelineEndSec: 9, + }), + ], + }, + }); + expect(setClipCrop(doc, "clip_b", crop).timeline.clips).toMatchObject([ + { id: "clip_a", sourceStartSec: 0, sourceEndSec: 9, cropRegion: crop }, + ]); + }); +}); diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index ba256fd3b..4cbe47869 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -4,7 +4,13 @@ // or a new document with updated clips. import { clampToBound } from "../../projectDefaults"; -import type { AxcutClip, AxcutDocument, AxcutTranscript, AxcutTrimRange } from "../schema"; +import type { + AxcutClip, + AxcutClipCropRegion, + AxcutDocument, + AxcutTranscript, + AxcutTrimRange, +} from "../schema"; /** * What `resolvePlaybackSegments` returns: a clip-shaped slice of playable film. @@ -992,8 +998,7 @@ export function moveClip( return withClipsChanged(document, reordered); } -// ponytail: duplicate a clip (preserves the original). Used for "split this -// clip into two" or "make a copy". Mirrors axcut's +// ponytail: duplicate a clip (preserves the original): "make a copy". Mirrors axcut's // apps/server/src/lib/timeline.ts#duplicateClip. // // The copy takes its own COPY of the original's trims, re-anchored to the new clip id. @@ -1053,6 +1058,49 @@ export function duplicateClip( ); } +/** + * The single mutator for "place a clip of this asset at this position": the media panel's drop + * onto the timeline and the agent's `insertClip`. `index` counts the clips before it (0 first, + * `clips.length` last) and is clamped. The new clip goes through `withClipsChanged` like every + * other clip edit, so one whose media meets a neighbour's is folded into it. Pure. + */ +export function insertAssetClip( + document: AxcutDocument, + index: number, + clip: Pick & { + sourceEndSec: number; + }, +): AxcutDocument { + const length = clip.sourceEndSec - clip.sourceStartSec; + const clips = [...document.timeline.clips]; + clips.splice(Math.max(0, Math.min(clips.length, index)), 0, { + ...clip, + timelineStartSec: 0, + timelineEndSec: length, + wordRefs: [], + }); + return withClipsChanged(document, clips); +} + +/** + * The single mutator for a clip's crop: the clip's Edit modal and the agent's `setClipRange`. + * `null` clears it back to the full frame rather than storing the identity region. Through + * `withClipsChanged`, since crop is the one property that keeps two meeting clips apart. An + * unknown `clipId` is a no-op. Pure. + */ +export function setClipCrop( + document: AxcutDocument, + clipId: string, + cropRegion: AxcutClipCropRegion | null, +): AxcutDocument { + return withClipsChanged( + document, + document.timeline.clips.map((c) => + c.id === clipId ? { ...c, cropRegion: cropRegion ?? undefined } : c, + ), + ); +} + /** * The single mutator for "narrow/extend a clip's own source in/out" — the edit the * clip's Edit modal, the renderer op dispatcher, and the LLM's `setClipRange` tool all diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index b8a867f77..8312e600c 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -27,6 +27,7 @@ import { clearEditRegions, countEditRegions, duplicateClip as duplicateClipInDocument, + insertAssetClip, moveClip as moveClipInDocument, PLACEHOLDER_DURATION_SEC, type RegionKind, @@ -34,8 +35,8 @@ import { removeClip as removeClipInDocument, removeRegion as removeRegionInDocument, resequenceClips, + setClipCrop, setClipSourceRange, - withClipsChanged, } from "../document/timeline"; import type { AxcutAudioTrack, AxcutClipCropRegion, AxcutDocument } from "../schema"; import { appendAutoZoomSuggestions } from "../timeline/apply-auto-zooms"; @@ -81,8 +82,6 @@ interface RegionHandle { id: string; } -type Clip = AxcutDocument["timeline"]["clips"][number]; - /** * Patch the region `id` belongs to: every fragment of it across clip junctions, or the * pieces of one region would disagree and, by the merge rule, visibly split. Not the whole @@ -1223,18 +1222,7 @@ export function useTimeline() { const doc = useProjectStore.getState().document; if (!doc) return; const ranged = setClipSourceRange(doc, clipId, sourceStartSec, sourceEndSec); - const next: AxcutDocument = - cropRegion === undefined - ? ranged - : { - ...ranged, - timeline: { - ...ranged.timeline, - clips: ranged.timeline.clips.map((c) => - c.id === clipId ? { ...c, cropRegion: cropRegion ?? undefined } : c, - ), - }, - }; + const next = cropRegion === undefined ? ranged : setClipCrop(ranged, clipId, cropRegion); await saveDocument(next, { history: true }); }, [saveDocument], @@ -1339,22 +1327,15 @@ export function useTimeline() { // durationSec we use it; otherwise we fall back to the placeholder // and let the background probe correct it. const knownDuration = asset.durationSec ?? PLACEHOLDER_DURATION_SEC; - const newClip: Clip = { + const newClip = { id: createId("clip"), assetId, sourceStartSec: 0, sourceEndSec: knownDuration, - timelineStartSec: 0, - timelineEndSec: knownDuration, - wordRefs: [], - origin: "user", + origin: "user" as const, reason: "Inserted from media panel", }; - const oldClips = currentDoc.timeline.clips; - const arr = [...oldClips]; - const at = Math.max(0, Math.min(arr.length, index)); - arr.splice(at, 0, newClip); - const finalDoc = withClipsChanged(currentDoc, arr); + const finalDoc = insertAssetClip(currentDoc, index, newClip); if (!(await saveDocument(finalDoc, { history: true }))) return; setClipSelection(newClip.id); From 6e6b5f3bf195722df2c8b37e6cda87e903821585 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:20:57 +0200 Subject: [PATCH 2/5] feat(mcp): insertClip, duplicateClip, addAutoZooms and per-clip crop insertClip places any video asset, duplicateClip copies a clip, addAutoZooms runs the timeline wand's plan through the runtime cursor reader, and setClipRange takes a cropRegion that getCurrentDocument reads back. All go through the editor's own mutators, on both transports and in the in-app chat. Part of #1180. --- electron/ai-edition/agent-tools.test.ts | 225 ++++++++++++++- electron/ai-edition/agent-tools.ts | 260 +++++++++++++++++- .../ai-edition/deep-agent/service.test.ts | 46 ++++ electron/ai-edition/deep-agent/service.ts | 55 +++- 4 files changed, 565 insertions(+), 21 deletions(-) diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index f9811fa81..af629ae70 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -8,7 +8,12 @@ import { } from "../../src/lib/ai-edition/schema"; import { ZOOM_DEPTH_SCALES } from "../../src/lib/ai-edition/timeline/zoom-scale"; import { zoomScaleLimit } from "../../src/native/sceneDescription"; -import { executeAgentTool, isMutatingTool, MUTATING_TOOL_NAMES } from "./agent-tools"; +import { + type AgentToolOptions, + executeAgentTool, + isMutatingTool, + MUTATING_TOOL_NAMES, +} from "./agent-tools"; import { measureTextInShippedFonts } from "./text-measure"; /** ponytail: every fixture in this file starts from the SAME instant. @@ -192,6 +197,9 @@ describe("the mutating-tool table", () => { "addTrims", "addZoom", "addZooms", + "addAutoZooms", + "duplicateClip", + "insertClip", "moveClip", "removeClip", "removeModifier", @@ -2491,7 +2499,7 @@ describe("addAudio / setAudio", () => { it("refuses a video asset, pointing at the tool that does place footage", () => { const result = place(withAudioAsset(), { assetId: "asset_1", startSec: 0, endSec: 4 }); expect(result.ok).toBe(false); - expect(result.resultJson).toContain("replaceTimeline"); + expect(result.resultJson).toContain("insertClip"); }); it("setAudio re-levels and re-lanes the track it names", () => { @@ -3475,3 +3483,216 @@ describe("region tools reach the inspector's settings (#1180)", () => { ).toBe(false); }); }); + +describe("clip structure (#1180)", () => { + /** Two recordings, one not probed yet, and a music file; one clip of the first, cut once. */ + function twoTakes(): AxcutDocument { + const base = createEmptyDocument({ + title: "Takes", + projectId: "proj_takes", + createdAt: FIXTURE_CREATED_AT, + }); + return documentSchema.parse({ + ...base, + project: { ...base.project, primaryAssetId: "asset_1" }, + assets: [ + { + id: "asset_1", + kind: "video", + label: "Take 1", + originalPath: "C:/v/1.mp4", + durationSec: 20, + }, + { + id: "asset_2", + kind: "video", + label: "Take 2", + originalPath: "C:/v/2.mp4", + durationSec: 8, + }, + { id: "asset_3", kind: "video", label: "Unprobed", originalPath: "C:/v/3.mp4" }, + { + id: "audio_1", + kind: "audio", + label: "Music", + originalPath: "C:/v/m.mp3", + durationSec: 30, + }, + ], + timeline: { + ...base.timeline, + clips: [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: 0, + timelineEndSec: 10, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + trimRanges: [ + { + id: "trim_1", + assetId: "asset_1", + clipId: "clip_1", + startSec: 2, + endSec: 3, + reason: "", + origin: "user", + }, + ], + }, + }); + } + + const run = (document: AxcutDocument, name: string, args: unknown) => + executeAgentTool(document, name, JSON.stringify(args)); + + it("insertClip places another asset's footage before the clip it names", () => { + const result = run(twoTakes(), "insertClip", { assetId: "asset_2", beforeClipId: "clip_1" }); + expect(result.ok).toBe(true); + const clips = (result.document as AxcutDocument).timeline.clips; + const { clipId } = JSON.parse(result.resultJson); + expect(clips.map((c) => c.id)).toEqual([clipId, "clip_1"]); + expect(clips[0]).toMatchObject({ assetId: "asset_2", sourceStartSec: 0, sourceEndSec: 8 }); + // Back to back: the clip after it moved along the ruler. + expect(clips[1]).toMatchObject({ timelineStartSec: 8, timelineEndSec: 18 }); + }); + + it("insertClip folds footage that continues the clip before it into that clip", () => { + const result = run(twoTakes(), "insertClip", { + assetId: "asset_1", + sourceStartSec: 10, + sourceEndSec: 15, + }); + expect(JSON.parse(result.resultJson)).toMatchObject({ + clipId: null, + joinedIntoNeighbour: true, + }); + expect((result.document as AxcutDocument).timeline.clips).toMatchObject([ + { id: "clip_1", sourceStartSec: 0, sourceEndSec: 15 }, + ]); + }); + + it("insertClip refuses what is not footage, and says what to use", () => { + expect(run(twoTakes(), "insertClip", { assetId: "audio_1" }).resultJson).toContain("addAudio"); + const unknown = run(twoTakes(), "insertClip", { assetId: "nope" }); + expect(unknown.ok).toBe(false); + expect(unknown.resultJson).toContain("asset_2 (Take 2)"); + const nowhere = run(twoTakes(), "insertClip", { assetId: "asset_2", beforeClipId: "clip_x" }); + expect(nowhere.ok).toBe(false); + }); + + it("insertClip asks for an end when the asset's length is unknown, and clamps to a known one", () => { + const unprobed = run(twoTakes(), "insertClip", { assetId: "asset_3" }); + expect(unprobed.ok).toBe(false); + expect(unprobed.resultJson).toContain("sourceEndSec"); + expect(run(twoTakes(), "insertClip", { assetId: "asset_3", sourceEndSec: 4 }).ok).toBe(true); + const clamped = run(twoTakes(), "insertClip", { assetId: "asset_2", sourceEndSec: 99 }); + expect(JSON.parse(clamped.resultJson).sourceEndSec).toBe(8); + expect(run(twoTakes(), "insertClip", { assetId: "asset_2", sourceStartSec: 9 }).ok).toBe(false); + }); + + it("duplicateClip puts the copy right after the original, with its own copy of the cuts", () => { + const result = run(twoTakes(), "duplicateClip", { clipId: "clip_1" }); + const payload = JSON.parse(result.resultJson); + const document = result.document as AxcutDocument; + expect(document.timeline.clips.map((c) => c.id)).toEqual(["clip_1", payload.clipId]); + expect(payload.copiedTrimIds).toHaveLength(1); + expect(document.timeline.trimRanges.map((t) => t.clipId)).toEqual(["clip_1", payload.clipId]); + expect(run(twoTakes(), "duplicateClip", { clipId: "clip_x" }).ok).toBe(false); + }); + + it("setClipRange crops a clip without touching its range, and the snapshot reads it back", () => { + const crop = { x: 0.1, y: 0.2, width: 0.5, height: 0.6 }; + const result = run(twoTakes(), "setClipRange", { clipId: "clip_1", cropRegion: crop }); + const document = result.document as AxcutDocument; + expect(document.timeline.clips[0]).toMatchObject({ + sourceStartSec: 0, + sourceEndSec: 10, + cropRegion: crop, + }); + const snapshot = JSON.parse(run(document, "getCurrentDocument", {}).resultJson); + expect(snapshot.clips[0].cropRegion).toEqual(crop); + // Absent, not null, on a clip that shows the whole frame. + const plain = JSON.parse(run(twoTakes(), "getCurrentDocument", {}).resultJson); + expect(plain.clips[0]).not.toHaveProperty("cropRegion"); + }); + + it("setClipRange clears a crop with null or with the whole frame", () => { + const cropped = run(twoTakes(), "setClipRange", { + clipId: "clip_1", + cropRegion: { x: 0, y: 0, width: 0.5, height: 0.5 }, + }).document as AxcutDocument; + for (const cropRegion of [null, { x: 0, y: 0, width: 1, height: 1 }]) { + const cleared = run(cropped, "setClipRange", { clipId: "clip_1", cropRegion }); + expect((cleared.document as AxcutDocument).timeline.clips[0].cropRegion).toBeUndefined(); + } + }); + + it("setClipRange refuses a crop outside the frame, and a call that changes nothing", () => { + const outside = run(twoTakes(), "setClipRange", { + clipId: "clip_1", + cropRegion: { x: 0.6, y: 0, width: 0.6, height: 1 }, + }); + expect(outside.ok).toBe(false); + expect(outside.resultJson).toContain("inside the frame"); + expect(run(twoTakes(), "setClipRange", { clipId: "clip_1" }).ok).toBe(false); + }); + + it("setClipRange moves one edge and keeps the other", () => { + const result = run(twoTakes(), "setClipRange", { clipId: "clip_1", sourceEndSec: 6 }); + expect((result.document as AxcutDocument).timeline.clips[0]).toMatchObject({ + sourceStartSec: 0, + sourceEndSec: 6, + }); + }); + + describe("addAutoZooms", () => { + const suggestion = { + span: { start: 3500, end: 5500 }, + focus: { cx: 0.4, cy: 0.6 }, + depth: 3 as const, + }; + const withPlan = (autoZooms: AgentToolOptions["autoZooms"]) => + executeAgentTool(twoTakes(), "addAutoZooms", "{}", { autoZooms }); + + it("adds the planned zooms, as the wand does", () => { + const result = withPlan({ suggestions: [suggestion], telemetry: { asset_1: "ok" } }); + expect(result.ok).toBe(true); + expect(JSON.parse(result.resultJson)).toMatchObject({ + added: 1, + zooms: [{ startSec: 3.5, endSec: 5.5, focus: { cx: 0.4, cy: 0.6 } }], + telemetry: { asset_1: "ok" }, + }); + expect((result.document as AxcutDocument).zoomRanges).toMatchObject([ + { focusMode: "auto", depth: 3, clipId: "clip_1" }, + ]); + }); + + it("answers added 0 without writing when no click calls for a zoom", () => { + const result = withPlan({ suggestions: [], telemetry: { asset_1: "no-sidecar" } }); + expect(result.ok).toBe(true); + expect(result.document).toBeUndefined(); + expect(JSON.parse(result.resultJson)).toMatchObject({ + added: 0, + telemetry: { asset_1: "no-sidecar" }, + }); + }); + + it("refuses, as a limit of its own, when the clicks could not be read", () => { + for (const plan of [ + undefined, + { suggestions: [], telemetry: { asset_1: "unavailable" as const } }, + ]) { + const result = withPlan(plan); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("could not be read"); + } + }); + }); +}); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index e6bf21585..a2420cb9f 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -32,12 +32,15 @@ import { import { createId } from "../../src/lib/ai-edition/document/ids"; import { resolveAspectRatioValue } from "../../src/lib/ai-edition/document/outputFormat"; import { + duplicateClip, + insertAssetClip, moveClip, planTimelineReplacement, type RegionKind, removeClip, removeRegion, replaceTimeline, + setClipCrop, setClipSourceRange, } from "../../src/lib/ai-edition/document/timeline"; import { setDocumentWordText } from "../../src/lib/ai-edition/document/transcript"; @@ -45,9 +48,11 @@ import { type AxcutAnnotationRegion, type AxcutDocument, blurDataSchema, + clipCropRegionSchema, figureDataSchema, } from "../../src/lib/ai-edition/schema"; import { getEditorSettings } from "../../src/lib/ai-edition/store/editorSettings"; +import { appendAutoZoomSuggestions } from "../../src/lib/ai-edition/timeline/apply-auto-zooms"; import { hasAnyClipWithCamera } from "../../src/lib/ai-edition/timeline/camera"; import { isGeneratedAssetId } from "../../src/lib/ai-edition/timeline/clip-parts"; import { @@ -72,6 +77,7 @@ import { ZOOM_DEPTH_LEGEND, zoomLevelFor, } from "../../src/lib/ai-edition/timeline/zoom-scale"; +import type { AutoZoomSuggestion } from "../../src/lib/ai-edition/timeline/zoom-suggestions"; import { chainedPans, transitionCutsMs, @@ -664,12 +670,38 @@ export const setTrimArgs = z.object({ endSec: secondsSchema, }); +/** A crop the frame can hold: a non-empty rectangle inside it, in 0–1 fractions. */ +const cropRegionArgs = clipCropRegionSchema.refine( + (c) => c.width > 0 && c.height > 0 && c.x + c.width <= 1 + 1e-6 && c.y + c.height <= 1 + 1e-6, + "cropRegion must be a non-empty rectangle inside the frame: width and height above 0, x + width and y + height at most 1.", +); + export const setClipRangeArgs = z.object({ clipId: z.string().min(1), - sourceStartSec: secondsSchema, - sourceEndSec: secondsSchema, + sourceStartSec: secondsSchema.optional(), + sourceEndSec: secondsSchema.optional(), + cropRegion: cropRegionArgs + .nullable() + .optional() + .describe( + "The part of the source frame this clip shows, as 0–1 fractions of it ({x, y, width, height}); null shows the whole frame again.", + ), +}); + +/** `beforeClipId`, never an index, for the reason `moveClipArgs` gives. */ +export const insertClipArgs = z.object({ + assetId: z.string().min(1), + beforeClipId: z.string().min(1).nullish(), + sourceStartSec: secondsSchema.default(0), + sourceEndSec: secondsSchema.optional(), }); +export const duplicateClipArgs = z.object({ + clipId: z.string().min(1), +}); + +export const addAutoZoomsArgs = z.object({}); + export const setAspectRatioArgs = z.object({ aspectRatio: z .string() @@ -1016,9 +1048,12 @@ export const OPENSCREEN_TOOL_NAMES = [ "setTrim", "setClipRange", "moveClip", + "insertClip", + "duplicateClip", "replaceTimeline", "addZoom", "addZooms", + "addAutoZooms", "setZoom", "addSpeed", "setSpeed", @@ -1091,8 +1126,11 @@ export const MUTATING_TOOL_NAMES: ReadonlySet = new Set([ "setTrim", "setClipRange", "moveClip", + "insertClip", + "duplicateClip", "replaceTimeline", "addZoom", + "addAutoZooms", "setZoom", "addSpeed", "setSpeed", @@ -1301,6 +1339,8 @@ export function documentSnapshotForModel( sourceEndSec: c.sourceEndSec ?? null, timelineStartSec: c.timelineStartSec, timelineEndSec: c.timelineEndSec, + // Only when set: absent is the whole frame. + ...(c.cropRegion ? { cropRegion: c.cropRegion } : {}), })), trimRanges: document.timeline.trimRanges.map((s) => ({ id: s.id, @@ -1522,6 +1562,16 @@ export interface AgentToolOptions { /** Whether this machine can cut out the camera background, asked only by the two tools that * report it. Absent where nothing can ask the compositor: reported as not checked. */ cameraSegmentation?: () => SegmentationSupport; + /** `addAutoZooms`' plan, made by the async wrapper from the recorded clicks. Absent means + * no cursor reader is wired here, which the tool reports as "could not look". */ + autoZooms?: AutoZoomPlan; +} + +/** What the timeline wand would add, and what each recording's cursor read gave. */ +export interface AutoZoomPlan { + suggestions: AutoZoomSuggestion[]; + /** assetId → the read's verdict, for every video asset with a clip on the timeline. */ + telemetry: Record; } /** @@ -2284,31 +2334,211 @@ export function executeAgentTool( case "setClipRange": { const parsed = setClipRangeArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); - const { clipId } = parsed.data; - if (!document.timeline.clips.some((c) => c.id === clipId)) { - return failure(`Unknown clip: ${clipId}`); + const { clipId, cropRegion } = parsed.data; + const clip = document.timeline.clips.find((c) => c.id === clipId); + if (!clip) return failure(`Unknown clip: ${clipId}. ${clipRoster(document)}`); + const ranged = + parsed.data.sourceStartSec !== undefined || parsed.data.sourceEndSec !== undefined; + if (!ranged && cropRegion === undefined) { + return failure( + "Pass sourceStartSec / sourceEndSec, cropRegion, or both: nothing to change.", + ); + } + const start = parsed.data.sourceStartSec ?? clip.sourceStartSec; + const end = + parsed.data.sourceEndSec ?? + clip.sourceEndSec ?? + document.assets.find((a) => a.id === clip.assetId)?.durationSec; + if (end === undefined) { + return failure(`Clip ${clipId} has no known end yet: pass sourceEndSec too.`); } - const sourceStartSec = Math.min(parsed.data.sourceStartSec, parsed.data.sourceEndSec); - const sourceEndSec = Math.max(parsed.data.sourceStartSec, parsed.data.sourceEndSec); - // One shared mutator with the modale + op dispatcher: recomputes the clip's - // width from the new source window AND clamps/drops the anchored pills the - // trim removed. Hand-rolling it here is exactly what left this façade orphaning - // stale pills the other two didn't. - const next = setClipSourceRange(document, clipId, sourceStartSec, sourceEndSec); + const sourceStartSec = Math.min(start, end); + const sourceEndSec = Math.max(start, end); + // The clip's Edit modal, as one document: the range through the one shared mutator + // (which recomputes the clip's width and clamps / drops the anchored pills the new + // window no longer covers), then the crop. `null` clears it, and so does the whole + // frame: an untouched clip does not store the identity region. + const wholeFrame = + cropRegion?.x === 0 && + cropRegion.y === 0 && + cropRegion.width === 1 && + cropRegion.height === 1; + const afterRange = ranged + ? setClipSourceRange(document, clipId, sourceStartSec, sourceEndSec) + : document; + const next = + cropRegion === undefined + ? afterRange + : setClipCrop(afterRange, clipId, wholeFrame ? null : cropRegion); const dropped = droppedByEdit(document, next); const casualties = dropped.droppedModifierIds.length + dropped.droppedTrimIds.length; + const after = next.timeline.clips.find((c) => c.id === clipId); return { ok: true, document: next, - resultJson: JSON.stringify({ clipId, sourceStartSec, sourceEndSec, ...dropped }), + resultJson: JSON.stringify({ + clipId, + sourceStartSec, + sourceEndSec, + cropRegion: after?.cropRegion ?? null, + // Its media now continues the clip before it, with the same framing: one clip. + ...(after + ? {} + : { joinedIntoNeighbour: true, clipOrder: next.timeline.clips.map((c) => c.id) }), + ...dropped, + }), summary: - `trimmed clip to ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)}` + + (ranged + ? `trimmed clip to ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)}` + : `cropped clip ${clipId}`) + + (ranged && cropRegion !== undefined ? ", cropped" : "") + (casualties > 0 ? ` — dropped ${[...dropped.droppedModifierIds, ...dropped.droppedTrimIds].join(", ")}` : ""), }; } + case "insertClip": { + const parsed = insertClipArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { assetId } = parsed.data; + const asset = document.assets.find((a) => a.id === assetId); + const footage = document.assets.filter( + (a) => a.kind === "video" && !isGeneratedAssetId(a.id), + ); + const roster = footage.length + ? ` Footage in this project: ${footage.map((a) => `${a.id} (${a.label})`).join(", ")}.` + : " This project has no footage to place."; + if (!asset) return failure(`Unknown asset: ${assetId}.${roster}`); + if (asset.kind === "audio") { + return failure( + `Asset ${assetId} is audio, not footage: lay it over the recording with addAudio.`, + ); + } + if (isGeneratedAssetId(assetId)) { + return failure(`Asset ${assetId} is generated media, not footage you can place.${roster}`); + } + const beforeClipId = parsed.data.beforeClipId ?? null; + const clips = document.timeline.clips; + const index = + beforeClipId === null ? clips.length : clips.findIndex((c) => c.id === beforeClipId); + if (index < 0) return failure(`Unknown clip: ${beforeClipId}. ${clipRoster(document)}`); + const durationSec = asset.durationSec; + const requestedEnd = parsed.data.sourceEndSec ?? durationSec; + if (requestedEnd === undefined) { + return failure( + `The length of ${assetId} is not known yet: pass sourceEndSec, in its source seconds, to say where the clip ends.`, + ); + } + const bound = (sec: number) => (durationSec === undefined ? sec : Math.min(sec, durationSec)); + const sourceStartSec = bound(parsed.data.sourceStartSec); + const sourceEndSec = bound(requestedEnd); + if (sourceEndSec <= sourceStartSec) { + return failure( + `The clip would be empty: ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)} of ${assetId}` + + (durationSec === undefined ? "." : `, which lasts ${formatSec(durationSec)}.`), + ); + } + const clipId = createId("clip"); + // The media panel's drop, through the same mutator. + const next = insertAssetClip(document, index, { + id: clipId, + assetId, + sourceStartSec, + sourceEndSec, + origin: "agent", + reason: "", + }); + const placed = next.timeline.clips.find((c) => c.id === clipId); + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + clipId: placed ? clipId : null, + assetId, + sourceStartSec, + sourceEndSec, + ...(placed + ? { timelineStartSec: placed.timelineStartSec, timelineEndSec: placed.timelineEndSec } + : // Its media continues the clip before it, with the same framing: one clip. + { joinedIntoNeighbour: true }), + clipOrder: next.timeline.clips.map((c) => c.id), + }), + summary: + `inserted ${asset.label} ${formatSec(sourceStartSec)} – ${formatSec(sourceEndSec)} ` + + (beforeClipId ? `before ${beforeClipId}` : "at the end"), + }; + } + + case "duplicateClip": { + const parsed = duplicateClipArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + const { clipId } = parsed.data; + const index = document.timeline.clips.findIndex((c) => c.id === clipId); + if (index < 0) return failure(`Unknown clip: ${clipId}. ${clipRoster(document)}`); + // The editor's Duplicate: the copy right after the original, with its own copy of + // the original's cuts and anchored effects. + const next = duplicateClip(document, clipId, "agent", ""); + const copy = next.timeline.clips[index + 1]; + return { + ok: true, + document: next, + resultJson: JSON.stringify({ + clipId: copy.id, + of: clipId, + timelineStartSec: copy.timelineStartSec, + timelineEndSec: copy.timelineEndSec, + clipOrder: next.timeline.clips.map((c) => c.id), + copiedTrimIds: next.timeline.trimRanges + .filter((t) => t.clipId === copy.id) + .map((t) => t.id), + }), + summary: `duplicated ${clipId}`, + }; + } + + case "addAutoZooms": { + const parsed = addAutoZoomsArgs.safeParse(args); + if (!parsed.success) return failure(parsed.error.message); + if (document.timeline.clips.length === 0) { + return failure("The timeline has no clips, so there is nothing to zoom on."); + } + const plan = options?.autoZooms; + const reads = Object.values(plan?.telemetry ?? {}); + if (!plan || (reads.length > 0 && reads.every((status) => status === "unavailable"))) { + return failure( + "The recorded clicks could not be read from here, so no zoom was planned. This says nothing about whether the recording has any: report the limit as yours.", + ); + } + // The timeline wand's write: zooms on the clicks, clear of the zooms already there. + const next = appendAutoZoomSuggestions(document, plan.suggestions); + const before = new Set(document.zoomRanges.map((z) => z.id)); + const added = coalesceForAgent(next.zoomRanges).filter((z) => !before.has(z.id)); + return { + ok: true, + ...(added.length ? { document: next } : {}), + resultJson: JSON.stringify({ + added: added.length, + zooms: added.map((z) => ({ + id: z.id, + startSec: roundSec(z.startMs), + endSec: roundSec(z.endMs), + renderedScale: effectiveZoomScale(z), + focus: z.focus, + })), + telemetry: plan.telemetry, + ...(added.length + ? cutTransitionsReport(document, next) + : { + note: "No click on the timeline calls for a new zoom: none was recorded, each one is already under a zoom, or they fall in the first seconds or on the stop.", + }), + }), + ...(added.length + ? { summary: `added ${added.length} automatic zoom${added.length === 1 ? "" : "s"}` } + : {}), + }; + } + case "moveClip": { const parsed = moveClipArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); @@ -2980,7 +3210,7 @@ export function executeAgentTool( } if (asset.kind !== "audio") { return failure( - `Asset ${assetId} is video, not audio. addAudio plays an imported audio file over the recording; to place footage use replaceTimeline.`, + `Asset ${assetId} is video, not audio. addAudio plays an imported audio file over the recording; to place footage on the timeline use insertClip.`, ); } const durationSec = asset.durationSec ?? 0; diff --git a/electron/ai-edition/deep-agent/service.test.ts b/electron/ai-edition/deep-agent/service.test.ts index a718552ff..30d9734a7 100644 --- a/electron/ai-edition/deep-agent/service.test.ts +++ b/electron/ai-edition/deep-agent/service.test.ts @@ -38,7 +38,9 @@ import { anthropicCachingMiddleware, buildSystemPrompt, buildTools, + type CursorTelemetryReader, type OpenScreenAgentSink, + runDocumentTool, SYSTEM_PROMPT, TOOL_DESCRIPTIONS, } from "./service"; @@ -67,9 +69,13 @@ const ARGS: Record = { setTrim: { trimRangeId: "trim_1", startSec: 1, endSec: 2 }, setClipRange: { clipId: "clip_1", sourceStartSec: 0, sourceEndSec: 10 }, moveClip: { clipId: "clip_1", beforeClipId: null }, + insertClip: { assetId: "asset_1", sourceStartSec: 2, sourceEndSec: 4 }, + duplicateClip: { clipId: "clip_1" }, replaceTimeline: { intervals: [{ startSec: 0, endSec: 10 }] }, addZoom: { startSec: 1, endSec: 2 }, addZooms: { regions: [{ startSec: 1, endSec: 2 }] }, + // No cursor reader in these tests: the refusal that says it could not look. + addAutoZooms: {}, setZoom: { zoomId: "zoom_nope" }, addSpeed: { startSec: 1, endSec: 2 }, setSpeed: { speedId: "speed_nope" }, @@ -707,3 +713,43 @@ describe("what the descriptions say about a zoom's focus", () => { expect(SYSTEM_PROMPT).not.toMatch(/cursorAnchor/); }); }); + +describe("addAutoZooms reads the clicks through the runtime's cursor reader", () => { + const click = (timeMs: number, interactionType?: string) => ({ + timeMs, + cx: 0.4, + cy: 0.6, + ...(interactionType ? { interactionType } : {}), + }); + + it("plans on the recorded clicks, as the timeline wand does", async () => { + const reads: string[] = []; + const cursor: CursorTelemetryReader = { + read: async ({ assetId, originalPath }) => { + reads.push(`${assetId}:${originalPath}`); + return { + status: "ok", + assetId, + samples: [click(3700), click(4000, "click"), click(4300, "mouseup"), click(20000)], + }; + }, + }; + const result = await runDocumentTool(fixtureDocument(), "addAutoZooms", {}, true, { cursor }); + expect(reads).toEqual(["asset_1:/tmp/rec.mp4"]); + expect(result.ok).toBe(true); + const payload = JSON.parse(result.resultJson); + expect(payload).toMatchObject({ added: 1, telemetry: { asset_1: "ok" } }); + expect(result.document?.zoomRanges).toMatchObject([{ focusMode: "auto" }]); + }); + + it("reports a reader that throws as a read it could not make", async () => { + const cursor: CursorTelemetryReader = { + read: async () => { + throw new Error("disk gone"); + }, + }; + const result = await runDocumentTool(fixtureDocument(), "addAutoZooms", {}, true, { cursor }); + expect(result.ok).toBe(false); + expect(result.resultJson).toContain("could not be read"); + }); +}); diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index fa610515f..408a2aa4d 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -19,8 +19,10 @@ import { anthropicPromptCachingMiddleware, createAgent, tool } from "langchain"; import { z } from "zod"; +import type { CursorTelemetryPoint } from "../../../src/components/video-editor/types"; import { AUDIO_FADE_MAX_MS } from "../../../src/lib/ai-edition/document/audioTracks"; import type { AxcutDocument } from "../../../src/lib/ai-edition/schema"; +import { collectAutoZoomSuggestionsForDocument } from "../../../src/lib/ai-edition/timeline/apply-auto-zooms"; // ponytail: the legend is DERIVED from the depth→scale table, not retyped. Both // tool descriptions used to assert "depth 1–6 maps to 1.0×–3.5×" — a formula // (`depth/2 + 0.5`) that was purged from two rendering sites and survived here, @@ -35,8 +37,10 @@ import type { SegmentationSupport } from "../../../src/native/contracts"; import { AGENT_EDITS_SETTING, type AgentToolExecution, + type AutoZoomPlan, addAnnotationArgs, addAudioArgs, + addAutoZoomsArgs, addCameraFullscreenArgs, addSpeedArgs, addTrimArgs, @@ -44,10 +48,12 @@ import { addZoomArgs, addZoomsArgs, type CursorTelemetryLoad, + duplicateClipArgs, executeAgentTool, getCursorTrackArgs, getTranscriptArgs, getTranscriptWordsArgs, + insertClipArgs, isMutatingTool, moveClipArgs, removeClipArgs, @@ -132,7 +138,8 @@ const BASE_SYSTEM_PROMPT = [ "How the tools map to intent — pick the most specific one, and prefer the smallest edit that satisfies the request:", "- Silences, pauses and dead stretches are removed as trims INSIDE the placed clip. Send them together with addTrims once you know the ranges; addTrim is for a single cut or a correction. The placed clip stays the canonical cut; it is not rebuilt to drop them.", "- Cutting specific spoken words (fillers, a repeated or flubbed phrase, a false start, a duplicate take) is removeWords: read getTranscriptWords, choose the occurrences from their surrounding speech, then pass that assetId and ONLY those word IDs. Word IDs restart for each asset, so keep the read's assetId. Consecutive words become one cut. Do not cut every occurrence of a word just because its text matches one you were asked to remove.", - "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim.", + "- Changing where a clip starts or ends within its source is setClipRange — the clip's in/out, distinct from a trim. It also crops the clip (cropRegion).", + "- insertClip places footage of any video asset at a position on the timeline; duplicateClip copies a clip right after itself. addAutoZooms plans zooms on the recorded clicks, as the timeline's Automatic zooms does.", `- addZoom takes a virtual-timeline span (depth is an ordinal 1–6 selecting from a fixed table — ${ZOOM_DEPTH_LEGEND} — never a multiplier; focus in 0–1 frame fractions). addSpeed changes pacing over a span. addAnnotation puts text, an arrow or a privacy blur on screen: a blur is how an email address, a token or any private detail the recording shows is hidden. addCameraFullscreen enlarges the webcam, and only does something where assets[].hasCameraTrack is true.`, "- setEditorSettings sets the look of the whole video: the background (wallpaper, its motion and blur), padding, rounded corners, shadow, a device frame, motion blur, depth of field, the shape of the video (aspectRatio: auto follows the recording) and how a recording of another shape fills it, the camera's layout (picture-in-picture, side by side, stacked, its shape, size, position, framing and background cutout), the cursor, and the master volume. getCurrentDocument returns all of them under settings, in the shape this tool takes, and settingsContext.inactive names each one the editor hides in the current state and why: setting one of those changes nothing, so say so instead of reporting the look as changed. setAspectRatio is the same write for the shape alone.", "- A zoom animates in just before its span and out just after it (zoomInFromSec, zoomOutUntilSec in getCurrentDocument). Leave those windows untrimmed, or the export jumps at the cut.", @@ -199,13 +206,19 @@ export const TOOL_DESCRIPTIONS: Record = { addTrims: `Add MANY trim ranges in one call: \`ranges\` is a list, each entry taking exactly the fields addTrim takes. Use this whenever you have more than one cut to make — 'remove the silences' on a half-hour recording is hundreds of cuts, and sending them one at a time costs one round trip each. Each range stands or falls ALONE: one that cannot be placed is refused by itself and listed in \`refused\` with its index and the reason, while every other range is still applied. Nothing is rolled back, so a single bad bound never costs you the rest. The result leads with requested / appliedCount / refusedCount so you can see a partial outcome without re-reading the document — report what was refused rather than claiming the whole list landed. ${ZOOM_TRANSITIONS_NOTE} Each applied entry carries the same \`cutTransitions\` warning addTrim reports.`, setTrim: `Move or resize an existing trim range by id. Times are source-time seconds. The cut follows to whichever clip the new range lands in, when that clip is unambiguous. ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, setClipRange: - "Set a clip's in/out points (source-time seconds) to shorten its head or tail — distinct from a trim (which cuts a span inside the clip). All clips are re-laid back-to-back afterwards, so downstream clips shift automatically. Use this ONLY when the user explicitly asks to shorten or extend a user-placed clip. Do NOT use this for 'remove silences' or 'cut pauses' — for those, use addTrim.", + "Set a clip's in/out points (source-time seconds) to shorten its head or tail — distinct from a trim (which cuts a span inside the clip) — and/or its crop. Only the fields you pass are changed. All clips are re-laid back-to-back afterwards, so downstream clips shift automatically. Change the range ONLY when the user explicitly asks to shorten or extend a clip. Do NOT use this for 'remove silences' or 'cut pauses' — for those, use addTrim. cropRegion is the part of the source frame the clip shows, {x, y, width, height} in 0–1 fractions of it, per clip as in the editor's Edit clip; null shows the whole frame again. getCurrentDocument reports it on the clip, absent when the clip shows the whole frame. Two neighbouring clips of the same asset whose source ranges meet and whose crops match are one clip: the result says joinedIntoNeighbour when an edit made them so.", moveClip: "Reorder a placed clip: move `clipId` so it plays just before `beforeClipId` (pass null, or omit it, to move it last). Ids come from getCurrentDocument, where each clip carries its `index` and its label in `reason`. This preserves every clip id, every source range, every trim, and the zooms / speed regions / annotations anchored to each clip. This is the tool for 'swap these clips', 'put X first' and 'change the clip order' — replaceTimeline cannot reorder anything.", + insertClip: + "Place a clip of a video asset on the timeline, just before `beforeClipId` (null, or omitted, places it last): another take, or footage imported in the editor. Any video asset of getCurrentDocument works, the primary one too. sourceStartSec/sourceEndSec pick the stretch of the asset in its source seconds, the whole asset by default. Clips stay back to back, so the clips after it shift. Its media continuing the clip before it makes the two one clip (joinedIntoNeighbour). An audio asset is not footage: use addAudio. Nothing here imports a file from disk.", + duplicateClip: + "Copy a clip, placed right after it: the same footage, crop and source range, with its own copy of the clip's trims and of the zooms, speed regions, annotations and camera-fullscreen regions on it. Edits to one no longer touch the other. Returns the copy's clipId; moveClip puts it somewhere else.", replaceTimeline: - "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.", + "Replace the whole timeline with the given kept intervals of the primary asset's source time. Everything outside the intervals becomes a trim. The intervals are SORTED, so this can never reorder clips — use moveClip for that. DO NOT use this for 'cut silences' or 'remove pauses' — the user has likely placed clips on the timeline that you'd be discarding. Use this ONLY when the user explicitly asks you to rebuild the timeline from scratch (e.g. 'start over with the kept intervals from the transcript'). To place another asset's footage, use insertClip. It is refused when it would merge away, shorten or drop an existing clip; the refusal names them and the tool to use instead.", addZoom: `Add a zoom-in over a span of the edited timeline (virtual seconds). depth is an ORDINAL 1–6, not a factor: it selects a magnification from a fixed table (${ZOOM_DEPTH_LEGEND}), so the default depth 3 renders at 1.80×. The result reports renderedScale — quote that, never the depth, when telling the user how strong the zoom is. focus is the zoom centre in 0–1 fractions of the frame (default centre). When the recording's pointer telemetry can be read for the footage under the span, the result also carries \`cursorAnchor\`: \`focus\` echoes the value this call used (including the default, if you left it out), \`cursor\` is where the pointer ACTUALLY was over the span the zoom landed on — the median of the recorded samples, \`spread\` being how far the farthest one strays from it — and \`offset\` is the distance between the two, in frame fractions. It is a measurement, not a correction: nothing is moved and no call is refused over it, and a zoom framing a slide, a face, or a region the pointer never enters is a legitimate choice. \`available:false\` names what it found instead (\`no-samples\`, \`trimmed-out\`). Its ABSENCE means no telemetry was read for that footage — never that the recording has none; assets[].hasCursorTelemetry and getCursorTrack are what answer that. Use for 'zoom in on …' and the smart-zoom pass. ${ZOOM_BEHAVIOUR_NOTE} ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, addZooms: `Add MANY zooms in one call: \`regions\` is a list, each entry taking exactly the fields addZoom takes (same depth table, ${ZOOM_DEPTH_LEGEND}). Use this for the smart-zoom pass, where you have decided every zoom before emitting the first one — sending them one at a time costs one round trip each. Each region stands or falls ALONE: one that covers no clip is refused by itself and listed in \`refused\` with its index and the reason, while the others are still applied. The result leads with requested / appliedCount / refusedCount, and each applied entry carries its renderedScale — quote that, never the depth — plus the same \`cursorAnchor\` addZoom reports, whenever the footage under that region has readable pointer telemetry, and the same \`cutTransitions\` warning. ${ZOOM_TRANSITIONS_NOTE}`, + addAutoZooms: + "Plan zooms on the clicks recorded with each take on the timeline and add them, as the timeline's Automatic zooms does: clicks close in time and on screen share one zoom, the first seconds and the click that stopped the take get none, and no zoom lands on one already there. Takes no arguments. Use it after cutting or rearranging a project, or on one whose zooms were removed. The result lists the zooms added, and telemetry gives each recording's read: ok, no-sidecar (it has no recorded clicks), or unavailable (it could not be read from here, which says nothing about the recording). added 0 is an answer, not a failure.", setZoom: `Move, resize, or restyle an existing zoom by id (virtual-timeline seconds). Only the fields you pass are changed. depth selects from the same table (${ZOOM_DEPTH_LEGEND}); if the zoom carries a customScale (getCurrentDocument shows it as depthIsOverridden), that custom value is what renders, and passing depth clears it so the depth takes effect — the result says so. The result reports the resulting renderedScale, and — when the footage under the span has readable pointer telemetry — the same \`cursorAnchor\` addZoom reports, measured against the zoom's EFFECTIVE focus, so a call that moved only the span still learns what its unchanged focus is now looking at. ${ZOOM_BEHAVIOUR_NOTE} ${ZOOM_CAMERA_NOTE} ${ZOOM_TRANSITIONS_NOTE} ${CUT_TRANSITIONS_RESULT}`, addSpeed: "Add a speed-change region over a span of the edited timeline (virtual seconds). speed > 1 fast-forwards, < 1 slows down, from 0.25× to 16× (default 1.5×). Use to speed through slow stretches without cutting them.", @@ -342,13 +355,36 @@ export async function runDocumentTool( const load = TOOLS_READING_CURSOR.has(name) ? await loadCursorTelemetry(document, args, runtime) : undefined; + const autoZooms = + name === "addAutoZooms" && editsAllowed ? await planAutoZooms(document, runtime) : undefined; return executeAgentTool(document, name, JSON.stringify(args ?? {}), { editsAllowed, cursorTelemetry: { availableByAssetId: runtime.availableByAssetId, load }, cameraSegmentation: runtime.cameraSegmentation, + autoZooms, }); } +/** The timeline wand's plan (`collectAutoZoomSuggestionsForDocument`), each recording's + * clicks read through the runtime's cursor reader, and what each read gave. */ +async function planAutoZooms( + document: AxcutDocument, + runtime: ToolRuntime, +): Promise { + const cursor = runtime.cursor; + if (!cursor) return undefined; + const telemetry: AutoZoomPlan["telemetry"] = {}; + const suggestions = await collectAutoZoomSuggestionsForDocument(document, async (videoPath) => { + const asset = document.assets.find((a) => a.originalPath === videoPath); + if (!asset) return []; + const load = await readCursor(cursor, asset.id, videoPath); + telemetry[asset.id] = load.status; + // The sidecar's samples, read as the renderer's `getTelemetry` reads them. + return load.status === "ok" ? (load.samples as CursorTelemetryPoint[]) : []; + }); + return { suggestions, telemetry }; +} + /** Reads the sidecar for whichever asset the call names, defaulting to the * primary one — the same resolution the executor will use to report it. */ async function loadCursorTelemetry( @@ -363,8 +399,16 @@ async function loadCursorTelemetry( return { status: "unavailable", assetId }; } const asset = document.assets.find((a) => a.id === assetId); + return readCursor(runtime.cursor, assetId, asset?.originalPath ?? null); +} + +async function readCursor( + cursor: CursorTelemetryReader, + assetId: string, + originalPath: string | null, +): Promise { try { - return await runtime.cursor.read({ assetId, originalPath: asset?.originalPath ?? null }); + return await cursor.read({ assetId, originalPath }); } catch (error) { // A reader that throws is still "we could not look" — never "there is // none". Collapsing the two is the exact failure this tool exists to stop. @@ -414,9 +458,12 @@ export const TOOL_ARG_SCHEMAS: ReadonlyArray = [ ["setTrim", setTrimArgs], ["setClipRange", setClipRangeArgs], ["moveClip", moveClipArgs], + ["insertClip", insertClipArgs], + ["duplicateClip", duplicateClipArgs], ["replaceTimeline", replaceTimelineArgs], ["addZoom", addZoomArgs], ["addZooms", addZoomsArgs], + ["addAutoZooms", addAutoZoomsArgs], ["setZoom", setZoomArgs], ["addSpeed", addSpeedArgs], ["setSpeed", setSpeedArgs], From c91ab403f5176c8202fecb43fb33de9d7fcf2d81 Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:21:00 +0200 Subject: [PATCH 3/5] docs(mcp): describe the clip tools, and stop promising to split a clip The timeline model makes two meeting clips of one asset and one framing a single clip, so a bare split cannot exist; the chat welcome now promises rearranging clips. Part of #1180. --- skills/openscreen-demo/SKILL.md | 3 ++- src/i18n/locales/ar/editor.json | 2 +- src/i18n/locales/cs/editor.json | 2 +- src/i18n/locales/de/editor.json | 2 +- src/i18n/locales/en/editor.json | 2 +- src/i18n/locales/es/editor.json | 2 +- src/i18n/locales/fr/editor.json | 2 +- src/i18n/locales/it/editor.json | 2 +- src/i18n/locales/ja-JP/editor.json | 2 +- src/i18n/locales/ko-KR/editor.json | 2 +- src/i18n/locales/pt-BR/editor.json | 2 +- src/i18n/locales/ru/editor.json | 2 +- src/i18n/locales/tr/editor.json | 2 +- src/i18n/locales/vi/editor.json | 2 +- src/i18n/locales/zh-CN/editor.json | 2 +- src/i18n/locales/zh-TW/editor.json | 2 +- technical-documentation/architecture/mcp-server.md | 2 +- website/docs/ai-editing.md | 2 +- .../de/docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../es/docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../fr/docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../ja/docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- .../zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md | 2 +- 25 files changed, 26 insertions(+), 25 deletions(-) diff --git a/skills/openscreen-demo/SKILL.md b/skills/openscreen-demo/SKILL.md index 19c3025bd..3918a420b 100644 --- a/skills/openscreen-demo/SKILL.md +++ b/skills/openscreen-demo/SKILL.md @@ -82,7 +82,8 @@ Use the document tools, always with the `projectId`. Read the project first with - **Format first:** `setAspectRatio` for where the video will be shown, `16:9` for X. Do it before placing text. - **Cut dead time:** the setup at the start, the stop at the end, any wait. Use `addTrims`, or `addSpeed` to fast-forward a wait instead of cutting it. -- **Zoom on the action:** clicks close in time and on screen share one zoom, and the first 2.5 s stay wide. Check them, then add or move zooms with `addZooms` and `setZoom` where a result deserves its own. Hold a zoom at least 1.5 s. +- **Zoom on the action:** clicks close in time and on screen share one zoom, and the first 2.5 s stay wide. Check them, then add or move zooms with `addZooms` and `setZoom` where a result deserves its own. Hold a zoom at least 1.5 s. After a cut that removed some, `addAutoZooms` plans them again on the clicks left, clear of the zooms still there. +- **Frame the app:** when the take shows more than the app, `setClipRange` with a `cropRegion` crops the clip to it. - **Callouts:** one `addAnnotation` per beat, short, placed clear of the action. Text sits on the output frame and stays put when the picture zooms. Use its look for hierarchy: a large title with `animation: "typewriter"`, step callouts with `animation: "pop"`. An arrow, `type: "figure"`, points at a control the text names. `setAnnotation` moves or restyles one. - **Hide what is private:** an email address, a token or a name that shows on screen gets `addAnnotation` with `type: "blur"` over the span it is visible. Its x, y, width and height are percent of the recording's picture, not of the frame, so it stays on what it hides when a zoom moves the picture. Check it with `getFrames`. - **Music or a voiceover:** `importMedia` with the file's absolute path adds it to the project's media, then `addAudio` plays it over a span. `removeMedia` takes back a file the timeline no longer uses. diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index da053d2d7..157133a4b 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -273,7 +273,7 @@ "subtitle": "تعمل المحادثة على تعديل الفيديو عبر التحدث إلى نموذج لغوي. اختر المزوّد الذي تثق به واربط مفتاح API للبدء.", "feature1": "اقطع الصمت، اضغط الوقفات، أزل الكلمات الحشوية", "feature2": "أضف ترجمات، كبّر الكاميرا، أنشئ عنواناً", - "feature3": "أعد كتابة قسم، أو أعِد صياغته، أو قسّم مقطعاً عند الطلب", + "feature3": "أعد كتابة قسم، أو أعِد صياغته، أو أعِد ترتيب المقاطع عند الطلب", "cta": "إعداد مزوّد", "disclaimer": "سيُرسَل نصّ فيديوك إلى المزوّد الذي تختاره. لا يُشارَك شيء قبل أن تربط مزوّداً." }, diff --git a/src/i18n/locales/cs/editor.json b/src/i18n/locales/cs/editor.json index 086d3f74b..f4720fa9c 100644 --- a/src/i18n/locales/cs/editor.json +++ b/src/i18n/locales/cs/editor.json @@ -273,7 +273,7 @@ "subtitle": "Chat upravuje vaše video prostřednictvím komunikace s jazykovým modelem. Vyberte poskytovatele, kterému důvěřujete, a připojte API klíč pro začátek.", "feature1": "Vystřihnout ticho, zkrátit pauzy, odstranit výplňová slova", "feature2": "Přidat titulky, přiblížit kameru, generovat název", - "feature3": "Přepsat sekci, přeformulovat nebo rozdělit klip na žádost", + "feature3": "Přepsat sekci, přeformulovat nebo přeskládat klipy na žádost", "cta": "Nastavit poskytovatele", "disclaimer": "Přepis vašeho videa bude odeslán vámi zvolenému poskytovateli. Nic není sdíleno, dokud ho nepřipojíte." }, diff --git a/src/i18n/locales/de/editor.json b/src/i18n/locales/de/editor.json index eb9a7fdf9..c452cd77e 100644 --- a/src/i18n/locales/de/editor.json +++ b/src/i18n/locales/de/editor.json @@ -273,7 +273,7 @@ "subtitle": "Der Chat bearbeitet dein Video, indem er mit einem Sprachmodell spricht. Wähle einen Anbieter, dem du vertraust, und verbinde einen API-Schlüssel, um loszulegen.", "feature1": "Stille schneiden, Pausen straffen, Füllwörter entfernen", "feature2": "Untertitel hinzufügen, auf die Kamera zoomen, einen Titel erzeugen", - "feature3": "Einen Abschnitt umschreiben, umformulieren oder auf Wunsch einen Clip teilen", + "feature3": "Einen Abschnitt umschreiben, umformulieren oder auf Wunsch Clips neu anordnen", "cta": "Anbieter einrichten", "disclaimer": "Das Transkript deines Videos wird an den Anbieter gesendet, den du wählst. Es wird nichts geteilt, bevor du einen verbindest." }, diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index 8e6d253d9..8c0db248e 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -273,7 +273,7 @@ "subtitle": "The chat edits your video by talking to a language model. Pick a provider you trust and connect an API key to get started.", "feature1": "Cut silences, tighten pauses, drop filler words", "feature2": "Add captions, zoom on the camera, generate a title", - "feature3": "Rewrite a section, rephrase, or split a clip on request", + "feature3": "Rewrite a section, rephrase, or rearrange clips on request", "cta": "Set up a provider", "disclaimer": "Your video's transcript will be sent to the provider you choose. Nothing is shared until you connect one." }, diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 680632caa..f11ca3013 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -273,7 +273,7 @@ "subtitle": "El chat edita tu vídeo hablando con un modelo de lenguaje. Elige el proveedor que prefieras y conecta una clave de API para empezar.", "feature1": "Corta silencios, aprieta pausas, elimina muletillas", "feature2": "Añade subtítulos, hace zoom a la cámara, genera un título", - "feature3": "Reescribe una sección, reformula o divide un clip bajo pedido", + "feature3": "Reescribe una sección, reformula o reordena los clips bajo pedido", "cta": "Configurar un proveedor", "disclaimer": "La transcripción de tu vídeo se enviará al proveedor que elijas. No se comparte nada hasta que conectes uno." }, diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 6fd774cd6..b624ccafc 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -273,7 +273,7 @@ "subtitle": "Le chat édite votre vidéo en discutant avec un modèle de langage. Choisissez le fournisseur auquel vous faites confiance et connectez une clé API pour commencer.", "feature1": "Couper les silences, resserrer les pauses, supprimer les tics de langage", "feature2": "Ajouter des sous-titres, zoomer sur la caméra, générer un titre", - "feature3": "Réécrire une section, reformuler ou découper un clip à la demande", + "feature3": "Réécrire une section, reformuler ou réorganiser les clips à la demande", "cta": "Configurer un fournisseur", "disclaimer": "La transcription de votre vidéo sera envoyée au fournisseur que vous choisissez. Rien n'est partagé tant que vous n'en connectez pas un." }, diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index 27fd5cf6c..d51b0c87a 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -273,7 +273,7 @@ "subtitle": "La chat modifica il tuo video parlando con un modello linguistico. Scegli il fornitore di cui ti fidi e collega una chiave API per iniziare.", "feature1": "Taglia i silenzi, stringi le pause, elimina le parole superflue", "feature2": "Aggiungi sottotitoli, ingrandisci la camera, genera un titolo", - "feature3": "Riscrivi una sezione, riformula o dividi un clip su richiesta", + "feature3": "Riscrivi una sezione, riformula o riordina i clip su richiesta", "cta": "Configura un fornitore", "disclaimer": "La trascrizione del tuo video verrà inviata al fornitore che scegli. Nulla viene condiviso finché non ne colleghi uno." }, diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 3de6f0a77..14fc00c54 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -273,7 +273,7 @@ "subtitle": "チャットは言語モデルと会話しながら動画を編集します。信頼できるプロバイダーを選んで API キーを接続すれば始められます。", "feature1": "無音部分をカットし、間を詰め、つなぎ言葉を削除", "feature2": "字幕を追加し、カメラをズームし、タイトルを生成", - "feature3": "セクションを言い換えたり、書き直したり、クリップを分割", + "feature3": "セクションを言い換えたり、書き直したり、クリップを並べ替え", "cta": "プロバイダーを設定", "disclaimer": "動画の文字起こしは、選択したプロバイダーに送信されます。接続するまでは何も共有されません。" }, diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index 77731a1d3..47b8fc67d 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -273,7 +273,7 @@ "subtitle": "채팅은 언어 모델과 대화하며 영상을 편집합니다. 신뢰하는 공급자를 골라 API 키를 연결하면 시작할 수 있습니다.", "feature1": "무음을 잘라내고, 끊김을 줄이며, 군더더기 말을 지웁니다", "feature2": "자막을 추가하고, 카메라를 확대하며, 제목을 만듭니다", - "feature3": "원하는 대로 구간을 다시 쓰고, 표현을 바꾸거나 클립을 나눕니다", + "feature3": "원하는 대로 구간을 다시 쓰고, 표현을 바꾸거나 클립 순서를 바꿉니다", "cta": "공급자 설정", "disclaimer": "영상 전사본은 선택한 공급자에게 전송됩니다. 공급자를 연결하기 전까지는 아무것도 공유되지 않습니다." }, diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 55e1fa5bf..8dbac331a 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -273,7 +273,7 @@ "subtitle": "O chat edita o seu vídeo conversando com um modelo de linguagem. Escolha o fornecedor em que confia e conecte uma chave de API para começar.", "feature1": "Corta silêncios, aperta pausas, remove vícios de linguagem", "feature2": "Adiciona legendas, aplica zoom na câmera, gera um título", - "feature3": "Reescreve uma seção, reformula ou divide um clipe sob pedido", + "feature3": "Reescreve uma seção, reformula ou reorganiza os clipes sob pedido", "cta": "Configurar um fornecedor", "disclaimer": "A transcrição do seu vídeo será enviada ao fornecedor que escolher. Nada é compartilhado até você conectar um." }, diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index c89607df1..05b049cba 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -273,7 +273,7 @@ "subtitle": "Чат редактирует видео через языковую модель. Выберите провайдера, которому доверяете, и подключите API-ключ, чтобы начать.", "feature1": "Вырезает тишину, сокращает паузы, убирает слова-паразиты", "feature2": "Добавляет субтитры, зум на камеру, генерирует заголовок", - "feature3": "Переписывает раздел, перефразирует или разрезает клип по запросу", + "feature3": "Переписывает раздел, перефразирует или переставляет клипы по запросу", "cta": "Настроить провайдера", "disclaimer": "Транскрипция вашего видео будет отправлена выбранному провайдеру. Пока вы не подключите провайдера, ничего не отправляется." }, diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index f052f504e..8f6fe686a 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -273,7 +273,7 @@ "subtitle": "Sohbet, videonuzu bir dil modeliyle konuşarak düzenler. Güvendiğiniz sağlayıcıyı seçin ve başlamak için bir API anahtarı bağlayın.", "feature1": "Sessizlikleri keser, duraklamaları sıkılaştırır, dolgu kelimeleri kaldırır", "feature2": "Altyazı ekler, kameraya yakınlaştırır, başlık üretir", - "feature3": "Bir bölümü yeniden yazar, ifadeyi değiştirir veya klibi isteğe göre böler", + "feature3": "Bir bölümü yeniden yazar, ifadeyi değiştirir veya klipleri isteğe göre yeniden sıralar", "cta": "Sağlayıcı kur", "disclaimer": "Videonuzun transkripsiyonu seçtiğiniz sağlayıcıya gönderilecek. Bir sağlayıcı bağlamadan hiçbir şey paylaşılmaz." }, diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index f85a6e755..d484dc4bd 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -273,7 +273,7 @@ "subtitle": "Chat chỉnh sửa video của bạn bằng cách trò chuyện với một mô hình ngôn ngữ. Hãy chọn nhà cung cấp bạn tin tưởng và kết nối khóa API để bắt đầu.", "feature1": "Cắt khoảng lặng, rút ngắn nhịp nghỉ, loại bỏ từ đệm", "feature2": "Thêm phụ đề, zoom vào camera, tạo tiêu đề", - "feature3": "Viết lại một đoạn, diễn đạt khác hoặc tách clip theo yêu cầu", + "feature3": "Viết lại một đoạn, diễn đạt khác hoặc sắp xếp lại clip theo yêu cầu", "cta": "Thiết lập nhà cung cấp", "disclaimer": "Bản chép lời video sẽ được gửi đến nhà cung cấp bạn chọn. Không có gì được chia sẻ cho đến khi bạn kết nối một nhà cung cấp." }, diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 940a28963..5a7413051 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -273,7 +273,7 @@ "subtitle": "聊天通过与语言模型对话来编辑你的视频。选择你信任的提供方并连接一个 API 密钥即可开始。", "feature1": "剪掉静音、收紧停顿、去除口头禅", "feature2": "添加字幕、为镜头变焦、生成标题", - "feature3": "按需重写章节、换种说法或拆分片段", + "feature3": "按需重写章节、换种说法或重新排列片段", "cta": "设置提供方", "disclaimer": "你视频的转录文本将发送到你选择的提供方。在你连接之前不会共享任何内容。" }, diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index 54fe4028e..5019e3f07 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -273,7 +273,7 @@ "subtitle": "聊天透過與語言模型對話來編輯你的影片。選擇你信任的提供者並連接一個 API 金鑰即可開始。", "feature1": "剪輯靜音、收緊停頓、去除口頭禪", "feature2": "新增字幕、為鏡頭變焦、產生標題", - "feature3": "依需求重寫段落、換種說法或分割片段", + "feature3": "依需求重寫段落、換種說法或重新排列片段", "cta": "設定提供者", "disclaimer": "你影片的轉錄文字會送到你選擇的提供者。在你連接之前不會分享任何內容。" }, diff --git a/technical-documentation/architecture/mcp-server.md b/technical-documentation/architecture/mcp-server.md index 555ebfd7d..e8ebb85f1 100644 --- a/technical-documentation/architecture/mcp-server.md +++ b/technical-documentation/architecture/mcp-server.md @@ -34,7 +34,7 @@ The same tools are served over two transports, by two different processes: ## One tool surface, two agents -Nothing about the tools is reimplemented. The server registers `TOOL_ARG_SCHEMAS` (names and zod schemas) and `TOOL_DESCRIPTIONS`, and builds its `instructions` around `buildSystemPrompt`, all exported from [`deep-agent/service.ts`](../../electron/ai-edition/deep-agent/service.ts), where `buildTools` builds the in-app agent from the same table. Every call goes through `runDocumentTool`, the function the in-app agent's `documentTool` also calls: the cursor-telemetry read the zoom tools need, then `executeAgentTool`. +Nothing about the tools is reimplemented. The server registers `TOOL_ARG_SCHEMAS` (names and zod schemas) and `TOOL_DESCRIPTIONS`, and builds its `instructions` around `buildSystemPrompt`, all exported from [`deep-agent/service.ts`](../../electron/ai-edition/deep-agent/service.ts), where `buildTools` builds the in-app agent from the same table. Every call goes through `runDocumentTool`, the function the in-app agent's `documentTool` also calls: the cursor-telemetry read the zoom tools need (for `addAutoZooms`, the timeline wand's plan, `collectAutoZoomSuggestionsForDocument`, over every take on the timeline), then `executeAgentTool`. So a tool added to the agent appears over MCP with no further work, and the MCP test asserts the listed tools equal `listProjects`, then `OPENSCREEN_TOOL_NAMES`, then the three project tools, then the two checkpoint tools, then the [media tools](#the-media-tools). What the server adds: diff --git a/website/docs/ai-editing.md b/website/docs/ai-editing.md index 2a6d13204..7e76f2ca9 100644 --- a/website/docs/ai-editing.md +++ b/website/docs/ai-editing.md @@ -41,7 +41,7 @@ The ChatGPT and GitHub Copilot sign-in options were **removed in 1.8.0**. They w ## Using the agent -Describe the edit in plain language — "cut the dead air in the intro", "zoom in when I open the terminal". The agent works through real, undoable timeline operations, not a re-render: it can add and adjust trims, zooms, speed regions, annotations (text, arrows, and blurs that hide private details on screen), imported audio tracks with their fades, and Full Camera segments, edit clip in/out points, reorder or remove clips, and read the transcript to find what you're referring to. It also sets the look of the video: background, padding, rounded corners, shadow, device frame, format, the camera layout, the cursor and the master volume. +Describe the edit in plain language — "cut the dead air in the intro", "zoom in when I open the terminal". The agent works through real, undoable timeline operations, not a re-render: it can add and adjust trims, zooms, speed regions, annotations (text, arrows, and blurs that hide private details on screen), imported audio tracks with their fades, and Full Camera segments, add automatic zooms on your clicks, edit a clip's in/out points and crop, place another source's footage, duplicate, reorder or remove clips, and read the transcript to find what you're referring to. It also sets the look of the video: background, padding, rounded corners, shadow, device frame, format, the camera layout, the cursor and the master volume. The panel around it: diff --git a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md index ca10e9f4d..0e5bed04e 100644 --- a/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/de/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Die Anmeldeoptionen für ChatGPT und GitHub Copilot wurden **in 1.8.0 entfernt** ## Den Agenten nutzen {#using-the-agent} -Beschreibe die Änderung in Alltagssprache, etwa „schneide die Stille im Intro heraus“ oder „zoome hinein, wenn ich das Terminal öffne“. Der Agent arbeitet mit echten, rückgängig machbaren Operationen auf der Zeitleiste, nicht mit einem neuen Rendering: Er kann Schnitte, Zooms, Geschwindigkeitsbereiche, Annotationen (Text, Pfeile und Unschärfen, die private Details auf dem Bildschirm verbergen), importierte Audiospuren samt Ein- und Ausblendung und Full-Camera-Segmente hinzufügen und anpassen, Start- und Endpunkte von Clips ändern, Clips umsortieren oder entfernen und das Transkript lesen, um herauszufinden, was du meinst. Außerdem stellt er das Aussehen des Videos ein: Hintergrund, Innenabstand, abgerundete Ecken, Schatten, Geräterahmen, Format, das Kamera-Layout, den Cursor und die Gesamtlautstärke. +Beschreibe die Änderung in Alltagssprache, etwa „schneide die Stille im Intro heraus“ oder „zoome hinein, wenn ich das Terminal öffne“. Der Agent arbeitet mit echten, rückgängig machbaren Operationen auf der Zeitleiste, nicht mit einem neuen Rendering: Er kann Schnitte, Zooms, Geschwindigkeitsbereiche, Annotationen (Text, Pfeile und Unschärfen, die private Details auf dem Bildschirm verbergen), importierte Audiospuren samt Ein- und Ausblendung und Full-Camera-Segmente hinzufügen und anpassen, automatische Zooms auf deine Klicks setzen, Start- und Endpunkte und den Zuschnitt eines Clips ändern, Material einer anderen Quelle platzieren, Clips duplizieren, umsortieren oder entfernen und das Transkript lesen, um herauszufinden, was du meinst. Außerdem stellt er das Aussehen des Videos ein: Hintergrund, Innenabstand, abgerundete Ecken, Schatten, Geräterahmen, Format, das Kamera-Layout, den Cursor und die Gesamtlautstärke. Der Bereich drumherum: diff --git a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md index 02162af31..a31d46c26 100644 --- a/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/es/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Las opciones de inicio de sesión con ChatGPT y GitHub Copilot se **eliminaron e ## Usar el agente {#using-the-agent} -Describe la edición con tus propias palabras: "corta el tiempo muerto de la introducción", "haz zoom cuando abro la terminal". El agente trabaja con operaciones reales de la línea de tiempo, que se pueden deshacer, y no con un nuevo renderizado: puede agregar y ajustar recortes, zooms, regiones de velocidad, anotaciones (texto, flechas y desenfoques que ocultan datos privados en pantalla), pistas de audio importadas con sus fundidos y segmentos de cámara a pantalla completa, editar los puntos de entrada y salida de los clips, reordenar o quitar clips, y leer la transcripción para encontrar aquello a lo que te refieres. También ajusta el aspecto del vídeo: fondo, margen, esquinas redondeadas, sombra, marco de dispositivo, formato, la disposición de la cámara, el cursor y el volumen general. +Describe la edición con tus propias palabras: "corta el tiempo muerto de la introducción", "haz zoom cuando abro la terminal". El agente trabaja con operaciones reales de la línea de tiempo, que se pueden deshacer, y no con un nuevo renderizado: puede agregar y ajustar recortes, zooms, regiones de velocidad, anotaciones (texto, flechas y desenfoques que ocultan datos privados en pantalla), pistas de audio importadas con sus fundidos y segmentos de cámara a pantalla completa, agregar zooms automáticos en tus clics, editar los puntos de entrada y salida y el recorte de un clip, colocar el material de otra fuente, duplicar, reordenar o quitar clips, y leer la transcripción para encontrar aquello a lo que te refieres. También ajusta el aspecto del vídeo: fondo, margen, esquinas redondeadas, sombra, marco de dispositivo, formato, la disposición de la cámara, el cursor y el volumen general. El panel que lo rodea: diff --git a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md index 4c7bacfd7..e7f7e5066 100644 --- a/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/fr/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ Les options de connexion ChatGPT et GitHub Copilot ont été **supprimées dans ## Utiliser l'agent {#using-the-agent} -Décrivez la modification en langage courant : « coupe les temps morts de l'intro », « zoome quand j'ouvre le terminal ». L'agent travaille avec de vraies opérations de timeline annulables, pas avec un nouveau rendu : il peut ajouter et ajuster des coupes, des zooms, des régions de vitesse, des annotations (texte, flèches et flous qui masquent une information privée à l'écran), des pistes audio importées avec leurs fondus, et des segments Caméra plein écran, modifier les points d'entrée et de sortie des clips, réordonner ou retirer des clips, et lire la transcription pour trouver ce dont vous parlez. Il règle aussi l'apparence de la vidéo : fond, marge, coins arrondis, ombre, cadre d'appareil, format, disposition de la caméra, curseur et volume général. +Décrivez la modification en langage courant : « coupe les temps morts de l'intro », « zoome quand j'ouvre le terminal ». L'agent travaille avec de vraies opérations de timeline annulables, pas avec un nouveau rendu : il peut ajouter et ajuster des coupes, des zooms, des régions de vitesse, des annotations (texte, flèches et flous qui masquent une information privée à l'écran), des pistes audio importées avec leurs fondus, et des segments Caméra plein écran, ajouter des zooms automatiques sur vos clics, modifier les points d'entrée et de sortie et le recadrage d'un clip, placer les images d'une autre source, dupliquer, réordonner ou retirer des clips, et lire la transcription pour trouver ce dont vous parlez. Il règle aussi l'apparence de la vidéo : fond, marge, coins arrondis, ombre, cadre d'appareil, format, disposition de la caméra, curseur et volume général. Le panneau qui l'entoure : diff --git a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md index bbc2f3ceb..ba14ba492 100644 --- a/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/ja/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT と GitHub Copilot のサインインは、**1.8.0 で削除されまし ## エージェントを使う {#using-the-agent} -行いたい編集を、普段の言葉で伝えます（「イントロの無音部分をカットして」「ターミナルを開いたところでズームして」など）。エージェントは再レンダリングではなく、元に戻せる実際のタイムライン操作で作業します。トリム、ズーム、再生速度の範囲、注釈（テキスト、矢印、画面上の個人情報を隠すぼかし）、取り込んだオーディオトラックとそのフェード、フルスクリーンカメラのセグメントの追加と調整、クリップのイン点とアウト点の編集、クリップの並べ替えと削除ができ、文字起こしを読んで、あなたが指している箇所を見つけます。背景、余白、角の丸み、影、デバイスフレーム、フォーマット、カメラのレイアウト、カーソル、全体の音量といった動画の見た目も設定できます。 +行いたい編集を、普段の言葉で伝えます（「イントロの無音部分をカットして」「ターミナルを開いたところでズームして」など）。エージェントは再レンダリングではなく、元に戻せる実際のタイムライン操作で作業します。トリム、ズーム、再生速度の範囲、注釈（テキスト、矢印、画面上の個人情報を隠すぼかし）、取り込んだオーディオトラックとそのフェード、フルスクリーンカメラのセグメントの追加と調整、クリック位置への自動ズームの追加、クリップのイン点とアウト点やクロップの編集、別のソースの映像の配置、クリップの複製・並べ替え・削除ができ、文字起こしを読んで、あなたが指している箇所を見つけます。背景、余白、角の丸み、影、デバイスフレーム、フォーマット、カメラのレイアウト、カーソル、全体の音量といった動画の見た目も設定できます。 エージェントを囲むパネル： diff --git a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md index 41591063a..fe091b6c8 100644 --- a/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/pt-BR/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ As opções de login do ChatGPT e do GitHub Copilot foram **removidas na 1.8.0** ## Como usar o agente {#using-the-agent} -Descreva a edição em linguagem natural — "corte o tempo morto da introdução", "dê zoom quando eu abrir o terminal". O agente trabalha com operações reais da linha do tempo, que podem ser desfeitas, e não com uma nova renderização: ele pode adicionar e ajustar recortes, zooms, regiões de velocidade, anotações (texto, setas e desfoques que escondem dados privados na tela), faixas de áudio importadas com seus fades e segmentos de Câmera em Tela Cheia, editar os pontos de entrada/saída dos clipes, reordenar ou remover clipes e ler a transcrição para encontrar o trecho a que você se refere. Ele também ajusta a aparência do vídeo: fundo, margem, cantos arredondados, sombra, moldura de dispositivo, formato, o layout da câmera, o cursor e o volume geral. +Descreva a edição em linguagem natural — "corte o tempo morto da introdução", "dê zoom quando eu abrir o terminal". O agente trabalha com operações reais da linha do tempo, que podem ser desfeitas, e não com uma nova renderização: ele pode adicionar e ajustar recortes, zooms, regiões de velocidade, anotações (texto, setas e desfoques que escondem dados privados na tela), faixas de áudio importadas com seus fades e segmentos de Câmera em Tela Cheia, adicionar zooms automáticos nos seus cliques, editar os pontos de entrada/saída e o corte de um clipe, posicionar as imagens de outra fonte, duplicar, reordenar ou remover clipes e ler a transcrição para encontrar o trecho a que você se refere. Ele também ajusta a aparência do vídeo: fundo, margem, cantos arredondados, sombra, moldura de dispositivo, formato, o layout da câmera, o cursor e o volume geral. O painel em volta dele: diff --git a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md index 785c6d617..f34020ff1 100644 --- a/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-CN/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT 和 GitHub Copilot 登录选项已**在 1.8.0 中移除**。它们的工 ## 使用 AI 代理 {#using-the-agent} -用日常语言描述你想要的编辑，例如“剪掉开头的空白”“我打开终端时放大画面”。AI 代理通过真实、可撤销的时间轴操作来完成编辑，而不是重新渲染：它可以添加和调整删减区间、缩放、速度区间、标注（文字、箭头，以及遮住屏幕上隐私信息的模糊）、导入的音轨及其淡入淡出和全屏摄像头区间，编辑片段的入点/出点，重新排序或移除片段，还会读取转录文本，找到你所指的内容。它还能设置视频的外观：背景、边距、圆角、阴影、设备边框、画面比例、摄像头布局、光标和总音量。 +用日常语言描述你想要的编辑，例如“剪掉开头的空白”“我打开终端时放大画面”。AI 代理通过真实、可撤销的时间轴操作来完成编辑，而不是重新渲染：它可以添加和调整删减区间、缩放、速度区间、标注（文字、箭头，以及遮住屏幕上隐私信息的模糊）、导入的音轨及其淡入淡出和全屏摄像头区间，在你的点击处添加自动缩放，编辑片段的入点/出点和裁剪，放入另一个素材的画面，复制、重新排序或移除片段，还会读取转录文本，找到你所指的内容。它还能设置视频的外观：背景、边距、圆角、阴影、设备边框、画面比例、摄像头布局、光标和总音量。 它周围的面板包括： diff --git a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md index b6518c020..8d4e69f9f 100644 --- a/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md +++ b/website/i18n/zh-TW/docusaurus-plugin-content-docs/current/ai-editing.md @@ -41,7 +41,7 @@ ChatGPT 與 GitHub Copilot 的登入選項已**在 1.8.0 中移除**。它們的 ## 使用代理 {#using-the-agent} -用一般的語言描述你要的剪輯，例如「剪掉開頭的空檔」、「我打開終端機時放大畫面」。代理會透過真實、可復原的時間軸操作來完成工作，而不是重新算繪：它可以新增與調整剪輯、縮放、速度區域、標註（文字、箭頭，以及遮住畫面上隱私資訊的模糊）、匯入的音軌及其淡入淡出與全螢幕攝影機片段，編輯片段的入點與出點，重新排序或移除片段，也會讀取逐字稿來找出你指的是哪一段。它也能設定影片的外觀：背景、邊距、圓角、陰影、裝置外框、畫面比例、攝影機版面、游標與整體音量。 +用一般的語言描述你要的剪輯，例如「剪掉開頭的空檔」、「我打開終端機時放大畫面」。代理會透過真實、可復原的時間軸操作來完成工作，而不是重新算繪：它可以新增與調整剪輯、縮放、速度區域、標註（文字、箭頭，以及遮住畫面上隱私資訊的模糊）、匯入的音軌及其淡入淡出與全螢幕攝影機片段，在你的點擊處加入自動縮放，編輯片段的入點與出點和裁切，放入另一個來源的畫面，複製、重新排序或移除片段，也會讀取逐字稿來找出你指的是哪一段。它也能設定影片的外觀：背景、邊距、圓角、陰影、裝置外框、畫面比例、攝影機版面、游標與整體音量。 它周圍的面板包括： From 209e8207d50fef9963c89f968fd8fd966b00bf9f Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 15:32:32 +0200 Subject: [PATCH 4/5] fix(mcp): name the duplicated clip by id, and give every asset on a file its cursor read Part of #1180. --- electron/ai-edition/agent-tools.test.ts | 12 ++++++++++++ electron/ai-edition/agent-tools.ts | 11 ++++++++--- electron/ai-edition/deep-agent/service.ts | 9 +++++---- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index af629ae70..98487a3d5 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -3607,6 +3607,18 @@ describe("clip structure (#1180)", () => { expect(run(twoTakes(), "duplicateClip", { clipId: "clip_x" }).ok).toBe(false); }); + it("duplicateClip names the copy even when the clips before it fold together", () => { + // clip_1 and clip_2 already meet, so the pass after the copy makes them one clip and + // the copy is no longer at the original's index + 1. + const result = run(fixtureDocument(), "duplicateClip", { clipId: "clip_2" }); + const { clipId } = JSON.parse(result.resultJson); + expect(result.ok).toBe(true); + expect((result.document as AxcutDocument).timeline.clips.map((c) => c.id)).toEqual([ + "clip_1", + clipId, + ]); + }); + it("setClipRange crops a clip without touching its range, and the snapshot reads it back", () => { const crop = { x: 0.1, y: 0.2, width: 0.5, height: 0.6 }; const result = run(twoTakes(), "setClipRange", { clipId: "clip_1", cropRegion: crop }); diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index a2420cb9f..83ea23f9b 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -2474,12 +2474,17 @@ export function executeAgentTool( const parsed = duplicateClipArgs.safeParse(args); if (!parsed.success) return failure(parsed.error.message); const { clipId } = parsed.data; - const index = document.timeline.clips.findIndex((c) => c.id === clipId); - if (index < 0) return failure(`Unknown clip: ${clipId}. ${clipRoster(document)}`); + if (!document.timeline.clips.some((c) => c.id === clipId)) { + return failure(`Unknown clip: ${clipId}. ${clipRoster(document)}`); + } // The editor's Duplicate: the copy right after the original, with its own copy of // the original's cuts and anchored effects. const next = duplicateClip(document, clipId, "agent", ""); - const copy = next.timeline.clips[index + 1]; + // By id, not position: the pass every clip edit ends with can fold clips that + // already met, which moves the positions. + const before = new Set(document.timeline.clips.map((c) => c.id)); + const copy = next.timeline.clips.find((c) => !before.has(c.id)); + if (!copy) return failure(`Clip ${clipId} could not be copied. Nothing was modified.`); return { ok: true, document: next, diff --git a/electron/ai-edition/deep-agent/service.ts b/electron/ai-edition/deep-agent/service.ts index 408a2aa4d..e003a7460 100644 --- a/electron/ai-edition/deep-agent/service.ts +++ b/electron/ai-edition/deep-agent/service.ts @@ -375,10 +375,11 @@ async function planAutoZooms( if (!cursor) return undefined; const telemetry: AutoZoomPlan["telemetry"] = {}; const suggestions = await collectAutoZoomSuggestionsForDocument(document, async (videoPath) => { - const asset = document.assets.find((a) => a.originalPath === videoPath); - if (!asset) return []; - const load = await readCursor(cursor, asset.id, videoPath); - telemetry[asset.id] = load.status; + // The plan asks by path; every asset on that file gets the verdict of its one read. + const assets = document.assets.filter((a) => a.originalPath === videoPath); + if (assets.length === 0) return []; + const load = await readCursor(cursor, assets[0].id, videoPath); + for (const asset of assets) telemetry[asset.id] = load.status; // The sidecar's samples, read as the renderer's `getTelemetry` reads them. return load.status === "ok" ? (load.samples as CursorTelemetryPoint[]) : []; }); From b322087700496aa684d3eb86648e8f35e056481e Mon Sep 17 00:00:00 2001 From: EtienneLescot Date: Tue, 13 Oct 2026 16:24:00 +0200 Subject: [PATCH 5/5] fix(timeline): apply a clip's range and crop in one pass A range applied first could fold the clip into a neighbour its new crop tells apart, and the crop was then lost. setClipSourceRange takes the crop, for the Edit clip dialog and setClipRange alike. Part of #1180. --- electron/ai-edition/agent-tools.test.ts | 22 ++++++++++++++++++++++ electron/ai-edition/agent-tools.ts | 12 +++++------- src/lib/ai-edition/document/timeline.ts | 17 +++++++++++++++-- src/lib/ai-edition/store/useTimeline.ts | 4 +--- 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/electron/ai-edition/agent-tools.test.ts b/electron/ai-edition/agent-tools.test.ts index 98487a3d5..26fd98cb5 100644 --- a/electron/ai-edition/agent-tools.test.ts +++ b/electron/ai-edition/agent-tools.test.ts @@ -3656,6 +3656,28 @@ describe("clip structure (#1180)", () => { expect(run(twoTakes(), "setClipRange", { clipId: "clip_1" }).ok).toBe(false); }); + it("setClipRange weighs a new range and a new crop together", () => { + // clip_2 is moved to meet clip_1 in the media AND given its own crop in one call: the + // crop keeps them apart, so neither edit is lost to a fold. + const placed = run(twoTakes(), "insertClip", { + assetId: "asset_1", + sourceStartSec: 12, + sourceEndSec: 16, + }); + const { clipId } = JSON.parse(placed.resultJson); + const crop = { x: 0, y: 0, width: 0.5, height: 0.5 }; + const result = run(placed.document as AxcutDocument, "setClipRange", { + clipId, + sourceStartSec: 10, + sourceEndSec: 16, + cropRegion: crop, + }); + expect((result.document as AxcutDocument).timeline.clips).toMatchObject([ + { id: "clip_1", sourceEndSec: 10 }, + { id: clipId, sourceStartSec: 10, sourceEndSec: 16, cropRegion: crop }, + ]); + }); + it("setClipRange moves one edge and keeps the other", () => { const result = run(twoTakes(), "setClipRange", { clipId: "clip_1", sourceEndSec: 6 }); expect((result.document as AxcutDocument).timeline.clips[0]).toMatchObject({ diff --git a/electron/ai-edition/agent-tools.ts b/electron/ai-edition/agent-tools.ts index 83ea23f9b..2df9ab491 100644 --- a/electron/ai-edition/agent-tools.ts +++ b/electron/ai-edition/agent-tools.ts @@ -2363,13 +2363,11 @@ export function executeAgentTool( cropRegion.y === 0 && cropRegion.width === 1 && cropRegion.height === 1; - const afterRange = ranged - ? setClipSourceRange(document, clipId, sourceStartSec, sourceEndSec) - : document; - const next = - cropRegion === undefined - ? afterRange - : setClipCrop(afterRange, clipId, wholeFrame ? null : cropRegion); + const crop = cropRegion === undefined ? undefined : wholeFrame ? null : cropRegion; + // One pass for both, as the dialog's Apply: the fold weighs range and crop together. + const next = ranged + ? setClipSourceRange(document, clipId, sourceStartSec, sourceEndSec, crop) + : setClipCrop(document, clipId, crop ?? null); const dropped = droppedByEdit(document, next); const casualties = dropped.droppedModifierIds.length + dropped.droppedTrimIds.length; const after = next.timeline.clips.find((c) => c.id === clipId); diff --git a/src/lib/ai-edition/document/timeline.ts b/src/lib/ai-edition/document/timeline.ts index 4cbe47869..4e1a75141 100644 --- a/src/lib/ai-edition/document/timeline.ts +++ b/src/lib/ai-edition/document/timeline.ts @@ -1083,7 +1083,8 @@ export function insertAssetClip( } /** - * The single mutator for a clip's crop: the clip's Edit modal and the agent's `setClipRange`. + * The single mutator for a clip's crop alone (the agent's crop-only `setClipRange`); with a range, + * `setClipSourceRange` takes the crop in the same pass. * `null` clears it back to the full frame rather than storing the identity region. Through * `withClipsChanged`, since crop is the one property that keeps two meeting clips apart. An * unknown `clipId` is a no-op. Pure. @@ -1113,19 +1114,31 @@ export function setClipCrop( * its source length), lay everything back-to-back, then `rederiveRegionMs` clamps every * anchored pill to the clip's kept window (dropping what the trim removed) and refreshes the * derived ms of the clips that reflowed. An unknown `clipId` is a no-op. + * + * `cropRegion` is the Edit modal's crop, set in the same pass: `undefined` leaves it, `null` + * clears it. One pass, because the fold decides on range AND crop together: a range applied + * first can fold the clip into a neighbour whose crop the new one would have told apart. */ export function setClipSourceRange( document: AxcutDocument, clipId: string, sourceStartSec: number, sourceEndSec: number, + cropRegion?: AxcutClipCropRegion | null, ): AxcutDocument { const clamp = (n: number) => (Number.isFinite(n) ? Math.max(0, n) : 0); const lo = Math.min(clamp(sourceStartSec), clamp(sourceEndSec)); const hi = Math.max(clamp(sourceStartSec), clamp(sourceEndSec)); const arr = document.timeline.clips.map((c) => c.id === clipId - ? { ...c, sourceStartSec: lo, sourceEndSec: hi, timelineStartSec: 0, timelineEndSec: 0 } + ? { + ...c, + sourceStartSec: lo, + sourceEndSec: hi, + timelineStartSec: 0, + timelineEndSec: 0, + ...(cropRegion === undefined ? {} : { cropRegion: cropRegion ?? undefined }), + } : c, ); return withClipsChanged(document, arr); diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index 8312e600c..7c173f870 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -35,7 +35,6 @@ import { removeClip as removeClipInDocument, removeRegion as removeRegionInDocument, resequenceClips, - setClipCrop, setClipSourceRange, } from "../document/timeline"; import type { AxcutAudioTrack, AxcutClipCropRegion, AxcutDocument } from "../schema"; @@ -1221,8 +1220,7 @@ export function useTimeline() { ) => { const doc = useProjectStore.getState().document; if (!doc) return; - const ranged = setClipSourceRange(doc, clipId, sourceStartSec, sourceEndSec); - const next = cropRegion === undefined ? ranged : setClipCrop(ranged, clipId, cropRegion); + const next = setClipSourceRange(doc, clipId, sourceStartSec, sourceEndSec, cropRegion); await saveDocument(next, { history: true }); }, [saveDocument],