Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions electron/ai-edition/agent-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1235,7 +1235,7 @@ export function documentSnapshotForModel(
timeBaseNote:
"clips and trims are in source-time seconds; zooms, speedRegions, annotations, cameraFullscreenRegions and audioTracks are in virtual (edited-timeline) seconds: the editor's ruler, where a trim leaves a hole instead of shortening it and a speed region stretches nothing. The exported video's clock skips those holes and plays sped-up spans faster, so it differs from the ruler once there are any.",
audioNote:
"audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio'; nothing here can import a file from disk or record one, so if the project has no audio asset, say so rather than inventing an id.",
"audioTracks are imported voiceover / music files laid over the recording. They are clip-anchored like every other region, so they travel with their clip through reorder and trim, and they play at 1x whatever a speed region does to the picture under them. addAudio places an EXISTING asset of kind 'audio': if the project has none, a file has to be imported first (importMedia, where it is offered), so never invent an id.",
zoomNote:
`renderedScale is what the viewer sees (depth is an ordinal, not a factor: ${ZOOM_DEPTH_LEGEND}). ` +
"When a zoom carries customScale it wins over depth and depthIsOverridden is true — " +
Expand Down Expand Up @@ -2960,7 +2960,7 @@ export function executeAgentTool(
`Unknown asset: ${assetId}.` +
(available.length
? ` Imported audio in this project: ${available.map((a) => `${a.id} (${a.label})`).join(", ")}.`
: " This project has no imported audio; a file can only be imported or recorded from the editor, not from here."),
: " This project has no imported audio: a file has to be imported into it first (importMedia, where it is offered)."),
);
}
if (asset.kind !== "audio") {
Expand Down
2 changes: 1 addition & 1 deletion electron/ai-edition/deep-agent/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {
"Add a camera-fullscreen region over a span of the edited timeline (virtual seconds): the webcam fills the frame for that span. This only does something when the footage under that span comes from an asset with a linked webcam — check assets[].hasCameraTrack (or hasAnyCamera) in getCurrentDocument first. On footage with no camera the call is refused rather than storing a region that would render nothing; say so instead of retrying.",
setCameraFullscreen:
"Move or resize an existing camera-fullscreen region by id (virtual-timeline seconds). Only the fields you pass are changed. Refused if the new span lands on footage with no linked webcam.",
addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them; nothing here can import a file from disk or record one, so if there is none, say so instead of guessing an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`,
addAudio: `Lay an ALREADY-IMPORTED audio file over the recording across a span of the edited timeline (virtual seconds): a voiceover, or a music bed. assetId must name an asset whose kind is 'audio' — getCurrentDocument lists them. If there is none, a file has to be imported first (importMedia, where it is offered): never guess an id. Omit endSec to play the whole file from offsetSec. kind picks the lane ('voiceover' or 'music'). offsetSec is where in the FILE playback starts, gainDb its level (0 unchanged, negative ducks it), fadeInMs / fadeOutMs its fades (0 to ${AUDIO_FADE_MAX_MS} ms, default none; a track too short for both shares its span between them). A voiceover-lane track is also what gets transcribed, so the lane is not only cosmetic.`,
setAudio: `Move, resize, re-level, re-lane, mute, loop, fade or re-point an existing audio track by id (virtual-timeline seconds). Only the fields you pass are changed. Use it to duck a bed under narration (gainDb), to fade it in or out (fadeInMs / fadeOutMs, 0 to ${AUDIO_FADE_MAX_MS} ms), to shift what part of the file plays (offsetSec), or to move it between the voiceover and music lanes (kind). The whole track is edited, not one fragment of it, so a track split across a cut stays one thing.`,
removeTrim:
"Delete a trim range by id — the cut is undone and that span plays/exports again. This is how you 'remove a trim'; never re-add a trim to undo one.",
Expand Down
137 changes: 74 additions & 63 deletions electron/ai-edition/document-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,75 @@ function isSupportedAudioPath(filePath: string): boolean {
return SUPPORTED_AUDIO_EXTENSIONS.has(ext);
}

/** Why `input` cannot be filed as its kind of asset, or `null` when it can. */
export function unsupportedAssetPath(input: Pick<AddAssetInput, "path" | "kind">): string | null {
if (!input.path) return "Asset path is required.";
if ((input.kind ?? "video") === "audio") {
return isSupportedAudioPath(input.path)
? null
: `Unsupported audio extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_AUDIO_EXTENSIONS].join(", ")})`;
}
return isSupportedVideoPath(input.path)
? null
: `Unsupported video extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_VIDEO_EXTENSIONS].join(", ")})`;
}

/** The kind a media file is imported as, from its extension: audio only when it cannot be a video. */
export function assetKindForPath(filePath: string): "video" | "audio" {
return !isSupportedVideoPath(filePath) && isSupportedAudioPath(filePath) ? "audio" : "video";
}

/** `doc` with `asset` added to its media, as an import adds it. */
export function documentWithAsset(doc: AxcutDocument, asset: AxcutAsset): AxcutDocument {
// An audio import is an overlay, never the thing the timeline is built
// around, so it must not claim the empty primaryAssetId slot — otherwise the
// first file dropped into a fresh project (a BGM track) would become its
// primary asset and the editor would try to lay out clips from a file with
// no video.
const claimsPrimary = asset.kind !== "audio" && !doc.project.primaryAssetId;
return {
...doc,
assets: [...doc.assets, asset],
project: {
...doc.project,
...(claimsPrimary ? { primaryAssetId: asset.id } : {}),
updatedAt: new Date().toISOString(),
},
};
}

/** `doc` without the asset, its clips, trims and audio tracks. */
export function documentWithoutAsset(doc: AxcutDocument, assetId: string): AxcutDocument {
const assets = doc.assets.filter((a) => a.id !== assetId);
// Primary is the thing the timeline is built around, so it must fall to the
// next VIDEO asset — never an audio overlay (issue #350), which can't be
// primary (see addAsset). Falling back to `assets[0]` would hand primary to
// an audio asset when the removed one was the last video.
const primaryAssetId =
doc.project.primaryAssetId === assetId
? (assets.find((a) => a.kind !== "audio")?.id ?? undefined)
: doc.project.primaryAssetId;
const withoutAssetClips = doc.timeline.clips
.filter((clip) => clip.assetId === assetId)
.reduce((current, clip) => removeClip(current, clip.id), doc);
return {
...withoutAssetClips,
assets,
// Drop imported audio tracks that referenced the removed asset — they
// would otherwise dangle, pointing at an asset the document no longer has.
audioTracks: withoutAssetClips.audioTracks.filter((t) => t.assetId !== assetId),
timeline: {
...withoutAssetClips.timeline,
trimRanges: withoutAssetClips.timeline.trimRanges.filter((r) => r.assetId !== assetId),
},
project: {
...withoutAssetClips.project,
primaryAssetId,
updatedAt: new Date().toISOString(),
},
};
}

function safeProjectId(raw: string): string {
// ponytail: project ids are uuid-prefixed strings (e.g. "proj_<uuid>"). Reject
// anything that smells like path traversal before we ever touch the disk.
Expand Down Expand Up @@ -532,23 +601,8 @@ export class DocumentService {

async addAsset(projectId: string, input: AddAssetInput): Promise<AxcutDocument> {
const doc = await this.getProject(projectId);
if (!input.path) {
throw new ProjectFileError("Asset path is required.", projectId);
}
const kind = input.kind ?? "video";
if (kind === "audio") {
if (!isSupportedAudioPath(input.path)) {
throw new ProjectFileError(
`Unsupported audio extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_AUDIO_EXTENSIONS].join(", ")})`,
projectId,
);
}
} else if (!isSupportedVideoPath(input.path)) {
throw new ProjectFileError(
`Unsupported video extension: ${path.extname(input.path)} (supported: ${[...SUPPORTED_VIDEO_EXTENSIONS].join(", ")})`,
projectId,
);
}
const unsupported = unsupportedAssetPath(input);
if (unsupported) throw new ProjectFileError(unsupported, projectId);
const absolutePath = path.isAbsolute(input.path) ? input.path : path.resolve(input.path);
// P3.1 — capture the file size at import. Non-fatal: a stat failure
// (network drive, permissions) just leaves sizeBytes undefined.
Expand All @@ -560,64 +614,21 @@ export class DocumentService {
}
const asset: AxcutAsset = {
id: createId("asset"),
kind,
kind: input.kind ?? "video",
label: input.label?.trim() || path.basename(absolutePath),
originalPath: absolutePath,
sizeBytes,
cameraTrack: null,
};
// An audio import is an overlay, never the thing the timeline is built
// around, so it must not claim the empty primaryAssetId slot — otherwise the
// first file dropped into a fresh project (a BGM track) would become its
// primary asset and the editor would try to lay out clips from a file with
// no video.
const claimsPrimary = kind !== "audio" && !doc.project.primaryAssetId;
const next: AxcutDocument = {
...doc,
assets: [...doc.assets, asset],
project: {
...doc.project,
...(claimsPrimary ? { primaryAssetId: asset.id } : {}),
updatedAt: new Date().toISOString(),
},
};
return this.saveProject(next);
return this.saveProject(documentWithAsset(doc, asset));
}

async removeAsset(projectId: string, assetId: string): Promise<AxcutDocument> {
const doc = await this.getProject(projectId);
if (!doc.assets.some((a) => a.id === assetId)) {
throw new ProjectFileError(`Asset ${assetId} not found in project ${projectId}.`, projectId);
}
const assets = doc.assets.filter((a) => a.id !== assetId);
// Primary is the thing the timeline is built around, so it must fall to the
// next VIDEO asset — never an audio overlay (issue #350), which can't be
// primary (see addAsset). Falling back to `assets[0]` would hand primary to
// an audio asset when the removed one was the last video.
const primaryAssetId =
doc.project.primaryAssetId === assetId
? (assets.find((a) => a.kind !== "audio")?.id ?? undefined)
: doc.project.primaryAssetId;
const withoutAssetClips = doc.timeline.clips
.filter((clip) => clip.assetId === assetId)
.reduce((current, clip) => removeClip(current, clip.id), doc);
const next: AxcutDocument = {
...withoutAssetClips,
assets,
// Drop imported audio tracks that referenced the removed asset — they
// would otherwise dangle, pointing at an asset the document no longer has.
audioTracks: withoutAssetClips.audioTracks.filter((t) => t.assetId !== assetId),
timeline: {
...withoutAssetClips.timeline,
trimRanges: withoutAssetClips.timeline.trimRanges.filter((r) => r.assetId !== assetId),
},
project: {
...withoutAssetClips.project,
primaryAssetId,
updatedAt: new Date().toISOString(),
},
};
return this.saveProject(next);
return this.saveProject(documentWithoutAsset(doc, assetId));
}

/**
Expand Down
8 changes: 8 additions & 0 deletions electron/ipc/handlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@ const ALLOWED_IMPORT_AUDIO_EXTENSIONS = new Set([
".aac",
".flac",
".ogg",
".oga",
".opus",
]);

Expand Down Expand Up @@ -5059,6 +5060,13 @@ export function registerIpcHandlers(
editsAllowed: () => mcpSettings.getSettings().allowEdits,
cursor: agentCursorTelemetryReader,
version: app.getVersion(),
// importMedia: the editor's own probe, and the grant a picked file gets, so the editor
// can play a file a client imported into the project it has open.
openVideo: openVideoInHiddenWindow,
approveMedia: (filePath) => {
const media = normalizeVideoSourcePath(filePath);
if (media && hasAllowedImportMediaExtension(media)) approveFilePath(media);
},
};
// One runner, so one queue and one checkpoint store, for the in-app server and every
// `openscreen mcp` the app answers through its channel (app-channel.ts, started by main.ts).
Expand Down
3 changes: 2 additions & 1 deletion electron/mcp/app-channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import {
type McpToolDeps,
type McpToolRunner,
} from "./openscreen-mcp-server";
import { PROJECT_EDIT_TOOLS } from "./project-tools";

const DISCOVERY_FILE = "mcp-app.json";
/** Connect, token and MCP handshake: the app answers in milliseconds when it is alive. */
Expand Down Expand Up @@ -260,7 +261,7 @@ export async function connectToApp(
function isReadOnly(name: string): boolean {
if (name === LIST_PROJECTS_TOOL) return true;
if (name === "createCheckpoint" || name === "restoreCheckpoint") return false;
return !isMutatingTool(name);
return !isMutatingTool(name) && !PROJECT_EDIT_TOOLS.has(name);
}

/**
Expand Down
47 changes: 46 additions & 1 deletion electron/mcp/media-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,48 @@ describe("recording", () => {
});
});

describe("createProject", () => {
it("creates an empty project the document tools can edit at once", async () => {
const created = await call("createProject", { title: " Blank " });
expect(created.isError).toBeFalsy();
const { projectId, checkpointId } = JSON.parse(textOf(created));
expect(checkpointId).toEqual(expect.any(String));
const document = await documents.getProject(projectId);
expect(document.project.title).toBe("Blank");
expect(document.assets).toEqual([]);
// Its checkpoint lets an edit land on the file straight away.
const renamed = await call("setProjectInfo", { projectId, title: "Named" });
expect(renamed.isError).toBeFalsy();
expect((await documents.getProject(projectId)).project.title).toBe("Named");
});

it("lays a video out as one clip, titled after the file, with the zooms of its clicks", async () => {
await fs.writeFile(videoPath, "video");
const created = await call("createProject", { videoPath });
expect(created.isError).toBeFalsy();
const result = JSON.parse(textOf(created));
expect(result).toMatchObject({ durationSec: 12 });
expect(result.autoZooms).toBeGreaterThan(0);
const document = await documents.getProject(result.projectId);
expect(document.project.title).toBe("take");
expect(document.timeline.clips).toHaveLength(1);
expect(document.assets[0]).toMatchObject({ originalPath: videoPath, durationSec: 12 });
});

it("creates nothing from a path it cannot use", async () => {
const missing = await call("createProject", { videoPath: path.join(dir, "gone.mp4") });
expect(missing.isError).toBe(true);
expect(textOf(missing)).toContain("No file");
const relative = await call("createProject", { videoPath: "take.mp4" });
expect(textOf(relative)).toContain("must be absolute");
await fs.writeFile(videoPath, "video");
openVideoFails = true;
const unreadable = await call("createProject", { videoPath });
expect(textOf(unreadable)).toContain("media error 4");
expect(await documents.listProjects()).toEqual([]);
});
});

describe("pointerSamples", () => {
const startedAt = 10_000;
const samples = pointerSamples(
Expand Down Expand Up @@ -716,7 +758,7 @@ describe("the media tools in the app", () => {
allowed = false;
const app = await connectApp();
expect(app.getInstructions()).toContain(
"startRecording and exportVideo are refused too while 'MCP edits'",
"startRecording, exportVideo and createProject are refused too while 'MCP edits'",
);
const started = await callOn(app, "startRecording", {});
expect(started.isError).toBe(true);
Expand All @@ -728,6 +770,9 @@ describe("the media tools in the app", () => {
});
expect(textOf(exported)).toContain("'MCP edits'");
expect(runners.started).toEqual([]);
const created = await callOn(app, "createProject", {});
expect(textOf(created)).toContain("'MCP edits'");
expect(await documents.listProjects()).toEqual([]);
// Reading stays open: listing what could be recorded changes nothing.
runners.results.sources = {
success: true,
Expand Down
Loading
Loading