diff --git a/scripts/run-browser-suite.mjs b/scripts/run-browser-suite.mjs index d129f2345..ec575c587 100644 --- a/scripts/run-browser-suite.mjs +++ b/scripts/run-browser-suite.mjs @@ -21,6 +21,10 @@ const workflows = [ name: "generation reference", script: "tests/browser/canvas_generation_reference.mjs", }, + { + name: "tts reading references", + script: "tests/browser/tts_reading_references.mjs", + }, { name: "minimax h3", script: "tests/browser/minimax_h3.mjs" }, { name: "provider settings", script: "tests/browser/provider_settings.mjs" }, { name: "created node focus", script: "tests/browser/created_node_focus.mjs" }, diff --git a/src/backend/rules.ts b/src/backend/rules.ts index ad3429cfb..0dd1a026d 100644 --- a/src/backend/rules.ts +++ b/src/backend/rules.ts @@ -215,6 +215,10 @@ export function backendReferenceIssue( ): string | null { const kind = node.data.kind; const rules = config.inputRules; + const unusedInReading = + settings.mode === "audio" && !isMusicModel(config) + ? tNow("generation.issue.audioModeReference") + : null; if ( settings.mode === "video" && settings.videoMode === "first-last" && @@ -241,7 +245,7 @@ export function backendReferenceIssue( if (kind === "text") return rules.allowedKinds.includes("text") ? node.data.text?.trim() - ? null + ? (unusedInReading ?? null) : tNow("errors.rules.textEmpty") : tNow("errors.rules.textReference"); const continuation = @@ -254,6 +258,7 @@ export function backendReferenceIssue( return tNow("errors.rules.kindReference", { kind: tNow(LABELS[kind] ?? "errors.rules.kindImage"), }); + if (unusedInReading) return unusedInReading; return node.data.src && node.data.assetState !== "missing" ? null : tNow("errors.rules.mediaMissing"); diff --git a/src/canvas/NodeComposer.tsx b/src/canvas/NodeComposer.tsx index 1b404c77e..b03407af2 100644 --- a/src/canvas/NodeComposer.tsx +++ b/src/canvas/NodeComposer.tsx @@ -52,6 +52,7 @@ import { frameImageLimitIssue, frameImageLimitMessage, } from "./frameReferences.ts"; +import { composerReferenceVisibility } from "./composerReferenceVisibility.ts"; export interface NodeComposerProps { selectedNode: CanvasNode | null; @@ -159,6 +160,14 @@ export function NodeComposer({ // YuE2-style music models can follow one connected recording. const musicReferences = music && Boolean(config?.inputRules.allowedKinds.includes("audio")); + const referenceVisibility = composerReferenceVisibility({ + firstLast, + reading, + music, + musicReferences, + incomingCount: incomingNodes.length, + mode, + }); const followsReference = Boolean( config && hasMusicReference(settings, incomingNodes, config), ); @@ -388,37 +397,39 @@ export function NodeComposer({ )} - {!firstLast && - !reading && - (!music || musicReferences) && - (mode === "video" || incomingNodes.length > 0) && ( -
- mentions.current?.openUpload()} - aria-label={t("generation.reference.add")} - label={t("generation.reference.add")} - > - - - {incomingNodes.map((node) => ( - insertReference(node)} - onRemove={() => mentions.current?.remove(node)} - disabled={running || Boolean(referenceIssue(node))} - removalDisabled={running} - title={`${t("generation.reference.title", { - role: generationReferenceRole(node, t), - title: node.data.title, - })}${referenceIssue(node) ? ` · ${referenceIssue(node)}` : ""}`} - /> - ))} -
- )} + {referenceVisibility.showStrip && ( +
+ mentions.current?.openUpload()} + aria-label={t("generation.reference.add")} + label={t("generation.reference.add")} + > + + + {incomingNodes.map((node) => ( + insertReference(node)} + onRemove={() => mentions.current?.remove(node)} + disabled={running || Boolean(referenceIssue(node))} + removalDisabled={running} + title={`${t("generation.reference.title", { + role: generationReferenceRole(node, t), + title: node.data.title, + })}${referenceIssue(node) ? ` · ${referenceIssue(node)}` : ""}`} + /> + ))} +
+ )} + {referenceVisibility.showReadingContextNote && ( +

+ {t("generation.reference.readingContext")} +

+ )} )} - {!firstLast && - !reading && - (!music || musicReferences) && - incomingNodes.length === 0 && ( - mentions.current?.openUpload()} - > - - - )} + {referenceVisibility.showEmptyAdd && ( + mentions.current?.openUpload()} + > + + + )} {mode === "video" && ( )} - {!reading && !music && ( + {referenceVisibility.showMention && ( 0), + showEmptyAdd: canManage && input.incomingCount === 0 && !input.reading, + showMention: !input.reading && !input.music, + showReadingContextNote: input.reading && input.incomingCount > 0, + }; +} diff --git a/src/canvas/generation-composer.css b/src/canvas/generation-composer.css index 6468bbe55..6075983e8 100644 --- a/src/canvas/generation-composer.css +++ b/src/canvas/generation-composer.css @@ -51,6 +51,15 @@ box-sizing: border-box; } +.cw-reference-context-note { + flex: 0 0 auto; + margin: 0; + max-width: 100%; + font-size: 12px; + line-height: 1.4; + color: var(--cw-prompt-dim, #7d8291); +} + .cw-reference-strip { display: flex; gap: 6px; diff --git a/src/canvas/generationCatalog.ts b/src/canvas/generationCatalog.ts index 155984719..ca752742b 100644 --- a/src/canvas/generationCatalog.ts +++ b/src/canvas/generationCatalog.ts @@ -219,11 +219,6 @@ export function generationReferenceIssue( settings: GenerationSettings, t: Translate = defaultTranslate, ): string | null { - if ( - settings.mode === "audio" && - (settings.audioCreationType === "music" || settings.audioMode === "tts") - ) - return t("generation.issue.audioModeReference"); if ( settings.mode === "video" && settings.videoMode === "first-last" && @@ -256,6 +251,11 @@ export function generationReferenceIssue( : ["audio"]; if (!allowed.includes(node.data.kind)) return t("generation.issue.unsupportedKind"); + if ( + settings.mode === "audio" && + (settings.audioCreationType === "music" || settings.audioMode === "tts") + ) + return t("generation.issue.audioModeReference"); if (node.type === "text") return node.data.text?.trim() ? null : t("generation.issue.textEmpty"); return node.data.src && node.data.assetState !== "missing" diff --git a/src/i18n/messages/en-US/generation.ts b/src/i18n/messages/en-US/generation.ts index dcea08a65..542afa478 100644 --- a/src/i18n/messages/en-US/generation.ts +++ b/src/i18n/messages/en-US/generation.ts @@ -148,6 +148,7 @@ export const generation = { "generation.reference.connected": "Reference connected node", "generation.reference.upload": "Upload node reference", "generation.reference.title": "{role}: {title}", + "generation.reference.readingContext": "Text-to-speech does not send these references to the model. You can still inspect or remove them.", "generation.action.cancel": "Cancel generation", "generation.action.submit": "Generate", diff --git a/src/i18n/messages/zh-CN/generation.ts b/src/i18n/messages/zh-CN/generation.ts index bf276ba5d..4e6184e66 100644 --- a/src/i18n/messages/zh-CN/generation.ts +++ b/src/i18n/messages/zh-CN/generation.ts @@ -139,6 +139,7 @@ export const generation = { "generation.reference.connected": "引用已连接节点", "generation.reference.upload": "上传节点参考", "generation.reference.title": "{role}:{title}", + "generation.reference.readingContext": "文本朗读不把这些参考作为模型输入,仍可查看或移除。", "generation.action.cancel": "取消生成", "generation.action.submit": "提交生成", diff --git a/tests/backendRules.test.ts b/tests/backendRules.test.ts index 8707d17ee..68d2322df 100644 --- a/tests/backendRules.test.ts +++ b/tests/backendRules.test.ts @@ -329,3 +329,40 @@ test("FastH3 preserves supported reference mode for text-only generation", () => assert.equal(backendVideoMode(settings, [], fast), "reference"); assert.equal(backendBlocker(settings, [], [fast]), null); }); + +test("reading service keeps unsupported kinds distinct from unused allowed references", () => { + const speech = { + ...config("audio", { + profile: "openai-speech", + allowedKinds: [], + maxImages: 0, + maxVideos: 0, + maxAudios: 0, + videoModes: [], + }), + protocol: "openai" as const, + modelId: "seed-tts-2.0", + }; + const settings = { + ...generationDefaults("audio"), + modelConfigId: speech.id, + audioMode: "tts" as const, + prompt: "朗读这段文字", + }; + const image = media("image"); + const video = media("video"); + assert.match(backendReferenceIssue(image, settings, speech)!, /不接收/); + assert.doesNotMatch(backendReferenceIssue(image, settings, speech)!, /不使用该参考/); + assert.match(backendReferenceIssue(video, settings, speech)!, /视频参考/); + assert.doesNotMatch(backendReferenceIssue(video, settings, speech)!, /不使用该参考/); + assert.deepEqual(backendReferences(settings, [image, video], speech), []); + + const speechWithImage = { + ...speech, + inputRules: { ...speech.inputRules, allowedKinds: ["image" as const], maxImages: 1 }, + }; + assert.match(backendReferenceIssue(image, settings, speechWithImage)!, /不使用该参考/); + assert.match(backendReferenceIssue(video, settings, speechWithImage)!, /视频参考/); + assert.doesNotMatch(backendReferenceIssue(video, settings, speechWithImage)!, /不使用该参考/); + assert.deepEqual(backendReferences(settings, [image], speechWithImage), []); +}); diff --git a/tests/browser/manifest.json b/tests/browser/manifest.json index ea78287e6..6991bdec6 100644 --- a/tests/browser/manifest.json +++ b/tests/browser/manifest.json @@ -18,7 +18,9 @@ }, { "file": "kling_video.mjs", - "lanes": ["extendedFixture"], + "lanes": [ + "extendedFixture" + ], "note": "Uses isolated creation fixtures for bilingual Kling configuration, default preservation and image-based generation." }, { @@ -482,6 +484,13 @@ "extended" ] }, + { + "file": "tts_reading_references.mjs", + "lanes": [ + "gate" + ], + "note": "TTS reading panels keep connected, context-only references visible and removable. Isolated fixture only; does not submit generation or call a paid model." + }, { "file": "voice_library.mjs", "previewSafe": true, diff --git a/tests/browser/tts_reading_references.mjs b/tests/browser/tts_reading_references.mjs new file mode 100644 index 000000000..51eabc7b0 --- /dev/null +++ b/tests/browser/tts_reading_references.mjs @@ -0,0 +1,201 @@ +/** Isolated fixture only: TTS panels keep connected references visible and removable. */ +import assert from "node:assert/strict"; +const { chromium } = await import( + process.env.PLAYWRIGHT_MODULE ?? "playwright-core" +); +const base = process.env.CREATION_WEB_URL ?? "http://127.0.0.1:5179"; +const browser = await chromium.launch({ + headless: true, + ...(process.env.CHROMIUM_PATH + ? { executablePath: process.env.CHROMIUM_PATH } + : {}), + args: ["--no-sandbox"], +}); +const page = await browser.newPage({ + locale: "zh-CN", + viewport: { width: 1600, height: 1000 }, +}); +// Skip the first-run model setup welcome; it covers the canvas. +await page.addInitScript(() => { + localStorage.setItem("opendirector.modelSetup.v1", "dismissed"); + localStorage.setItem("opendirector.locale.v1", "zh-CN"); +}); +page.setDefaultTimeout(20000); +const errors = []; +page.on("pageerror", (error) => errors.push(error.message)); +const png = + "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAGUlEQVR4nGNsYGhgIAUwkaR6VMOohiGlAQBCPQEgiSD+iQAAAABJRU5ErkJggg=="; + +try { + const health = await ( + await page.request.get(base + "/api/creation-test-health", { + headers: { "X-OpenDirector-Request": "1" }, + }) + ).json(); + assert.equal(health.isolated, true, "Use the isolated fixture server"); + const bootstrap = (url) => + url.origin === new URL(base).origin && url.pathname === "/"; + await page.route(bootstrap, (route) => + route.fulfill({ contentType: "text/html", body: "" }), + ); + await page.goto(base + "/"); + const fixture = await page.evaluate(async (png) => { + const { request } = await import("/src/backend/api.ts"); + const { createDocument, createNode } = await import("/src/canvas/model.ts"); + const { importWorkspaceDocument } = + await import("/src/backend/workspace.ts"); + const { creationClient } = await import("/src/backend/creationClient.ts"); + const { releaseBackendProject } = await import("/src/backend/projects.ts"); + const models = await request("/model-configs"); + const speech = models.find( + (model) => + model.kind === "audio" && model.protocol !== "vllm_omni_music", + ); + const source = createNode( + "image", + { x: 160, y: 180 }, + { + title: "朗读参考图", + src: png, + mimeType: "image/png", + mediaRole: "asset", + assetState: "ready", + }, + ); + const audio = createNode( + "audio", + { x: 720, y: 200 }, + { + title: "文本朗读节点", + prompt: "请朗读这段文字", + settings: speech + ? { + mode: "audio", + model: speech.modelId, + modelConfigId: speech.id, + audioMode: "tts", + audioCreationType: "audio", + prompt: "请朗读这段文字", + } + : { + mode: "audio", + model: "SeedAudio 1.0", + audioMode: "tts", + audioCreationType: "audio", + prompt: "请朗读这段文字", + }, + }, + ); + const saved = await importWorkspaceDocument({ + ...createDocument("TTS reading references"), + nodes: [source, audio], + edges: [ + { + id: crypto.randomUUID(), + source: source.id, + target: audio.id, + sourceHandle: "out", + targetHandle: "in", + }, + ], + viewport: { x: 0, y: 0, zoom: 0.9 }, + }); + const client = creationClient(saved.id); + await client.refresh(); + const references = client.snapshot.references.filter( + (reference) => reference.active, + ); + const jobs = client.snapshot.jobs.length; + client.close(); + await releaseBackendProject(); + return { + id: saved.id, + audioId: audio.id, + sourceId: source.id, + sourceTitle: source.data.title, + usedServiceModel: Boolean(speech), + savedReferences: references.length, + jobs, + }; + }, png); + assert.equal(fixture.savedReferences, 1); + assert.equal(fixture.jobs, 0); + await page.unroute(bootstrap); + + const openAudioComposer = async () => { + await page + .locator(".react-flow__node") + .filter({ hasText: "文本朗读节点" }) + .click(); + await page.getByRole("textbox", { name: "朗读文本", exact: true }).waitFor(); + }; + const expectVisibleReferences = async () => { + const strip = page.locator(".cw-reference-strip"); + await strip.waitFor({ state: "visible" }); + assert.equal( + await page.getByRole("button", { name: "引用素材:朗读参考图" }).count(), + 1, + ); + assert.equal( + await page.getByRole("button", { name: "移除参考:朗读参考图" }).count(), + 1, + ); + await page + .locator('[data-reference-role="reading-context"]') + .waitFor({ state: "visible" }); + assert.match( + await page.locator('[data-reference-role="reading-context"]').innerText(), + /不把这些参考作为模型输入/, + ); + assert.equal( + await page.getByRole("button", { name: "引用已连接节点" }).count(), + 0, + ); + }; + + await page.goto(`${base}/ai-tool/ai-canvas/${fixture.id}`); + await page.getByText("已保存到本机", { exact: true }).waitFor(); + await openAudioComposer(); + await expectVisibleReferences(); + + await page.reload({ waitUntil: "domcontentloaded" }); + await page.getByText("已保存到本机", { exact: true }).waitFor(); + await openAudioComposer(); + await expectVisibleReferences(); + + await page.getByRole("button", { name: "移除参考:朗读参考图" }).click(); + await page.getByText("已保存到本机", { exact: true }).waitFor(); + await page.locator(".cw-reference-strip").waitFor({ state: "hidden" }); + assert.equal( + await page.locator('[data-reference-role="reading-context"]').count(), + 0, + ); + assert.equal( + await page.getByRole("button", { name: "添加参考素材", exact: true }).count(), + 0, + ); + + const after = await page.evaluate(async (projectId) => { + const { creationClient } = await import("/src/backend/creationClient.ts"); + const client = creationClient(projectId); + await client.refresh(); + const active = client.snapshot.references.filter( + (reference) => reference.active, + ); + const jobs = client.snapshot.jobs.length; + client.close(); + return { active: active.length, jobs }; + }, fixture.id); + assert.equal(after.active, 0); + assert.equal(after.jobs, 0); + assert.deepEqual(errors, []); + console.log( + JSON.stringify({ + passed: true, + usedServiceModel: fixture.usedServiceModel, + jobs: after.jobs, + }), + ); +} finally { + await browser.close(); +} diff --git a/tests/composerReferenceVisibility.test.ts b/tests/composerReferenceVisibility.test.ts new file mode 100644 index 000000000..9f5dd32b7 --- /dev/null +++ b/tests/composerReferenceVisibility.test.ts @@ -0,0 +1,107 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { composerReferenceVisibility } from "../src/canvas/composerReferenceVisibility.ts"; + +test("reading mode shows connected references and hides the empty add control", () => { + const withRef = composerReferenceVisibility({ + firstLast: false, + reading: true, + music: false, + musicReferences: false, + incomingCount: 1, + mode: "audio", + }); + assert.deepEqual(withRef, { + showStrip: true, + showEmptyAdd: false, + showMention: false, + showReadingContextNote: true, + }); + + const empty = composerReferenceVisibility({ + firstLast: false, + reading: true, + music: false, + musicReferences: false, + incomingCount: 0, + mode: "audio", + }); + assert.deepEqual(empty, { + showStrip: false, + showEmptyAdd: false, + showMention: false, + showReadingContextNote: false, + }); +}); + +test("music and first-last visibility stay unchanged, and mentions stay off in reading and music", () => { + assert.deepEqual( + composerReferenceVisibility({ + firstLast: false, + reading: false, + music: true, + musicReferences: false, + incomingCount: 1, + mode: "audio", + }), + { + showStrip: false, + showEmptyAdd: false, + showMention: false, + showReadingContextNote: false, + }, + ); + const music = composerReferenceVisibility({ + firstLast: false, + reading: false, + music: true, + musicReferences: true, + incomingCount: 1, + mode: "audio", + }); + assert.equal(music.showStrip, true); + assert.equal(music.showEmptyAdd, false); + assert.equal(music.showMention, false); + assert.equal(music.showReadingContextNote, false); + + const frames = composerReferenceVisibility({ + firstLast: true, + reading: false, + music: false, + musicReferences: false, + incomingCount: 2, + mode: "video", + }); + assert.equal(frames.showStrip, false); + assert.equal(frames.showEmptyAdd, false); + assert.equal(frames.showMention, true); + assert.equal(frames.showReadingContextNote, false); +}); + +test("image and video panels still show the manager and mention control", () => { + const image = composerReferenceVisibility({ + firstLast: false, + reading: false, + music: false, + musicReferences: false, + incomingCount: 1, + mode: "image", + }); + assert.deepEqual(image, { + showStrip: true, + showEmptyAdd: false, + showMention: true, + showReadingContextNote: false, + }); + const video = composerReferenceVisibility({ + firstLast: false, + reading: false, + music: false, + musicReferences: false, + incomingCount: 0, + mode: "video", + }); + assert.equal(video.showStrip, true); + assert.equal(video.showEmptyAdd, true); + assert.equal(video.showMention, true); +}); diff --git a/tests/generationCatalog.test.ts b/tests/generationCatalog.test.ts index 3ca8df474..5d56bceba 100644 --- a/tests/generationCatalog.test.ts +++ b/tests/generationCatalog.test.ts @@ -158,6 +158,29 @@ test("music selection resets incompatible audio parameters and validates 30–12 assert.equal(spoken.durationSeconds, 5); }); + +test("reading mode keeps unsupported kinds distinct from unused audio references", () => { + const audio = createNode( + "audio", + { x: 0, y: 0 }, + { src: "audio.wav", assetState: "ready" }, + ); + const video = createNode( + "video", + { x: 0, y: 0 }, + { src: "clip.mp4", assetState: "ready" }, + ); + const image = createNode( + "image", + { x: 0, y: 0 }, + { src: "still.png", assetState: "ready" }, + ); + const reading = { ...generationDefaults("audio"), audioMode: "tts" as const }; + assert.match(generationReferenceIssue(audio, reading)!, /不使用该参考/); + assert.match(generationReferenceIssue(video, reading)!, /未核实支持该素材类型/); + assert.match(generationReferenceIssue(image, reading)!, /未核实支持该素材类型/); +}); + test("reading needs text and voice; audio references cannot silently replace reading text", () => { const audio = createNode( "audio",