fix: fit the chat agent's text annotations to their words, and refit them on a format change

electron/ai-edition/agent-tools.test.ts

@@ -1,4 +1,5 @@
-import { describe, expect, it } from "vitest";
+import path from "node:path";
+import { describe, expect, it, vi } from "vitest";
 import {
 	type AxcutAnnotationRegion,
 	type AxcutDocument,
@@ -7,6 +8,7 @@ import {
 } from "../../src/lib/ai-edition/schema";
 import { ZOOM_DEPTH_SCALES } from "../../src/lib/ai-edition/timeline/zoom-scale";
 import { executeAgentTool, isMutatingTool, MUTATING_TOOL_NAMES } from "./agent-tools";
+import { measureTextInShippedFonts } from "./text-measure";
 
 /** ponytail: every fixture in this file starts from the SAME instant.
  *  `createEmptyDocument` reads the wall clock when a caller hands it no
@@ -843,6 +845,60 @@ describe("executeAgentTool", () => {
 		).toBe(false);
 	});
 
+	it("fits a title to its words in the shipped font, in 16:9 and once switched to 9:16", () => {
+		// The e2e pass's turn: a bold Inter title at 64 px, then the vertical format. Measured with
+		// the estimate Node used to fall back on, the box held 554 px of text for 587 px of words,
+		// and the 9:16 export showed "Demo" with "OpenScreen" wrapped out of the box.
+		vi.stubEnv("OPENSCREEN_FONTS_DIR", path.resolve(__dirname, "../../public/fonts"));
+		try {
+			/** The room the box leaves its words, inside the plate, against their measured width. */
+			const room = (ann: AxcutAnnotationRegion | undefined, aspect: number) => {
+				if (!ann) throw new Error("no annotation");
+				const box = (ann.size.width / 100) * 1080 * aspect - 0.4 * ann.style.fontSize;
+				return box - measureTextInShippedFonts(ann.content ?? "", ann.style).width;
+			};
+			let document = executeAgentTool(
+				fixtureDocument(),
+				"setAspectRatio",
+				JSON.stringify({ aspectRatio: "16:9" }),
+			).document as AxcutDocument;
+			const added = executeAgentTool(
+				document,
+				"addAnnotation",
+				JSON.stringify({ startSec: 1, endSec: 4, text: "Demo OpenScreen", fontSize: 64, x: 50 }),
+			);
+			const id = JSON.parse(added.resultJson).annotationId;
+			document = added.document as AxcutDocument;
+			const wide = document.annotations.find((a) => a.id === id);
+			expect(wide?.style.fontSize).toBe(64);
+			expect(room(wide, 16 / 9)).toBeGreaterThan(0);
+
+			document = executeAgentTool(
+				document,
+				"setAspectRatio",
+				JSON.stringify({ aspectRatio: "9:16" }),
+			).document as AxcutDocument;
+			const tall = document.annotations.find((a) => a.id === id);
+			expect(room(tall, 9 / 16)).toBeGreaterThan(0);
+			expect(tall?.size.width).toBeLessThanOrEqual(100);
+			// 64 px of Inter bold does not fit a portrait frame's width: brought down just enough.
+			expect(tall?.style.fontSize).toBeLessThan(64);
+			expect(tall?.style.fontSize).toBeGreaterThan(55);
+			expect(centreOf(tall)).toEqual({ x: expect.closeTo(50, 5), y: expect.closeTo(50, 5) });
+
+			// Added straight into the portrait frame, the same title fits too.
+			const direct = executeAgentTool(
+				document,
+				"addAnnotation",
+				JSON.stringify({ startSec: 5, endSec: 6, text: "Demo OpenScreen", fontSize: 64 }),
+			).document?.annotations.at(-1);
+			expect(room(direct, 9 / 16)).toBeGreaterThan(0);
+			expect(direct?.size.width).toBeLessThanOrEqual(100);
+		} finally {
+			vi.unstubAllEnvs();
+		}
+	});
+
 	it("refuses a set that would run into another region, instead of landing it elsewhere", () => {
 		let document = fixtureDocument();
 		const add = (name: string, args: Record<string, unknown>) => {

electron/ai-edition/agent-tools.ts

@@ -17,7 +17,7 @@
 
 import { z } from "zod";
 import { DEFAULT_TEXT_PLATE } from "../../src/lib/ai-edition/annotations/background";
-import { fitTextBox } from "../../src/lib/ai-edition/annotations/placement";
+import { fitTextInFrame } from "../../src/lib/ai-edition/annotations/placement";
 import {
 	collapseTracksToPills,
 	patchAudioTrack,
@@ -67,6 +67,7 @@ import {
 } from "../../src/lib/ai-edition/timeline/zoom-transitions";
 import { SETTING_BOUNDS } from "../../src/lib/projectDefaults";
 import { ASPECT_RATIO_PRESETS } from "../../src/utils/aspectRatioUtils";
+import { measureTextInShippedFonts } from "./text-measure";
 
 export interface AgentToolExecution {
 	ok: boolean;
@@ -200,7 +201,8 @@ function applyAnnotationLook(
 /**
  * A text annotation on the output frame, its box sized to its text around `center` (frame
  * percentages): what the editor's "add text" does, so the agent's text sits where the editor's
- * would, is measured the same, and stays put when the picture zooms.
+ * would, is measured in the same fonts, and stays put when the picture zooms. A size too large for
+ * the frame's width is brought down to fit on its line, as a format change does.
  */
 function placeTextOnFrame<T extends AxcutAnnotationRegion>(
 	region: T,
@@ -214,7 +216,7 @@ function placeTextOnFrame<T extends AxcutAnnotationRegion>(
 		position: center,
 		size: { width: 0, height: 0 },
 	};
-	return { ...centred, ...fitTextBox(centred, frameAspect) };
+	return fitTextInFrame(centred, frameAspect, measureTextInShippedFonts);
 }
 
 // v5: a modifier is stored as clip-anchored fragment(s), and adjacent regions with the
@@ -1461,8 +1463,13 @@ export function executeAgentTool(
 			const parsed = setAspectRatioArgs.safeParse(args);
 			if (!parsed.success) return failure(parsed.error.message);
 			const previous = getEditorSettings(document).aspectRatio;
-			// The editor's own write: the frame-placed annotations keep their shape and centre.
-			const next = applySettingsPatch(document, { aspectRatio: parsed.data.aspectRatio });
+			// The editor's own write: the frame-placed annotations keep their centre, and a text is
+			// refitted to its words in the new frame, measured in the shipped fonts.
+			const next = applySettingsPatch(
+				document,
+				{ aspectRatio: parsed.data.aspectRatio },
+				measureTextInShippedFonts,
+			);
 			return {
 				ok: true,
 				document: next,

electron/ai-edition/text-measure.ts

@@ -0,0 +1,50 @@
+// Text measured in the main process, where the agent's tools run.
+//
+// The editor sizes a text box with a canvas, in the fonts the compositor draws. Node has no
+// canvas, so the agent's `fitTextBox` fell back to an average advance that is too narrow for
+// Inter bold, and the compositor wrapped the last word of an agent's title out of its box. This
+// reads the same font files instead (`public/fonts` in dev, `<resources>/fonts` packaged, the
+// directory `OPENSCREEN_FONTS_DIR` names once the compositor is loaded).
+
+import fs from "node:fs";
+import path from "node:path";
+import {
+	type FontMetrics,
+	measureWithFontTables,
+	readFontMetrics,
+} from "../../src/lib/ai-edition/annotations/fontTables";
+import { textFontFile } from "../../src/lib/textFonts";
+
+/** The directories the shipped font files may be in, most specific first. */
+function fontDirs(): string[] {
+	return [
+		process.env.OPENSCREEN_FONTS_DIR,
+		process.env.VITE_PUBLIC && path.join(process.env.VITE_PUBLIC, "fonts"),
+		process.resourcesPath && path.join(process.resourcesPath, "fonts"),
+	].filter((dir): dir is string => typeof dir === "string" && dir.length > 0);
+}
+
+const parsed = new Map<string, FontMetrics | null>();
+
+function readFont(file: string): FontMetrics | null {
+	if (!parsed.has(file)) {
+		let metrics: FontMetrics | null = null;
+		try {
+			metrics = readFontMetrics(fs.readFileSync(file));
+		} catch {
+			// Unreadable: measured by the estimate, like a missing file.
+		}
+		parsed.set(file, metrics);
+	}
+	return parsed.get(file) ?? null;
+}
+
+/** The words of a text annotation, measured in the font files the compositor draws them in. */
+export const measureTextInShippedFonts = measureWithFontTables((family, weight) => {
+	const name = path.basename(textFontFile(family, weight));
+	for (const dir of fontDirs()) {
+		const file = path.join(dir, name);
+		if (fs.existsSync(file)) return readFont(file);
+	}
+	return null;
+});

src/lib/ai-edition/annotations/fontTables.test.ts

@@ -0,0 +1,104 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import { TEXT_FONT_FAMILIES, textFontFile } from "@/lib/textFonts";
+import { type FontMetrics, measureWithFontTables, readFontMetrics } from "./fontTables";
+
+const FONTS = path.resolve(__dirname, "../../../../public");
+
+function shipped(family: (typeof TEXT_FONT_FAMILIES)[number], weight: "normal" | "bold") {
+	const metrics = readFontMetrics(readFileSync(path.join(FONTS, textFontFile(family, weight))));
+	if (!metrics) throw new Error(`${family} ${weight} unreadable`);
+	return metrics;
+}
+
+const style = (
+	overrides: Partial<Parameters<ReturnType<typeof measureWithFontTables>>[1]> = {},
+) => ({
+	color: "#ffffff",
+	backgroundColor: "rgba(0, 0, 0, 0.7)",
+	fontSize: 64,
+	fontFamily: "Inter",
+	fontWeight: "bold" as const,
+	fontStyle: "normal" as const,
+	textDecoration: "none" as const,
+	textAlign: "center" as const,
+	textAnimation: "none" as const,
+	...overrides,
+});
+
+describe("readFontMetrics", () => {
+	it("reads every shipped font", () => {
+		for (const family of TEXT_FONT_FAMILIES) {
+			for (const weight of ["normal", "bold"] as const) {
+				const font = shipped(family, weight);
+				expect(font.unitsPerEm).toBeGreaterThan(0);
+				expect(font.lineHeight).toBeGreaterThan(font.unitsPerEm);
+				expect(font.advanceOf("A".codePointAt(0) ?? 0)).toBeGreaterThan(0);
+			}
+		}
+	});
+
+	it("knows a glyph the font lacks from one it has", () => {
+		const inter = shipped("Inter", "bold");
+		expect(inter.advanceOf(" ".codePointAt(0) ?? 0)).toBeGreaterThan(0);
+		// A CJK ideograph: not in Inter.
+		expect(inter.advanceOf(0x4e2d)).toBeNull();
+	});
+
+	it("refuses bytes that are not a font", () => {
+		expect(readFontMetrics(new Uint8Array([1, 2, 3, 4]))).toBeNull();
+	});
+});
+
+describe("measureWithFontTables", () => {
+	const measure = measureWithFontTables(shipped);
+
+	it("measures Inter bold at its real width, wider than the old estimate", () => {
+		// What the chat agent stored for "Demo OpenScreen" at 64 px: 15 × 0.56 em, 538 px. Inter
+		// bold draws it 586 px wide, and the compositor wrapped the second word out of the box.
+		const block = measure("Demo OpenScreen", style());
+		expect(block.width).toBeCloseTo(586.5, 1);
+		expect(block.width).toBeGreaterThan(15 * 0.56 * 64 + 40);
+	});
+
+	it("tells the weights and the families apart, and scales with the size", () => {
+		const bold = measure("Demo OpenScreen", style());
+		expect(measure("Demo OpenScreen", style({ fontWeight: "normal" })).width).toBeLessThan(
+			bold.width,
+		);
+		expect(measure("Demo OpenScreen", style({ fontFamily: "Oswald" })).width).toBeLessThan(
+			bold.width,
+		);
+		expect(measure("Demo OpenScreen", style({ fontSize: 32 })).width).toBeCloseTo(
+			bold.width / 2,
+			6,
+		);
+	});
+
+	it("takes the longest line, and one line height per line", () => {
+		const one = measure("Demo", style());
+		const two = measure("Demo\nOpenScreen", style());
+		expect(two.width).toBeCloseTo(measure("OpenScreen", style()).width, 6);
+		expect(two.height).toBeCloseTo(2 * one.height, 6);
+		// Inter's hhea line is about 1.21 em.
+		expect(one.height / 64).toBeGreaterThan(1.1);
+		expect(one.height / 64).toBeLessThan(1.35);
+	});
+
+	it("counts a character the font lacks as an em, and falls back without the font", () => {
+		expect(measure("中", style()).width).toBeCloseTo(64, 6);
+		const none: (f: string, w: string) => FontMetrics | null = () => null;
+		const fallback = measureWithFontTables(none)("Demo OpenScreen", style());
+		expect(fallback.width).toBeCloseTo(15 * 0.56 * 64, 6);
+		// A font whose advances cannot be read (a truncated `hmtx`) is measured the same way.
+		const truncated: FontMetrics = {
+			unitsPerEm: 1000,
+			lineHeight: 1200,
+			advanceOf: () => {
+				throw new RangeError("Offset is outside the bounds of the DataView");
+			},
+		};
+		expect(measureWithFontTables(() => truncated)("Demo OpenScreen", style())).toEqual(fallback);
+	});
+});