From e096a0116019ce0eb0c8370500c73b751b310e3e Mon Sep 17 00:00:00 2001 From: umiyosh Date: Sat, 24 Oct 2026 20:44:16 +0900 Subject: [PATCH] feat(render): add html films with a layout check --- package-lock.json | 17 ++ package.json | 2 + quality-baseline.json | 19 +- skills/motion-render/SKILL.md | 38 +++- skills/motion-render/kit/base.css | 59 ++++++ skills/motion-render/kit/player.js | 125 ++++++++++++ skills/motion-render/package-lock.json | 17 ++ skills/motion-render/package.json | 8 +- skills/motion-render/references/html-film.md | 144 ++++++++++++++ skills/motion-render/scripts/cli.ts | 101 +++++++--- skills/motion-render/scripts/kit.ts | 52 +++++ skills/motion-render/scripts/layout.ts | 197 +++++++++++++++++++ skills/motion-render/scripts/render.ts | 162 ++++++++++----- skills/motion-render/scripts/snapshot.js | 174 ++++++++++++++++ tests/html-film.test.ts | 130 ++++++++++++ tests/layout.test.ts | 95 +++++++++ 16 files changed, 1239 insertions(+), 101 deletions(-) create mode 100644 skills/motion-render/kit/base.css create mode 100644 skills/motion-render/kit/player.js create mode 100644 skills/motion-render/references/html-film.md create mode 100644 skills/motion-render/scripts/kit.ts create mode 100644 skills/motion-render/scripts/layout.ts create mode 100644 skills/motion-render/scripts/snapshot.js create mode 100644 tests/html-film.test.ts create mode 100644 tests/layout.test.ts diff --git a/package-lock.json b/package-lock.json index bf02657..5291298 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,9 @@ "name": "motion-render-plugin", "version": "0.5.0", "dependencies": { + "@fontsource/noto-sans-jp": "5.2.8", "ffmpeg-static": "5.2.0", + "gsap": "3.13.0", "playwright-core": "1.56.1" }, "devDependencies": { @@ -746,6 +748,15 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@fontsource/noto-sans-jp": { + "version": "5.2.8", + "resolved": "https://registry.npmjs.org/@fontsource/noto-sans-jp/-/noto-sans-jp-5.2.8.tgz", + "integrity": "sha512-ngWXPtSV3XH8ZmNX9hnfeQ+x/3mCoaAOA3BVDcyS5jQdh4FYqKjf/S9tEUEYDPwGa2pULJVXTO2+oFg42gWvSw==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -3352,6 +3363,12 @@ "dev": true, "license": "MIT" }, + "node_modules/gsap": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/gsap/-/gsap-3.13.0.tgz", + "integrity": "sha512-QL7MJ2WMjm1PHWsoFrAQH/J8wUeqZvMtHO58qdekHpCfhvhSL4gSiz6vJf5EeMP0LOn3ZCprL2ki/gjED8ghVw==", + "license": "Standard 'no charge' license: https://gsap.com/standard-license." + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", diff --git a/package.json b/package.json index ac601d3..efd8031 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,9 @@ "node": ">=22.18" }, "dependencies": { + "@fontsource/noto-sans-jp": "5.2.8", "ffmpeg-static": "5.2.0", + "gsap": "3.13.0", "playwright-core": "1.56.1" }, "scripts": { diff --git a/quality-baseline.json b/quality-baseline.json index 87a5cfb..66e661c 100644 --- a/quality-baseline.json +++ b/quality-baseline.json @@ -2,22 +2,5 @@ "version": 2, "policy": "Every current quality violation is a repayment target. The baseline exists to prevent increases and track reductions.", "thresholds": {}, - "issues": [ - { - "id": "eslint|complexity|skills/motion-render/scripts/cli.ts|157|1", - "tool": "eslint", - "category": "complexity", - "path": "skills/motion-render/scripts/cli.ts", - "line": 157, - "column": 1, - "detail": "Async function 'main' has a complexity of 22. Maximum allowed is 20.", - "metric": { - "name": "complexity", - "value": 22, - "limit": 20, - "direction": "lowerIsBetter" - }, - "stableKey": "eslint|complexity|skills/motion-render/scripts/cli.ts|function:main" - } - ] + "issues": [] } diff --git a/skills/motion-render/SKILL.md b/skills/motion-render/SKILL.md index 920e844..6f7f17c 100644 --- a/skills/motion-render/SKILL.md +++ b/skills/motion-render/SKILL.md @@ -1,12 +1,13 @@ --- name: motion-render -description: テーマを渡されたら、それを説明するアニメーション動画を Claude Motion で作り、ナレーション（Gemini TTS）とテーマに合う BGM（Lyria）、アニメーションに合わせた効果音（ElevenLabs）を付けて MP4 にする。「〇〇を説明する動画を作って」「〇〇の紹介動画」「/motion-render 〇〇」、既存の Motion の動画を MP4 にしたいときに使う。 +description: テーマを渡されたら、それを説明するアニメーション動画を Claude Motion（使えなければ HTML）で作り、ナレーション（Gemini TTS）とテーマに合う BGM（Lyria）、アニメーションに合わせた効果音（ElevenLabs）を付けて MP4 にする。「〇〇を説明する動画を作って」「〇〇の紹介動画」「/motion-render 〇〇」、既存の Motion の動画を MP4 にしたいときに使う。 argument-hint: "<動画のテーマ。長さ・出力先などの指定があれば一緒に>" --- # Motion Render `/motion-render <テーマ>` の一言で、テーマを説明するアニメーション動画を作り、ナレーションと BGM を付けて MP4 にする。 +映像は Claude Motion で作る。Motion が使えないアカウントでは、映像を 1 枚の HTML として書く（以下「html の映像」）。 **ナレーションと BGM は必ず付ける。** 音のない動画は作らない。既存の Motion の動画を MP4 にするときも、 場面に合わせてナレーションを書き、BGM を作る。効果音は `ELEVENLABS_API_KEY` があるときだけ付ける（下の「効果音」）。 @@ -59,7 +60,7 @@ Motion の Artifact 型は作成時に Style・Colors・Length を質問する script.json # 原稿（Claude が書く） timing.json # 場面の秒数（narrate が書く） audio/ # <場面id>.wav、bgm.mp3、mix.m4a、sfx/（効果音と試聴用の preview.m4a） - film/ # Motion から取得した映像データ（prepare が展開する） + film/ # 映像。Motion から取得したもの（prepare が展開する）か、html の映像（kit が用意し、Claude が index.html を書く） ``` `script.json` の形: @@ -107,9 +108,14 @@ motion-render narrate `timing.json` に各場面の開始と秒数が出る。Motion の場面の `dur` は、この値をそのまま使う。 -### 3. Motion の動画を作る +### 3. 映像を作る -1. `Artifact` の `quickstart`（`intent: "other"`）か `list`（`scope: "types"`）で Motion 型の `type_url` を見つけ、`title` を付けて作る +`Artifact` の `quickstart`（`intent: "other"`）か `list`（`scope: "types"`）で Motion 型を探す。 +見つかれば 3A、無ければ（Artifact ツールが無い場合も）3B に進む。どちらを使ったかを報告に書く。 + +#### 3A. Motion で作る + +1. Motion 型の `type_url` に `title` を付けて作る 2. 返ってきた Motion 型の指示に従って場面を作る。ただし: - 質問はしない（上の「決め方」） - `scenes.json` の場面の `id`・`dur` は `timing.json` と同じにする @@ -119,13 +125,25 @@ motion-render narrate 4. 効果音を付けるときは、場面ごとに「何秒目に何が出るか」を決めた値で `script.json` の `sfx` を書き、 `motion-render sfx ` で作る（下の「効果音」） -### 4. 映像データを取得して確認する +#### 3B. html の映像を作る + +1. `motion-render kit ` で `/film/` にプレーヤー・フォント・GSAP・場面の秒数を置く +2. [references/html-film.md](references/html-film.md) を読み、その決まりに従って `/film/index.html` を書く。 + 場面は `
`、動きは `film.build` の中で `film.at(場面id, 秒)` に置く +3. 並行して BGM を作ってよい: `motion-render bgm ` +4. 効果音を付けるときは、`script.json` の `sfx` の `at` に、その動きに渡した `film.at` の秒を書き、 + `motion-render sfx ` で作る。タイトルの場面に `🎧:11.ai` を小さく入れる + +### 4. 映像を確認する + +html の映像（3B）は手順 4 の 1〜3 を飛ばし、先に `motion-render check /film` を `layout: ok` になるまで直す +（出力の読み方は references/html-film.md の「確認」）。そのあと 4・5 で静止画を見る。 1. `Artifact` の `read` で `paths: ["index.html", "artifact-type/app.js", "artifact-type/app.css"]`、`out_dir: /film` 2. `ArtifactData` の `list`（`collection: "modules"`）と `get`（`film`/`script`）を `out_dir: /store` で保存 3. `motion-render prepare /store /film` 4. `motion-render stills /film /stills <各場面の中ほどの秒をカンマ区切り>` で全場面を見る。 - 見出しの不自然な改行、はみ出し、重なりがあれば Motion 側を直し、手順 4 をやり直す + 見出しの不自然な改行、はみ出し、重なりがあれば Motion 側（html の映像なら index.html）を直し、手順 4 をやり直す 5. 効果音があれば、各効果音の秒（場面の開始 + `at`）の静止画も書き出し、鳴らす対象がその時点で画面に出ているかを見る。 ずれていれば `at` を直して `motion-render sfx` をやり直す（作り直されるのは変えたものだけ） @@ -136,12 +154,13 @@ motion-render build [<出力.mp4>] ``` ナレーションを各場面の頭に置き、BGM を一定の音量で重ね、効果音をそれぞれの秒に置いて MP4 にする。 -ナレーションか BGM、または `script.json` にある効果音が無いと止まる。 +ナレーションか BGM、または `script.json` にある効果音が無いと止まる。html の映像は、`check` と同じ問題が +残っていても止まる。 ### 6. 確認して報告する - `ffprobe` で、長さと映像・音声の両トラックがあることを見る -- 報告: 出力のパス、長さ、場面の流れ、BGM の雰囲気、Motion の URL。耳で聞いていないことは聞いていないと書く +- 報告: 出力のパス、長さ、場面の流れ、BGM の雰囲気、Motion の URL（html の映像なら `/film/index.html` のパス）。耳で聞いていないことは聞いていないと書く - 効果音を付けたときは、試聴用の `audio/sfx/preview.m4a` のパスと、その中の順番（`motion-render sfx` の出力）も報告し、 合わない音があれば教えてほしいと頼む。Claude には音の良し悪しが判定できないため、ここは人が聞いて決める @@ -201,5 +220,8 @@ URL を渡された場合も音を付ける。映像データを取得し（手 - `scripts/sfx.ts`: ElevenLabs による効果音の生成（`sfx`）、音量の測定とそろえ方、試聴用ファイル - `scripts/work.ts`: `script.json` の検証、`timing.json` の計算、ナレーションを動画の場面の開始時刻に置く処理 - `scripts/render.ts`: プレーヤーを開いてコマを返す。描画中は外部への通信を遮断し、Chromium の sandbox を有効にする +- `scripts/kit.ts`・`kit/`: html の映像のプレーヤー（`player.js`）と土台の CSS（`base.css`）を、フォントと GSAP と一緒に置く（`kit`） +- `scripts/layout.ts`・`scripts/snapshot.js`: html の映像の配置の確認（`check`。`stills`・`build` も実行する） +- `references/html-film.md`: html の映像の書き方と配置の決まり - `scripts/json.ts`: ファイル・API・プレーヤーから来る JSON を、形を確かめながら読む - `scripts/store.ts`: Motion の store（ArtifactData の保存先）を映像データのフォルダに展開する（`prepare`） diff --git a/skills/motion-render/kit/base.css b/skills/motion-render/kit/base.css new file mode 100644 index 0000000..31155d5 --- /dev/null +++ b/skills/motion-render/kit/base.css @@ -0,0 +1,59 @@ +/* Base of an html film: a 1920x1080 frame, Japanese typesetting, and fixed layers. */ +:root { + --safe-x: 144px; /* 7.5% action-safe at 1920 */ + --safe-y: 81px; +} +* { + margin: 0; + padding: 0; + box-sizing: border-box; +} +html, +body { + width: 1920px; + height: 1080px; + overflow: hidden; +} +body { + font-family: "Noto Sans JP", sans-serif; + font-feature-settings: "palt"; + -webkit-font-smoothing: antialiased; +} +:lang(ja) { + word-break: auto-phrase; + line-break: strict; +} +#root { + position: relative; + width: 1920px; + height: 1080px; + overflow: hidden; +} +section.scene { + position: absolute; + inset: 0; +} +.num { + font-variant-numeric: tabular-nums; +} + +/* Stacking by role, lowest first. Give every positioned element one of these instead of a + z-index of its own, so what is drawn on top never depends on the order in the HTML. */ +.layer-bg { + z-index: 0; +} +.layer-card { + z-index: 10; +} +.layer-line { + z-index: 20; +} +.layer-chip { + z-index: 30; +} +.layer-mark { + z-index: 40; +} +.layer-caption { + z-index: 50; +} diff --git a/skills/motion-render/kit/player.js b/skills/motion-render/kit/player.js new file mode 100644 index 0000000..2176488 --- /dev/null +++ b/skills/motion-render/kit/player.js @@ -0,0 +1,125 @@ +// The player of an html film. Scene times come from kit/timing.js (written by "motion-render kit" +// from timing.json), so the page never hard-codes them. The page builds one paused GSAP timeline in +// film.build(); render.ts seeks it through window.__film. +(function () { + var T = window.__timing; + if (!T || !window.gsap) + throw new Error("load kit/gsap.min.js and kit/timing.js before kit/player.js"); + var FAMILY = '"Noto Sans JP"'; + var scenes = {}; + T.scenes.forEach(function (s) { + scenes[s.id] = s; + }); + var timeline = gsap.timeline({ paused: true }); + var builders = []; + + function el(x) { + var e = typeof x === "string" ? document.querySelector(x) : x; + if (!e) throw new Error("no element " + x); + return e; + } + // An element's laid-out box in frame px, ignoring transforms (GSAP's x/y/scale). + function layoutBox(e) { + var x = 0, + y = 0; + for (var p = e; p && p.id !== "root"; p = p.offsetParent) { + x += p.offsetLeft; + y += p.offsetTop; + } + return { x: x, y: y, w: e.offsetWidth, h: e.offsetHeight }; + } + + window.film = { + scenes: scenes, + total: T.total, + timeline: timeline, + // Seconds into the film of `dt` seconds into scene `id`. + at: function (id, dt) { + if (!scenes[id]) throw new Error('scene "' + id + '" is not in timing.json'); + return scenes[id].start + (dt || 0); + }, + // `dt` seconds before scene `id` ends. + end: function (id, dt) { + return window.film.at(id, scenes[id] ? scenes[id].dur - (dt || 0) : 0); + }, + // Runs fn(timeline) once the fonts are in, so text has its real size when the page measures it. + build: function (fn) { + builders.push(fn); + }, + // GSAP x/y that put the center of `what` on the center of `target`, moved by (dx, dy). + // Use it to send a chip to a card instead of working out pixel offsets by hand. + spot: function (what, target, dx, dy) { + var a = layoutBox(el(what)), + b = layoutBox(el(target)); + return { + x: b.x + b.w / 2 - (a.x + a.w / 2) + (dx || 0), + y: b.y + b.h / 2 - (a.y + a.h / 2) + (dy || 0), + }; + }, + }; + + function sections() { + return Array.prototype.slice.call(document.querySelectorAll("section.scene")); + } + // A scene's section is shown over [start, start + dur); everything outside sections stays. + function show(t) { + sections().forEach(function (el) { + var s = scenes[el.id]; + el.style.visibility = t >= s.start && t < s.start + s.dur ? "" : "hidden"; + }); + } + // Fonts load per unicode-range subset on first use, so a scene's first frame could fall back. + // Load every glyph the film shows before the first frame, and fail if any is missing. + async function loadFonts() { + var text = document.body.textContent.replace(/\s+/g, "") + "0123456789"; + var weights = ["400", "700"]; + try { + await Promise.all( + weights.map(function (w) { + return document.fonts.load(w + " 48px " + FAMILY, text); + }) + ); + } catch (e) { + throw new Error( + "font " + FAMILY + " failed to load (" + e.message + '). Run "motion-render kit" again.' + ); + } + await document.fonts.ready; + weights.forEach(function (w) { + if (!document.fonts.check(w + " 48px " + FAMILY, text)) + throw new Error( + "font " + FAMILY + " " + w + ' did not load. Run "motion-render kit" again.' + ); + }); + } + + window.__film = { + load: async function () { + var unknown = sections() + .map(function (el) { + return el.id; + }) + .filter(function (id) { + return !scenes[id]; + }); + if (unknown.length) + throw new Error("section.scene ids not in timing.json: " + unknown.join(", ")); + await loadFonts(); + builders.forEach(function (fn) { + fn(timeline); + }); + window.__film.seek(0); + return { + duration: T.total, + parts: T.scenes.map(function (s) { + return { key: s.id, a0: s.start, dur: s.dur }; + }), + }; + }, + seek: function (t) { + show(t); + // suppressEvents=false so onUpdate callbacks (count-ups) run on every seek. + timeline.seek(t, false); + }, + }; +})(); diff --git a/skills/motion-render/package-lock.json b/skills/motion-render/package-lock.json index cfa6227..6292d74 100644 --- a/skills/motion-render/package-lock.json +++ b/skills/motion-render/package-lock.json @@ -8,7 +8,9 @@ "name": "motion-render", "version": "0.5.0", "dependencies": { + "@fontsource/noto-sans-jp": "5.2.8", "ffmpeg-static": "5.2.0", + "gsap": "3.13.0", "playwright-core": "1.56.1" }, "bin": { @@ -33,6 +35,15 @@ "node": ">=6.0.0" } }, + "node_modules/@fontsource/noto-sans-jp": { + "version": "5.2.8", + "resolved": "https://registry.npmjs.org/@fontsource/noto-sans-jp/-/noto-sans-jp-5.2.8.tgz", + "integrity": "sha512-ngWXPtSV3XH8ZmNX9hnfeQ+x/3mCoaAOA3BVDcyS5jQdh4FYqKjf/S9tEUEYDPwGa2pULJVXTO2+oFg42gWvSw==", + "license": "OFL-1.1", + "funding": { + "url": "https://github.com/sponsors/ayuhito" + } + }, "node_modules/@types/node": { "version": "10.17.60", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", @@ -120,6 +131,12 @@ "node": ">=16" } }, + "node_modules/gsap": { + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/gsap/-/gsap-3.13.0.tgz", + "integrity": "sha512-QL7MJ2WMjm1PHWsoFrAQH/J8wUeqZvMtHO58qdekHpCfhvhSL4gSiz6vJf5EeMP0LOn3ZCprL2ki/gjED8ghVw==", + "license": "Standard 'no charge' license: https://gsap.com/standard-license." + }, "node_modules/http-response-object": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/http-response-object/-/http-response-object-3.0.2.tgz", diff --git a/skills/motion-render/package.json b/skills/motion-render/package.json index d3d1ed2..24015f5 100644 --- a/skills/motion-render/package.json +++ b/skills/motion-render/package.json @@ -1,7 +1,7 @@ { "name": "motion-render", "version": "0.5.0", - "description": "Make an animated explainer video from a theme with Claude Motion, add narration (Gemini TTS), BGM (Lyria) and sound effects (ElevenLabs), and render it to MP4.", + "description": "Make an animated explainer video from a theme with Claude Motion (or as an HTML page when Motion is not available), add narration (Gemini TTS), BGM (Lyria) and sound effects (ElevenLabs), and render it to MP4.", "repository": "github:umiyosh/motion-render", "type": "module", "bin": { @@ -9,13 +9,17 @@ }, "files": [ "SKILL.md", - "scripts/" + "scripts/", + "kit/", + "references/" ], "engines": { "node": ">=22.18" }, "dependencies": { + "@fontsource/noto-sans-jp": "5.2.8", "ffmpeg-static": "5.2.0", + "gsap": "3.13.0", "playwright-core": "1.56.1" } } diff --git a/skills/motion-render/references/html-film.md b/skills/motion-render/references/html-film.md new file mode 100644 index 0000000..213c145 --- /dev/null +++ b/skills/motion-render/references/html-film.md @@ -0,0 +1,144 @@ +# html の映像の作り方 + +Claude Motion が使えないときに、映像を 1 枚の HTML として書く。`motion-render kit ` が +`/film/kit/` にプレーヤー・フォント（Noto Sans JP）・GSAP・場面の秒数を置くので、 +Claude は `/film/index.html` だけを書く。描画中はネットワークが遮断されるので、CDN や Google Fonts は使えない。 + +## ページの骨組み + +```html + + + + + + + + + + + + +
+ +
…
+ +
…
+
…
+
+ + + +``` + +プレーヤーが用意するもの: + +| 名前 | 意味 | +| ------------------------------- | ---------------------------------------------------------------------------------- | +| `film.build(fn)` | フォントの読み込みが終わってから `fn(tl)` を呼ぶ。アニメーションは全部この中に書く | +| `film.at(id, 秒)` | 場面 `id` の頭から何秒目か（動画の先頭からの秒に直す） | +| `film.end(id, 秒)` | 場面 `id` の終わりの何秒前か | +| `film.spot(要素, 相手, dx, dy)` | `要素` の中心を `相手` の中心へ（`dx`・`dy` だけずらして）動かす GSAP の `x`・`y` | +| `section.scene` | その場面の間だけ見える。`#stage` のように section の外に置いたものはずっと見える | + +## 配置の決まり + +`motion-render check` はこれを破った所を見つけて止める。最初から守って書く。 + +1. **関係のあるものは、親子にしてレイアウトで並べる。** カードの中身（見出し、説明、チェック項目、 + 中に入るチップ）はカードの子要素にし、`display: flex` / `grid` と `gap` で並べる。 + 画面全体の座標（`position: absolute; left: …px`）で 1 つずつ置くのは、図の大きな部品（カード、 + 字幕、流れ図）だけにする +2. **印は、判定する相手の中に置く。** ✕・✓・バッジ・番号は、それが付く要素の子にする + （`
OIDC トークン✕
`）。2 つの要素の間の + 空いた所に浮かせない。拒否を見せるときは、チップを相手にぶつけて跳ね返し、チップに ✕ を出し、 + 相手の枠を一瞬赤くする +3. **重なり順は役割で決める。** 位置を持つ要素には `kit/base.css` の層のクラスを 1 つ付ける。 + HTML に書いた順番に頼らない + + | クラス | 使うもの | + | --------------- | ------------------------------ | + | `layer-bg` | 背景、グラデーション | + | `layer-card` | カード、パネル、枠 | + | `layer-line` | 矢印、つなぐ線、グラフの補助線 | + | `layer-chip` | 動くチップ、トークン | + | `layer-mark` | ✕・✓・強調の輪 | + | `layer-caption` | 字幕 | + +4. **チップの行き先は `film.spot` で決める。** 止まる位置は、カードの中に収まるか、どのカードとも + 重ならない所にする。カードの枠線をまたいで止めない +5. **箱は親の中に収める。** 親より大きくなりそうな中身は、文字を短くするか親を広げる。 + はみ出しを `overflow: hidden` で隠さない +6. **前の場面の残りを片付ける。** 次の場面で使わない部品は、場面の頭で消すか薄くし、新しい部品と + 同じ場所に重ならないようにする + +意図した重なり（グラフの基準線が棒を横切る、軸のラベルが目盛りの外に出る、わざと斜めに押す判子）は、 +その要素に `data-layout-ok` を付けると `check` が数えない。間違いを黙らせるためには使わない。 + +## 確認 + +```bash +motion-render check /film +``` + +0.2 秒ごとに全体を調べ、止まっている部品について次を出す。動いている途中の重なりは数えない。 + +| 出力 | 意味と直し方 | +| --------------------------- | ------------------------------------------------------------------- | +| `A and B partly overlap` | 2 つが中途半端に重なっている。片方をもう片方の中に入れるか、離す | +| `A sticks out of B` | 子の A が親の B からはみ出している。中身を減らすか B を広げる | +| `A is cut off by the frame` | 画面の端で切れている。安全域（左右 144px、上下 81px）の中へ入れる | +| `A is hidden under B` | A の大部分が B の下に隠れている。層のクラスを見直すか、位置をずらす | + +`layout: ok` になるまで直してから `stills` で見る。`build` は問題が残っていると止まる。 + +## 動き + +- 役割で緩急を決める: 出るとき `expo.out`（0.4〜0.7 秒）、移動 `power2.inOut`（0.6〜0.9 秒）、 + 消えるとき `power3.in`（0.2〜0.3 秒）。並んだものは 0.05〜0.1 秒ずらして出す +- 1 場面 1 つの出来事。図が動き終わってから字幕を出す。字幕は場面の終わりの 0.25 秒前に消す +- 場面の最後の動きのあと、1.5 秒以上止めて読ませる +- 秒数は `film.at` / `film.end` だけで書く。ナレーションを作り直しても `build` が秒数を読み直す + +## 文字 + +- `` と `kit/base.css` で、見出しは文節の切れ目で改行される。長いカタカナ語 + （セキュリティトークンサービス）は途中で割れるので、短い言葉か英語の表記にする +- 本文 28px 以上、字幕 52〜60px、見出し 40px 以上。数字には `class="num"`（等幅の数字） +- 色は `:root` の変数で決め、意味で使い分ける（正常・強調・危険） + +## 決まった絵になるように + +- 動きはすべて `film.build` の中の GSAP のタイムラインで書く。CSS の `animation`・`transition`、 + `Date`・`Math.random`・`setTimeout`・`requestAnimationFrame` は使わない +- 数え上げは `onUpdate` で文字を書き換える（プレーヤーがコマごとに呼ぶ） +- 画像を使うときは `/film/` の中に置き、相対パスで読む + +## 効果音 + +`script.json` の `sfx` の `at` には、その音を鳴らす動きに渡した `film.at(場面, 秒)` の秒をそのまま書く。 +例えば `tl.fromTo("#key", …, film.at("problem", 1.6))` に合わせる音は、場面 `problem` の `"at": 1.6`。 +効果音を付けた動画は、タイトルの場面に `🎧:11.ai` を小さく入れる。 diff --git a/skills/motion-render/scripts/cli.ts b/skills/motion-render/scripts/cli.ts index c737699..8fe657d 100755 --- a/skills/motion-render/scripts/cli.ts +++ b/skills/motion-render/scripts/cli.ts @@ -1,12 +1,14 @@ #!/usr/bin/env node -// motion-render: テーマから作った Claude Motion の動画に、ナレーションと BGM を付けて MP4 にする CLI。 +// motion-render: テーマから作った動画（Claude Motion か html）に、ナレーションと BGM を付けて MP4 にする CLI。 import { writeFileSync, existsSync, mkdirSync } from "node:fs"; import { join, resolve, basename, dirname } from "node:path"; import { spawn } from "node:child_process"; import { once } from "node:events"; import { parseArgs } from "node:util"; import { fileURLToPath } from "node:url"; -import { openFilm, checkFilmDir, type Film } from "./render.ts"; +import { openFilm, checkFilmDir, isHtmlFilm, type Film } from "./render.ts"; +import { checkLayout, formatIssue } from "./layout.ts"; +import { writeKit, writeKitTiming } from "./kit.ts"; import { narrate, bgm, bgmFile, mix, findFfmpeg, encodeArgs } from "./audio.ts"; import { loadScript, writeTiming, loadTiming, placeOnFilm, placeSfx } from "./work.ts"; import { checkSfx, makeSfx, narrationLevel, previewSfx, sfxTrack } from "./sfx.ts"; @@ -21,9 +23,14 @@ const USAGE = `usage (one work dir per video; see SKILL.md for its layout): /audio/sfx/preview.m4a to listen to them all in order. motion-render prepare /film Expand a Motion film's store (ArtifactData out_dir) into a film dir. + motion-render kit Set up /film for an html film: player, fonts, GSAP and + the scene times from timing.json (write film/index.html after). + motion-render check Check an html film's layout every 0.2s: overlaps, things + sticking out of their box or the frame, things hidden under others. motion-render stills - Write PNG stills at the given seconds. + Write PNG stills at the given seconds (and run check). motion-render build [out.mp4] Mix narration + BGM + sound effects and render /film to MP4. + Stops if check finds a layout issue in an html film. Without out.mp4: $MOTION_RENDER_OUT_DIR or the current dir, .mp4. env: @@ -47,6 +54,15 @@ function runPrepare(storeDir: string, filmDir: string): void { console.log(`modules: ${modules.join(", ")}`); } +// Prints the layout issues and returns how many there are (0 for a Motion film, which is not checked). +async function reportLayout(film: Film): Promise { + const issues = await checkLayout(film); + if (!issues) return 0; + for (const i of issues) console.error(`layout: ${formatIssue(i)}`); + console.error(issues.length ? `layout: ${issues.length} issues` : "layout: ok"); + return issues.length; +} + async function stills(filmDir: string, outDir: string, list: string): Promise { const times = list.split(",").map(Number); if (times.some((t) => !Number.isFinite(t))) die(`times must be numbers: ${list}`); @@ -55,6 +71,7 @@ async function stills(filmDir: string, outDir: string, list: string): Promise { + if (!isHtmlFilm(filmDir)) + die("check works on html films only (Motion films draw in their own player)"); + const film = await openFilm(filmDir); + let n: number; + try { + n = await reportLayout(film); + } finally { + await film.close(); + } + if (n) die(`${n} layout issues in ${filmDir}`); + console.log(`layout ok: ${filmDir}`); +} + +function runKit(work: string): void { + loadTiming(work); + console.log(`kit: ${join(writeKit(work), "kit")} (write ${join(work, "film", "index.html")})`); +} + async function runNarrate(work: string): Promise { const script = loadScript(work); await narrate(work, script); @@ -124,9 +160,14 @@ async function build(work: string, out: string, fps: number): Promise { checkSfx(work, script); const filmDir = join(work, "film"); checkFilmDir(filmDir); + if (isHtmlFilm(filmDir)) writeKitTiming(work); const film = await openFilm(filmDir); try { + if (await reportLayout(film)) + throw new Error( + `fix the layout issues above in ${join(filmDir, "index.html")}, then build again` + ); const warn = (w: string) => console.error(`warning: ${w}`); const scenes = placeOnFilm(timing, film.meta, script, warn); const narration = await narrationLevel(work, script); @@ -143,11 +184,28 @@ async function build(work: string, out: string, fps: number): Promise { } } -// Commands that take just the work dir. -const WORK_COMMANDS = new Map Promise>([ - ["narrate", runNarrate], - ["bgm", runBgm], - ["sfx", runSfx], +interface Command { + args: number[]; // accepted argument counts + run(args: string[], fps: number): Promise | void; +} + +function runBuild([a, b]: string[], fps: number): Promise { + const work = resolve(a!); + const out = resolve( + b ?? join(process.env.MOTION_RENDER_OUT_DIR || process.cwd(), `${basename(work)}.mp4`) + ); + return build(work, out, fps); +} + +const COMMANDS = new Map([ + ["narrate", { args: [1], run: ([w]) => runNarrate(resolve(w!)) }], + ["bgm", { args: [1], run: ([w]) => runBgm(resolve(w!)) }], + ["sfx", { args: [1], run: ([w]) => runSfx(resolve(w!)) }], + ["kit", { args: [1], run: ([w]) => runKit(resolve(w!)) }], + ["prepare", { args: [2], run: ([s, f]) => runPrepare(s!, f!) }], + ["stills", { args: [3], run: ([f, o, t]) => stills(f!, o!, t!) }], + ["check", { args: [1], run: ([f]) => check(f!) }], + ["build", { args: [1, 2], run: runBuild }], ]); function version(): string { @@ -165,27 +223,12 @@ async function main(): Promise { }, }); const [cmd, ...rest] = positionals; - if (values.version) { - console.log(version()); - return; - } - if (values.help || !cmd) { - console.log(USAGE); - return; - } - const [a, b, c] = rest; - const onWork = WORK_COMMANDS.get(cmd); - if (onWork && a && rest.length === 1) return onWork(resolve(a)); - if (cmd === "prepare" && a && b && rest.length === 2) return runPrepare(a, b); - if (cmd === "stills" && a && b && c && rest.length === 3) return stills(a, b, c); - if (cmd === "build" && a && (rest.length === 1 || rest.length === 2)) { - const work = resolve(a); - const out = resolve( - b ?? join(process.env.MOTION_RENDER_OUT_DIR || process.cwd(), `${basename(work)}.mp4`) - ); - return build(work, out, Number(values.fps)); - } - die(`unknown command or wrong arguments\n\n${USAGE}`, 2); + if (values.version) return console.log(version()); + if (values.help || !cmd) return console.log(USAGE); + const command = COMMANDS.get(cmd); + if (!command || !command.args.includes(rest.length) || rest.some((a) => !a)) + die(`unknown command or wrong arguments\n\n${USAGE}`, 2); + return command.run(rest, Number(values.fps)); } main().catch((e: unknown) => die(errorMessage(e))); diff --git a/skills/motion-render/scripts/kit.ts b/skills/motion-render/scripts/kit.ts new file mode 100644 index 0000000..40094a8 --- /dev/null +++ b/skills/motion-render/scripts/kit.ts @@ -0,0 +1,52 @@ +// Sets up /film for an html film: the player, the base stylesheet, GSAP, the Japanese font +// and the scene times. Claude writes /film/index.html against these files. +import { copyFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; +import { loadTiming } from "./work.ts"; + +const require = createRequire(import.meta.url); +const KIT = fileURLToPath(new URL("../kit/", import.meta.url)); +// The two weights the player loads (kit/player.js). +const FONT_CSS = [ + require.resolve("@fontsource/noto-sans-jp/400.css"), + require.resolve("@fontsource/noto-sans-jp/700.css"), +]; + +// Fontsource's CSS for one weight, cut down to woff2 files under fonts/ and font-display: block, +// so a frame is never drawn in a fallback font. +function fontCss(file: string, out: string): string { + const files = join(dirname(file), "files"); + return readFileSync(file, "utf8") + .replace(/font-display:\s*swap/g, "font-display: block") + .replace(/,\s*url\([^)]*\.woff\) format\('woff'\)/g, "") + .replace(/url\(\.\/files\/([^)]+\.woff2)\)/g, (_, name: string) => { + copyFileSync(join(files, name), join(out, "fonts", name)); + return `url(fonts/${name})`; + }); +} + +// Rewrites kit/timing.js from timing.json. build runs this too, so the film always uses the +// scene times of the current narration. +export function writeKitTiming(work: string): void { + const { total, scenes } = loadTiming(work); + const data = { total, scenes: scenes.map(({ id, start, dur }) => ({ id, start, dur })) }; + writeFileSync( + join(work, "film", "kit", "timing.js"), + `window.__timing = ${JSON.stringify(data)};\n` + ); +} + +export function writeKit(work: string): string { + const film = join(work, "film"); + const kit = join(film, "kit"); + mkdirSync(join(kit, "fonts"), { recursive: true }); + for (const f of ["player.js", "base.css"]) copyFileSync(join(KIT, f), join(kit, f)); + const gsap = require.resolve("gsap/dist/gsap.min.js"); + copyFileSync(gsap, join(kit, basename(gsap))); + writeFileSync(join(kit, "fonts.css"), FONT_CSS.map((f) => fontCss(f, kit)).join("\n")); + writeKitTiming(work); + writeFileSync(join(film, "film.json"), JSON.stringify({ kind: "html" }) + "\n"); + return film; +} diff --git a/skills/motion-render/scripts/layout.ts b/skills/motion-render/scripts/layout.ts new file mode 100644 index 0000000..9090353 --- /dev/null +++ b/skills/motion-render/scripts/layout.ts @@ -0,0 +1,197 @@ +// Layout check of an html film: samples the page through the film and reports what a viewer would +// see as a mistake: two things partly on top of each other, something sticking out of the box it +// sits in or out of the frame, and something hidden under another thing. +// Only what stays still is reported, so a chip passing over a card on its way is not a mistake. +import type { Film } from "./render.ts"; +import { isRecord } from "./json.ts"; + +// A thing that paints on screen at one moment (snapshot.js). Coordinates are CSS px in the frame. +export interface LayoutBox { + key: number; // stable across samples: its index among the page's elements + label: string; + x: number; + y: number; + w: number; + h: number; + cut: boolean; // part of it lies outside the frame + anc: number[]; // keys of the painted boxes it sits inside in the DOM, nearest first + ok: boolean; // data-layout-ok on it or an ancestor: the overlap is intended + under: number | null; // key of a solid box drawn over most of it, if any +} + +export interface LayoutSample { + t: number; + boxes: LayoutBox[]; +} + +export interface LayoutIssue { + kind: "overlap" | "outside" | "frame" | "hidden"; + from: number; + to: number; + message: string; +} + +const SLACK = 2; // px of overlap that reads as touching, not overlapping +const MOVED = 1; // px a box may shift between samples and still count as still + +type Rect = Pick; + +function overlaps(a: Rect, b: Rect): boolean { + const w = Math.min(a.x + a.w, b.x + b.w) - Math.max(a.x, b.x); + const h = Math.min(a.y + a.h, b.y + b.h) - Math.max(a.y, b.y); + return w > SLACK && h > SLACK; +} + +function contains(outer: Rect, inner: Rect): boolean { + return ( + inner.x >= outer.x - SLACK && + inner.y >= outer.y - SLACK && + inner.x + inner.w <= outer.x + outer.w + SLACK && + inner.y + inner.h <= outer.y + outer.h + SLACK + ); +} + +function still(a: Rect, b: Rect | undefined): boolean { + if (!b) return false; + const d = [a.x - b.x, a.y - b.y, a.w - b.w, a.h - b.h]; + return d.every((v) => Math.abs(v) <= MOVED); +} + +interface Finding { + id: string; + kind: LayoutIssue["kind"]; + message: string; +} + +// What is wrong with one still box on its own: cut by the frame, out of its box, or hidden. +function boxFindings(b: LayoutBox, byKey: Map, quiet: Set): Finding[] { + const out: Finding[] = []; + if (b.cut) + out.push({ + id: `frame:${b.key}`, + kind: "frame", + message: `${b.label} is cut off by the frame`, + }); + if (b.ok) return out; + const parent = byKey.get(b.anc[0] ?? -1); + if (parent && !contains(parent, b)) + out.push({ + id: `outside:${b.key}`, + kind: "outside", + message: `${b.label} sticks out of ${parent.label}`, + }); + // Something resting inside b (a chip on a card) is on top of it, not hiding it. + const over = b.under === null ? undefined : byKey.get(b.under); + if (over && quiet.has(over.key) && !contains(b, over)) + out.push({ + id: `hidden:${b.key}:${over.key}`, + kind: "hidden", + message: `${b.label} is hidden under ${over.label}`, + }); + return out; +} + +// Two still boxes, neither inside the other in the DOM, partly on top of each other. A pair where +// one is drawn over the other is reported as hidden instead. +function pairOverlaps(a: LayoutBox, b: LayoutBox): boolean { + if (a.ok || b.ok || a.anc.includes(b.key) || b.anc.includes(a.key)) return false; + if (a.under === b.key || b.under === a.key) return false; + return overlaps(a, b) && !contains(a, b) && !contains(b, a); +} + +function findingsAt(boxes: LayoutBox[], prev: Map): Finding[] { + const quiet = boxes.filter((b) => still(b, prev.get(b.key))); + const keys = new Set(quiet.map((b) => b.key)); + const byKey = new Map(boxes.map((b) => [b.key, b])); + const out = quiet.flatMap((b) => boxFindings(b, byKey, keys)); + quiet.forEach((a, i) => { + for (const b of quiet.slice(i + 1)) + if (pairOverlaps(a, b)) + out.push({ + id: `overlap:${a.key}:${b.key}`, + kind: "overlap", + message: `${a.label} and ${b.label} partly overlap`, + }); + }); + return out; +} + +interface Seen extends LayoutIssue { + last: number; // index of the last sample it was seen in + run: number; // samples in a row up to `last` + best: number; // longest run +} + +// A finding counts when it holds in at least two samples in a row, so a single in-between frame of +// a transition is not reported. Each finding is reported once with the span it was seen over. +export function findLayoutIssues(samples: LayoutSample[]): LayoutIssue[] { + const seen = new Map(); + let prev = new Map(); + samples.forEach((s, i) => { + for (const f of findingsAt(s.boxes, prev)) { + const e = seen.get(f.id); + const run = e && e.last === i - 1 ? e.run + 1 : 1; + seen.set(f.id, { + kind: f.kind, + message: f.message, + from: e ? e.from : s.t, + to: s.t, + last: i, + run, + best: Math.max(e ? e.best : 0, run), + }); + } + prev = new Map(s.boxes.map((b) => [b.key, b])); + }); + return [...seen.values()] + .filter((e) => e.best >= 2) + .sort((a, b) => a.from - b.from) + .map(({ kind, from, to, message }) => ({ kind, from, to, message })); +} + +export function formatIssue(i: LayoutIssue): string { + const f = i.from.toFixed(1), + t = i.to.toFixed(1); + return `${f === t ? f : `${f}-${t}`}s: ${i.message}`; +} + +const num = (v: unknown): v is number => typeof v === "number" && Number.isFinite(v); + +function parseBox(v: unknown): LayoutBox { + if (isRecord(v)) { + const { key, label, x, y, w, h, cut, anc, ok, under } = v; + if ( + num(key) && + typeof label === "string" && + num(x) && + num(y) && + num(w) && + num(h) && + typeof cut === "boolean" && + typeof ok === "boolean" && + Array.isArray(anc) && + anc.every(num) && + (under === null || num(under)) + ) + return { key, label, x, y, w, h, cut, anc, ok, under }; + } + throw new Error("the layout snapshot of the film page is malformed"); +} + +// The snapshot comes from the film page, whose code is not ours, so its shape is checked here. +export function parseBoxes(raw: unknown): LayoutBox[] { + if (!Array.isArray(raw)) throw new Error("the layout snapshot of the film page is malformed"); + return raw.map(parseBox); +} + +// Samples the whole film every `step` seconds. Motion films draw inside their own player and +// cannot be inspected, so they get null. +export async function checkLayout(film: Film, step = 0.2): Promise { + if (!film.inspect) return null; + const samples: LayoutSample[] = []; + for (let i = 0; i * step < film.meta.duration; i++) { + const t = Math.round(i * step * 1000) / 1000; + samples.push({ t, boxes: await film.inspect(t) }); + } + return findLayoutIssues(samples); +} diff --git a/skills/motion-render/scripts/render.ts b/skills/motion-render/scripts/render.ts index fc8bd98..d6d6623 100644 --- a/skills/motion-render/scripts/render.ts +++ b/skills/motion-render/scripts/render.ts @@ -1,15 +1,17 @@ -// Motion のプレーヤーを headless Chromium で開き、任意の時刻のコマを PNG で返す。 +// 映像（Motion のプレーヤー、または html の映像）を headless Chromium で開き、任意の時刻のコマを PNG で返す。 // ブラウザは playwright-core の版に対応する Chromium を使い、無ければ初回に自動で取得する。 import { readFileSync, readdirSync, existsSync } from "node:fs"; import { join, resolve, extname, dirname } from "node:path"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { createRequire } from "node:module"; import { spawnSync } from "node:child_process"; -import { chromium, type Browser, type LaunchOptions } from "playwright-core"; -import { errorMessage, field, isRecord } from "./json.ts"; +import { chromium, type Browser, type LaunchOptions, type Page } from "playwright-core"; +import { errorMessage, field, isRecord, readJson } from "./json.ts"; +import { parseBoxes, type LayoutBox } from "./layout.ts"; const require = createRequire(import.meta.url); const PLAYWRIGHT_CLI = join(dirname(require.resolve("playwright-core")), "cli.js"); +const SNAPSHOT = fileURLToPath(new URL("snapshot.js", import.meta.url)); const KIND: Partial> = { ".css": "css", @@ -77,6 +79,8 @@ interface FilmMeta { export interface Film { meta: FilmMeta; frame(t: number): Promise; + // html films only: what paints on screen at t, for the layout check. + inspect?(t: number): Promise; warnings(): Promise; close(): Promise; } @@ -89,11 +93,28 @@ declare global { frame(t: number, opts: { scale: number }): unknown; consoleLog(): unknown; }; + // An html film's hook: the page lays itself out at time t and the runner screenshots it. + __film: { + load(): unknown; + seek(t: number): unknown; + // Optional: false while the page is still settling. Polled from Node, since a film page may + // control its own timers. + ready?(): unknown; + }; + // Added by snapshot.js for the layout check. + __layoutSnapshot(): unknown; } } +// A film dir is a Motion film unless its film.json says {"kind": "html"}. +export function isHtmlFilm(dir: string): boolean { + const f = join(dir, "film.json"); + return existsSync(f) && field(readJson(f), "kind") === "html"; +} + export function checkFilmDir(dir: string): void { - const missing = REQUIRED.filter((f) => !existsSync(join(dir, f))); + const need = isHtmlFilm(dir) ? ["index.html"] : REQUIRED; + const missing = need.filter((f) => !existsSync(join(dir, f))); if (missing.length) throw new Error( `${dir} is not a film dir (missing: ${missing.join(", ")}). Run "motion-render prepare" first.` @@ -164,50 +185,103 @@ async function launch(): Promise { } } +// Opens the film page with the network cut off: only files under the film dir and data:/blob: load. +async function openPage(browser: Browser): Promise { + const page = await browser.newPage({ viewport: { width: 1920, height: 1080 } }); + await page.context().addInitScript(() => { + for (const k of ["RTCPeerConnection", "webkitRTCPeerConnection"]) + Reflect.deleteProperty(globalThis, k); + }); + let here: string | null = null; + await page.context().route("**/*", (r) => { + const u = r.request().url(); + if (!here && r.request().isNavigationRequest()) here = u.slice(0, u.lastIndexOf("/") + 1); + return (here && u.startsWith(here)) || /^(data|blob):/.test(u) ? r.continue() : r.abort(); + }); + return page; +} + +async function motionFilm(page: Page, dir: string): Promise> { + await page.goto(pathToFileURL(resolve(dir, "index.html")).href + "#tools"); + await page.waitForFunction(() => typeof window.__animation === "object"); + const res = await page.evaluate((f) => window.__animation.load(f), readFilm(dir)); + const error = field(res, "error"); + if (error) { + const msg = field(error, "message"); + throw new Error( + "film failed to load: " + (typeof msg === "string" && msg ? msg : String(error)) + ); + } + return { + meta: parseMeta(field(res, "meta")), + async frame(t) { + const url = await page.evaluate((tt) => window.__animation.frame(tt, { scale: 1 }), t); + if (typeof url !== "string" || !url.includes(",")) + throw new Error(`the film player returned no frame at ${t}s`); + return Buffer.from(url.slice(url.indexOf(",") + 1), "base64"); + }, + async warnings() { + const log = await page.evaluate(() => window.__animation.consoleLog()); + if (!Array.isArray(log)) return []; + return log + .filter((l: unknown) => isRecord(l) && l.level !== "log") + .map((l: unknown) => `${String(field(l, "level"))}: ${String(field(l, "text"))}`); + }, + }; +} + +async function waitReady(page: Page): Promise { + const deadline = Date.now() + 30_000; + while (!(await page.evaluate(() => !window.__film.ready || window.__film.ready() !== false))) { + if (Date.now() > deadline) throw new Error("the film page did not become ready in 30s"); + await new Promise((r) => setTimeout(r, 20)); + } +} + +async function htmlFilm(page: Page, dir: string): Promise> { + const log: string[] = []; + page.on("console", (m) => { + if (m.type() === "warning" || m.type() === "error") log.push(`${m.type()}: ${m.text()}`); + }); + page.on("pageerror", (e) => log.push(`error: ${e.message}`)); + await page.goto(pathToFileURL(resolve(dir, "index.html")).href); + await page.waitForFunction(() => typeof window.__film === "object"); + let meta: unknown; + try { + await waitReady(page); + meta = await page.evaluate(() => window.__film.load()); + } catch (e) { + throw new Error("film failed to load: " + errorMessage(e)); + } + let injected = false; + const seek = async (t: number) => { + await page.evaluate((tt) => window.__film.seek(tt), t); + await waitReady(page); + }; + return { + meta: parseMeta(meta), + async frame(t) { + await seek(t); + return page.screenshot({ type: "png" }); + }, + async inspect(t) { + await seek(t); + if (!injected) await page.addScriptTag({ path: SNAPSHOT }); + injected = true; + return parseBoxes(await page.evaluate(() => window.__layoutSnapshot())); + }, + warnings: () => Promise.resolve(log), + }; +} + // Opens the film and returns its meta and a way to grab frames as PNG. export async function openFilm(dir: string): Promise { checkFilmDir(dir); const browser = await launch(); try { - const page = await browser.newPage({ viewport: { width: 1920, height: 1080 } }); - await page.context().addInitScript(() => { - for (const k of ["RTCPeerConnection", "webkitRTCPeerConnection"]) - Reflect.deleteProperty(globalThis, k); - }); - let here: string | null = null; - await page.context().route("**/*", (r) => { - const u = r.request().url(); - if (!here && r.request().isNavigationRequest()) here = u.slice(0, u.lastIndexOf("/") + 1); - return (here && u.startsWith(here)) || /^(data|blob):/.test(u) ? r.continue() : r.abort(); - }); - await page.goto(pathToFileURL(resolve(dir, "index.html")).href + "#tools"); - await page.waitForFunction(() => typeof window.__animation === "object"); - const res = await page.evaluate((f) => window.__animation.load(f), readFilm(dir)); - const error = field(res, "error"); - if (error) { - const msg = field(error, "message"); - throw new Error( - "film failed to load: " + (typeof msg === "string" && msg ? msg : String(error)) - ); - } - const meta = parseMeta(field(res, "meta")); - return { - meta, - async frame(t) { - const url = await page.evaluate((tt) => window.__animation.frame(tt, { scale: 1 }), t); - if (typeof url !== "string" || !url.includes(",")) - throw new Error(`the film player returned no frame at ${t}s`); - return Buffer.from(url.slice(url.indexOf(",") + 1), "base64"); - }, - async warnings() { - const log = await page.evaluate(() => window.__animation.consoleLog()); - if (!Array.isArray(log)) return []; - return log - .filter((l: unknown) => isRecord(l) && l.level !== "log") - .map((l: unknown) => `${String(field(l, "level"))}: ${String(field(l, "text"))}`); - }, - close: () => browser.close(), - }; + const page = await openPage(browser); + const film = await (isHtmlFilm(dir) ? htmlFilm : motionFilm)(page, dir); + return { ...film, close: () => browser.close() }; } catch (e) { await browser.close(); throw e; diff --git a/skills/motion-render/scripts/snapshot.js b/skills/motion-render/scripts/snapshot.js new file mode 100644 index 0000000..5ade3f1 --- /dev/null +++ b/skills/motion-render/scripts/snapshot.js @@ -0,0 +1,174 @@ +// Injected into an html film page for the layout check (render.ts). It lists what paints on screen +// now, as boxes in frame px; layout.ts decides which of them are mistakes. Plain JS because it runs +// in the page, not in Node. +(function () { + var SIDES = ["Top", "Right", "Bottom", "Left"]; + var FRAME = { left: 0, top: 0, right: 1920, bottom: 1080 }; + var POINTS = [ + [0.5, 0.5], + [0.25, 0.25], + [0.75, 0.25], + [0.25, 0.75], + [0.75, 0.75], + ]; + + function alpha(color) { + var m = /rgba?\(([^)]*)\)/.exec(color); + if (!m) return 0; + var p = m[1].split(/[\s,/]+/).filter(Boolean); + return p.length > 3 ? parseFloat(p[3]) : 1; + } + + function ownText(el) { + return Array.prototype.some.call(el.childNodes, function (n) { + return n.nodeType === 3 && n.textContent.trim() !== ""; + }); + } + + // How an element paints: "solid" (hides what is under it), "box" (border or faint fill), + // "text" (glyphs only), or null (nothing of its own). + function paint(el, cs) { + if (/^(img|svg|canvas|video)$/i.test(el.tagName)) return "solid"; + if (alpha(cs.backgroundColor) >= 0.5 || cs.backgroundImage !== "none") return "solid"; + var border = SIDES.some(function (s) { + return parseFloat(cs["border" + s + "Width"]) > 0 && alpha(cs["border" + s + "Color"]) > 0; + }); + if (border || alpha(cs.backgroundColor) > 0) return "box"; + return ownText(el) ? "text" : null; + } + + function opacity(el) { + var o = 1; + for (var e = el; e; e = e.parentElement) o *= parseFloat(getComputedStyle(e).opacity); + return o; + } + + function cross(a, b) { + return { + left: Math.max(a.left, b.left), + top: Math.max(a.top, b.top), + right: Math.min(a.right, b.right), + bottom: Math.min(a.bottom, b.bottom), + }; + } + + // The area an element's children may paint in: the frame cut by every overflow-clipping ancestor. + function clipper() { + var memo = new Map(); + return function clipOf(el) { + if (!el || el === document.body || el === document.documentElement) return FRAME; + if (memo.has(el)) return memo.get(el); + var area = clipOf(el.parentElement); + if (getComputedStyle(el).overflow !== "visible") + area = cross(area, el.getBoundingClientRect()); + memo.set(el, area); + return area; + }; + } + + // Text without a box of its own takes the space of its glyphs, not of its (often wide) block. + // A line's box is about 1.45em tall in Noto Sans JP while the glyphs fill about 1em of it, so + // 0.2em is trimmed off the top and bottom; large numbers would otherwise overlap their labels. + function rectOf(el, kind, cs) { + if (kind !== "text") return el.getBoundingClientRect(); + var range = document.createRange(); + range.selectNodeContents(el); + var r = range.getBoundingClientRect(); + var trim = Math.min(0.2 * parseFloat(cs.fontSize), (r.height - 1) / 2); + return new DOMRect(r.x, r.y + trim, r.width, r.height - 2 * trim); + } + + function label(el) { + var name = el.id + ? "#" + el.id + : el.tagName.toLowerCase() + (el.classList[0] ? "." + el.classList[0] : ""); + var text = el.textContent.replace(/\s+/g, " ").trim().slice(0, 24); + return text ? name + ' "' + text + '"' : name; + } + + function ancestors(el, keyOf) { + var anc = []; + for (var e = el.parentElement; e; e = e.parentElement) if (keyOf.has(e)) anc.push(keyOf.get(e)); + return anc; + } + + function collect(all) { + var clipOf = clipper(); + var keyOf = new Map(); + var boxes = []; + all.forEach(function (el, key) { + if (el === document.documentElement || el === document.body) return; + if (el instanceof SVGElement && !(el instanceof SVGSVGElement)) return; + var cs = getComputedStyle(el); + if (cs.visibility !== "visible" || cs.display === "none") return; + var kind = paint(el, cs); + if (!kind || opacity(el) < 0.05) return; + var r = rectOf(el, kind, cs); + var v = cross(r, clipOf(el.parentElement)); + if (v.right - v.left < 1 || v.bottom - v.top < 1) return; + keyOf.set(el, key); + boxes.push({ + key: key, + label: label(el), + x: v.left, + y: v.top, + w: v.right - v.left, + h: v.bottom - v.top, + cut: r.left < -2 || r.top < -2 || r.right > 1922 || r.bottom > 1082, + anc: ancestors(el, keyOf), + ok: !!el.closest("[data-layout-ok]"), + solid: kind === "solid", + under: null, + }); + }); + return { boxes: boxes, keyOf: keyOf }; + } + + // The topmost box at a point that is not b itself, its ancestor or its descendant, if it is + // solid; null when b (or something of its own) is on top there. + function coverAt(b, x, y, keyOf, byKey) { + var stack = document.elementsFromPoint(x, y); + for (var i = 0; i < stack.length; i++) { + var p = stack[i]; + while (p && !keyOf.has(p)) p = p.parentElement; + if (!p) continue; + var o = byKey.get(keyOf.get(p)); + if (o === b || o.anc.indexOf(b.key) >= 0 || b.anc.indexOf(o.key) >= 0) return null; + if (o.solid) return o; + } + return null; + } + + // Hidden: at three or more of five points inside b, the same solid box is drawn over it. + function markHidden(boxes, keyOf) { + var byKey = new Map( + boxes.map(function (b) { + return [b.key, b]; + }) + ); + boxes.forEach(function (b) { + var count = new Map(); + POINTS.forEach(function (pt) { + var o = coverAt(b, b.x + b.w * pt[0], b.y + b.h * pt[1], keyOf, byKey); + if (o) count.set(o.key, (count.get(o.key) || 0) + 1); + }); + count.forEach(function (n, k) { + if (n >= 3) b.under = k; + }); + }); + } + + window.__layoutSnapshot = function () { + var got = collect(Array.prototype.slice.call(document.querySelectorAll("*"))); + // Hit testing skips pointer-events: none, which says nothing about what is drawn on top. + var style = document.createElement("style"); + style.textContent = "* { pointer-events: auto !important; }"; + document.head.append(style); + try { + markHidden(got.boxes, got.keyOf); + } finally { + style.remove(); + } + return got.boxes; + }; +})(); diff --git a/tests/html-film.test.ts b/tests/html-film.test.ts new file mode 100644 index 0000000..8d38138 --- /dev/null +++ b/tests/html-film.test.ts @@ -0,0 +1,130 @@ +// html films in headless Chromium: the kit, scene times, frames, and the layout check. +import { describe, expect, it } from "vitest"; +import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { execFileSync } from "node:child_process"; +import { openFilm } from "../skills/motion-render/scripts/render.ts"; +import { checkLayout, formatIssue } from "../skills/motion-render/scripts/layout.ts"; +import { writeKit } from "../skills/motion-render/scripts/kit.ts"; +import { tempDir } from "./helpers.ts"; + +const CLI = join(import.meta.dirname, "..", "skills", "motion-render", "scripts", "cli.ts"); + +const TIMING = { + total: 2, + scenes: [ + { id: "intro", name: "Intro", start: 0, dur: 1, speech: 0.8 }, + { id: "deny", name: "Deny", start: 1, dur: 1, speech: 0.8 }, + ], +}; + +// The scene of the WIF film that had the mistake: a token turned away at a condition box. With +// `wrong`, the ✕ is placed on its own and half of it ends up under the box. +function page(wrong: boolean): string { + const mark = wrong + ? `
✕
+
OIDC トークン
` + : `
OIDC トークン✕
`; + return ` + + + + +
+

条件

+
+
assertion.repository == 'org/app'
+
+ ${mark} +
+
+ +`; +} + +function htmlFilm(wrong: boolean): string { + const work = tempDir("mr-html-"); + writeFileSync(join(work, "timing.json"), JSON.stringify(TIMING)); + const film = writeKit(work); + writeFileSync(join(film, "index.html"), page(wrong)); + return film; +} + +describe("html film", () => { + it("takes scene times from timing.json and draws the same frame twice", async () => { + const film = await openFilm(htmlFilm(false)); + try { + expect(film.meta).toEqual({ + duration: 2, + parts: [ + { key: "intro", a0: 0, dur: 1 }, + { key: "deny", a0: 1, dur: 1 }, + ], + }); + const a = await film.frame(0.5); + const b = await film.frame(1.5); + expect(b.equals(a)).toBe(false); + expect((await film.frame(0.5)).equals(a)).toBe(true); + // film.spot sent the chip to the center of #dock. + const boxes = (await film.inspect?.(1.9)) ?? []; + const center = (l: string) => { + const b = boxes.find((x) => x.label.startsWith(l)); + return b && [Math.round(b.x + b.w / 2), Math.round(b.y + b.h / 2)]; + }; + expect(center("div.chip")).toEqual(center("#dock")); + expect(await film.warnings()).toEqual([]); + } finally { + await film.close(); + } + }); + + it("finds the ✕ half under the condition box, and passes once the ✕ sits in its chip", async () => { + const wrong = await openFilm(htmlFilm(true)); + try { + expect((await checkLayout(wrong))?.map(formatIssue)).toEqual([ + `1.2-1.8s: #cond "assertion.repository == " and #deny "✕" partly overlap`, + ]); + } finally { + await wrong.close(); + } + const right = await openFilm(htmlFilm(false)); + try { + expect(await checkLayout(right)).toEqual([]); + } finally { + await right.close(); + } + }); + + it("fails to load with a pointer to the kit when the font is missing", async () => { + const dir = htmlFilm(false); + rmSync(join(dir, "kit", "fonts"), { recursive: true }); + await expect(openFilm(dir)).rejects.toThrow(/Noto Sans JP.*motion-render kit/); + }); + + it("is set up by the kit command and checked by the check command", () => { + const work = tempDir("mr-html-"); + writeFileSync(join(work, "timing.json"), JSON.stringify(TIMING)); + const run = (...args: string[]) => + execFileSync(process.execPath, [CLI, ...args], { encoding: "utf8", stdio: "pipe" }); + expect(run("kit", work)).toContain("kit: "); + expect(JSON.parse(readFileSync(join(work, "film", "film.json"), "utf8"))).toEqual({ + kind: "html", + }); + expect(existsSync(join(work, "film", "kit", "gsap.min.js"))).toBe(true); + writeFileSync(join(work, "film", "index.html"), page(true)); + expect(() => run("check", join(work, "film"))).toThrow(/#cond .* and #deny "✕" partly overlap/); + writeFileSync(join(work, "film", "index.html"), page(false)); + expect(run("check", join(work, "film"))).toContain("layout ok"); + }); +}); diff --git a/tests/layout.test.ts b/tests/layout.test.ts new file mode 100644 index 0000000..003a58f --- /dev/null +++ b/tests/layout.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import { + findLayoutIssues, + formatIssue, + parseBoxes, + type LayoutBox, + type LayoutSample, +} from "../skills/motion-render/scripts/layout.ts"; + +const box = ( + key: number, + [x = 0, y = 0, w = 0, h = 0]: number[], + more: Partial = {} +): LayoutBox => ({ + key, + label: `#b${key}`, + x, + y, + w, + h, + cut: false, + anc: [], + ok: false, + under: null, + ...more, +}); + +// The same boxes at t = 0, 0.2, 0.4 ... +const held = (boxes: LayoutBox[], n = 3): LayoutSample[] => + Array.from({ length: n }, (_, i) => ({ t: i / 5, boxes })); + +describe("findLayoutIssues", () => { + it("reports two still boxes that partly overlap, once, with the span", () => { + const issues = findLayoutIssues(held([box(1, [0, 0, 100, 50]), box(2, [80, 10, 100, 50])], 4)); + expect(issues).toEqual([ + { kind: "overlap", from: 0.2, to: 0.6, message: "#b1 and #b2 partly overlap" }, + ]); + expect(formatIssue(issues[0]!)).toBe("0.2-0.6s: #b1 and #b2 partly overlap"); + }); + + it("leaves alone a box inside another, boxes that only touch, and nested elements", () => { + const boxes = [ + box(1, [0, 0, 400, 200]), + box(2, [20, 20, 100, 50]), // drawn inside 1 without being its child + box(3, [400, 0, 100, 50]), // touches 1's right edge + box(4, [380, 150, 100, 100], { anc: [1] }), // a child sticking out is "outside", not overlap + ]; + expect(findLayoutIssues(held(boxes)).map((i) => i.kind)).toEqual(["outside"]); + }); + + it("does not report a box while it moves, nor a pose held for a single sample", () => { + const card = box(1, [0, 0, 400, 200]); + const chip = (x: number) => box(2, [x, 50, 100, 40]); + const samples = [0, 200, 340, 360, 380].map((x, i) => ({ t: i / 5, boxes: [card, chip(x)] })); + expect(findLayoutIssues(samples)).toEqual([]); + // Still for two samples in a row: reported. + samples.push({ t: 1, boxes: [card, chip(380)] }, { t: 1.2, boxes: [card, chip(380)] }); + expect(findLayoutIssues(samples)).toHaveLength(1); + }); + + it("reports a box sticking out of its parent, out of the frame, or hidden under another", () => { + const boxes = [ + box(1, [100, 100, 300, 200]), + box(2, [350, 120, 100, 40], { anc: [1] }), + box(3, [1880, 500, 100, 40], { cut: true }), + box(4, [600, 600, 100, 60], { under: 5 }), + box(5, [640, 580, 200, 120]), + ]; + expect(findLayoutIssues(held(boxes)).map(formatIssue)).toEqual([ + "0.2-0.4s: #b2 sticks out of #b1", + "0.2-0.4s: #b3 is cut off by the frame", + "0.2-0.4s: #b4 is hidden under #b5", + ]); + }); + + it("does not call a card hidden by a chip resting inside it", () => { + const boxes = [box(1, [0, 0, 600, 120], { under: 2 }), box(2, [340, 30, 240, 64])]; + expect(findLayoutIssues(held(boxes))).toEqual([]); + }); + + it("skips boxes marked data-layout-ok", () => { + const boxes = [box(1, [0, 0, 100, 50]), box(2, [80, 10, 100, 50], { ok: true })]; + expect(findLayoutIssues(held(boxes))).toEqual([]); + }); +}); + +describe("parseBoxes", () => { + it("accepts the page's snapshot and rejects anything else", () => { + const b = box(1, [0, 0, 10, 10], { anc: [0], under: 3 }); + expect(parseBoxes([b])).toEqual([b]); + expect(() => parseBoxes({})).toThrow("malformed"); + expect(() => parseBoxes([{ ...b, x: "0" }])).toThrow("malformed"); + expect(() => parseBoxes([{ ...b, anc: ["1"] }])).toThrow("malformed"); + }); +});