diff --git a/.clusterfuzzlite/Dockerfile b/.clusterfuzzlite/Dockerfile new file mode 100644 index 00000000..f5cd127d --- /dev/null +++ b/.clusterfuzzlite/Dockerfile @@ -0,0 +1,6 @@ +# ClusterFuzzLite build image. The base is pinned by digest (Scorecard Pinned-Dependencies) and +# already carries atheris and PyInstaller; nothing is installed from a package index here. +FROM gcr.io/oss-fuzz-base/base-builder-python@sha256:21f6fab17ace4dc76e1c56de91ee95d3d03da021df94b41056ba036f6c6f55e9 +COPY . $SRC/ffmpeg-skill +WORKDIR $SRC/ffmpeg-skill +COPY .clusterfuzzlite/build.sh $SRC/ diff --git a/.clusterfuzzlite/build.sh b/.clusterfuzzlite/build.sh new file mode 100755 index 00000000..5bc5b0cc --- /dev/null +++ b/.clusterfuzzlite/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash -eu +# Build every fuzz/*_fuzzer.py into a standalone fuzzer. scripts/ is added to the bundle's import +# path so `_common` resolves inside the PyInstaller build exactly as it does from the checkout. +for fuzzer in "$SRC"/ffmpeg-skill/fuzz/*_fuzzer.py; do + compile_python_fuzzer "$fuzzer" --paths "$SRC/ffmpeg-skill/scripts" --paths "$SRC/ffmpeg-skill/fuzz" + name=$(basename "$fuzzer" .py) + seeds="$SRC/ffmpeg-skill/fuzz/seeds/${name%_fuzzer}" + if [ -d "$seeds" ]; then + (cd "$seeds" && zip -q -r "$OUT/${name}_seed_corpus.zip" .) + fi +done diff --git a/.clusterfuzzlite/project.yaml b/.clusterfuzzlite/project.yaml new file mode 100644 index 00000000..d1ad0ae5 --- /dev/null +++ b/.clusterfuzzlite/project.yaml @@ -0,0 +1 @@ +language: python diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1d7e0c5a..2aa9570d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,4 +1,5 @@ -# Keeps GitHub Actions pins (actions/checkout@v4, release-drafter@v6, etc.) and package.json's +# Keeps GitHub Actions pins (each `uses:` is a commit SHA with its tag in a comment; Dependabot +# updates both) and package.json's # npm metadata current, opening a PR for each update instead of letting either go stale silently. # There's no requirements.txt/pyproject.toml to watch here -- scripts/ is deliberately Python # 3.10+ standard library only, no third-party runtime dependencies (see README). diff --git a/.github/scripts/publish_plugin_branch.sh b/.github/scripts/publish_plugin_branch.sh index 98b1442b..00e604b8 100755 --- a/.github/scripts/publish_plugin_branch.sh +++ b/.github/scripts/publish_plugin_branch.sh @@ -113,7 +113,7 @@ README, the changelog and the tests are in the repository: https://github.com/ka Needs FFmpeg 6.1 or later and Python 3.10 or later. The tools themselves open no network connection and read no API key. The one exception is transcription (`caption.py --transcribe`, -`silence.py --filler --transcribe`), which runs a speech engine you installed yourself +`silence.py --transcribe` with `--filler` or a sentence flag), which runs a speech engine you installed yourself (whisper.cpp, faster-whisper, openai-whisper or parakeet): on its first run such an engine may download its model, and a Hugging Face client uses a token you are logged in with. PLUGIN_README diff --git a/.github/workflows/cflite.yml b/.github/workflows/cflite.yml new file mode 100644 index 00000000..489b327f --- /dev/null +++ b/.github/workflows/cflite.yml @@ -0,0 +1,42 @@ +# ClusterFuzzLite: build the fuzz/ targets and fuzz them for a few minutes on each pull request +# that touches Python, and longer once a week (Scorecard Fuzzing). Each target documents what it +# accepts as a refusal; anything else is a crash and fails the job. +# https://google.github.io/clusterfuzzlite/running-clusterfuzzlite/github-actions/ +name: ClusterFuzzLite +on: + pull_request: + paths: + - "scripts/**.py" + - "fuzz/**" + - ".clusterfuzzlite/**" + - ".github/workflows/cflite.yml" + schedule: + - cron: '17 3 * * 2' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + fuzz: + runs-on: ubuntu-latest + steps: + - name: Build fuzzers + id: build + uses: google/clusterfuzzlite/actions/build_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1 + with: + language: python + github-token: ${{ secrets.GITHUB_TOKEN }} + sanitizer: address + - name: Run fuzzers + uses: google/clusterfuzzlite/actions/run_fuzzers@884713a6c30a92e5e8544c39945cd7cb630abcd1 # v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + fuzz-seconds: ${{ github.event_name == 'schedule' && 1800 || 300 }} + mode: ${{ github.event_name == 'pull_request' && 'code-change' || 'batch' }} + sanitizer: address + output-sarif: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9dad33fa..0e20f10c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,14 +47,15 @@ jobs: runs-on: ${{ matrix.python == '3.10' && startsWith(matrix.os, 'ubuntu') && 'ubuntu-24.04' || matrix.os }} name: test (${{ matrix.os }}, py${{ matrix.python }}, ${{ matrix.part }}/3) steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + persist-credentials: false # Full history (not the default shallow depth-1 clone): test_contract.py's # test_changelog_mentions_every_closed_issue_since_last_tag walks git log back to the # latest tag to check CHANGELOG.md against "Closes #N" in commit bodies -- a shallow # clone would make that check silently a no-op (no tag reachable to diff against). fetch-depth: 0 - - uses: actions/setup-python@v7 + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: ${{ matrix.python }} - name: Install ffmpeg (Linux) @@ -106,7 +107,7 @@ jobs: # that is reported, not hidden, but the listings must still be uploaded python scripts/_contract.py doctor --json > listings/doctor.json || echo "doctor exit $?" cat listings/doctor.json - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: always() with: name: ffmpeg-listings-${{ matrix.os }}-py${{ matrix.python }}-${{ matrix.part }} @@ -139,8 +140,9 @@ jobs: apt-get update -qq DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ffmpeg python3 fonts-dejavu-core git nodejs ca-certificates git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + persist-credentials: false fetch-depth: 0 - name: ffmpeg version, listings and doctor run: | @@ -152,7 +154,7 @@ jobs: ffmpeg -hide_banner -bsfs > listings/ffmpeg_bsfs.txt python3 scripts/_contract.py doctor --json > listings/doctor.json || echo "doctor exit $?" cat listings/doctor.json - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: always() with: name: ffmpeg-listings-debian-trixie-ffmpeg7-${{ matrix.part }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e2944036..8b73e94b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -19,24 +19,30 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +# Read-only at the top; the one write the analysis needs is granted to its job (Scorecard +# Token-Permissions). permissions: contents: read - security-events: write jobs: analyze: runs-on: ubuntu-latest + permissions: + contents: read + security-events: write # upload the SARIF to code scanning strategy: fail-fast: false matrix: language: [python, javascript] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - - uses: github/codeql-action/init@v4 + - uses: github/codeql-action/init@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3 with: languages: ${{ matrix.language }} - - uses: github/codeql-action/analyze@v4 + - uses: github/codeql-action/analyze@33d7b85da1a04171e211befe2c739303b9a4c066 # v4.39.3 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml index 66ff2f8f..66d8e25c 100644 --- a/.github/workflows/dependabot-auto-merge.yml +++ b/.github/workflows/dependabot-auto-merge.yml @@ -11,17 +11,19 @@ on: types: [opened, reopened, synchronize] permissions: - contents: write - pull-requests: write + contents: read jobs: auto-merge: if: github.actor == 'dependabot[bot]' runs-on: ubuntu-latest + permissions: + contents: write # enable auto-merge + pull-requests: write # approve the PR steps: - name: Fetch Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v3 + uses: dependabot/fetch-metadata@25dd0e34f4fe68f24cc83900b1fe3fe149efef98 # v3.1.0 with: github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 4029656a..c2528f79 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -20,8 +20,6 @@ on: permissions: contents: read - pages: write - id-token: write concurrency: group: pages @@ -31,22 +29,27 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Assemble the site run: python3 .github/scripts/build_pages.py _pages - - uses: actions/configure-pages@v6 - - uses: actions/jekyll-build-pages@v1 + - uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + - uses: actions/jekyll-build-pages@44a6e6beabd48582f863aeeb6cb2151cc1716697 # v1.0.13 with: source: _pages destination: _site - - uses: actions/upload-pages-artifact@v5 + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 deploy: needs: build runs-on: ubuntu-latest + permissions: + pages: write # publish the artifact + id-token: write # verify the deployment's origin environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: - id: deployment - uses: actions/deploy-pages@v5 + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index d26ab554..af82f49c 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -34,7 +34,7 @@ jobs: pull-requests: write runs-on: ubuntu-latest steps: - - uses: release-drafter/release-drafter@v7 + - uses: release-drafter/release-drafter@978bb0b30c5ee8cdd332ced253758157150d43f0 # v7.10.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -48,6 +48,6 @@ jobs: # The dedicated autolabeler sub-action (not the main release-drafter action used above) -- # applies just the `autolabeler` rules from release-drafter.yml to this PR, with nothing # else running (no draft-release read/write against a PR ref that isn't main). - - uses: release-drafter/release-drafter/autolabeler@v7 + - uses: release-drafter/release-drafter/autolabeler@978bb0b30c5ee8cdd332ced253758157150d43f0 # v7.10.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7d62dcf3..62202e94 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,15 +59,17 @@ concurrency: cancel-in-progress: false permissions: - contents: write - pull-requests: read # resolve_version.py reads merged PRs' labels through gh api - id-token: write # required for `npm publish --provenance` + contents: read jobs: release: runs-on: ubuntu-latest + permissions: + contents: write # bump commit, tag, GitHub release, plugin branch + pull-requests: read # resolve_version.py reads merged PRs' labels through gh api + id-token: write # required for `npm publish --provenance` steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # The tip of main, not the SHA that triggered this run: with two merges in quick # succession the concurrency group queues the second run, and the first must bump on @@ -266,14 +268,14 @@ jobs: - name: Create tag and GitHub Release if: steps.check.outputs.already_released == 'false' - uses: softprops/action-gh-release@v3 + uses: softprops/action-gh-release@8a0cd651e5b2365bd640854a3fa5dfc70e4875f3 # v3.0.4 with: tag_name: ${{ steps.check.outputs.tag }} target_commitish: ${{ steps.check.outputs.sha }} body_path: release_notes.md generate_release_notes: true - - uses: actions/setup-node@v7 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 if: steps.check.outputs.already_released == 'false' with: node-version: "20" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7788962e..fa67cc99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,158 @@ ## Unreleased -(nothing yet) +### Security + +- **Every GitHub Action is pinned to a commit.** A tag can be moved; the release job's + third-party step ran on one after the release token was in the checkout. Each `uses:` now names + the commit of the tag it replaced (Dependabot keeps them current), and a contract test refuses + a tag reference. +- **Write scopes are granted per job.** No workflow grants a write scope at the top level any + more; CodeQL, Pages, Dependabot auto-merge and the release each grant theirs to the one job + that needs it, and checkouts that never push no longer keep the token. +- **The landing page loads its icon stylesheet with Subresource Integrity.** The unpkg script that + added six stylesheets at run time is replaced by the one stylesheet the page uses, pinned by + hash. +- **The input parsers are fuzzed.** `fuzz/` holds atheris targets for the time grammar, the SRT + reader and the hardened timeline XML reader; ClusterFuzzLite fuzzes them on each pull request + that changes `scripts/**.py`, `fuzz/` or the fuzzing setup, and for half an hour each week, and the contract suite runs every target on + its seed inputs so a target that stops importing fails CI. +- `SECURITY.md` links the private advisory form by its full address, which also resolves outside + GitHub's file view. +- **MCP annotations now match what a call can write.** `probe`, `check` and `cropdetect` were + annotated `readOnlyHint: true`, yet `plan` + `overwrite` let them replace any existing file, and + every tool said `destructiveHint: false`. Over MCP the server now runs the three read-only tools + with `FFMPEG_SKILL_READ_ONLY=1`, which refuses `--plan` structured or in any raw `argv` + spelling, and their schemas say so (`kind: input`, `error.reason: read_only_plan_refused`). Every tool + that writes is now `destructiveHint: true`. The CLI is unchanged. +- **A structured value is never read as an option.** MCP arguments and render.py's stage commands + pass a value that starts with `-` as one `--flag=value` token, so overlay text `-50%` or a title + `--help` works instead of failing with "expected one argument". Positionals that start with `-` + follow `--`, so an input named `--plan=x` can no longer set `--plan`. + +### Fixed + +- **`batch.py` project recipes read the project's side files beside the project.** Each item's + copy of the project was written into its work directory with its relative paths unchanged, and + `render.py` reads those against the project file's directory, so a project naming + `"music": "bed.wav"`, a brand, captions or a chapters file failed every item after its earlier + stages had encoded (the dry run said it would complete). The copy now carries them absolute. + The same list of path keys is what `render.py` applies to its own project, so + `audiogram.image` and `overlays[].font_file` are read against the project file too, not the + working directory, and `multicam.py --write-project` writes its clip paths absolute, so the + project renders from wherever it is written. +- **`batch.py --watch` no longer processes a source that is still being written.** A file copied + into the watched folder was processed as the fraction already there and reported `ok` (and + `verified` for MKV and TS); once the copy finished, every later pass refused that output with + `output_exists`, so the truncated file stayed. A source is processed once its size and mtime + are unchanged across one poll (first seen: last modified at least one interval ago); until then + its row is `skipped: "settling"`. A file an earlier pass of the same watch delivered (the + output and any side file a step wrote, such as an `.srt`), and nobody changed since, is + replaced when its source changes. Each file is checked on its own, by the output check of the + step that writes it (batch passes no `--overwrite`) or under the output lock when batch moves + the output into place, so an edited sidecar (also one edited while the same item was still + encoding: each file is recorded as it was when written), or an output another process put there + meanwhile, is still refused without `--overwrite`. +- **Text symbols are no longer taken for emoji.** `♪ ✓ ★` (not emoji in Unicode's emoji-data) + and `♥ ✔ ☺` (emoji only when VS16 follows) were counted as emoji because the whole Misc + Symbols, Dingbats and Misc Symbols and Arrows blocks were. A `mode: none` run (auto on macOS, or + on Linux with no colour emoji face) then cut SDH music notes and check marks out of the burned + captions and said no face could draw them. In those blocks only Emoji_Presentation characters + start an emoji on their own now, and a text-default one does with VS16 or a skin tone; the + playing cards and mahjong tiles other than 🀄 🃏 are text too. The same rule reaches + `graphics.py` and `overlay.py`, where such a title no longer switched to libass or reported + `emoji: mono`. +- **A wrap pass that moves a break keeps the space that stood there.** The orphan, weak-line and + rebalance passes guessed whether a space stood at a break from the script on each side, so a + moved break in Korean, in Japanese or Chinese written with spaces, next to an emoji, or between + Latin and CJK ran two words together (`늦지마세요`, `したよろしく`) while the result said + `text_unchanged: true`. The greedy fill now records each break's space and every pass carries + it. Korean is wrapped as the spaced script it is: at the spaces between words, never inside one + unless the word alone is wider than the line (`broken_inside_word`). Shared by `caption.py`, + `graphics.py` and `overlay.py --text`. +- **`caption.py` styles SRT markup on the routes where it writes its own ASS.** `--animate`, + `--karaoke`, an emoji-forced ASS and the tiktok/reels/shorts templates drew ``, ``, + `` and `{\an8}` as text, and counted them as width in the wrap and the size fit. + They become the overrides ffmpeg's own SRT route gives them (`` and `` too); the wrap, + the fit, the emoji placement and the blank-cue count measure the words only, and a cue split + across `--max-lines` keeps its italics and position on every part. Any other tag or `{...}` + block stays escaped text, as before; under `--karaoke` a `` is dropped for the + highlight's colours, and `--karaoke-style word` no longer resets the cue's style (`{\r}`) after + the active word. A tag pair around whitespace alone (`A B`) + stays around it through the wrap, rather than styling the next word, and a `{\an8}` that a + `--max-lines` split puts in a later part places the earlier parts too. +- **`speedramp.py`: a sped-up segment no longer leaves the rest of the picture late.** `setpts` + kept each sped-up frame's unscaled duration, and the concat started the next segment after it: + a 2x segment followed by a 1x one showed every later frame two frames (67 ms) after its sound + and wrote 92 frames for 90. Each segment's frames now get the source rate's durations back + (`fps` after `setpts`, as `reverse` and `freeze` do); `0-4:2` writes 60 frames, `0-4:0.5` 240. +- **`sync.py --trim-second` no longer stream-copies a video source file.** The copy started at + the keyframe before the trim point and showed the frames from it, so the "synced" file started + early by up to a GOP (with a 5 s GOP nothing was trimmed at all) and still reported `verified: + true`; even a keyframe on the trim point moved the picture by its B-frame delay. A video source + is now re-encoded from the frame at the trim point and re-measures at 0; a sound-only source is + still copied. +- **`cut.py`: an upper-case `.MOV`/`.MP4` output is an MPEG-4 output.** The join's container + checks compared the suffix case-sensitively, so a `--segments` re-cut of more than 32 ranges + (and every `silence.py` render keeping that many) into `IMG_0001_tight.MOV` wrote HEVC tagged + `hev1`, which Apple players refuse, with the index at the end, and a copy join into `.MP4` lost + `+faststart`. A `.TS` part list is also recognised as MPEG-TS for the copy join's check. +- **The start timecode moves with the start of the picture.** `cut.py` (copy and re-encode, one + range), `fit.py --method trim` and `sync.py --trim-second` wrote the source's start timecode + unchanged on a file that starts later (a cut from 1 s stamped 01:00:00:00 on frame 30), and so + did a `render.py` clip with an `in`, disagreeing with the render's own `--export-timeline`. They + now write the label of the frame the output starts on (drop-frame where the source is; a padded + head counts back). A join of several ranges writes no start timecode, and a stream copy out of + MXF writes none, both listed under `source_carry.dropped`. `source_carry` compares the + timecode's value, not only its presence. The label is counted at the rate the output is + delivered at (`fit.py --fps 50` on a 25 fps source starting 01:00:00:12 writes 01:00:00:24, and + a `render.py` clip reaching `frame.fps` the same), a cut that starts before a picture which + starts after the file's origin is stamped with that picture's first frame, and `sync.py + --trim-second` and `fit.py --method trim` seek a second early and drop the margin on the output + side, as `cut.py` does, so a B-frame HEVC camera does not lose the frames before a keyframe. + The source label and the offset to the first frame are rounded once, at the delivered rate, + and a delivered rate below 1 fps (where no frame count names a frame) writes no label instead + of failing. +- **`audio.py --downmix` no longer clips, and uses the weights the docs name.** pan's `=` summed + 2.7× per side (3.2× on 7.1) and mixed the LFE in at 0.5, so a 5.1 with every channel at + -6 dBFS came out hard-clipped (flat factor 20+) while the run said verified, and the weights + were not the ITU ones `references/scripts.md` claimed. The fold-down is now ITU-R BS.775 + (centre and surrounds at -3 dB, LFE dropped), scaled with pan's `<` so each side's gains sum to + 1: in-phase channels cannot clip, and a front channel alone is 7.7 dB lower (9.9 dB on 7.1). +- **`color.py --to-sdr` takes the source peak to SDR white; `look.py`'s HDR frames match it.** + The linearising zscale used `npl=--peak`, while `tonemap` reads linear 1.0 as 100 nits, so no + HDR pixel reached white: PQ at 1000 nits and HLG at 100 % came out at Y 151 of 235, HLG reference + white at 91, and raising `--peak` to a brighter master's darkened the picture further. It now + linearises at `npl=100` and passes `peak=--peak/100` (HLG: always 1000 nits, the peak zimg + decodes it to), so a PQ pixel at `--peak` nits and HLG at 100 % land on Y 235. `look.py` used + the same `npl=1000` chain for HDR sheets and tone-mapped a BT.2020 SDR source too; it now uses + `--to-sdr`'s chains at their defaults. `look.py --compare` converts each picture by its own + tags: the first input's tone map ran on both, so an SDR clip beside a PQ one was read as PQ + (and an HDR clip given second was shown untone-mapped). +- **`overlay.py --video --opacity` below 1 no longer shifts the colours of the whole frame.** The + alpha was scaled with `colorchannelmixer`, which takes RGB only, so FFmpeg composited the main + picture in RGB and it came back BT.601 under its BT.709 tag (the defect 2.10.2 removed from + `--image`). The alpha is now scaled in YUV (`lut`). This covers `render.py` video layers with an + `opacity` too. An untagged PiP over a BT.709-tagged main (or the other way round) is labelled BT.709 too: + FFmpeg 8.0's overlay converted the tagged picture to the untagged one's "unknown" (BT.601) + matrix, so the main picture shifted away from the PiP at any opacity. +- **Overlays onto a 10-bit HDR or ProRes source keep its 10 bits.** overlay's default format is + 8-bit, `format=auto` picks 8-bit beside an 8-bit layer and `drawbox` is 8-bit only on 6.1, so + `broll.py`, `redact.py` (blur, pixelate), `graphics.py` (bars, boxes, emoji), `caption.py`'s + emoji and `overlay.py --video` took the whole picture to 8 bits before the Main10 encoder, and + `overlay.py --image` took it through 8-bit RGB, shifting its colours (Y/Cb/Cr 402/602/422 came + out 399/608/418). These overlays now run at 10 bits in the source's chroma subsampling + (`overlay_format()`), graphics.py draws its boxes as layers there, and a 10-bit PiP stays + 10-bit. +- **The sentence flags from 3.6.0 are named wherever transcription is.** `silence.py + --transcribe` with `--sentence-min-silence` or `--sentence-margin-tail` runs the speech engine + (under `--dry-run` too, and it may download a model on first use), but the contract's + capabilities, refusal reasons and model-download statement, `PRIVACY.md`, the plugin README and + `README.md` named only `--filler --transcribe`. README's `silence.py` row and SKILL.md's routing + table now carry the 3.5.0 and 3.6.0 flags. +- **`silence.py`'s sentence lookup is a binary search.** Placing the word before each silence + scanned the whole transcript once per silence: about 1.2 s for an hour of speech and 11 s for + three hours. It is now proportional to the number of silences times the log of the words. ## 3.6.1 diff --git a/PRIVACY.md b/PRIVACY.md index 934d731e..da0dfba6 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -8,7 +8,8 @@ they do with data, as of the version it ships with. - **Your media stays local.** Video, audio, subtitles and project files are read from and written to the paths you give. Nothing is uploaded. - **No network use by the tools themselves**, and no API key is read. One exception: - transcription (`caption.py --transcribe`, `silence.py --filler --transcribe`) runs a speech + transcription (`caption.py --transcribe`, `silence.py --transcribe` with `--filler` or a sentence + flag) runs a speech engine you installed yourself (whisper.cpp, faster-whisper, openai-whisper or parakeet). On its first run such an engine may download its model from its own provider, and a Hugging Face client uses a token you are logged in with. What those engines and providers do is covered by diff --git a/README.md b/README.md index 0fc36ced..08b59d1d 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ An agent that "knows FFmpeg" still guesses: it assumes a frame rate, picks a cod - **Structured tools, not shell strings.** Each operation is a script with typed arguments. Nothing runs through a shell; no filter graph is accepted from the caller. - **A contract the agent can read.** `contract --json` states, for every tool, what it takes, what it writes, which FFmpeg components it needs and how the result is verified. The MCP surface is derived from it. - **Verification after execution.** The result is probed, checked against the destination's spec and, when the picture changed, looked at as a contact sheet. -- **Local first.** No cloud, no API keys, no Python dependencies. Optional local transcription is used when a speech engine (Parakeet or a whisper) is installed, never required — by `caption.py --transcribe` and, since 1.17, by `silence.py --filler --transcribe`; both take a transcript you already have instead, and both refuse with the install lines rather than guessing. Both take `--engine` and `--model`; `FFMPEG_SKILL_ASR_ENGINE` and `FFMPEG_SKILL_ASR_MODEL` set this machine's defaults (e.g. `large-v3-turbo` where that whisper.cpp model is installed; without either, auto and `base`). The engine runs on your machine and the audio is never uploaded; parakeet-mlx, faster-whisper and openai-whisper download their model weights the first time they run (parakeet-mlx's default from Hugging Face). +- **Local first.** No cloud, no API keys, no Python dependencies. Optional local transcription is used when a speech engine (Parakeet or a whisper) is installed, never required — by `caption.py --transcribe` and, since 1.17, by `silence.py --transcribe` (with `--filler`, or since 3.6 with `--sentence-min-silence` / `--sentence-margin-tail`); both take a transcript you already have instead, and both refuse with the install lines rather than guessing. Both take `--engine` and `--model`; `FFMPEG_SKILL_ASR_ENGINE` and `FFMPEG_SKILL_ASR_MODEL` set this machine's defaults (e.g. `large-v3-turbo` where that whisper.cpp model is installed; without either, auto and `base`). The engine runs on your machine and the audio is never uploaded; parakeet-mlx, faster-whisper and openai-whisper download their model weights the first time they run (parakeet-mlx's default from Hugging Face). ## Paper @@ -223,7 +223,7 @@ These are the rules the skill file gives the agent and the code enforces. |---|---| | `cut.py` | In/out or multi-segment cuts, lossless `-c copy` first, re-encode fallback, `--accurate` for frame-exact video and sample-exact audio, `--smart` for a frame-exact H.264 cut that re-encodes only the partial GOPs at the cut points (matched to the source, verified against its packets and frames) and copies the rest, an SDR HEVC source re-encodes as HEVC (`--no-keep-hevc` for x264), a single `.mp4`/`.mov` copy keeps its edit list, so the picture starts at `--start` unless a keyframe lands later (`start_snapped` then says so; `--no-edit-list` for 2.x's shifted start), `--vfr-guard` to choose the VFR check (`sampled` by default); reports `precision`; `--snap beats` moves the in/out points onto a measured beat, or refuses when there is no measurable pulse ; `--remove A-B,...` drops ranges and keeps the rest | | `join.py` | Concatenate clips with xfade transitions, normalising size, fps, sample rate and channel layout (the widest clip's, or `--channels`); audio-only inputs are joined as audio; `--list parts.txt` reads generated segments (TTS lines) and checks every one first — missing, empty or unreadable parts are all named in one refusal, or skipped and reported with `--on-missing skip` ; `--copy --transition none` joins clips with matching streams by stream copy, measured afterwards, and refuses a mismatch by name | -| `silence.py` | Detect and remove dead air (jump cuts) with a margin around speech (`--margin`, 0.2 s each side); list or export the cut list, or a source-to-output time map (`--timemap`, JSON or CSV); reports the level at every cut and warns where one may land in a breath or a word's tail; `--filler` also removes filler words, but only where a speech engine timed them; `--speech-aware` keeps a breath inside a sentence and cuts only at sentence-boundary pauses, composing with `--filler` through the same `keep_ranges()` | +| `silence.py` | Detect and remove dead air (jump cuts) with a margin around speech (`--margin`, 0.2 s each side); list or export the cut list, or a source-to-output time map (`--timemap`, JSON or CSV); reports the level at every cut and warns where one may land in a breath or a word's tail; `--filler` also removes filler words, but only where a speech engine timed them; `--speech-aware` keeps a breath inside a sentence and cuts only at sentence-boundary pauses, composing with `--filler` through the same `keep_ranges()`; `--margin-lead` / `--margin-tail` keep different amounts before and after speech, and with word timings `--sentence-min-silence` / `--sentence-margin-tail` give the pause after a sentence end (the transcript's punctuation) its own floor and tail | | `fit.py` | Fit to a duration (pitch-preserving speed change or trim, smooth slow-mo) and/or aspect ratio (pad, crop or `--fit blur`'s blurred, darkened fill, with `--crop-x`/`--crop-y` to keep an off-centre subject) and/or exact `--width`/`--height`; rotate 90/180/270, flip h/v; force constant fps | | `crop.py` | Crop to an exact pixel rectangle (`--x --y --width --height`) — distinct from `fit.py --fit crop`, which crops to an aspect ratio it computes itself | | `cropdetect.py` | Measure existing black letterbox/pillarbox bars and report the `crop.py`-ready rectangle that removes them — analysis only, writes no file; `--motion-centre` reports the per-second motion centroid instead, for a 9:16 reframe — report only, the calling agent picks the crop, no auto-reframing | @@ -364,6 +364,8 @@ On Windows, `python3` is only on PATH if Python was installed from the Microsoft No `inputSchema` lists `json` or `progress`: they are transport flags the server sets itself, not tool arguments (2.0; 1.x listed them unless `FFMPEG_SKILL_MCP_LEAN=1`). A client that still sends them is not refused. +A structured value is never read as an option: one that starts with `-` (overlay text `-50%`) is passed as `--flag=value`, and positionals that start with `-` follow `--`. `probe`, `check` and `cropdetect` are annotated `readOnlyHint` and write no file over MCP (their `plan` is refused, structured or in a raw `argv`); every other tool is `destructiveHint`, because with `overwrite` it replaces an existing file. See `docs/contract.md` § MCP relationship. + The server also advertises a `prompts` capability: five canned workflow recipes an agent can ask for by name instead of composing the individual tool calls itself — `reel`, `podcast`, `multicam`, `delivery_check`, `hdr`. `prompts/list` returns each one's name, description and arguments; `prompts/get {name, arguments}` fills the matching template and returns one text message naming the actual command lines to run and what to check afterward. These are recipes, not new tool calls — every command line a prompt names is one `tools/call` (or the CLI) can already run. ### Capability detection @@ -458,7 +460,7 @@ They are the most used tools here, and they are tested on every pull request and `color.py --to-sdr` converts PQ (HDR10), HLG and iPhone Dolby Vision 8.4 to SDR BT.709 with real tone mapping (`zscale` and `tonemap`, `hable` by default; `mobius`, `reinhard`, `clip`, `linear` and `gamma` with `--tonemap`). BT.2020 footage on an SDR transfer gets a gamut conversion only. Watch for four things: - It needs `zscale` (libzimg). The plain Homebrew `ffmpeg` lacks it, so use `ffmpeg-full` (keg-only: see [Install](#install) for putting it on `PATH`); `doctor` reports whether it is there. -- `--peak` (default 1000 nits) should match how bright the source was mastered. A wrong value makes highlights dull or clipped. +- `--peak` (default 1000 nits) is the PQ source's brightest level, the one that becomes SDR white: set it to how bright the source was mastered (its MaxCLL or mastering peak). Too high dims the picture; too low clips the highlights. HLG always peaks at 1000 nits, so `--peak` does not apply to it. - Tone mapping is a judgement, and the result says so (`grade_look` in `unverified`). Compare frames with `look.py`, and try another `--tonemap` if skin or skies look off. - An HDR source that only needs to play everywhere may not need converting at all. `--strip-dovi` removes the Dolby Vision layer from an iPhone clip, keeping its HLG picture, with a stream copy. @@ -468,7 +470,7 @@ ffmpeg-skill edits footage you already have, on this machine, with FFmpeg tools ### Does it validate the filter graph before running? -No. `--dry-run` prints the ffmpeg command a writing tool would run and does not start ffmpeg, so ffmpeg never parses that filter graph. ffprobe still runs when the tool probes an input that exists. Unless `--overwrite` is passed, an existing output is refused (`kind: input`) before ffmpeg. An output that is the same file as an input is refused either way. No media file is written, and `verified` is false; `--plan FILE` does write the plan JSON. A path that is not on disk yet is not measured: the probe is a stub (0×0, duration 0), so a number taken from it is not a measurement. The measuring tools (`probe`, `check`, `sync`, `multicam`, `scenes`, `cropdetect`, `report`, `silence`, `loudness`, `stabilize`) still run their measurements; only the artifact is skipped, and `silence` with `--filler --transcribe` also runs the speech engine, which may download a model. `overlay` measures the lines of a multi-line `--text` with one drawtext on a generated frame on FFmpeg 8.0, 8.0.1 and 8.1.0, where it draws them one by one. `verify` accepts `--dry-run` and ignores it: its steps run. `--plan FILE` is that dry run written down — the planned argv, the command lines, and a fingerprint of each existing input (size, plus a sha256 of the first and last 8 MiB). `render.py` and `verify.py` refuse `--plan`. `render.py FILE` runs a plan only when those fingerprints still match (a mismatch is refused with each input's expected and actual fingerprint in `error.details`, and logged to `.audit.jsonl`), by running the tool again with the planned argv, not by executing the recorded ffmpeg lines; `--dry-run` on that run only prints them. `doctor` reports whether the filters, encoders and bitstream filters the contract names are in this build (`ffmpeg -filters`, `-encoders`, `-bsfs`) and which fonts are installed. A name in that listing is not a check of the graph this command would build, and a listed GPU encoder is not a successful encode. +No. `--dry-run` prints the ffmpeg command a writing tool would run and does not start ffmpeg, so ffmpeg never parses that filter graph. ffprobe still runs when the tool probes an input that exists. Unless `--overwrite` is passed, an existing output is refused (`kind: input`) before ffmpeg. An output that is the same file as an input is refused either way. No media file is written, and `verified` is false; `--plan FILE` does write the plan JSON. A path that is not on disk yet is not measured: the probe is a stub (0×0, duration 0), so a number taken from it is not a measurement. The measuring tools (`probe`, `check`, `sync`, `multicam`, `scenes`, `cropdetect`, `report`, `silence`, `loudness`, `stabilize`) still run their measurements; only the artifact is skipped, and `silence` with `--transcribe` (with `--filler` or a sentence flag) also runs the speech engine, which may download a model. `overlay` measures the lines of a multi-line `--text` with one drawtext on a generated frame on FFmpeg 8.0, 8.0.1 and 8.1.0, where it draws them one by one. `verify` accepts `--dry-run` and ignores it: its steps run. `--plan FILE` is that dry run written down — the planned argv, the command lines, and a fingerprint of each existing input (size, plus a sha256 of the first and last 8 MiB). `render.py` and `verify.py` refuse `--plan`. `render.py FILE` runs a plan only when those fingerprints still match (a mismatch is refused with each input's expected and actual fingerprint in `error.details`, and logged to `.audit.jsonl`), by running the tool again with the planned argv, not by executing the recorded ffmpeg lines; `--dry-run` on that run only prints them. `doctor` reports whether the filters, encoders and bitstream filters the contract names are in this build (`ffmpeg -filters`, `-encoders`, `-bsfs`) and which fonts are installed. A name in that listing is not a check of the graph this command would build, and a listed GPU encoder is not a successful encode. ### Can it remove burned-in subtitles or logos? @@ -590,7 +592,7 @@ Where it can be installed from today. Each route is spelled out below the table. | MCP Registry | clients that install from the registry | `io.github.kajisho5/ffmpeg-skill` ([entry](https://registry.modelcontextprotocol.io/v0/servers/io.github.kajisho5%2Fffmpeg-skill/versions/latest)); the release workflow publishes a new version there after npm, and a failed registry step leaves the entry on the previous version | | GitHub release | a pinned copy | [releases](https://github.com/kajisho5/ffmpeg-skill/releases) | -Not listed in a directory yet: the Claude plugin directory, the Cursor Marketplace, the Gemini CLI extension gallery and Glama (each is submitted and waiting on its review or crawl). This table gains a row when a listing goes live; until then the routes above are the whole list. +Not listed in a directory yet. The Claude plugin directory accepts a plugin that ships scripts or programs only from an organization Anthropic has marked Verified, and there is no way to ask for that yet, so the submission is on hold (the tools are Python files; a text-only build would install nothing that runs). The Cursor Marketplace, the Gemini CLI extension gallery and Glama are submitted and waiting on their review or crawl. This table gains a row when a listing goes live; until then the routes above are the whole list. ```bash npx ffmpeg-skill # Claude Code → ~/.claude/skills/ffmpeg-skill diff --git a/SECURITY.md b/SECURITY.md index 6407793c..ad4f69d6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,7 +2,7 @@ ## Reporting a vulnerability -Please report security issues privately using GitHub's [Report a vulnerability](../../security/advisories/new) +Please report security issues privately using GitHub's [Report a vulnerability](https://github.com/kajisho5/ffmpeg-skill/security/advisories/new) flow (Security tab → Advisories → "Report a vulnerability") rather than a public issue or PR. That opens a private advisory only maintainers can see until a fix is ready. diff --git a/SKILL.md b/SKILL.md index f306d677..ae53c4f1 100644 --- a/SKILL.md +++ b/SKILL.md @@ -102,6 +102,7 @@ Variants, caveats and extra flags per row: `references/requests.md`. | "cut out the pauses", "jump cuts" | `silence.py in.mp4` | | "cut the ums and uhs" | `silence.py in.mp4 --filler --words words.json` | | "just the real pauses, not inside a sentence" | `silence.py in.mp4 --speech-aware` | +| "tighter inside a sentence, more room after one" | `silence.py in.mp4 --words w.json --min-silence 0.2 --sentence-min-silence 0.3 --sentence-margin-tail 0.3` | | "a 60 s highlight from this hour" | `scenes.py long.mp4 --highlights 6 --target 60 --edl picks.txt` (then `cut.py --segments` with picks.txt's lines joined by commas) | | "cut on the beat" | `cut.py in.mp4 --segments ... --snap beats --snap-source beats.json` (`scenes.py --beats`; a WAV/MP3 music bed too) | | "speed up here, slow-mo there" | `speedramp.py in.mp4 --segment 0-3:1.0 --segment 3-4:0.25` | diff --git a/demos/build.py b/demos/build.py index 2f7f7005..86c5d3f1 100644 --- a/demos/build.py +++ b/demos/build.py @@ -1699,7 +1699,7 @@ def demo_switch_energy(ctx): "Two showwavespic plots: the quiet input on the left, the same programme brought up to broadcast level on the right without clipping.", demo_loudness, "wave", None), ("downmix_51", AUDIO, "5.1 to stereo downmix", - "Six discrete tones folded into two channels at the standard coefficients -- the centre and LFE are still audible.", + "Six discrete tones folded into two channels with the ITU-R BS.775 weights -- the centre and surrounds are still audible, the LFE is dropped, and nothing clips.", demo_downmix_51, "wave", None), ("bgm_ducking", AUDIO, "Music bed with ducking", "The bed drops by 12 dB whenever the speech-like track is active and comes back up in the pauses.", diff --git a/docs/contract.md b/docs/contract.md index dc23b6e3..c67dcc57 100644 --- a/docs/contract.md +++ b/docs/contract.md @@ -446,7 +446,7 @@ source is refused (`color.py --to-sdr` first). `filter:boxblur` stays required on `redact`, because `--mode blur` is the default. `external:whisper` is **optional** for two tools since 1.17: `caption.py` -(`--transcribe`) and `silence.py` (`--filler --transcribe`). Neither requires +(`--transcribe`) and `silence.py` (`--transcribe` with `--filler`, or since 3.6 with a sentence flag). Neither requires it — both take a transcript the caller already has (`--srt`/`--words`), and both refuse with the same install lines when asked to make one with no engine present. Whisper is never a dependency of this skill. @@ -637,6 +637,14 @@ The mapping is stated in `invocation.structured.argument_mapping`: positionals i arrays repeated, `output` → `-o`, `loudness.lufs` → `-I`. `--json` is appended for every tool except `probe` (JSON by default) and `look`. +A value is always a value. One that starts with `-` is passed as a single `--long-flag=value` +token (`{"text": "-50%"}` → `--text=-50%`, `{"lufs": -16}` → `--lufs=-16`), because argparse +reads a separate token that starts with `-` as the next option. When any positional starts with +`-`, every positional follows `--` after all the options, so `{"inputs": ["--plan=x"]}` names a +file called `--plan=x` instead of setting `--plan`. `render.py` builds its stage commands the +same way (`_common.option_args`), so an overlay or a title that starts with `-` works in a +project too. + The MCP server also accepts `{"argv": [...]}` for CLI compatibility. That path is marked `canonical: false`: it is still bound to the named script and never reaches a shell, but an agent ecosystem should use the structured form. No tool, CLI or MCP, @@ -723,7 +731,7 @@ Per-tool keys added after 2.2.2, all additive: | `silent` | `waveform.py` (every run) | `true` when the input audio's whole-file peak is at or below `--silence-threshold` (default -50 dBFS), which draws a flat line; `--on-silent warn` (default) renders it anyway with a `notes` line, `fail` refuses (`kind: input`) before ffmpeg runs. `false` when audible, `null` under `--dry-run` (nothing measured). `verified` is unaffected | | `encoder`, `hw`, `notes` | every re-encoding tool, `export.py`, `render.py`, `batch.py` (and its rows), `waveform.py` | `encoder`: the video encoder the last command ran (after any GPU→CPU fallback; under `--dry-run`, the planned one), `copy` for a stream copy; absent when no command encoded video. A tool that runs other tools reports its stages': `render.py` its stages and an executed plan's tool, `waveform.py` the title/caption stage that writes its file (its `commands` include that stage's). `hw` is present when VideoToolbox was asked for (`--hw`, `FFMPEG_SKILL_HW=1`, or a stage or `batch.py` item that reported it), on `export.py` when `FFMPEG_SKILL_HW=1` is set but its delivery preset does not take it, and when an encode ran on VideoToolbox because the build has no `libx264`/`libx265`: `{requested, source: flag\|env\|auto, used, fallback, notes}`. `requested` says whether this run's encodes were asked onto VideoToolbox (`false` for that export, and for `source: "auto"`, where nobody asked: `notes` names the missing CPU encoder and the one that ran, and `--no-hw` is not offered as a remedy since there is no CPU encoder to go back to; a job VideoToolbox refuses there is a plain `kind: ffmpeg` failure, not retried), `used` whether the encoder that ran last is VideoToolbox (`null` under `--dry-run`, which ran nothing, and when the tool encoded nothing itself: `batch.py`, whose top-level `hw` gathers its items' `requested`/`source`/`fallback`/`notes` while each `results` row carries the item's own `encoder`, and `hw`/`notes` when VideoToolbox was asked for), `fallback` whether a job VideoToolbox refused was re-encoded on the CPU, and `notes` why. Only a VideoToolbox failure (the encoder could not open, or its compression session failed) is retried on the CPU; any other failure is reported as it happened, once. Every VideoToolbox encode adds a top-level `notes` line: its quality is not CRF-equivalent (`--quality` / a preset's CRF maps to `-q:v` approximately), and `verified` stays what it is for every tool, the measured properties of the output. A `kind: ffmpeg` failure under `--hw` / `FFMPEG_SKILL_HW` carries `encoder` and `hw` for the command that ran last (a stage's, when `render.py`/`waveform.py` re-raise it); when the command itself failed inside the run (not a tool's own check afterwards, such as `cut.py`'s), its message also names that encoder | | `transcription` | `caption.py --transcribe` (every `--mode`); inside `filler` for `silence.py --filler --transcribe` | `{routing, detected_language?, engine, model, language}`: the engine and model that made the transcript; `language` is the language the engine was told -- the one named (`--language`, `--filler-lang`), else the one auto's detector named -- and, when nothing was named, the language the engine decoded where it reports one (whisper.cpp's word-timing JSON, faster-whisper, openai-whisper's word-timing JSON), else `null`; `detected_language` is present when auto's whisper.cpp detector named one; `routing` says why `--engine auto` chose the engine (`"requested"` for a named engine), ending `, no Parakeet engine transcribed it` when auto chose Parakeet and a Whisper engine made the transcript after every Parakeet engine failed | -| `notes` | `silence.py --filler --transcribe` (new for `silence.py`; `caption.py` already had it) | present when `--engine auto` ran the English-only Parakeet model on English *assumed* (no language given or detectable, and no Whisper engine installed or none that succeeded): the transcript is wrong if the speech is not English. `caption.py --transcribe` puts the same line in its `notes` | +| `notes` | `silence.py --transcribe` with `--filler` or a sentence flag (new for `silence.py`; `caption.py` already had it) | present when `--engine auto` ran the English-only Parakeet model on English *assumed* (no language given or detectable, and no Whisper engine installed or none that succeeded): the transcript is wrong if the speech is not English. `caption.py --transcribe` puts the same line in its `notes` | | `reencode_reason`, `segment_precision`, `least_exact_precision`, `start_snapped` | `cut.py` (every run) | `reencode_reason`: why anything was re-encoded, distinct values in first-seen order, `[]` for a clean copy: `requested` (`--accurate`), `codec` (`--codec`), `vfr`, `vfr_inconclusive`, `pcm_container`, `copy_failed`, `stream_lost` (the copy wrote no video or no audio: an MPEG-TS copy whose window holds no video keyframe, #367), `tolerance`, `concat_fallback` (a `--segments` join that could not stay a stream copy, which now also sets `reencoded: true` and `mode: hybrid`; it reported `copy` before). `segment_precision`: each `--segments` segment's precision (`null` for one segment); `least_exact_precision`: the least exact of them, on every run. Since 3.0.0 `precision` is the least exact segment's, equal to `least_exact_precision`, and `keyframe_snapped` equals `start_snapped` (2.x: `precision` was the whole run's, `packet` only when nothing re-encoded, and `keyframe_snapped` was `precision == "packet"`, a stream copy end to end; see "What 3.0 changed"). `start_snapped`: measured, `true` when a copied segment's picture starts more than one frame from its requested start (or its start could not be measured: an unknown start is never claimed exact), `false` when every start is where asked or everything re-encoded, `null` under `--dry-run`. `requested_segments`/`requested_duration` keep their meaning, the request clamped to the media's duration: a segment ended with the video (less than a frame of sound trimmed) shows in `duration_delta_seconds` and a `notes` line | | `edit_list`, `stored_preroll_seconds`, `av_start_skew_seconds`, `notes` | `cut.py` (every run) | `edit_list`: `true` only for a single-segment `.mp4`/`.m4v`/`.mov` stream copy, which by default since 3.0.0 keeps the MP4 edit list the copy writes (`--no-edit-list` restores 2.x's `-avoid_negative_ts make_zero`): the keyframe's pre-roll is stored but hidden, so the picture starts at `--start`. `stored_preroll_seconds`: with `edit_list`, the seconds decoded from the keyframe the copy's seek lands on (measured with the seek ffmpeg makes) up to the first presented frame; `null` otherwise. With an edit list a copy is judged by its video's length (a `notes` line says so when the container is longer), and one that starts where asked but ends past `--tolerance` re-encodes (`tolerance`) without a `lossless_alternative`. `av_start_skew_seconds`: the output's audio start minus its video start (`null` when either is missing or under `--dry-run`); on a stream copy, when it is more than max(2 frames, 0.1 s) away from the source's own audio/video offset where the copy's picture starts (the keyframe it began from; `--start` with an edit list), a `notes` line names `--accurate`, and the edit list for a single `.mp4`/`.mov` copy whose start snapped under `--no-edit-list`. A re-encode carries no such note. The skew is reported, never repaired. `notes`: what the run did that a caller should know, in plain sentences (`[]` when nothing) | | `video.start_time`, `audio.start_time` | `probe.py` (and every `probe` block a tool returns) | where each stream starts, in seconds on the file's own clock (ffprobe's stream `start_time`; `null` when unknown). A video stream that starts after its audio, or the reverse, is how a lossless cut's A/V skew shows; `cut.py` reads them for `av_start_skew_seconds` | @@ -817,7 +825,7 @@ Per-tool keys added after 2.10.11 ("nothing happens silently"), all additive: | `ass_shifted`, `ass_unshifted`, `ass_malformed` | `caption.py --ass` (inside `caption`) | with `--offset`: the Dialogue lines moved, and those whose timestamps could not be read (copied as they were); every run: Dialogue lines that could not be read (too few fields, an unreadable start/end) and are not counted as cues. A count above 0 is a warning | | `matched_untimed`, `matched_too_long` | `silence.py --filler` (inside `filler`) | listed words that were not cut: no usable timing in the transcript (also a `filler_words_untimed` warning), or longer than a filler (`FILLER_MAX_WORD`, a held vowel) | -`caption.py --transcribe` and `silence.py --filler --transcribe`: when a local speech engine ran +`caption.py --transcribe` and `silence.py --transcribe` (with `--filler` or a sentence flag): when a local speech engine ran and produced no cue (for `--filler`: a Whisper transcript with no text at all; text without word timings is a build without them, refused as such), the refusal is `" found no speech in "` with `kind: input`, `reason: "no_speech"` and `engine` in the failure document (the @@ -926,8 +934,13 @@ says it is kept without the step that was stopped, and the document carries it a besides the picture and the sound with the file it wrote, and says so in `source_carry: {kept: [...], dropped: [...]}`. The items are `subtitles`, `chapters`, `cover_art`, `audio_channels` (a surround source only: more than two), `creation_time`, `audio_language` and -`timecode`. The key is present only when the source had at least one of them, so a result for a -plain source is unchanged. It is absent under `--dry-run`, for a still or GIF output, and when two +`timecode`. A timecode is compared by its value: it is kept when the written start timecode +labels the frame the output starts on, which is the source's own label or, where the tool moved +the picture's start (`cut` from a later frame, `fit --method trim`, `sync --trim-second`, a +`render.py` clip with an `in`), the label of the source frame the output now starts on. A join of +several ranges (`cut --segments`, `--remove` leaving two or more) writes no start timecode, since +no single start labels the frames after the first join. The key is present only when the source +had at least one of them, so a result for a plain source is unchanged. It is absent under `--dry-run`, for a still or GIF output, and when two or more inputs have a picture or sound (which one's items the output should carry is the tool's choice). An item bound to a stream the output does not have at all is not compared (a sound-only extract has no subtitle track to lose). It is a report, not a check: `verified` does not depend on @@ -977,7 +990,8 @@ failed row's `error: {step, tool, kind, message, code, reason, details, hint, ex failing step's own document, and `cache_miss` when a cache entry was not served because its output changed or could not be checked; an interrupted batch has a row for every input (`interrupted: true`, or `skipped: "interrupted"` when it had not started) and `total` counts them -all. `verify.py`'s steps keep the tool's `kind`, `reason`, `message` and `exit_code`, and the steps +all; under `--watch` a source that has not settled since the previous poll is `skipped: "settling"` +(`ok: false`, not cached) and processed on a later pass. `verify.py`'s steps keep the tool's `kind`, `reason`, `message` and `exit_code`, and the steps that do not apply are listed under each file's `skipped` with a reason. **Audit log.** Events a later reader must be able to reconstruct are appended as JSON lines, @@ -1040,6 +1054,20 @@ caller chooses. 1.10.0-1.26 listed them unless `FFMPEG_SKILL_MCP_LEAN=1`; 2.0 le good (see "What 2.0 changed"). A client that still sends either is not refused: the server maps it like any other key. +### Annotations: read-only means it writes no file + +Each `tools/list` entry carries MCP `annotations` (`_contract.mcp_annotations`). `readOnlyHint` is +true for the three tools that write no artifact: `probe`, `check` and `cropdetect`. On the CLI +they still take `--plan FILE` and `--overwrite` like every tool. Over MCP the two property names +stay in their `inputSchema` (3.x removes no property) with descriptions that say they are refused +or have no effect (`_contract.MCP_READ_ONLY_REFUSED`), and the server runs these three tools +with `FFMPEG_SKILL_READ_ONLY=1`. With that variable set, `--plan` is refused after argparse, +however a raw `argv` spells it (`kind: input`, `error.reason: read_only_plan_refused`), and no +file is written. Every other tool is annotated `destructiveHint: true`: none edits its input, +but with `overwrite` it replaces an existing file at its output or plan path, which MCP does not +count as an additive update. The hints are hints: a client must not treat them as a security +boundary. + ### Default `tools/list` surface: core 12, opt-in for all 42 By default `tools/list` returns only the core dozen in `_contract.MCP_CORE_TOOLS` (`render`, diff --git a/docs/demos.md b/docs/demos.md index eb0e50e3..bcc74cc4 100644 --- a/docs/demos.md +++ b/docs/demos.md @@ -647,7 +647,7 @@ python3 scripts/loudness.py demos/out/loudness_before.mp4 -I -14 --tp -1 -o demo python3 scripts/audio.py demos/out/fixtures/surround.mp4 --downmix -o demos/out/downmix_51_after.mp4 ``` -**Look for:** Six discrete tones folded into two channels at the standard coefficients -- the centre and LFE are still audible. +**Look for:** Six discrete tones folded into two channels with the ITU-R BS.775 weights -- the centre and surrounds are still audible, the LFE is dropped, and nothing clips. ### Music bed with ducking diff --git a/docs/demos/downmix_51.gif b/docs/demos/downmix_51.gif index e56d02a5..7f6a0786 100644 Binary files a/docs/demos/downmix_51.gif and b/docs/demos/downmix_51.gif differ diff --git a/docs/design-decisions.md b/docs/design-decisions.md index 3459ff92..e5deef3b 100644 --- a/docs/design-decisions.md +++ b/docs/design-decisions.md @@ -74,6 +74,8 @@ exists. When a decision changes, edit the entry in the same PR. - [After 3.3.0 — a declared change scope for `check.py --compare` (#439)](#after-330--a-declared-change-scope-for-checkpy---compare-439) - [After 3.3.0 — non-finite numbers are refused in one place](#after-330--non-finite-numbers-are-refused-in-one-place) - [After 3.4.3 — the MCP Registry entry is published by the release, not by hand](#after-343--the-mcp-registry-entry-is-published-by-the-release-not-by-hand) +- [After 3.6.0 — a fold-down, a tone map and an overlay keep the source's levels and depth](#after-360--a-fold-down-a-tone-map-and-an-overlay-keep-the-sources-levels-and-depth) +- [After 3.6.0 — MCP annotations and argument values (audit A9, A10)](#after-360--mcp-annotations-and-argument-values-audit-a9-a10) ## Time limits @@ -190,9 +192,10 @@ exists. When a decision changes, edit the entry in the same PR. - **`escape_drawtext()` drops `'` and `%` from burnt-in text** instead of escaping them. Both characters have no reliable escape across the FFmpeg versions in CI; a missing apostrophe is a known limitation, a broken filter graph is not. Test: `test_drawtext_semicolon_and_quote_render_as_inert_literal_text`. -- **`grid.py` and `look.py` composite an HDR input into an 8-bit SDR picture without a tone map.** Both - produce comparison/inspection artefacts, not deliverables; `grid.py` prints a note when an input is - HDR and points to `color.py --to-sdr` for a graded conversion. `join.py` and `broll.py`, whose output +- **`grid.py` composites an HDR input into an 8-bit SDR picture without a tone map.** It produces + a comparison/inspection artefact, not a deliverable; it prints a note when an input is HDR and + points to `color.py --to-sdr` for a graded conversion. (`look.py` shows an HDR frame through + `--to-sdr`'s own tone map: "After 3.6.0" below.) `join.py` and `broll.py`, whose output is a deliverable, keep the HDR (10-bit HEVC) path. Test: `test_grid_composites_cols_rows_with_labels`. - **`look.py -o` is used verbatim only for a single `--at` with an image extension**; several frames, or `-o` given as a stem, produce `_s.png` names. Test: `test_hdr_source_stays_hdr_through_reencodes`. @@ -413,7 +416,10 @@ exists. When a decision changes, edit the entry in the same PR. `batch.py`'s cross-filesystem delivery) writes the descriptor that created the temp (`copy_to_sibling()`); ffmpeg opens its temp by name, so a process that may delete entries in the output directory could still swap the file between the create and that open, which a - sticky shared directory (`/tmp`) and a directory only the caller can write both rule out. A lock's default audit log beside the output + sticky shared directory (`/tmp`) and a directory only the caller can write both rule out. A private + subdirectory for those temps would not close it either (whoever can rename entries in the output + directory can swap the subdirectory for a link, and ffmpeg cannot open relative to a directory + descriptor), so this residual case is accepted (#456). A lock's default audit log beside the output is opened without following a symlink; a log named with `--audit-log` is opened as named. Code: `_common._stage_existing_output()`, `runner.reserve_sibling()`, `emit.audit()`. Test: `test_hidden_temp_names_are_never_written_through_a_planted_link`. @@ -743,6 +749,16 @@ Each item is recorded here so it is not re-proposed from scratch. 1.14.0, so the demos, the golden frames and every existing test do not move (the filter string itself did change: the label moved from `text=` into `textfile=…:expansion=none`) — and the new route carries no risk for the 95 % of jobs that never needed it. +- **SRT markup on `caption.py`'s own ASS route is the set ffmpeg's SRT reader styles, minus + free override blocks.** ` `, `` and a `{\anN}` become the overrides + ffmpeg writes for them, so `--animate`/`--karaoke` draw what the plain `--srt` route draws. Any + other `{\...}` block stays escaped text there, although ffmpeg's SRT route would apply it: cue + text (an SRT, `--text`, a transcript) must not be able to `\pos`, `\t` or `\fscx` itself, which + 1.15 decided and this keeps. An unknown `` stays text too (ffmpeg removes it), so a + caption that says `press ` is not shortened. Under `--karaoke`, `` is + dropped: the highlight is the primary colour, and the cue's colour would replace it. Tests: + `test_srt_markup_becomes_ass_overrides_not_text`, + `test_caption_ass_dialogue_text_cannot_forge_override_blocks`. ## 1.16.0 — long-form delivery @@ -2129,6 +2145,43 @@ under it; the rest of the silent paths are converted onto the same infrastructur warning. Code: `batch.cached_output_changed()`, `recipe_key()`. Tests: `test_batch_serves_a_cached_output_only_while_it_is_unchanged_and_records_a_reset_cache`, `test_batch_recipe_key_records_an_unreadable_project`. +- **`batch.py --watch` processes a source only once it has settled, and replaces only its own + earlier output.** A pass took every file that existed, so a file still being copied into the + folder was delivered as the fraction already written (`ok`, and `verified` for an MKV or TS, + whose checks compare with the input as read), and once the copy finished every later pass met + that output as someone else's and refused it (`output_exists`). A source is processed once it + has the same size and mtime as at the previous poll, or, first seen, was last modified at least + one interval ago; until then its row is `skipped: "settling"` (`ok: false`, not cached), and a + watch stopped there fails with that row, since the file was not processed. One poll interval is + the whole rule: a writer that pauses longer than the interval is read early, and the pass after + its next write processes the file again. A file an earlier pass of the same watch delivered -- + the final output and every side file a step wrote, such as caption.py's `.srt` beside the + video -- still with the size, mtime and inode it had then, is this run's own file under the + overwrite rule above and is replaced without `--overwrite`; one replaced or edited since, or + left by an earlier run, is refused as before (`output_exists`); so is one another item of the + same run delivered (two items naming one fixed side file). The permission is per item and per + path, and is checked where the write happens, not when the item starts: batch.py never hands a step a + global `--overwrite` for it (that let `caption.py --text` replace a hand-corrected sidecar + because the video was batch's own), it gives each step a snapshot of the files it delivered + (`runner.OWNED_ENV`, internal to batch.py), and the step's own output check + (`refuse_existing_outputs()`) passes an existing target only while it is still that file and + reports back each path it was allowed to write, with the identity each had when it was written + (`runner.report_committed()`, called where a write lands); `deliver()` compares the final path's + identity again under the output lock and records the one it leaves. An SRT edited while the same + item's video was still encoding no longer matches what was written and is refused next pass. A file another process puts there between that check and the + write itself is the same window every tool's own `--overwrite` check has. A side file a step + writes without that check, or one of an item that failed, is not recorded, so the next pass + refuses it. A plain batch (no `--watch`) has no previous poll and processes what is there. + Code: `batch.settle_poll()`, `is_own_output()`, `deliver()`, `owned_config()`, + `runner.owned_replaceable()`. Tests: + `test_batch_watch_waits_for_a_source_to_settle_and_replaces_only_its_own_output`, + `test_batch_watch_rechecks_its_own_output_where_it_replaces_it`, + `test_batch_watch_step_refuses_its_own_output_replaced_before_its_check`, + `test_batch_watch_replaces_its_own_side_files_but_not_an_edited_one`, + `test_batch_watch_does_not_adopt_a_side_file_edited_while_the_step_ran`, + `test_batch_item_never_replaces_a_side_file_another_item_delivered`, + `test_a_step_replaces_only_the_files_batch_delivered_unchanged`, + `test_batch_watch_reports_a_source_still_settling_when_it_stops`. - **`verify.py` keeps each tool's own failure.** Steps run with `--json`; a failed step keeps the tool's `kind`, `reason`, `message`, `details` and `exit_code` (`step_timeout`, `step_output_unparsed` where the tool printed none), and the probe of an unreadable file keeps @@ -3119,6 +3172,26 @@ Tests: `SmartCutTests` in `tests/test_cut_copy.py`. output does not have (a sound-only extract has no subtitle track or timecode to lose). - **A chapter list kept but not retimed counts as kept.** The comparison is a count; whether the chapters still point at the right moments is a later, per-tool measurement. +- **A start timecode is compared by its value, and moves with the start of the picture.** FFmpeg + copies the video stream's `timecode` tag and the MOV/MP4 muxer rebuilds a tmcd track from it, so + a cut from 1 s was stamped with the source's 01:00:00:00 on frame 30 and listed as kept. A tool + that starts the output on a later frame writes that frame's label (drop-frame where the source + is): a single-range `cut` (its first frame at or after `--start` for a re-encode or an + edit-listed copy, the keyframe for a copy that shows its pre-roll), `fit --method trim`, + `sync --trim-second` (a padded head counts back before the source's start) and so a + `render.py` clip with an `in`. A join of several ranges writes none: a timecode counts on from + its first frame, so after the first join every label would name the wrong source frame, and a + conform by timecode would land there. A stream copy out of MXF writes none either: its demuxer + lands a seek on another keyframe than `cut.seek_keyframe` reads. `source_carry` then lists the + timecode as dropped (an MXF, which always writes one, reads 00:00:00:00). The label is read at + the source's rate and written at the rate the output is delivered at (`fit --fps`, a VFR source + conformed): 01:00:00:12 at 25 fps is 01:00:00:24 at 50, and a fit that changes the rate writes + it even without a trim, leaving the source's tmcd track behind. A start before a picture that + starts after the file's origin is that picture's first frame; only a padded head counts back. + Code: `_common.timeline.output_timecode_args()`, `cut.first_frame_from()`. Tests: + `test_a_tool_that_moves_the_start_moves_the_start_timecode`, + `test_the_start_timecode_is_counted_at_the_rate_fit_delivers`, + `test_a_cut_before_a_late_picture_keeps_the_start_timecode_of_its_first_frame`. Code: `_common.emit.source_carry()`, `_common.probe.carry_inventory()`. Test: `test_source_carry_says_what_the_output_lost_from_its_source`. @@ -3280,3 +3353,74 @@ Tests: `SmartCutTests` in `tests/test_cut_copy.py`. a last step turns it red, the same shape as the plugin branch: a missing registry entry must not undo a release. The first run was the merge of the PR that added the steps (#445): it listed 3.4.3, a version released before they existed. Test: `test_mcp_registry_manifest_tracks_the_package_version`. + +## After 3.6.0 — a fold-down, a tone map and an overlay keep the source's levels and depth + +- **`audio.py --downmix` is quieter than its input, by design.** It folds 5.1/7.1 with the + ITU-R BS.775 weights (centre and every surround at 0.707, LFE dropped) and pan's `<` scales + each side so its gains sum to 1 over the channels the input has: a front channel alone comes out + 7.7 dB lower on 5.1, 9.9 dB on 7.1, and in-phase channels at full level cannot clip. Through + 3.6.0 pan's `=` summed 2.7× per side (3.2× on 7.1) with the LFE at 0.5, so a loud 5.1 clipped + (flat factor 20+) while the run said verified. A limiter instead of the scale would change the + programme's dynamics, and the loudness is `loudness.py`'s job. `--stereo`/`--mono` on more than + two channels already went through swresample, which normalises the same way (measured on 6.1: + s16, f32 and AAC sources fold in-phase -6 dBFS channels to -6 dBFS). Code: + `audio.DOWNMIX_PAN`. Tests: `DownmixLevelTests`. +- **`--to-sdr` linearises at 100 nits per unit and gives `tonemap` the source peak; `--peak` is a + PQ setting.** FFmpeg's `tonemap` reads linear 1.0 as 100 nits (SDR white) and takes `peak` in + those units, so the zscale before it uses `npl=100` and the filter gets `peak=--peak/100`: a PQ + pixel at `--peak` nits and HLG at 100 % land on Y 235. Through 3.6.0 the zscale used + `npl=--peak`, so a 1000-nit pixel arrived as 1.0 (100 nits to the filter): nothing reached white + (PQ 1000 nits and HLG 100 % at Y 151, HLG reference white at 91), and raising `--peak` to a + brighter master's darkened everything. zimg decodes HLG for a 1000-nit display, so an HLG + picture always peaks at 1000 nits there and `--peak` is not applied to it. `look.py` shows an HDR + source through the same chain at `--to-sdr`'s defaults (hable, 1000 nits), and a BT.2020 SDR one + through the gamut conversion only, so the sheet shows what the conversion would give; with + `--compare`, each picture by its own tags (a BT.709 SDR one gets none). Code: + `_common/color.py` (`hdr_to_sdr_chain`), `look.display_conversion`. Test: + `test_to_sdr_takes_the_source_peak_to_sdr_white`, + `test_look_shows_an_hdr_sheet_at_the_to_sdr_levels`, + `test_look_compare_converts_each_input_by_its_own_tags`. +- **An overlay onto a picture deeper than 8 bits, on a line that keeps more than 8 bits, runs at + 10 bits.** overlay's default format is 8-bit `yuv420`, `format=auto` picks an 8-bit one as soon + as a layer is 8-bit, and `drawbox` is 8-bit only on 6.1: through 3.6.0 a cutaway, a redaction, + a graphics layer or box, an emoji and a PiP took a 10-bit HDR or ProRes picture to 8 bits before + the 10-bit encoder, and `overlay --image`'s RGBA logo took it through 8-bit RGB (Y 402 / Cb 602 + / Cr 422 came out 400 / 603 / 423, through RGB 399 / 608 / 418). `overlay_format()` names + `yuv420p10` (`yuv422p10`/`yuv444p10` for those sources) for such a picture, an 8-bit layer is + widened to meet it (bit depth only, no matrix), and graphics.py draws its boxes as layers there. + Not changed: an 8-bit main, an SDR H.264 line (its output is 8-bit; a 10-bit SDR source is + reduced at the encoder as before), the alpha line (its tool picks the format), and the colour of + an RGB layer on an HDR line, which is still FFmpeg's own conversion ("Not converted ... an HDR + line" above). `overlay --video --opacity` scales the layer's alpha with `lut` in YUV: the RGB-only + `colorchannelmixer` it used made FFmpeg composite the whole main picture in RGB, which came back + BT.601 under the BT.709 tag (#333's defect, on the video path; C03028 read 188/42/35 at opacity 1 + and 199/55/32 at 0.999). Code: `_common/decision.py` (`overlay_format`), `broll`, `redact`, + `graphics`, `overlay`, `caption`'s emoji. Tests: `test_an_overlay_onto_a_10_bit_main_keeps_its_low_bits`, + `test_a_pip_with_opacity_leaves_the_main_picture_alone`. + +## After 3.6.0 — MCP annotations and argument values (audit A9, A10) + +- **Over MCP, `probe`, `check` and `cropdetect` refuse `plan`; the property stays listed.** They + are annotated `readOnlyHint`, and a client may run a read-only tool without asking; `--plan FILE` + was the one file they could still write, and with `--overwrite` it replaced any existing file. + The CLI keeps both flags (every tool gets `--plan`, above). The MCP schema keeps both names, + because the 3.x promise removes no `inputSchema` property (`mcp_tools.json` pins it), and says in + their descriptions that `plan` is refused and `overwrite` has no effect + (`_contract.MCP_READ_ONLY_REFUSED`). The server runs these three with + `FFMPEG_SKILL_READ_ONLY=1`, which refuses `--plan` after argparse (`read_only_plan_refused`), + so a raw `argv` cannot write one under any spelling either. Every other tool is + `destructiveHint: true`: its `overwrite` replaces an existing file, which MCP does not count as + additive. Code: `_contract.mcp_annotations()`, `_common.apply_common()`. Tests: + `test_mcp_annotations_match_what_the_schema_lets_a_call_write`, + `test_mcp_read_only_tools_write_no_plan_in_any_spelling`. +- **A value that starts with `-` is passed as `--flag=value`; others keep `--flag value`.** + argparse reads a separate token that starts with `-` as an option, so `{"text": "-50%"}` was + refused and an overlay in a project could not start with a minus. Only those values change + form, so a call without one records the same argv as before (a plan's `argv`, a `--cache` key); + a render stage with a negative number (`loudness.lufs`, `audio.gain`) gets a new cache key once. + Positionals that start with `-` follow `--`, after every option. The joined form needs a long + flag, so `output` and `loudness.lufs` use `--output` / `--lufs` there. A raw `argv` is passed + as given: its caller chose the tokens. Code: `_common.option_args()`, `mcp/server.build_argv()`, + render.py's stage commands. Tests: `test_mcp_value_starting_with_a_dash_is_a_value_for_every_tool`, + `test_dash_leading_text_over_mcp_and_in_a_render_project`. diff --git a/fuzz/_harness.py b/fuzz/_harness.py new file mode 100644 index 00000000..6c488716 --- /dev/null +++ b/fuzz/_harness.py @@ -0,0 +1,25 @@ +"""Shared set-up for the fuzz targets: put scripts/ on sys.path and run a target either under +atheris (ClusterFuzzLite) or, without it, over the seed inputs given on the command line, which is +how tests/test_contract.py exercises every target on each CI run.""" +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPTS = os.path.join(os.path.dirname(HERE), "scripts") +if os.path.isdir(SCRIPTS): + sys.path.insert(0, SCRIPTS) +else: # PyInstaller bundle built by ClusterFuzzLite: the modules are next to the target + sys.path.insert(0, HERE) + + +def main(test_one_input): + try: + import atheris + except ImportError: + for path in sys.argv[1:]: + with open(path, "rb") as f: + test_one_input(f.read()) + return + atheris.instrument_all() + atheris.Setup(sys.argv, test_one_input) + atheris.Fuzz() diff --git a/fuzz/parse_time_fuzzer.py b/fuzz/parse_time_fuzzer.py new file mode 100644 index 00000000..89c3a789 --- /dev/null +++ b/fuzz/parse_time_fuzzer.py @@ -0,0 +1,24 @@ +"""Fuzz the time grammar every --start/--end/--at flag goes through (seconds, mm:ss, +hh:mm:ss.ms, SRT commas, hh:mm:ss:ff with an fps or an @fps suffix). A ValueError is the +documented refusal; anything else, or a non-finite result, is a bug. A negative value is a +valid answer here: offsets (`sync.py --offset -0.5`) go through the same grammar, and each tool +refuses a negative start or end itself.""" +import math + +import _harness # noqa: F401 (sets sys.path) +from _common.decision import parse_time + + +def test_one_input(data: bytes) -> None: + text = data.decode("utf-8", "replace") + for fps in (None, 25.0, 29.97): + try: + value = parse_time(text, fps) + except ValueError: + continue + if not math.isfinite(value): + raise AssertionError(f"parse_time({text!r}, {fps}) returned {value!r}") + + +if __name__ == "__main__": + _harness.main(test_one_input) diff --git a/fuzz/seeds/parse_time/mmss b/fuzz/seeds/parse_time/mmss new file mode 100644 index 00000000..58a212ee --- /dev/null +++ b/fuzz/seeds/parse_time/mmss @@ -0,0 +1 @@ +1:30 \ No newline at end of file diff --git a/fuzz/seeds/parse_time/negative b/fuzz/seeds/parse_time/negative new file mode 100644 index 00000000..ea40e229 --- /dev/null +++ b/fuzz/seeds/parse_time/negative @@ -0,0 +1 @@ +-1.5 \ No newline at end of file diff --git a/fuzz/seeds/parse_time/seconds b/fuzz/seeds/parse_time/seconds new file mode 100644 index 00000000..7ef6ffee --- /dev/null +++ b/fuzz/seeds/parse_time/seconds @@ -0,0 +1 @@ +12.5 \ No newline at end of file diff --git a/fuzz/seeds/parse_time/smpte b/fuzz/seeds/parse_time/smpte new file mode 100644 index 00000000..4e017073 --- /dev/null +++ b/fuzz/seeds/parse_time/smpte @@ -0,0 +1 @@ +00:00:01:15@29.97 \ No newline at end of file diff --git a/fuzz/seeds/parse_time/srt b/fuzz/seeds/parse_time/srt new file mode 100644 index 00000000..291e34cf --- /dev/null +++ b/fuzz/seeds/parse_time/srt @@ -0,0 +1 @@ +00:01:30,250 \ No newline at end of file diff --git a/fuzz/seeds/srt/bom_crlf.srt b/fuzz/seeds/srt/bom_crlf.srt new file mode 100644 index 00000000..b7e0c8e5 --- /dev/null +++ b/fuzz/seeds/srt/bom_crlf.srt @@ -0,0 +1,3 @@ +1 +00:00:00,000 --> 00:00:01,000 +BOM diff --git a/fuzz/seeds/srt/two_cues.srt b/fuzz/seeds/srt/two_cues.srt new file mode 100644 index 00000000..37ba8326 --- /dev/null +++ b/fuzz/seeds/srt/two_cues.srt @@ -0,0 +1,7 @@ +1 +00:00:00,000 --> 00:00:01,500 +Hello + +2 +00:00:01,500 --> 00:00:03,000 +world diff --git a/fuzz/seeds/timeline_xml/entity.xml b/fuzz/seeds/timeline_xml/entity.xml new file mode 100644 index 00000000..08b97c74 --- /dev/null +++ b/fuzz/seeds/timeline_xml/entity.xml @@ -0,0 +1,3 @@ + +]> +&a; diff --git a/fuzz/seeds/timeline_xml/fcpxml.xml b/fuzz/seeds/timeline_xml/fcpxml.xml new file mode 100644 index 00000000..1e306899 --- /dev/null +++ b/fuzz/seeds/timeline_xml/fcpxml.xml @@ -0,0 +1,3 @@ + + + diff --git a/fuzz/srt_fuzzer.py b/fuzz/srt_fuzzer.py new file mode 100644 index 00000000..32b864e7 --- /dev/null +++ b/fuzz/srt_fuzzer.py @@ -0,0 +1,27 @@ +"""Fuzz the SRT reader that caption.py, burn and the transcript path use. A file it cannot read +ends the run through die() (SystemExit with a JSON error); any other exception is a bug.""" +import os +import tempfile + +import _harness # noqa: F401 (sets sys.path) +from _common import asr + + +def test_one_input(data: bytes) -> None: + fd, path = tempfile.mkstemp(suffix=".srt") + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + try: + cues = asr.parse_srt(path) + except SystemExit: + return + for start, end, text in cues: + if not isinstance(text, str) or start != start or end != end: + raise AssertionError(f"bad cue {(start, end, text)!r}") + finally: + os.remove(path) + + +if __name__ == "__main__": + _harness.main(test_one_input) diff --git a/fuzz/timeline_xml_fuzzer.py b/fuzz/timeline_xml_fuzzer.py new file mode 100644 index 00000000..987d68fd --- /dev/null +++ b/fuzz/timeline_xml_fuzzer.py @@ -0,0 +1,18 @@ +"""Fuzz the hardened XML reader that checks an exported FCPXML before it is reported written: +it must refuse a DOCTYPE/entity and malformed input with ParseError, never expand an entity or +raise anything else.""" +from xml.etree.ElementTree import ParseError + +import _harness # noqa: F401 (sets sys.path) +from _common import timeline + + +def test_one_input(data: bytes) -> None: + try: + timeline._xml_root_tag(data) + except ParseError: + pass + + +if __name__ == "__main__": + _harness.main(test_one_input) diff --git a/mcp/server.py b/mcp/server.py index 456ffe3f..d4b99327 100755 --- a/mcp/server.py +++ b/mcp/server.py @@ -64,30 +64,39 @@ def build_argv(name: str, args: Dict[str, Any]) -> List[str]: return argv spec = specs().get(name) exceptions = spec["mcp"]["argument_exceptions"] if spec else {} - argv: List[str] = [] + props = spec["input_schema"]["properties"] if spec else {} + positionals: List[str] = [] args = dict(args) for key in POSITIONAL[name]: val = args.pop(key, None) if val is None: continue if isinstance(val, list): - argv += [str(v) for v in val] + positionals += [str(v) for v in val] else: - argv.append(str(val)) + positionals.append(str(val)) + options: List[str] = [] for key, val in args.items(): if val is None or val is False: continue flag = exceptions.get(key, "--" + key.replace("_", "-")) if val is True: - argv.append(flag) - elif isinstance(val, list): - for v in val: - argv += [flag, str(v)] - else: - argv += [flag, str(val)] - if name not in _contract.MCP_JSON_EXEMPT and "--json" not in argv and "--help" not in argv: - argv.append("--json") - return argv + options.append(flag) + continue + # A value is always a value: one starting with '-' goes as `--flag=value` (option_args), + # which needs the long spelling of a short exception (-o -> --output, -I -> --lufs). + cli = props.get(key, {}).get("cli") + longs = [f for f in cli if f.startswith("--")] if isinstance(cli, list) else [] + long_flag = flag if flag.startswith("--") or not longs else longs[0] + for v in (val if isinstance(val, list) else [val]): + options += _common.option_args(long_flag, v) if str(v).startswith("-") else [flag, str(v)] + if name not in _contract.MCP_JSON_EXEMPT and "--json" not in options and "--help" not in options: + options.append("--json") + # A positional starting with '-' would be read as an option (`--plan=x` given as an input path + # wrote a plan through a read-only tool): such positionals go after `--`, after every option. + if any(p.startswith("-") for p in positionals): + return options + ["--"] + positionals + return positionals + options def _timeout_in_argv(argv: List[str]) -> Optional[float]: @@ -150,7 +159,13 @@ def call_tool(name: str, args: Dict[str, Any]) -> Dict[str, Any]: # `--timeout` inside a raw argv list sets both (0 = no limit); otherwise # FFMPEG_SKILL_TIMEOUT / the 1800 s default. per_call = timeout_from_call(args or {}) - proc = _common.run_tool([str(script)] + argv, per_call=per_call) + # A tool annotated readOnlyHint must write nothing: its schema has no `plan`, and this makes + # the tool itself refuse --plan however a raw argv spells it. Every other tool runs without + # the variable, even when the server's own environment has it. + env = {k: v for k, v in os.environ.items() if k != _contract.READ_ONLY_ENV} + if _contract.mcp_read_only(specs()[name]): + env[_contract.READ_ONLY_ENV] = "1" + proc = _common.run_tool([str(script)] + argv, per_call=per_call, env=env) stdout = proc.stdout.strip() text = stdout structured = None diff --git a/references/gotchas.md b/references/gotchas.md index 74168470..3e2167ca 100644 --- a/references/gotchas.md +++ b/references/gotchas.md @@ -56,7 +56,8 @@ at source colour: `export.py --preset prores` does not retag. What else the source carried is compared on every write: `source_carry: {kept, dropped}` lists subtitles, chapters, a cover picture, surround channels, `creation_time`, the audio language tag -and the start timecode, and a lost subtitle track, chapter list, cover picture or surround layout +and the start timecode (kept only when it labels the output's first frame: a cut from a later +frame moves it there, a join of several ranges writes none), and a lost subtitle track, chapter list, cover picture or surround layout is a `source_items_dropped` warning. Read it before telling the user the file is complete; to carry the items over, use the tool that keeps them (`export.py` keeps subtitle tracks, `metadata.py` writes chapters and tags). diff --git a/references/scripts.md b/references/scripts.md index ba17910e..619a93e7 100644 --- a/references/scripts.md +++ b/references/scripts.md @@ -1376,7 +1376,18 @@ above the cap is clamped with a note, not refused, and both `jobs` and row has `written: false` (`output` keeps its path string), a note says so, and `ok` follows the steps' exit codes. Final names that differ only in case are a collision (they are one file on macOS and Windows). -- `--watch` composes: each pass is parallel, passes are sequential. +- `--watch` composes: each pass is parallel, passes are sequential. A pass processes a + source only once it has settled: the same size and mtime as at the previous poll, or, the + first time it is seen, last modified at least one interval ago. Until then its row is + `"skipped": "settling"` (`ok: false`, not cached), so a file still being copied, downloaded + or recorded into the folder is not delivered as a fraction of itself; a writer that pauses + longer than the interval can still be read early, and the pass after it changes again + processes it again. An output an earlier pass of the same watch delivered, unchanged since, + is that pass's own and is replaced without `--overwrite`; one someone else replaced or edited, + or one an earlier run left, is still refused (`output_exists`). +- A `project` recipe's relative paths (`clips[].src`, `audio.music`, `brand`, `captions.srt`, + `overlays[].image`, `chapters`, ...) are read against the project file's directory, as + `render.py` reads them: each item's copy of the project carries them absolute. ### caption.py --transcribe — optional local speech-to-text If `parakeet-mlx`, `parakeet-cli` (parakeet.cpp, with a `.gguf`), `whisper-cli` (whisper.cpp), @@ -1400,8 +1411,8 @@ English `tdt-0.6b-v2`. A Parakeet model in `FFMPEG_SKILL_ASR_MODEL` goes to the only (the Whisper engines keep `base`); one given with `--model` goes to whichever engine runs. A non-English `--language` on an English-only Parakeet model is refused; a multilingual (v3) model assumes no language, so it gets no English-assumed note. The result's -`transcription` names the engine, model and routing. `silence.py --filler --transcribe` takes -the same `--engine` and `--model`. +`transcription` names the engine, model and routing. `silence.py --transcribe` (with `--filler` +or a sentence flag) takes the same `--engine` and `--model`. Nothing is required: without an engine it prints install hints and the user can supply `--text` cues instead. When engines are installed but none transcribed, the refusal names each and why instead (`reason: engine_failed`, or `english_only` for a Parakeet engine auto passed over for @@ -1705,7 +1716,11 @@ Lines without a time run for `--auto-seconds` (3 s) after the previous cue. `|` SRT/text cues: `pop` is the short-form "bouncy" entrance, `--karaoke` fills each word from `--color` to `--highlight-color` across the cue; `--karaoke-timing energy` (default) follows the speech loudness in the audio, `even` splits the -cue equally (word timing is derived, not transcribed). The ASS is kept next to the +cue equally (word timing is derived, not transcribed). SRT markup in the cues is styled the +way ffmpeg's own SRT route styles it: `` `` `` `` and `` become +ASS overrides and a `{\anN}` keeps its position, measured as no width; any other tag or `{...}` +stays text (escaped), and under `--karaoke` `` is dropped for the highlight's +colours. The ASS is kept next to the user can hand-tune timings and re-run with `--ass`. Emoji (1.15): `--emoji-assets DIR` is a directory of PNGs named by code point (`1f389.png`, `1f1ef-1f1f5.png`, `1f469-200d-1f4bb.png` — the Twemoji/Noto @@ -1715,7 +1730,9 @@ with an invisible placeholder reserving the emoji's box and each PNG is composited on top (`--emoji-scale` sizes the box, `--emoji-max` caps the count). Without one the run still succeeds and says `emoji: {"mode": "mono"}` plus a warning; `--emoji none` strips them; `--emoji color` insists on a colour-capable -libass and refuses otherwise. Nothing is ever downloaded. What this machine can +libass and refuses otherwise. An emoji is what Unicode draws as one by default: a text +symbol such as `♪ ✓ ★` is never one, and `♥ ✔ ☺` only with VS16 (`♥️`) or a skin tone +after them; the font draws the rest. Nothing is ever downloaded. What this machine can do: `doctor --json` → `.fonts.emoji`. Details: `references/gotchas.md#emoji`. Readable by default (1.12, rebalanced in 1.15): every cue is wrapped to the safe area — the @@ -1724,9 +1741,10 @@ frame width minus the two side margins below, 90 % of it without a `--platform` full em per character, Latin per character from a table read off DejaVu Sans (so an all-caps line measures as wide as it draws), Cyrillic/Greek about 0.55, Arabic/Hebrew 0.6, Devanagari 0.7, and a combining mark nothing at all — -breaking between characters for CJK (a katakana word stays whole) and at spaces -otherwise, but never between a character and the combining marks that belong -to it (Devanagari matras, Arabic and Hebrew points). Thai (1.16.1) is never +breaking between characters for Chinese and Japanese (a katakana word stays whole) and at +spaces otherwise (Korean included: it is written with spaces between words, and a space +that stood at a break is kept when a later pass moves the break), but never between a +character and the combining marks that belong to it (Devanagari matras, Arabic and Hebrew points). Thai (1.16.1) is never broken inside a run: it writes no space inside a phrase and the wrapper has no dictionary, so the break goes where you put a space or a `|`, and a run with none stays long on its own line — `caption.py` counts such lines as `overlong` @@ -1898,7 +1916,10 @@ Cross-correlates loudness envelopes: coarse FFT search (20 ms), then a direct video with the source file's audio aligned (video stream copied); the output keeps the reference's full length -- a shorter or head-trimmed source file is padded with silence, never allowed to cut the picture. -`--trim-second` writes the source file shifted to the reference timeline. +`--trim-second` writes the source file shifted to the reference timeline. A video source file is +re-encoded from the frame at the trim point (a stream copy would start at the keyframe before it); +a sound-only one is stream-copied (to the packet: within an AAC frame). Its start timecode moves +with the trim or the padded head. `--fix-drift` measures the offset again near the end of the overlap, reports the clock difference in ppm, and resamples the source file so a 60-minute take stays in sync (typical consumer devices drift 20-500 ppm = up to 1.8 s/h). @@ -1982,9 +2003,11 @@ iPhone "HDR" video is Dolby Vision profile 8.4 on an HLG base layer: `probe.py` reports `hdr_format: Dolby Vision profile 8` and `--to-sdr` tone-maps it from the HLG base layer. When the user wants to keep HDR but players mis-render the DV layer, `--strip-dovi` removes it losslessly. -`--to-sdr` does a real conversion: linearise (zscale, PQ or HLG), tone-map -(default `hable`, `mobius` keeps more highlight detail; FFmpeg's CPU -`tonemap` filter has no BT.2390 curve), then BT.709 gamma + matrix. Refuses when probe says the +`--to-sdr` does a real conversion: linearise (zscale, PQ or HLG, 1.0 = 100 nits), +tone-map the source peak to SDR white (`--peak` nits for PQ, default 1000; HLG +always peaks at 1000; default `hable`, `mobius` keeps more highlight detail; FFmpeg's CPU +`tonemap` filter has no BT.2390 curve), then BT.709 gamma + matrix. `look.py` shows an +HDR source through the same chain at its defaults. Refuses when probe says the input is not HDR unless `--force`. `--lut` applies a 3D .cube with tetrahedral interpolation (Log→709 conversions, creative looks); blend with `--lut-strength`. Everything else in the skill assumes SDR BT.709, so run this @@ -2053,8 +2076,11 @@ needs a real stereo source: it scales the side signal (L−R), so a mono track duplicated to two channels has nothing to scale. A 1-channel input is refused (`kind: input`) — `--stereo` duplicates it but does not widen it — and more than two channels are refused unless `--downmix` is given too, in which case the -widening runs on the stereo fold-down. `--downmix` uses the -ITU centre/LFE weights for 5.1/7.1 → stereo. `--mono` averages a stereo pair, +widening runs on the stereo fold-down. `--downmix` folds 5.1/7.1 to stereo with +the ITU-R BS.775 weights (centre and surrounds at -3 dB, LFE dropped), scaled so +each side's gains sum to 1: in-phase channels cannot clip, and a front channel +alone comes out 7.7 dB lower than it went in (9.9 dB on 7.1; follow with `loudness.py`). +`--mono` averages a stereo pair, leaves a 1-channel input untouched and downmixes >2 channels through swresample. Video is always stream-copied, and so is a subtitle/data track when the container can hold it (`dropped_non_av_streams` says when it could not). diff --git a/scripts/_ass_overlay.py b/scripts/_ass_overlay.py index ea66a9c4..758345ad 100644 --- a/scripts/_ass_overlay.py +++ b/scripts/_ass_overlay.py @@ -20,7 +20,8 @@ milliseconds relative to the line's own start; `fade` is (in_ms, out_ms); `scale_t` is (t1_ms, t2_ms, percent) for a pop. """ -from typing import Any, Dict, List, Optional, Sequence +import re +from typing import Any, Dict, List, Optional, Sequence, Tuple # One sentinel character stands in drawn text where an emoji was, so the placeholder is inserted @@ -105,6 +106,238 @@ def ass_text(text: str) -> str: return ass_escape(text) +# --- SRT markup ------------------------------------------------------------------------------- +# The tags ffmpeg's own SRT reader turns into styling (libavcodec/htmlsubtitles.c): +# , , and a `{\anN}` position block. caption.py's plain --srt route hands the +# file to ffmpeg, which draws `Hello` in italics; the routes where caption.py writes its own +# ASS (--animate, --karaoke, an emoji-forced ASS, the tiktok/reels/shorts templates) escaped the +# same cue and drew the tags as text, and the wrapper measured them as width. Only these tags are +# markup here: an unknown `` and every other `{\...}` block stay text, escaped as before +# (a caption still cannot open \pos or \t from its cue text). +_SRT_TAG_RE = re.compile( + r"<\s*(/?)\s*(i|b|u|s)\s*>" # 1, 2: simple style tags + r"|<\s*font\b([^<>]*)>" # 3: + r"|<\s*/\s*font\s*>" # closing font + r"|\{\\an([1-9])\}", # 4: position block + re.IGNORECASE) +_FONT_COLOR_RE = re.compile(r"""\bcolor\s*=\s*["']?\s*([#\w]+)""", re.IGNORECASE) +_NAMED_COLORS = { + "white": "ffffff", "black": "000000", "red": "ff0000", "lime": "00ff00", "green": "008000", + "blue": "0000ff", "yellow": "ffff00", "cyan": "00ffff", "aqua": "00ffff", + "magenta": "ff00ff", "fuchsia": "ff00ff", "silver": "c0c0c0", "gray": "808080", + "grey": "808080", "maroon": "800000", "olive": "808000", "navy": "000080", + "purple": "800080", "teal": "008080", "orange": "ffa500"} + + +def srt_markup_tokens(text: str) -> "List[Tuple[bool, str]]": + """`text` cut into (is_markup, piece) runs, in order; the pieces concatenate back to `text`.""" + out: "List[Tuple[bool, str]]" = [] + pos = 0 + for m in _SRT_TAG_RE.finditer(text or ""): + if m.start() > pos: + out.append((False, text[pos:m.start()])) + out.append((True, m.group(0))) + pos = m.end() + if pos < len(text or ""): + out.append((False, text[pos:])) + return out + + +def strip_srt_markup(text: str) -> str: + """The words a viewer reads: `text` with the SRT markup tags removed.""" + return "".join(piece for markup, piece in srt_markup_tokens(text) if not markup) + + +def srt_alignment(text: str) -> "Optional[int]": + """The `{\\anN}` a cue asks for, or None.""" + for markup, piece in srt_markup_tokens(text): + m = _SRT_TAG_RE.fullmatch(piece) if markup else None + if m and m.group(4): + return int(m.group(4)) + return None + + +def _font_colour(attrs: str) -> "Optional[str]": + m = _FONT_COLOR_RE.search(attrs or "") + if not m: + return None + value = m.group(1).lower() + value = _NAMED_COLORS.get(value, value) + if value.startswith("0x"): + value = value[2:] + value = value.lstrip("#") + if re.fullmatch(r"[0-9a-f]{6}", value): + return "&H%s%s%s&" % (value[4:6].upper(), value[2:4].upper(), value[0:2].upper()) + return None # ffmpeg ignores a colour it cannot read (#f00 included); so does this + + +def _markup_to_ass(tag: str, colours: "List[str]", colour: bool = True) -> str: + m = _SRT_TAG_RE.fullmatch(tag) + if not m: + return "" + if not colour and not m.group(2) and not m.group(4): + return "" # a tag where the caller owns the colours (karaoke): dropped + if m.group(2): + return "{\\%s%d}" % (m.group(2).lower(), 0 if m.group(1) else 1) + if m.group(4): + return "{\\an%s}" % m.group(4) + if m.group(0).lstrip("< \t").lower().startswith("/"): + if colours: + colours.pop() + return "{\\c%s}" % colours[-1] if colours else "{\\c}" + colour = _font_colour(m.group(3)) + colours.append(colour or (colours[-1] if colours else "")) + return "{\\c%s}" % colour if colour else "" + + +def ass_escape_markup(text: str, colour: bool = True) -> str: + """ass_escape() for cue text that may carry SRT markup: the tags ffmpeg's SRT reader styles + become the same ASS overrides it writes for them (`` -> `{\\i1}`, `` + -> `{\\c&H00FFFF&}`, `{\\an8}` kept), and everything else is escaped exactly as ass_escape() + escapes it. `colour=False` drops the tags instead: karaoke's highlight IS the primary + colour, and a cue colour would replace it.""" + out: "List[str]" = [] + colours: "List[str]" = [] + for markup, piece in srt_markup_tokens(text): + if markup: + out.append(_markup_to_ass(piece, colours, colour)) + continue + escaped = ass_escape(piece) + if escaped.endswith("\\") and not escaped.endswith("\\​"): + escaped += "​" # a trailing backslash must not run into the override after it + out.append(escaped) + return "".join(out) + + +def reinsert_srt_markup(marked: str, lines: "Sequence[str]") -> "List[str]": + """Put `marked`'s markup back into `lines`, a wrap of strip_srt_markup(marked). + + A wrap keeps every visible character in order and only turns whitespace into a break (or puts a + break between two characters), so each tag is anchored to a character: a closing tag right + after the visible character before it, any other tag right before the visible character after + it. Lines that do not hold the stripped text in order come back unchanged (markup dropped): + tags would otherwise land in the wrong word. + + A tag pair that encloses no visible character (`A B`) has no character + to anchor to, and the rule above would put its closing tag after A and its opening tag before + B, colouring B. In a gap between two visible characters that holds such a pair, the tags keep + their source order instead: those before the pair's closing tag go right after the character + before the gap, the rest right before the character after it (both at the start of the next + line when the gap's whitespace became the break).""" + tokens = srt_markup_tokens(marked) + if not any(markup for markup, _p in tokens): + return list(lines) + # (line, column) of every non-space character of the wrapped lines, in reading order + slots = [(li, ci) for li, line in enumerate(lines) for ci, ch in enumerate(line) if not ch.isspace()] + if not slots: + return list(lines) + split_at = _empty_pair_splits(tokens) + inserts: "Dict[Tuple[int, int], List[str]]" = {} + seen = 0 # non-space visible characters passed so far + for index, (markup, piece) in enumerate(tokens): + if not markup: + for ch in piece: + if ch.isspace(): + continue + if seen >= len(slots) or lines[slots[seen][0]][slots[seen][1]] != ch: + return list(lines) + seen += 1 + continue + after_prev = (slots[seen - 1][0], slots[seen - 1][1] + 1) if seen else None + before_next = slots[seen] if seen < len(slots) else None + split = split_at.get(seen) + if split is not None: + # this gap holds a pair that encloses no visible character: keep the source order + if after_prev is None or before_next is None: + key = after_prev or before_next + elif after_prev[0] != before_next[0]: + key = before_next + else: + key = after_prev if index < split else before_next + elif piece.lstrip("< \t").startswith("/") and after_prev or before_next is None: + key = after_prev + else: + key = before_next + inserts.setdefault(key, []).append(piece) + if seen != len(slots): + return list(lines) + out = [] + for li, line in enumerate(lines): + cols = sorted(ci for (l2, ci) in inserts if l2 == li) + built, last = "", 0 + for ci in cols: + built += line[last:ci] + "".join(inserts[(li, ci)]) + last = ci + out.append(built + line[last:]) + return out + + +def _empty_pair_splits(tokens: "Sequence[Tuple[bool, str]]") -> "Dict[int, int]": + """For each gap between visible characters (keyed by the number of non-space characters + before it) that holds a tag pair enclosing none, the token index of the first such pair's + closing tag.""" + splits: "Dict[int, int]" = {} + seen = 0 + open_here: "List[str]" = [] # names of the tags opened in the current gap + for index, (markup, piece) in enumerate(tokens): + if not markup: + if piece.strip(): + seen += sum(1 for ch in piece if not ch.isspace()) + open_here = [] + continue + m = _SRT_TAG_RE.fullmatch(piece) + if not m or m.group(4): + continue + name = (m.group(2) or "font").lower() + if piece.lstrip("< \t").startswith("/"): + if name in open_here: + splits.setdefault(seen, index) + del open_here[len(open_here) - 1 - open_here[::-1].index(name)] + else: + open_here.append(name) + return splits + + +def carry_srt_markup(chunks: "Sequence[str]") -> "List[str]": + """Consecutive cues cut from one marked cue, each made to stand alone: a tag still open at the + end of a piece is closed there and opened again at the start of the next, and the cue's + `{\\anN}` is repeated on every piece -- the second half of an italic, top-placed cue stays + italic and at the top.""" + out: "List[str]" = [] + carried: "List[str]" = [] + # the cue's position, wherever in it the {\anN} stands: an earlier piece takes it too + align = next((a for a in (srt_alignment(c) for c in chunks) if a is not None), None) + for chunk in chunks: + lead = "".join(carried) + if align is not None and srt_alignment(chunk) is None: + lead = "{\\an%d}" % align + lead + opened: "List[str]" = list(carried) + for markup, piece in srt_markup_tokens(chunk): + if not markup: + continue + m = _SRT_TAG_RE.fullmatch(piece) + if not m or m.group(4): + continue + closing = piece.lstrip("< \t").startswith("/") + name = (m.group(2) or "font").lower() + if closing: + for k in range(len(opened) - 1, -1, -1): + if _tag_name(opened[k]) == name: + del opened[k] + break + else: + opened.append(piece) + tail = "".join("" % _tag_name(t) for t in reversed(opened)) + out.append(lead + chunk + tail) + carried = opened + return out + + +def _tag_name(tag: str) -> str: + m = _SRT_TAG_RE.fullmatch(tag) + return ((m.group(2) if m else None) or "font").lower() + + def text_overlay_ass(elements: Sequence[Dict[str, Any]], *, play_w: int, play_h: int, path: str, fonts_dir: Optional[str] = None) -> str: """Write `elements` as an ASS file at `path` and return the path.""" diff --git a/scripts/_common/__init__.py b/scripts/_common/__init__.py index ae2a4739..9c94efd1 100644 --- a/scripts/_common/__init__.py +++ b/scripts/_common/__init__.py @@ -52,7 +52,7 @@ # demos/ or evals/ has ever reached for through `_common`. from _common.runner import ( - add_common, apply_common, _check_existing_output, refuse_existing_outputs, guard_sidecar, _check_no_overwrite_input, _check_output_path, child_args, + add_common, apply_common, _check_existing_output, refuse_existing_outputs, guard_sidecar, _check_no_overwrite_input, _check_output_path, child_args, option_args, child_limit, _CHILDREN, _cleanup_partial_output, _cmdline, CODECS, Context, DEFAULT_TIMEOUT, _DRAWTEXT_PENDING, _DRAWTEXT_TMPDIR, _drawtext_tmpdir, dry_run_input_pending, _ENCODERS, _env_timeout, ERROR_CODE, ERROR_RETRYABLE, EVEN_SCALE, _execute, _fail, ffmpeg_base, ffmpeg_encoders, ffmpeg_exe, _FFMPEG_RELEASE, _FFMPEG_VERSION, @@ -85,7 +85,7 @@ _onset_strength, _pick_onsets, _autocorrelation_peak, _grid_score, brand_states_font, cfr_args, concat_list_line, db_to_linear, default_output, encoder_args, escape_filter_path, fmt_secs, fmt_smpte_time, fmt_srt_time, frame_size, is_audio_output, load_brand, MissingFpsError, pad_filters, parse_time, - signed_time_arg, source_codec_video_args, drawn_chain, draws_chroma, label_chain, layer_base_chain, matrix_chain, range_chain, rgb_chain_to_bt709, SVT_PRESET, time_arg, video_args, x264_args, _x264_raw + signed_time_arg, source_codec_video_args, drawn_chain, draws_chroma, label_chain, layer_base_chain, matrix_chain, overlay_format, range_chain, rgb_chain_to_bt709, SVT_PRESET, time_arg, video_args, x264_args, _x264_raw ) from _common.asr import ( ASR_ENGINES, ASR_INSTALL_HINT, _asr_run, die_no_engine, parse_srt, transcribe, _transcribe_in, @@ -183,7 +183,7 @@ def __delattr__(self, name): "aac_args", "add_common", "add_pad_fill_args", "ADVANCE_EM", "analyze_levels", "LEGAL_RANGE_CHOICES", "LEGAL_RANGES", "legal_limits", "measure_signal_range", "apply_common", "detect_scenes", "detect_silences", "SCORE_RE", "SIL_RE", "_aspect_string", "audio_codec_for", "AUDIO_CODECS", "BIDI_SCRIPTS", "_bit_depth", "brand_caption_style", "BRAND_DEFAULTS", "brand_states_font", "_brief", "_BRIEF_DROP", "_brief_summary", "bt709_matrix_args", "bt709_matrix_filter", "bt709_tag_args", "cfr_args", "_char_em", - "char_script", "_check_existing_output", "refuse_existing_outputs", "guard_sidecar", "_check_no_overwrite_input", "_check_output_path", "child_args", + "char_script", "_check_existing_output", "refuse_existing_outputs", "guard_sidecar", "_check_no_overwrite_input", "_check_output_path", "child_args", "option_args", "child_limit", "_CHILDREN", "_cleanup_partial_output", "_cmdline", "CODECS", "color_hex", "_COLOR_TOKEN_RE", "concat_list_line", "Context", "_CURRENT_CTX", "db_to_linear", "decode_pcm_mono", "description_block", "_evidence_rank", "fmt_chapter_time", "propose_chapters", "filler_spans", "FILLER_WORDS", "FILLER_AMBIGUOUS", "FILLER_DISCOURSE_MARKERS", @@ -216,7 +216,7 @@ def __delattr__(self, name): "SCRIPTS", "_sdr_bt709", "_set_current_ctx", "_SHAPING_BUILD_CACHE", "SHAPING_SCRIPTS", "shell_quote", "sibling_temp", "still_input_args", "_SIGNALS_INSTALLED", "signed_time_arg", "_stage_existing_output", "STATE", "SVT_PRESET", "text_width_em", "time_arg", "_timed_out", "_to_float", "_to_int", "_unwatch", "validate_color", "verify_output", - "source_codec_video_args", "drawn_chain", "draws_chroma", "drawn_stream_format", "is_neutral_color", "rgb_island", "label_chain", "layer_base_chain", "matrix_chain", "range_chain", "as_limited", "is_full_range", "limited_range_filter", "rgb_chain_to_bt709", "bt709_label_filter", "rgb_to_bt709_filter", "is_rgb", "video_args", "_VS15", "_VS16", "_watch", "WINDOWS_FONTS", "write_plan", "x264_args", "X264_PRESETS", + "source_codec_video_args", "drawn_chain", "draws_chroma", "drawn_stream_format", "is_neutral_color", "rgb_island", "label_chain", "layer_base_chain", "matrix_chain", "overlay_format", "range_chain", "as_limited", "is_full_range", "limited_range_filter", "rgb_chain_to_bt709", "bt709_label_filter", "rgb_to_bt709_filter", "is_rgb", "video_args", "_VS15", "_VS16", "_watch", "WINDOWS_FONTS", "write_plan", "x264_args", "X264_PRESETS", "_x264_raw", "_ZWJ", "absorb_stage", "audit", "note", "_plan_unbound_inputs", "record_fallback", "record_skip", "refuse", "warn", "ASR_ENGINES", "ASR_INSTALL_HINT", "_asr_run", "die_no_engine", "parse_srt", "transcribe", diff --git a/scripts/_common/asr.py b/scripts/_common/asr.py index 05a43138..f36af21e 100644 --- a/scripts/_common/asr.py +++ b/scripts/_common/asr.py @@ -207,6 +207,9 @@ def write_srt(cues: List[Tuple[float, float, str]], path: str) -> None: # newline so the cue's own text can never fake the format's block boundary. t = re.sub(r"\n{2,}", "\n", t).strip("\n") fh.write(f"{i}\n{fmt_srt_time(s)} --> {fmt_srt_time(e)}\n{t}\n\n") + # under batch.py: this SRT as written is the run's own (an edit made after it is not) + from _common.runner import report_committed + report_committed(path) def transcribe_result(video: str, out_srt: str, language: Optional[str], model: Optional[str], audio_stream: int = 0, diff --git a/scripts/_common/color.py b/scripts/_common/color.py index 0b61d580..ee6a4845 100644 --- a/scripts/_common/color.py +++ b/scripts/_common/color.py @@ -344,3 +344,58 @@ def validate_color(value: str, flag: str = "--color") -> str: if not _COLOR_TOKEN_RE.match(value): die(f"{flag} must be a plain colour (a name, 0xRRGGBB[AA], or #RRGGBB[AA], optionally @alpha), got '{value}'") return value + + +# The tonemap filter reads linear light in units of 100 nits (1.0 = SDR white; its default peak, +# 10, is 1000 nits), so the linearising zscale uses npl=100. Through 3.6.0 it used npl=--peak: +# a 1000-nit pixel came out as 1.0, which tonemap took as 100 nits, so nothing reached SDR white +# (PQ 1000 nits and HLG 100 % -> Y 151 of 235, HLG reference white 91) and raising --peak to a +# brighter master's darkened the picture further. zimg's HLG decode assumes a 1000-nit display, +# so an HLG picture always peaks at 1000 nits there, whatever --peak says. +TONEMAP_REFERENCE_NITS = 100.0 +HLG_PEAK_NITS = 1000.0 + + +def hdr_to_sdr_chain(meta: dict, tonemap: str, peak: float = 1000.0, desat: float = 0.0) -> str: + """PQ / HLG / Dolby Vision to BT.709 SDR: linearise at 100 nits per unit, tone-map the source + peak (`peak` nits for PQ, 1000 for HLG) to SDR white, re-encode as BT.709 limited range.""" + v = meta["video"] + trc = v.get("color_transfer") or "smpte2084" + prim = v.get("color_primaries") or "bt2020" + space = v.get("color_space") or "bt2020nc" + source_peak = HLG_PEAK_NITS if trc == "arib-std-b67" else peak + # zscale needs explicit input tags when the file lacks them + chain: List[str] = [ + f"zscale=tin={trc}:pin={prim}:min={space}:rin={v.get('color_range') or 'tv'}:t=linear" + f":npl={TONEMAP_REFERENCE_NITS:g}", + "format=gbrpf32le", + "zscale=p=bt709", + f"tonemap=tonemap={tonemap}:desat={desat:g}:peak={source_peak / TONEMAP_REFERENCE_NITS:g}", + "zscale=t=bt709:m=bt709:r=tv", + "format=yuv420p", + ] + return ",".join(chain) + + +def bt2020_sdr_to_bt709_chain(meta: dict) -> str: + """Gamut conversion only, for BT.2020 primaries on an SDR transfer (probe: `bt2020_or_hdr` + true, `hdr_signal` false). Such a file is already display-referred SDR: linearising it with + its own transfer, mapping the primaries to BT.709 and re-applying the BT.709 transfer keeps + white at white and grey at grey. The tone map the HDR chain adds would treat 100 % SDR white + as a fraction of a 1000-nit peak and darken the picture (white Y 235 -> 151, grey 126 -> 90).""" + v = meta["video"] + trc = v.get("color_transfer") or "bt709" + space = v.get("color_space") or "bt2020nc" + return ",".join([ + f"zscale=tin={trc}:pin=bt2020:min={space}:rin={v.get('color_range') or 'tv'}:t=linear", + "format=gbrpf32le", + "zscale=p=bt709", + "zscale=t=bt709:m=bt709:r=tv", + "format=yuv420p", + ]) + + +def is_bt2020_sdr(v: dict) -> bool: + """BT.2020 primaries on a non-HDR transfer and no Dolby Vision: gamut-only --to-sdr.""" + return bool(v.get("bt2020_or_hdr")) and not v.get("hdr_signal") and not v.get("dolby_vision") \ + and v.get("color_transfer") not in ("smpte2084", "arib-std-b67") diff --git a/scripts/_common/decision.py b/scripts/_common/decision.py index b8b0b8c5..d2efa61b 100644 --- a/scripts/_common/decision.py +++ b/scripts/_common/decision.py @@ -463,6 +463,20 @@ def cfr_args(meta: Optional[Dict[str, Any]], fps: Optional[float] = None) -> Lis return ["-fps_mode", "cfr", "-r", f"{rate:g}"] +def delivered_rate(meta: Optional[Dict[str, Any]], fps: Optional[float] = None) -> Optional[float]: + """The constant rate cfr_args() forces on the output (its -r), or None when it forces none and + the output keeps the source's rate -- what a start timecode is counted at + (timeline.output_timecode_args).""" + args = cfr_args(meta, fps) + return float(args[-1]) if args else None + + +# how far before a cut an input seek lands, so a start inside the B-frame reorder delay before a +# keyframe still decodes from the keyframe BEFORE it; the margin is dropped on the output side +# (cut.py, sync.py --trim-second) +SEEK_MARGIN = 1.0 + + # ------------------------------------------------------- VideoToolbox (--hw) # # CRF -> VideoToolbox -q:v (1-100, higher = better): at each CRF the highest -q:v whose SSIM matched @@ -864,6 +878,26 @@ def rgb_chain_to_bt709(line_meta: Optional[Dict[str, Any]], pix_fmt: str = "yuva return "," + rgb_to_bt709_filter(pix_fmt) +def overlay_format(meta: Optional[Dict[str, Any]]) -> str: + """":format=..." for an `overlay` whose main picture is the one `meta` probes, or "". + + overlay's default format is `yuv420`, which is 8-bit, and `format=auto` picks an 8-bit one as + soon as the layer is 8-bit, so through 3.6.0 a cutaway, a redaction, a graphics layer or a PiP + over a 10-bit HDR or ProRes source took the whole picture to 8 bits (and `overlay --image`'s + RGBA logo took it through 8-bit RGB) before the 10-bit encoder. On a line that keeps more than + 8 bits (HDR/BT.2020, --codec prores) and a main picture deeper than 8 bits the overlay runs at + 10 bits in the main's own chroma subsampling, and an 8-bit layer is widened to meet it (bit + depth only, no matrix). An alpha line keeps the format its tool chose.""" + v = (meta or {}).get("video") or {} + if STATE.alpha_line is not None or (v.get("bit_depth") or 8) <= 8: + return "" + if not (v.get("bt2020_or_hdr") or keeps_source_tags()): + return "" + pix = v.get("pix_fmt") or "" + sub = "444" if "444" in pix else "422" if "422" in pix else "420" + return f":format=yuv{sub}p10" + + def draws_chroma(colours: Sequence[Any], line_meta: Optional[Dict[str, Any]]) -> bool: """True when one of `colours` has chroma (is_neutral_color) and the line video_args(line_meta) tags the output BT.709 -- when a drawn colour needs drawn_chain() at all.""" @@ -1012,6 +1046,52 @@ def load_brand(path: Optional[str]) -> Dict[str, Any]: return brand +# Every key of a render.py project that names a file. render.py reads each one relative to the +# project file (its rel()); a copy of the project written anywhere else -- batch.py's per-item +# project in its work directory, multicam.py --write-project -- has to carry them absolute, or +# render looks for "bed.wav" beside the copy. A path is (section, key): section None is the +# project's top level, "x[]" each entry of the list x. tests/test_contract.py checks that every +# rel() in render.py reads one of these. +PROJECT_PATH_KEYS = ( + (None, "output"), (None, "brand"), (None, "chapters"), + ("clips[]", "src"), ("clips[].snap", "source"), ("snap", "source"), + ("audio", "music"), ("audio", "replace"), ("audio", "effects"), + ("captions", "text"), ("captions", "srt"), ("captions", "ass"), + ("overlays[]", "image"), ("overlays[]", "video"), ("overlays[]", "font_file"), + ("audiogram", "image"), +) + + +def rebase_project_paths(proj: Dict[str, Any], base: "os.PathLike[str] | str") -> Dict[str, Any]: + """A copy of a render.py project whose relative file paths (PROJECT_PATH_KEYS) are joined onto + `base`, the directory they were written relative to; absolute paths, URLs, non-strings, inline + chapter lists and keys the project does not have are left as they are.""" + import copy + out = copy.deepcopy(proj) + base = Path(base) + + def fix(obj: Any, key: str) -> None: + value = obj.get(key) if isinstance(obj, dict) else None + # a URL is left for the tool to refuse by what it is (waveform.py: no network access) + if isinstance(value, str) and value and not os.path.isabs(value) and "://" not in value: + obj[key] = str(base / value) + + def nodes(section: "Optional[str]") -> List[Any]: + found: List[Any] = [out] + for part in (section.split(".") if section else []): + name, many = (part[:-2], True) if part.endswith("[]") else (part, False) + nxt: List[Any] = [] + for node in found: + value = node.get(name) if isinstance(node, dict) else None + nxt.extend(value if many and isinstance(value, list) else [] if many else [value]) + found = nxt + return found + for section, key in PROJECT_PATH_KEYS: + for node in nodes(section): + fix(node, key) + return out + + def brand_states_font(brand: Dict[str, Any]) -> bool: """Did the brand FILE actually name a font (top-level `font`, `caption.font` or `styles.caption.font`)? BRAND_DEFAULTS always supplies one, so the merged document can never diff --git a/scripts/_common/emit.py b/scripts/_common/emit.py index fc57c00c..ca2194d9 100644 --- a/scripts/_common/emit.py +++ b/scripts/_common/emit.py @@ -209,6 +209,14 @@ def source_carry(output: Optional[str], ctx: "Context", extra: Dict[str, Any]) - continue if item in ("audio_channels", "audio_language") and not out["has_audio"]: continue + if item == "timecode": + # the value, not only its presence: a cut from 1 s that wrote the source's 01:00:00:00 + # on frame 30 kept a timecode that lies. A tool that moved the picture's start names + # the label it wrote (STATE.expected_timecode, "" for none); any other keeps the source's + want = getattr(STATE, "expected_timecode", None) + want = src.get("timecode_value") if want is None else want + (kept if want and out.get("timecode_value") == want else dropped).append(item) + continue (kept if out.get(item, 0) >= src[item] else dropped).append(item) if not kept and not dropped: return None diff --git a/scripts/_common/emoji.py b/scripts/_common/emoji.py index 574b7260..2cb5afe0 100644 --- a/scripts/_common/emoji.py +++ b/scripts/_common/emoji.py @@ -18,9 +18,9 @@ # separately from detect_script() so a cue's font resolution is still decided by its letters. EMOJI_RANGES = ( (0x1F300, 0x1FAFF), # symbols & pictographs, supplemental, extended-A - (0x1F000, 0x1F0FF), # mahjong/domino/playing cards - (0x2600, 0x27BF), # misc symbols + dingbats - (0x2B00, 0x2BFF), # misc symbols and arrows + (0x1F000, 0x1F0FF), # mahjong/domino/playing cards (only 🀄 and 🃏 are emoji) + (0x2600, 0x27BF), # misc symbols + dingbats (Emoji_Presentation only: _is_emoji_char) + (0x2B00, 0x2BFF), # misc symbols and arrows (likewise) (0xFE0F, 0xFE0F), # VS16 (emoji presentation selector) (0x1F1E6, 0x1F1FF), # regional indicators (flags) (0x20E3, 0x20E3), # combining enclosing keycap @@ -52,19 +52,80 @@ _KEYCAP_BASES = frozenset("0123456789#*") +def _cps(*spans) -> "frozenset": + out = set() + for span in spans: + lo, hi = span if isinstance(span, tuple) else (span, span) + out.update(range(lo, hi + 1)) + return frozenset(out) + + +# The text-symbol blocks in EMOJI_RANGES (U+2600-27BF, U+2B00-2BFF, U+1F000-1F0FF) are mostly +# *text*: ♪ U+266A, ✓ U+2713 and ★ U+2605 are not emoji at all in Unicode's emoji-data.txt, and +# ♥ U+2665, ✔ U+2714, ☺ U+263A are emoji only when VS16 asks for the picture -- their default +# presentation is the character, which the caption font draws. Counting the whole block made +# SDH music notes and check marks "emoji", and `--emoji none` (what macOS and a Linux box +# without a colour face get under auto) cut them out of the burned text. In those blocks only +# Emoji_Presentation=Yes characters start a cluster on their own; an Emoji=Yes one does when +# VS16 or a skin-tone modifier follows it. Pinned from emoji-data.txt (Unicode 15.1); the +# pictograph planes keep their block rule. +_EMOJI_PRESENTATION_SYMBOLS = _cps( + (0x2614, 0x2615), (0x2648, 0x2653), 0x267F, 0x2693, 0x26A1, (0x26AA, 0x26AB), + (0x26BD, 0x26BE), (0x26C4, 0x26C5), 0x26CE, 0x26D4, 0x26EA, (0x26F2, 0x26F3), 0x26F5, + 0x26FA, 0x26FD, 0x2705, (0x270A, 0x270B), 0x2728, 0x274C, 0x274E, (0x2753, 0x2755), 0x2757, + (0x2795, 0x2797), 0x27B0, 0x27BF, (0x2B1B, 0x2B1C), 0x2B50, 0x2B55, 0x1F004, 0x1F0CF) +_EMOJI_TEXT_DEFAULT_SYMBOLS = _cps( + (0x2600, 0x2604), 0x260E, 0x2611, 0x2618, 0x261D, 0x2620, (0x2622, 0x2623), 0x2626, 0x262A, + (0x262E, 0x262F), (0x2638, 0x263A), 0x2640, 0x2642, (0x265F, 0x2660), 0x2663, + (0x2665, 0x2666), 0x2668, 0x267B, 0x267E, (0x2692, 0x2697), 0x2699, (0x269B, 0x269C), + 0x26A0, 0x26A7, (0x26B0, 0x26B1), 0x26C8, 0x26CF, 0x26D1, 0x26D3, 0x26E9, (0x26F0, 0x26F1), + 0x26F4, (0x26F7, 0x26F9), 0x2702, (0x2708, 0x2709), (0x270C, 0x270D), 0x270F, 0x2712, 0x2714, + 0x2716, 0x271D, 0x2721, (0x2733, 0x2734), 0x2744, 0x2747, (0x2763, 0x2764), 0x27A1, + (0x2B05, 0x2B07)) +_SYMBOL_BLOCKS = ((0x2600, 0x27BF), (0x2B00, 0x2BFF), (0x1F000, 0x1F0FF)) + + +def _in_symbol_block(cp: int) -> bool: + return any(lo <= cp <= hi for lo, hi in _SYMBOL_BLOCKS) + + def _is_emoji_char(ch: str) -> bool: + """Is this character drawn as an emoji with nothing after it? In the text-symbol blocks only + the Emoji_Presentation characters are; ♪ ✓ ★ ♥ are text (see _EMOJI_PRESENTATION_SYMBOLS).""" cp = ord(ch) + if _in_symbol_block(cp): + return cp in _EMOJI_PRESENTATION_SYMBOLS return any(lo <= cp <= hi for lo, hi in EMOJI_RANGES) +def _can_be_emoji(ch: str) -> bool: + """Emoji=Yes: an emoji by default, or a text-default symbol VS16 can turn into one (♥ ✔).""" + return _is_emoji_char(ch) or ord(ch) in _EMOJI_TEXT_DEFAULT_SYMBOLS + + def _is_emoji_base(ch: str) -> bool: """Can this character START an emoji cluster? Pictographs and regional indicators can; the joiners and modifiers (ZWJ, VS16, keycap, skin tone) never can -- they only bind to an emoji base that already stands before them. Without this, a ZWJ or a VS16 sitting after an - ordinary letter turned that letter into "an emoji" and the PNG route replaced it with a gap.""" + ordinary letter turned that letter into "an emoji" and the PNG route replaced it with a gap. + A text-default symbol (♥ ✔ ☝) is a base only with the VS16 or modifier that follows it: + emoji_clusters() checks that, this function answers for the character alone.""" return ord(ch) not in _EMOJI_TAIL and _is_emoji_char(ch) +def _starts_emoji(text: str, i: int) -> bool: + """Does an emoji cluster start at text[i]? A default-emoji base does; a text-default symbol + does only when VS16 or a skin-tone modifier follows it (♥️, ☝🏽), the sequences Unicode + defines as emoji; ♥ alone, and ♪ ✓ ★ always, are characters the font draws.""" + ch = text[i] + if _is_emoji_base(ch): + return True + if ord(ch) in _EMOJI_TEXT_DEFAULT_SYMBOLS and i + 1 < len(text): + nxt = ord(text[i + 1]) + return nxt == _VS16 or 0x1F3FB <= nxt <= 0x1F3FF + return False + + def emoji_clusters(text: str) -> "List[Tuple[int, str]]": """(index in `text`, cluster) for every emoji in it, ZWJ sequences, VS16, keycaps, flag pairs and skin-tone modifiers kept together -- 👩‍💻 is one cluster, not three, and 1️⃣ starts at the @@ -74,7 +135,9 @@ def emoji_clusters(text: str) -> "List[Tuple[int, str]]": base (`0-9 # *`) that is actually followed by U+20E3. A ZWJ is glue *inside* a cluster, never a starter and never a tail on its own: `क्‍ष` (Hindi ka + virama + ZWJ + ssa) and `abc‍def` contain no emoji. A base explicitly marked with U+FE0E (VS15, text presentation) is likewise - not an emoji -- the author asked for the character, not the picture. + not an emoji -- the author asked for the character, not the picture. A text-default symbol + (♥ ✔ ☝) starts one only with the VS16 or skin-tone modifier after it, and a symbol that is + not emoji at all (♪ ✓ ★) never does: the caption font draws those. """ out: "List[Tuple[int, str]]" = [] i = 0 @@ -82,7 +145,7 @@ def emoji_clusters(text: str) -> "List[Tuple[int, str]]": while i < n: ch = text[i] start = i - if _is_emoji_base(ch): + if _starts_emoji(text, i): j = i + 1 if j < n and ord(text[j]) == _VS15: # text presentation requested: not an emoji i = j + 1 @@ -104,7 +167,9 @@ def emoji_clusters(text: str) -> "List[Tuple[int, str]]": if cp in (_VS16, _KEYCAP) or 0x1F3FB <= cp <= 0x1F3FF: j += 1 continue - if cp == _ZWJ and j + 1 < n and _is_emoji_base(text[j + 1]): + if cp == _ZWJ and j + 1 < n and ord(text[j + 1]) not in _EMOJI_TAIL \ + and _can_be_emoji(text[j + 1]): + # inside a ZWJ sequence a text-default symbol is emoji glue too: 🏃‍♀ ends in ♀ j += 2 continue if (j == start + 1 and ord(ch) in _EMOJI_REGIONAL and cp in _EMOJI_REGIONAL): @@ -345,7 +410,7 @@ def resolve_emoji_assets(flag: "Optional[str]" = None, project: "Optional[str]" -def emoji_filter_chain(plan, base_label, out_label, first_input=1, to_yuv="", base_chain=""): +def emoji_filter_chain(plan, base_label, out_label, first_input=1, to_yuv="", base_chain="", overlay_opts=""): """(chains, inputs) that composite the planned PNGs on top of `base_label`. `inputs` is a list of argv fragments, each ending in the asset path, to be appended to the @@ -355,6 +420,8 @@ def emoji_filter_chain(plan, base_label, out_label, first_input=1, to_yuv="", ba converted to the BT.709 the output is tagged with instead of by overlay's BT.601 default; `base_chain` (label_chain(meta, meta): an untagged source's BT.709 label) goes in front of the first overlay, so 7.1+ does not convert the two pictures to one another. + `overlay_opts` ends each overlay's options: the caller's overlay_format(meta), so a 10-bit + picture is not composited at overlay's 8-bit default. """ overlays = plan.get("overlays") or [] if not overlays: @@ -413,6 +480,6 @@ def _key(o): # exactly what shipped first). eof_action=repeat (the default) holds the still for the # whole timeline; enable= is what confines it to the cue's window. chains.append(f"[{cur}][{label}]overlay=x={x}:y={o['y']}:" - f"enable='between(t,{o['start']:.3f},{o['end']:.3f})'[{nxt}]") + f"enable='between(t,{o['start']:.3f},{o['end']:.3f})'{overlay_opts}[{nxt}]") cur = nxt return chains, inputs diff --git a/scripts/_common/probe.py b/scripts/_common/probe.py index bb1ec3cb..6e22ff95 100644 --- a/scripts/_common/probe.py +++ b/scripts/_common/probe.py @@ -412,6 +412,7 @@ def probe(path: str, role: str = "input") -> Dict[str, Any]: "creation_time": 1 if any(k.lower() == "creation_time" for k in (fmt.get("tags") or {})) else 0, "audio_language": 1 if lang and lang.lower() != "und" else 0, "timecode": 1 if out["timecode"].get("value") else 0, + "timecode_value": out["timecode"].get("value"), "audio_channels": (_to_int(audio.get("channels")) or 0) if audio else 0, "has_video": bool(video), "has_audio": bool(audio), } @@ -1281,7 +1282,7 @@ def signatures_match(sigs: list, ext: str) -> bool: return False else: codec = str(a.get("codec_name") or "") - if not (codec.startswith("pcm_") or codec in IN_BAND_CONFIG or ext in TS_EXTS): + if not (codec.startswith("pcm_") or codec in IN_BAND_CONFIG or ext.lower() in TS_EXTS): return False return True diff --git a/scripts/_common/runner.py b/scripts/_common/runner.py index 33e972b6..d7758b8a 100644 --- a/scripts/_common/runner.py +++ b/scripts/_common/runner.py @@ -332,7 +332,7 @@ class Context: __slots__ = ("dry_run", "json", "json_brief", "progress", "fast", "duration_hint", "commands", "timeout", "overwrite", "written", "preexisting", "plan", "plan_written", "plan_inputs", "codec", "alpha_line", "hw", "hw_source", "hw_notes", "hw_swaps", "hw_fallback", "hw_env_ignored", "hw_stages", "hw_auto", "notes", "warnings", "fallbacks", "skipped", "audit_log", "stage_verification", "first_input", "audio_inputs", "video_inputs", - "source_timecode", "smpte_hint", "delivered") + "source_timecode", "smpte_hint", "delivered", "expected_timecode") def __init__(self) -> None: self.reset() @@ -343,6 +343,7 @@ def reset(self) -> None: self.first_input = None # (path, audio track count) of the first input probed: emit() reports tracks the output left out self.video_inputs = set() # real paths of the inputs probed that have a picture: with two or more, which one the output shows is the tool's choice (sync --trim-second, a join) self.source_timecode = None # (start timecode, duration) of the first input probed that carries one (time_arg's SMPTE hint) + self.expected_timecode = None # the start timecode a tool that moved the picture's start wrote (timeline.output_timecode_args; "" for none): source_carry compares it self.smpte_hint = None # what die() says when an hh:mm:ss:ff flag read past the end looks like a source timecode (#411) self.delivered: List[str] = [] # deliverables written before a later step (export --normalize, render's chapters/check): a stop then keeps them (#415) self.audio_inputs = set() # real paths of the inputs probed that have sound: with two or more, whose tracks an output should keep is the tool's choice (grid --audio-from, a mix) @@ -481,6 +482,14 @@ def apply_common(args: "argparse.Namespace") -> None: STATE.progress = bool(getattr(args, "progress", False)) STATE.fast = bool(getattr(args, "fast", False)) STATE.overwrite = bool(getattr(args, "overwrite", False)) + if STATE.plan and os.environ.get("FFMPEG_SKILL_READ_ONLY", "") not in ("", "0"): + # mcp/server.py runs the tools it annotates readOnlyHint (probe, check, cropdetect) with + # this set: a plan is the one file they could write, so it is refused here, after argparse, + # whatever spelling of --plan a raw argv used. Refused before the exit hook can write it. + plan_path, STATE.plan = STATE.plan, None + refuse("input", "read_only_plan_refused", + f"--plan {plan_path!r}: this tool runs read-only here (FFMPEG_SKILL_READ_ONLY) and writes no file; " + "run it from the command line to write a plan", plan=plan_path) if STATE.plan: # the plan is a file this run writes itself: the side-file guards apply (#408), so it # never replaces an input, the output or an existing file without --overwrite @@ -1229,7 +1238,13 @@ def refuse_existing_outputs(paths: Sequence[str]) -> None: if output not in existing: existing.append(output) facts.append({"path": output, "resolved": real, "size": size, "mtime_ns": mtime_ns}) + # a file batch.py delivered on an earlier --watch pass, still exactly as it left it, is that + # run's own: replaced without --overwrite, per path (anything else existing is still refused) + if existing and not STATE.overwrite: + keep = [i for i, p in enumerate(existing) if not owned_replaceable(p)] + existing, facts = [existing[i] for i in keep], [facts[i] for i in keep] if not existing or STATE.overwrite: + report_owned_targets(paths) return hint = "pass --overwrite, or choose another -o path" if len(existing) == 1: @@ -1248,6 +1263,111 @@ def _remember_output(cmd: Sequence[str]) -> None: STATE.written.add(os.path.realpath(output)) except OSError: pass + report_committed(output) + + +# set only by batch.py for the steps it runs: the path of a JSON file {"replaceable": {real path: +# owned_identity()}, "report": directory}. "replaceable" is every file that batch process +# delivered (a --watch loop's earlier passes) as it left it; a step replaces one of those without +# --overwrite while it is still that file, and nothing else. Each process that passes the output +# check writes /.json: {"targets": the paths it was allowed to write, "committed": +# {real path: owned_identity()} taken when each of them was written (report_committed), so batch +# learns the side files (an .srt beside the video) it delivered as they were when written, not as +# they are when the item ends -- an edit made while a later step ran is not adopted. +OWNED_ENV = "_FFMPEG_SKILL_OWNED" +_OWNED_CACHE: Dict[str, Dict[str, Any]] = {} +_OWNED_TARGETS: Dict[str, List[str]] = {} +_OWNED_COMMITTED: Dict[str, Dict[str, Dict[str, int]]] = {} + + +def owned_identity(path: str) -> "Optional[Dict[str, int]]": + """{size, mtime_ns, ino, dev} of a regular file: what tells the file batch.py delivered from + one put at the same path since (a replacement with the same size and mtime is a new inode). + None when the path is not a regular file.""" + try: + st = os.stat(path) + except OSError: + return None + if not os.path.isfile(path): + return None + return {"size": st.st_size, "mtime_ns": st.st_mtime_ns, "ino": st.st_ino, "dev": st.st_dev} + + +def _owned_config(config: Optional[str] = None) -> Dict[str, Any]: + config = config if config is not None else os.environ.get(OWNED_ENV, "") + if not config: + return {} + if config not in _OWNED_CACHE: + try: + with open(config, encoding="utf-8") as fh: + doc = json.load(fh) + except (OSError, ValueError): + doc = {} + _OWNED_CACHE[config] = doc if isinstance(doc, dict) else {} + return _OWNED_CACHE[config] + + +def owned_replaceable(path: str, config: Optional[str] = None) -> bool: + """Whether `path` is a file the batch.py process running this step delivered, still with the + identity it had then (checked now, at the output check, not when the step was planned).""" + replaceable = _owned_config(config).get("replaceable") + if not isinstance(replaceable, dict): + return False + try: + real = os.path.realpath(path) + except OSError: + return False + expected = replaceable.get(real) + return isinstance(expected, dict) and owned_identity(real) == expected + + +def report_owned_targets(paths: Sequence[Optional[str]], config: Optional[str] = None) -> None: + """Under batch.py: record paths this process passed the output check for, in the report + directory its config names. Nothing outside batch.py; a report that cannot be written only + means the next pass refuses that file as someone else's.""" + config = config if config is not None else os.environ.get(OWNED_ENV, "") + report = _owned_config(config).get("report") + if not isinstance(report, str) or not report: + return + targets = _OWNED_TARGETS.setdefault(config, []) + for p in paths: + if p: + try: + real = os.path.realpath(p) + except OSError: + continue + if real not in targets: + targets.append(real) + _write_owned_report(config, report) + + +def _write_owned_report(config: str, report: str) -> None: + doc = {"targets": _OWNED_TARGETS.get(config, []), "committed": _OWNED_COMMITTED.get(config, {})} + try: + with open(os.path.join(report, f"{os.getpid()}.json"), "w", encoding="utf-8") as fh: + json.dump(doc, fh) + except OSError: + pass + + +def report_committed(path: Optional[str], config: Optional[str] = None) -> None: + """Under batch.py: record the identity `path` has right now, just after this process wrote it, + if it passed this process's output check. What batch later lets a step replace is that file + as it was written; one edited afterwards no longer matches and is refused.""" + config = config if config is not None else os.environ.get(OWNED_ENV, "") + report = _owned_config(config).get("report") + if not path or not isinstance(report, str) or not report: + return + try: + real = os.path.realpath(path) + except OSError: + return + if real not in _OWNED_TARGETS.get(config, []): + return + ident = owned_identity(real) + if ident: + _OWNED_COMMITTED.setdefault(config, {})[real] = ident + _write_owned_report(config, report) def remember_written(*paths: Optional[str]) -> None: @@ -1260,6 +1380,7 @@ def remember_written(*paths: Optional[str]) -> None: STATE.written.add(os.path.realpath(path)) except OSError: pass + report_committed(path) def _timed_out(cmd: Sequence[str], seconds: float) -> "None": @@ -1649,13 +1770,14 @@ def child_limit(per_call: Optional[float] = None) -> Optional[float]: return (limit * 4 + 60) if limit else None -def run_tool(argv: Sequence[str], *, per_call: Optional[float] = None) -> subprocess.CompletedProcess: +def run_tool(argv: Sequence[str], *, per_call: Optional[float] = None, + env: Optional[Dict[str, str]] = None) -> subprocess.CompletedProcess: """Run a sibling script (`argv[0]` is the script path) under child_limit(). On overrun the child is killed and a CompletedProcess is returned whose stdout is this skill's own failure document (kind timeout, exit 124), so callers that parse the child's --json see a timeout - exactly as they would from the child itself.""" + exactly as they would from the child itself. `env` replaces the inherited environment.""" limit = child_limit(per_call) - child = subprocess.Popen([sys.executable] + list(argv), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace") + child = subprocess.Popen([sys.executable] + list(argv), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, encoding="utf-8", errors="replace", env=env) _watch(child, []) # a sibling script removes its own partial output; there is none of ours to clean try: out, err = child.communicate(timeout=limit) @@ -1674,6 +1796,20 @@ def run_tool(argv: Sequence[str], *, per_call: Optional[float] = None) -> subpro return subprocess.CompletedProcess(list(argv), 124, json.dumps(doc), partial + f"\nerror: {msg}\n") +def option_args(flag: str, value: Any) -> List[str]: + """One option and its value for a sibling tool's argparse, as [flag, value] -- or as the single + token `flag=value` when the value starts with '-'. argparse reads a separate token that starts + with '-' (and is not a negative number) as the next option, so overlay text "-50%" was refused + as "expected one argument" and a value like "--plan=x" became an option of its own. The joined + form is read as the value whatever it holds. `flag` must be a long option: `-o=x` is not.""" + text = str(value) + if text.startswith("-"): + if not flag.startswith("--"): + raise ValueError(f"option_args: {flag!r} is not a long option") + return [f"{flag}={text}"] + return [flag, text] + + def child_args() -> List[str]: """The shared flags a tool that runs sibling scripts (render.py, batch.py) forwards to them, so one `--timeout`/`--overwrite`/`--fast`/`--dry-run` on the outer command governs every @@ -1749,7 +1885,7 @@ def subtitle_container_holds(codec: str, out_ext: str) -> bool: return True -def run_keeping_subtitles(cmd: List[str], output: str) -> bool: +def run_keeping_subtitles(cmd: List[str], output: str, relabel_timecode: bool = False) -> bool: """Run an ffmpeg command that already maps its video/audio, trying first to also stream-copy any subtitle/data streams the source has (`-map 0:s?`/`0:d?` are no-ops when there are none). A source whose subtitle codec cannot be copied into the target container @@ -1761,8 +1897,13 @@ def run_keeping_subtitles(cmd: List[str], output: str) -> bool: The retry is recorded with the first attempt's error (`diagnostics.fallbacks`, `non_av_streams_dropped`): the caller's `dropped_non_av_streams` says that streams went, the record says why, which is the only way to tell a container that cannot hold them from a first - attempt that failed for another reason and happened to succeed the second time.""" - first = run(cmd + ["-map", "0:s?", "-map", "0:d?", "-c:s", "copy", "-c:d", "copy", output], check=False) + attempt that failed for another reason and happened to succeed the second time. + + `relabel_timecode`: the command writes its own start timecode (timeline.output_timecode_args), + so a source's tmcd data track is not copied -- its label, at the source's rate, would win over + the new one; the MOV/MP4 muxer builds the track again from the label.""" + skip_tmcd = ["-map", "-0:d:m:timecode"] if relabel_timecode else [] + first = run(cmd + ["-map", "0:s?", "-map", "0:d?"] + skip_tmcd + ["-c:s", "copy", "-c:d", "copy", output], check=False) if first.returncode == 0: return False if classify_ffmpeg_failure(first.stderr or "") or not _has_non_av_streams(cmd): diff --git a/scripts/_common/timeline.py b/scripts/_common/timeline.py index 0c300039..51fa8068 100644 --- a/scripts/_common/timeline.py +++ b/scripts/_common/timeline.py @@ -153,6 +153,45 @@ def _tc_label(frames: int, rate: Fraction, drop: bool) -> str: return _tc(frames, nominal)[:-3] + ";" + _tc(frames, nominal)[-2:] +def output_timecode_args(meta: Dict[str, Any], first_frame: Optional[float], rate: Optional[float] = None) -> List[str]: + """The arguments that give an output the start timecode of the source frame it starts on: + `first_frame` seconds after the source picture's first frame (negative: a head the tool + added), or None to write no start timecode at all (a join, whose later frames no single start + label describes). A tool that moves the start of the picture used to write the source's own + start timecode unchanged: FFmpeg copies the video stream's `timecode` tag, and the MOV/MP4 + muxer rebuilds a tmcd track from it, so a cut from 1 s was stamped 01:00:00:00 for frame 30. + Both the stream and the container tag are set (Matroska keeps a stale stream tag otherwise; + MXF reads the container's). Nothing for a source without a start timecode. The label also + goes on STATE.expected_timecode, which source_carry compares the written file with: an + MXF, which always writes one, reads 00:00:00:00 when none is wanted and is listed as dropped. + `rate` is the frame rate the output is delivered at when the tool changes it (fit --fps, a + VFR source conformed to a constant rate), None for the source's own: the source's label is + read at the source's rate and the output's is counted at the output's, so 01:00:00:12 at 25 + fps (0.48 s past the hour) is 01:00:00:24 at 50.""" + from _common.runner import STATE + tc = (meta or {}).get("timecode") or {} + fps = ((meta or {}).get("video") or {}).get("fps") + if not tc.get("value") or not fps: + return [] + label = "" + if first_frame is not None: + src_rate = exact_rate(float(fps)) + out_rate = exact_rate(float(rate)) if rate else src_rate + drop = _drop_frame(tc.get("drop_frame"), out_rate) + nominal = round(float(out_rate)) + if nominal >= 1: + day = (nominal * 600 - (nominal // 15) * 9) * 144 if drop else nominal * 86400 + # the source label and the offset to the first frame, both in output frames, rounded + # once: 00:00:00:02 at 25 fps plus 0.08 s is 4.8 frames at 30, so frame 5, not 2 + 2 + pos = (Fraction(start_tc_frames(tc, src_rate)) / src_rate + + Fraction(first_frame).limit_denominator(1000000)) * out_rate + label = _tc_label(round(pos) % day, out_rate, drop) + # below 1 fps no frame count can name a frame: the label is removed, as when the first + # frame is unknown, rather than divide by a zero-length day + STATE.expected_timecode = label + return ["-metadata:s:v:0", f"timecode={label}", "-metadata", f"timecode={label}"] + + def build(proj: Dict[str, Any], probes: Dict[str, Dict[str, Any]], rel) -> Dict[str, Any]: """The editor-neutral timeline: every time in frames at `rate`. diff --git a/scripts/_common/wrap.py b/scripts/_common/wrap.py index 2db6bfc1..ef48f756 100644 --- a/scripts/_common/wrap.py +++ b/scripts/_common/wrap.py @@ -29,7 +29,11 @@ # a word. A Thai run is therefore one atom, broken only at the spaces (or the manual `|`) the # writer put there; an over-long run stays long on its own line, the rule long Latin words # already follow. -NO_SPACE_SCRIPTS = ("ja", "zh", "ko") +# Korean is NOT here: it is written with spaces between words (eojeol), so a Hangul run is a word +# like a Latin one -- broken at its spaces, and cut inside only by _slice_atom's escape hatch. +# Per-character atoms broke `시작|합니다` and, once a pass moved the break, joined the halves +# with no space (`늦지마세요`) while the result said text_unchanged. +NO_SPACE_SCRIPTS = ("ja", "zh") NO_BOUNDARY_SCRIPTS = ("th",) # per-character breaking would chop words: keep the run whole @@ -304,9 +308,11 @@ def _join(left: str, atom: str, spaced: bool) -> str: def _break_spaced(first: str, second: str) -> bool: - """Did a space stand at the break between these two wrapped lines? Only spaced scripts put one - there -- a CJK/Thai break sits between two characters that were written with nothing between - them, and re-joining them with a space would insert a character the cue never had.""" + """A guess at whether a space stood at the break between these two wrapped lines, from the + scripts on each side. The wrapper's own passes do not use it: the greedy fill records the real + answer per break (`gaps`) and every pass carries it along, because a script cannot tell -- a + Japanese cue written with spaces (`はい そうです`) lost one whenever a pass moved a break that + stood at a space. Kept as the fallback for a caller that has only the two strings.""" if not first or not second: return False return char_script(first[-1]) not in NO_SPACE_SCRIPTS and char_script(second[0]) not in NO_SPACE_SCRIPTS \ @@ -477,7 +483,18 @@ def best_break(atoms: "Sequence[Tuple[str, bool]]", max_em: float, return None if best is None else best[1] -def _fix_orphans(lines: "List[str]", max_em: float) -> "List[str]": +def _gaps(lines: "List[str]", gaps: "Optional[List[bool]]") -> "List[bool]": + """The per-break space flags a pass reads and updates: the caller's list (updated in place, so + the next pass sees the moved breaks), or the _break_spaced() guess when the caller has none.""" + if gaps is None: + return [_break_spaced(a, b) for a, b in zip(lines, lines[1:])] + if len(gaps) != max(len(lines) - 1, 0): + raise ValueError("gaps must hold one flag per break") + return gaps + + +def _fix_orphans(lines: "List[str]", max_em: float, + gaps: "Optional[List[bool]]" = None) -> "List[str]": """No last line that is a single stranded atom. Greedy wrapping leaves one character alone whenever the line before it filled exactly: eval 14 @@ -486,6 +503,7 @@ def _fix_orphans(lines: "List[str]", max_em: float) -> "List[str]": onto it -- but only while the result still fits and the line above does not become an orphan itself, so a two-word cue is never made worse.""" lines = list(lines) + gaps = _gaps(lines, gaps) for _ in range(len(lines)): if len(lines) < 2: break @@ -499,14 +517,16 @@ def _fix_orphans(lines: "List[str]", max_em: float) -> "List[str]": new_prev = "" for atom, sp in prev[:-1]: new_prev = _join(new_prev, atom, sp) - new_last = _join(moved, tail[0][0], _break_spaced(lines[-2], lines[-1])) + new_last = _join(moved, tail[0][0], gaps[-1]) if text_width_em(new_last) > max_em or text_width_em(new_prev) < ORPHAN_MIN_EM: break lines[-2], lines[-1] = new_prev, new_last + gaps[-1] = spaced return lines -def _fix_weak_lines(lines: "List[str]", max_em: float) -> "List[str]": +def _fix_weak_lines(lines: "List[str]", max_em: float, + gaps: "Optional[List[bool]]" = None) -> "List[str]": """R2, generalised: _fix_orphans run at *every* boundary, against _is_weak_line. 1.15 only ever looked at the last line, so a stranded digit or kana in the middle of a @@ -514,6 +534,7 @@ def _fix_weak_lines(lines: "List[str]", max_em: float) -> "List[str]": of the line above moves down onto it -- with 1.15's two guards intact (the result must still fit, and the line above must not itself become weak), so the line count never changes.""" lines = list(lines) + gaps = _gaps(lines, gaps) for i in range(len(lines) - 1, 0, -1): for _ in range(len(lines)): if not _is_weak_line(lines[i]): @@ -521,19 +542,21 @@ def _fix_weak_lines(lines: "List[str]", max_em: float) -> "List[str]": prev = _atoms(lines[i - 1]) if len(prev) < 2: break - moved, _spaced = prev[-1] + moved, spaced = prev[-1] new_prev = "" for atom, sp in prev[:-1]: new_prev = _join(new_prev, atom, sp) - new_last = _join(moved, lines[i], _break_spaced(lines[i - 1], lines[i])) + new_last = _join(moved, lines[i], gaps[i - 1]) if text_width_em(new_last) > max_em or _is_weak_line(new_prev) \ or text_width_em(new_prev) < ORPHAN_MIN_EM: break lines[i - 1], lines[i] = new_prev, new_last + gaps[i - 1] = spaced return lines -def _rebalance(lines: "List[str]", max_em: float) -> "List[str]": +def _rebalance(lines: "List[str]", max_em: float, + gaps: "Optional[List[bool]]" = None) -> "List[str]": """Move each break to the one that minimises the widest line of the pair, without changing the line count. @@ -544,11 +567,12 @@ def _rebalance(lines: "List[str]", max_em: float) -> "List[str]": if len(lines) < 2: return lines out = list(lines) + gaps = _gaps(out, gaps) for i in range(len(out) - 1): first, second = out[i], out[i + 1] tail_atoms = _atoms(second) if tail_atoms: - tail_atoms[0] = (tail_atoms[0][0], _break_spaced(first, second)) + tail_atoms[0] = (tail_atoms[0][0], gaps[i]) atoms = _atoms(first) + tail_atoms if not atoms or any(char_script(ch) in NO_SPACE_SCRIPTS for ch in first + second): continue @@ -568,13 +592,15 @@ def _rebalance(lines: "List[str]", max_em: float) -> "List[str]": continue key = (max(wa, wb), abs(wa - wb)) if best is None or key < best[0]: - best = (key, a, b) + best = (key, a, b, atoms[cut][1]) if best is not None: out[i], out[i + 1] = best[1], best[2] + gaps[i] = best[3] return out -def _rebalance_phrase(lines: "List[str]", max_em: float, lang: "Optional[str]") -> "Tuple[List[str], int]": +def _rebalance_phrase(lines: "List[str]", max_em: float, lang: "Optional[str]", + gaps: "Optional[List[bool]]" = None) -> "Tuple[List[str], int]": """_rebalance with R1-R4 deciding, for every script rather than spaced ones only. Returns the new lines and how many breaks a phrase rule moved away from the position 1.15's @@ -582,12 +608,13 @@ def _rebalance_phrase(lines: "List[str]", max_em: float, lang: "Optional[str]") if len(lines) < 2: return list(lines), 0 out = list(lines) + gaps = _gaps(out, gaps) moved = 0 for i in range(len(out) - 1): first, second = out[i], out[i + 1] tail_atoms = _atoms(second) if tail_atoms: - tail_atoms[0] = (tail_atoms[0][0], _break_spaced(first, second)) + tail_atoms[0] = (tail_atoms[0][0], gaps[i]) atoms = _split_hyphens(_atoms(first) + tail_atoms) if len(atoms) < 2: continue @@ -602,11 +629,13 @@ def _rebalance_phrase(lines: "List[str]", max_em: float, lang: "Optional[str]") if (a, b) != (first, second): moved += 1 out[i], out[i + 1] = a, b + gaps[i] = atoms[cut][1] return out, moved def _greedy_chunks(raw: str, max_em: float, *, slice_overlong: bool = False, - sliced: "Optional[List[int]]" = None) -> "List[str]": + sliced: "Optional[List[int]]" = None, + gaps: "Optional[List[bool]]" = None) -> "List[str]": """The greedy fill on its own: the line count every mode must keep. `slice_overlong` (off by default -- only caption.py's final burn-in pass turns it on, never @@ -615,47 +644,65 @@ def _greedy_chunks(raw: str, max_em: float, *, slice_overlong: bool = False, katakana run that already fits is completely untouched -- is hard-sliced by _slice_atom() instead of kept whole. `sliced`, when given, gets one entry (the number of extra lines that one atom produced) per atom actually sliced, which is how the caller counts - `broken_inside_word` without re-deriving it from the output lines.""" + `broken_inside_word` without re-deriving it from the output lines. + + `gaps`, when given, is filled with one flag per break: did a space stand there in `raw`? The + passes in _balance() carry it, so a break they move keeps (or drops) exactly that space.""" current = "" + current_gap = False # did a space stand before `current`'s first atom chunk: "List[str]" = [] - for atom, spaced in _atoms(raw): - if slice_overlong and not current and text_width_em(atom) > max_em: + breaks: "List[bool]" = [] + + def emit(line: str, gap_before: bool) -> None: + if chunk: + breaks.append(gap_before) + chunk.append(line) + + def start_line(atom: str, spaced: bool) -> "Tuple[str, bool]": + """The new current line opening with `atom`, sliced first when it is too wide alone.""" + if slice_overlong and text_width_em(atom) > max_em: pieces = _slice_atom(atom, max_em) if sliced is not None and len(pieces) > 1: sliced.append(len(pieces) - 1) - chunk.extend(pieces[:-1]) - current = pieces[-1] + if len(pieces) > 1: + emit(pieces[0], spaced) + for piece in pieces[1:-1]: + emit(piece, False) + return pieces[-1], False + return atom, spaced + + for atom, spaced in _atoms(raw): + if not current: + current, current_gap = start_line(atom, spaced) continue candidate = _join(current, atom, spaced) - if current and text_width_em(candidate) > max_em: - chunk.append(current) - if slice_overlong and text_width_em(atom) > max_em: - pieces = _slice_atom(atom, max_em) - if sliced is not None and len(pieces) > 1: - sliced.append(len(pieces) - 1) - chunk.extend(pieces[:-1]) - current = pieces[-1] - else: - current = atom + if text_width_em(candidate) > max_em: + emit(current, current_gap) + current, current_gap = start_line(atom, spaced) else: current = candidate if current: - chunk.append(current) + emit(current, current_gap) + if gaps is not None: + gaps[:] = breaks return chunk -def _balance(chunk: "List[str]", max_em: float, mode: str, lang: "Optional[str]") -> "List[str]": +def _balance(chunk: "List[str]", max_em: float, mode: str, lang: "Optional[str]", + gaps: "Optional[List[bool]]" = None) -> "List[str]": """The post-passes for one greedy chunk, in the mode's own order. Never changes the count: a pass that would is discarded, exactly as 1.15 did.""" if len(chunk) < 2: return chunk + gaps = list(gaps) if gaps is not None else None if mode == "measured": - fixed = _fix_orphans(chunk, max_em) - rebalanced = _rebalance(fixed, max_em) + fixed = _fix_orphans(chunk, max_em, gaps) + rebalanced = _rebalance(fixed, max_em, list(gaps) if gaps is not None else None) else: - fixed = _fix_weak_lines(_fix_orphans(chunk, max_em), max_em) - rebalanced, _moved = _rebalance_phrase(fixed, max_em, lang) - rebalanced = _fix_weak_lines(rebalanced, max_em) + fixed = _fix_weak_lines(_fix_orphans(chunk, max_em, gaps), max_em, gaps) + later = list(gaps) if gaps is not None else None + rebalanced, _moved = _rebalance_phrase(fixed, max_em, lang, later) + rebalanced = _fix_weak_lines(rebalanced, max_em, later) if len(rebalanced) == len(chunk): return rebalanced return fixed if len(fixed) == len(chunk) else chunk @@ -681,8 +728,9 @@ def wrap_text(text: str, max_em: float, *, balance: bool = True, mode: str = "ph for raw in text.split("\n"): if not raw.strip(): continue - chunk = _greedy_chunks(raw, max_em, slice_overlong=slice_overlong, sliced=sliced) - lines.extend(_balance(chunk, max_em, mode, lang) if balance else chunk) + gaps: "List[bool]" = [] + chunk = _greedy_chunks(raw, max_em, slice_overlong=slice_overlong, sliced=sliced, gaps=gaps) + lines.extend(_balance(chunk, max_em, mode, lang, gaps) if balance else chunk) return lines or [text] def wrap_variants(text: str, max_em: float, *, mode: str = "phrase", lang: "Optional[str]" = None, slice_overlong: bool = False, @@ -701,10 +749,12 @@ def wrap_variants(text: str, max_em: float, *, mode: str = "phrase", for raw in text.split("\n"): if not raw.strip(): continue - chunk = _greedy_chunks(raw, max_em, slice_overlong=slice_overlong, sliced=sliced) + gaps: "List[bool]" = [] + chunk = _greedy_chunks(raw, max_em, slice_overlong=slice_overlong, sliced=sliced, gaps=gaps) greedy.extend(chunk) - wrapped.extend(_balance(list(chunk), max_em, mode, lang)) - measured.extend(chunk if mode == "measured" else _balance(list(chunk), max_em, "measured", None)) + wrapped.extend(_balance(list(chunk), max_em, mode, lang, gaps)) + measured.extend(chunk if mode == "measured" + else _balance(list(chunk), max_em, "measured", None, gaps)) if not greedy: greedy = [text] return (wrapped or [text], greedy, measured or [text]) diff --git a/scripts/_contract.py b/scripts/_contract.py index a5f277d7..7642ced5 100644 --- a/scripts/_contract.py +++ b/scripts/_contract.py @@ -178,8 +178,8 @@ video_required=False, audio_only=True, visual=False, verify=["probe", "check"], produces_artifact=True, idempotency="content_equivalent", deterministic=True), "silence": dict(role="analysis_and_execution", inputs=["video or audio asset"], outputs=["silence list JSON (--list)", "artifact with silences removed", "EDL text (--edl)"], required=FF + ["filter:silencedetect"], optional=[{"capability": X264, "when": "removing silences from a video"}, HDR_X265, {"capability": AAC, "when": "removing silences from a video"}, - {"capability": "external:whisper", "when": "--filler --transcribe"}, - {"capability": "external:parakeet", "when": "--filler --transcribe (English; either engine family satisfies it)"}] + AUDIO_OUT, + {"capability": "external:whisper", "when": "--transcribe (with --filler or a sentence flag)"}, + {"capability": "external:parakeet", "when": "--transcribe (with --filler or a sentence flag; English; either engine family satisfies it)"}] + AUDIO_OUT, video_required=False, audio_only=True, visual=False, verify=["probe"], produces_artifact=True, idempotency="content_equivalent", deterministic=True), "join": dict(role="execution", inputs=["two or more video assets, or two or more audio-only assets"], outputs=["concatenated video artifact", "concatenated audio artifact (audio-only inputs, audio output extension)"], required=FF + ["filter:xfade", "filter:acrossfade"], @@ -289,7 +289,7 @@ "caption": dict(video="conditional", audio="conditional", note="--mode burn (default) always re-encodes both streams to render pixels; --mode mux copies video and audio untouched and only adds a subtitle stream"), "overlay": dict(video="always", audio="always"), "graphics": dict(video="always", audio="always"), - "sync": dict(video="conditional", audio="conditional", note="video is -c:v copy only for --trim-second when the second file started earlier (offset<0) and the copy succeeds; it is re-encoded whenever --replace-audio's stream copy fails, or in --trim-second when the second file started later (offset>=0, the common case) or --fix-drift is used"), + "sync": dict(video="conditional", audio="conditional", note="video is -c:v copy only for --replace-audio (the reference picture) while that copy succeeds; --trim-second always re-encodes a video second file, since a copy would start at the keyframe before the trim point; a sound-only second file that started earlier is stream-copied"), "multicam": dict(video="always", audio="always"), "audio": dict(video="never", audio="always", note="video stream is always -c:v copy when present; this tool's job is the audio"), "loudness": dict(video="never", audio="always"), @@ -421,6 +421,7 @@ def input_schema(parser: argparse.ArgumentParser) -> Dict[str, Any]: "input_incomplete": "kind ffmpeg: the input has no index (\"moov atom not found\"): a recording cut off or a copy not finished; --repair cannot rebuild it; details: {}", "input_unreadable": "kind ffmpeg: ffmpeg could not read the input as media (\"Invalid data found when processing input\"); details: {}", "ffmpeg_unsupported": "kind missing_tool: the ffmpeg/ffprobe on PATH reports a version older than 6.1, which 3.0.0 no longer supports (5.x was deprecated in 2.14.0); details: {version, required}", + "read_only_plan_refused": "--plan in a process run with FFMPEG_SKILL_READ_ONLY=1 (mcp/server.py sets it for the tools annotated readOnlyHint: probe, check, cropdetect), which must write no file; details: {plan}", "output_exists": "an output already exists and --overwrite was not given; details.outputs: [{path, resolved, size, mtime_ns}]", "output_is_input": "the output resolves to the same file as an input; details: {output, input, resolved}", "output_is_directory": "the output path is a directory; details: {output}", @@ -444,9 +445,9 @@ def input_schema(parser: argparse.ArgumentParser) -> Dict[str, Any]: "plan_argv_invalid": "render.py PLAN: argv is not a list; details: {plan, plan_sha256, checked_at, argv_type, audit_log}", "decoder_missing": "color.py / graphics.py / overlay.py: a VP9 source or --video layer stored with an alpha layer, on a build with no libvpx-vp9 decoder (FFmpeg's own VP9 decoder drops the alpha), kind missing_tool; details: {name}", "encoder_missing": "--codec av1/prores: the build's encoder list was read and has no encoder for the codec (kind missing_tool; details: {codec, encoders}); or ffmpeg itself reported \"Unknown encoder\" for the encoder the command named (kind ffmpeg; details: {name}; the hint points at doctor)", - "asr_engine_missing": "kind missing_tool: --transcribe (caption.py, silence.py --filler) found no local speech-to-text engine; the message lists the install lines; details: {}", + "asr_engine_missing": "kind missing_tool: --transcribe (caption.py; silence.py with --filler or a sentence flag) found no local speech-to-text engine; the message lists the install lines; details: {}", "encoder_list_unreadable": "--codec av1/prores: the build's encoder list could not be read, so whether it has the encoder is unknown (kind ffmpeg); details: {codec, encoders, error}", - "no_speech": "kind input: --transcribe (caption.py, silence.py --filler) ran a local speech engine that heard no speech in the input; details: {engine}; also top-level reason and engine", + "no_speech": "kind input: --transcribe (caption.py; silence.py with --filler or a sentence flag) ran a local speech engine that heard no speech in the input; details: {engine}; also top-level reason and engine", "engine_failed": "kind input: --transcribe: an installed speech engine ran and failed (a named --engine, or every engine --engine auto found); details: {engine, detail | engines: [{engine, reason, detail}]}; also top-level reason, engine and detail/engines", "english_only": "kind input: --transcribe under --engine auto found only Parakeet engines, which it runs for English speech only, and the language is not English; details: {engine, engines}; also top-level reason, engine and engines", } @@ -618,7 +619,7 @@ def output_schema(name: str, meta: Dict[str, Any], hw: bool = False) -> Dict[str "verified": {"type": "boolean", "description": "true only when the artifact was written, probed, and every self-check the tool ran (verification) met its target; false under --dry-run"}, "verification": {"type": "array", "items": {"type": "object", "properties": {"step": {"type": "string"}, "ok": {"type": "boolean"}, "stage": {"type": "string"}, "property": {"type": "string"}, "expected": {}, "measured": {}, "delta": {}, "tolerance": {}, "provenance": {"enum": ["specification", "user_specified", "independent_measurement", "input_premeasurement"]}, "attribution": {"enum": ["artifact", "reference", "instrument", "provenance", "unresolved"]}}}, "description": "what the tool itself verified: probe, plus loudness (loudness.py, export platform presets; ok false with reason not_measured when it could not be read), check (render), report.py's sections (loudness, compliance, contact_sheet) and batch.py's items (item). render.py also lists every stage's own steps beyond its artifact probe, tagged with `stage` (a cached stage's as recorded), so its `verified` is the conjunction of every stage's. Since 2.18 a step may carry the property it measured with expected, measured, delta, tolerance and the provenance of the expected value; cut, join and export add a `streams` step comparing the output's video/audio streams with the input's; a step with ok false carries `attribution` (unresolved until a tool knows the cause)"}, "unverified": {"type": "array", "items": {"type": "string"}, "description": "2.18: the properties of the output this tool did not measure (the unverified range), e.g. caption's rendered_placement; `verified` covers the measured properties only"}, - "source_carry": {"type": "object", "description": "what the first input carried besides its picture and sound, compared with the written file: {kept: [...], dropped: [...]} over subtitles, chapters, cover_art, audio_channels (a surround source only), creation_time, audio_language, timecode. Present only when the source had at least one of them; absent under --dry-run, for a still or GIF output, and when two or more inputs have a picture or sound. A report, not a check: `verified` does not depend on it. A dropped subtitles/chapters/cover_art/audio_channels also records a source_items_dropped warning"}, + "source_carry": {"type": "object", "description": "what the first input carried besides its picture and sound, compared with the written file: {kept: [...], dropped: [...]} over subtitles, chapters, cover_art, audio_channels (a surround source only), creation_time, audio_language, timecode (kept only when the written start timecode is the label of the frame the output starts on: the source's own, or moved there by a tool that moved the picture's start; a join of several ranges writes none). Present only when the source had at least one of them; absent under --dry-run, for a still or GIF output, and when two or more inputs have a picture or sound. A report, not a check: `verified` does not depend on it. A dropped subtitles/chapters/cover_art/audio_channels also records a source_items_dropped warning"}, "diagnostics": DIAGNOSTICS_SCHEMA} extra: Dict[str, Any] = {} if name == "check": @@ -666,7 +667,7 @@ def output_schema(name: str, meta: Dict[str, Any], hw: bool = False) -> Dict[str "hint": {"type": "string", "description": "when nothing sat below --threshold: the track's measured level and the --threshold that would find its pauses"}, "speech_aware": {"type": "object", "description": "--speech-aware (1.18): {min_silence, floor, breaths_kept, breaths_kept_seconds, breaths}"}, "filler": {"type": "object", "description": "--filler: {lang, source (whisper-json:FILE | whisper:ENGINE | parakeet:ENGINE), engine, transcription (with --transcribe: {engine, model, language, routing}), removed, removed_count, matched_untimed (listed words in the transcript with no usable timing, not removed), matched_too_long (listed words longer than a filler, kept), ...}"}, - "notes": {"type": "array", "items": {"type": "string"}, "description": "--filler --transcribe under --engine auto: present when the English-only Parakeet model ran on English assumed (no --filler-lang, no language detected, no Whisper engine installed or none that succeeded); the transcript is wrong if the speech is not English"}} + "notes": {"type": "array", "items": {"type": "string"}, "description": "--transcribe (with --filler or a sentence flag) under --engine auto: present when the English-only Parakeet model ran on English assumed (no --filler-lang, no language detected, no Whisper engine installed or none that succeeded); the transcript is wrong if the speech is not English"}} elif name == "sync": extra = {"reference": {"type": "string"}, "second": {"type": "string"}, "offset_seconds": {"type": "number"}, "confidence": {"type": "number"}, "meaning": {"type": "string"}, "drift": {"type": "object"}, "sources": {"type": "array", "description": "1.18: [{path, offset_s, confidence, drift_ppm}], one per SOURCE; the only per-source shape once more than one SOURCE is given; with --fix-drift each also carries drift_skipped_reason"}, @@ -694,7 +695,7 @@ def output_schema(name: str, meta: Dict[str, Any], hw: bool = False) -> Dict[str "files": {"type": "array", "description": "one entry per file: {file, probe, steps, skipped}. A step is {step, ok, seconds, error (one line, for the table)}, plus the tool's verified when it completed, and on a failure its own kind, reason, message, details and exit_code (reason step_timeout or step_output_unparsed when the tool printed no failure document); skipped lists the steps that do not apply as {step, reason: quick|no_video|no_audio|not_hdr|not_multichannel|input_unreadable, message}"}, "failed": {"type": "integer"}, "total": {"type": "integer"}} elif name == "batch": - extra = {"results": {"type": "array", "description": "one row per input, every input (an interrupted batch included): ok (every step completed), verified (every step also verified what it wrote; false under --dry-run) and unverified ([{step, tool, failed}] naming the steps that did not); a failed row's error {step, tool, kind, message, code, reason, details, hint, exit_code} from the failing step's own document; diagnostics from its steps (tagged with step); cache_miss {reason: output_missing|output_changed|entry_unverifiable, ...} when a cache entry was not served; skipped \"timeout\" or \"interrupted\" for an item that never started and interrupted true for one stopped while running. A row whose recipe ran cut.py carries cut_reencoded (each cut.py call's reencoded) and cut_reencode_reasons (each reason those calls named, once per call)"}, + extra = {"results": {"type": "array", "description": "one row per input, every input (an interrupted batch included): ok (every step completed), verified (every step also verified what it wrote; false under --dry-run) and unverified ([{step, tool, failed}] naming the steps that did not); a failed row's error {step, tool, kind, message, code, reason, details, hint, exit_code} from the failing step's own document; diagnostics from its steps (tagged with step); cache_miss {reason: output_missing|output_changed|entry_unverifiable, ...} when a cache entry was not served; skipped \"timeout\" or \"interrupted\" for an item that never started and interrupted true for one stopped while running; under --watch, skipped \"settling\" (ok false, not cached) for a source that changed since the previous poll, or, first seen, was modified within one poll interval: it is processed on a later pass. A row whose recipe ran cut.py carries cut_reencoded (each cut.py call's reencoded) and cut_reencode_reasons (each reason those calls named, once per call)"}, "processed": {"type": "integer"}, "total": {"type": "integer"}, "cut_stream_copy": {"type": ["object", "null"], "description": "cut.py calls across the folder: {calls, stream_copy, reencoded, stream_copy_rate}; null when no cut.py step ran"}, "cut_reencode_reasons": {"type": ["object", "null"], "description": "how many cut.py calls named each reencode_reason ({} when none re-encoded); a re-encoded call whose row came from a cache an earlier 2.x batch wrote, which recorded no reasons, counts as unknown; null when no cut.py step ran"}} @@ -1759,6 +1760,17 @@ def tool_spec(name: str, version: str) -> Dict[str, Any]: # them out of every inputSchema (1.10.0-1.x kept them unless FFMPEG_SKILL_MCP_LEAN=1). A client # that still sends them is not refused -- build_argv() maps them like any other key. MCP_TRANSPORT_FLAGS = ("json", "progress") +# what a read-only tool's (mcp_read_only: probe, check, cropdetect) MCP inputSchema says of the two +# keys that would write a file: `plan` is refused over MCP and `overwrite` only governed that plan. +# The names stay (3.x never removes an inputSchema property); the CLI keeps both working (every +# tool gets --plan, docs/design-decisions.md). +MCP_READ_ONLY_REFUSED = { + "plan": "refused over MCP (read-only tool; error.reason read_only_plan_refused): this tool writes no file here; run it from the CLI to write a plan", + "overwrite": "no effect over MCP: this read-only tool writes no file (it governed --plan, which is refused here)", +} +# set by mcp/server.py in the environment of a read-only tool's process: --plan is refused +# (kind input, error.reason read_only_plan_refused), however the raw argv spells it +READ_ONLY_ENV = "FFMPEG_SKILL_READ_ONLY" # default MCP tools/list surface (roadmap P1-7, shipped 1.18.3): the 42-tool schema dump costs # every MCP session context whether it needs it or not. These 12 are the tools eval iterations # 17-20's ground-truth `expect` lists actually name most often across the 118 agent prompts in @@ -1905,6 +1917,13 @@ def mcp_input_schema(spec: Dict[str, Any]) -> Dict[str, Any]: structured["anyOf"] = one_of[0] if len(one_of) == 1 else [{"allOf": [{"anyOf": g} for g in one_of]}] for dest in MCP_TRANSPORT_FLAGS: props.pop(dest, None) + if mcp_read_only(spec): + # a tool that writes no artifact is annotated readOnlyHint; --plan is the one file it could + # still write (and --overwrite only governs that file). mcp/server.py runs the tool with + # FFMPEG_SKILL_READ_ONLY=1, which refuses --plan however it is spelled; the descriptions say so + for dest, text in MCP_READ_ONLY_REFUSED.items(): + if dest in props: + props[dest]["description"] = text if structured.get("required"): structured["required"] = [d for d in structured["required"] if d not in MCP_TRANSPORT_FLAGS] if not structured["required"]: @@ -1925,14 +1944,23 @@ def mcp_tool(spec: Dict[str, Any]) -> Dict[str, Any]: "annotations": mcp_annotations(spec)} +def mcp_read_only(spec: Dict[str, Any]) -> bool: + """A tool that writes no artifact and does not touch its input: over MCP it writes nothing at + all (mcp/server.py refuses its `plan`, see mcp_input_schema), so it is annotated readOnlyHint.""" + return not (bool(spec.get("produces_artifact")) or bool(spec.get("mutates_input"))) + + def mcp_annotations(spec: Dict[str, Any]) -> Dict[str, Any]: """MCP ToolAnnotations (spec 2025-06-18) derived from the ToolSpec flags. Hints only: clients - must not treat them as a security boundary. Every tool runs locally (no open world); none edits - its input in place; a tool that writes nothing is read-only.""" - writes = bool(spec.get("produces_artifact")) or bool(spec.get("mutates_input")) + must not treat them as a security boundary. Every tool runs locally (no open world). A tool + that writes nothing is read-only (mcp_read_only: the server runs it with + FFMPEG_SKILL_READ_ONLY=1, which refuses `--plan` however it is spelled). Every other tool is + destructive: none edits its input in place, but with `overwrite` it replaces an existing file at its output or plan path, which is not an + additive update.""" + writes = not mcp_read_only(spec) return {"title": spec["name"], "readOnlyHint": not writes, - "destructiveHint": bool(spec.get("mutates_input")), + "destructiveHint": writes, "idempotentHint": spec.get("idempotency_hint") in ("bit_exact", "content_equivalent", "cached"), "openWorldHint": False} @@ -1980,7 +2008,7 @@ def build(detect: bool = True) -> Dict[str, Any]: # first time it runs; faster-whisper is a library, so its fetch is made from the skill's own # process. A caller that sandboxes the network must know that before a first --transcribe. "model_downloads": { - "when": "first use of a speech engine (--transcribe; silence.py --filler --transcribe runs it under --dry-run too, caption.py --transcribe does not) whose model is not cached yet; media is never uploaded", + "when": "first use of a speech engine (--transcribe; silence.py --transcribe, with --filler or a sentence flag, runs it under --dry-run too, caption.py --transcribe does not) whose model is not cached yet; media is never uploaded", "in_process": {"faster-whisper": "Hugging Face (Systran/faster-whisper-)"}, "child_process": {"parakeet-mlx": "Hugging Face (mlx-community/parakeet-*)", "openai-whisper": "OpenAI (its model checkpoint)"}, @@ -1994,8 +2022,8 @@ def build(detect: bool = True) -> Dict[str, Any]: "canonical": True, "transports": ["cli", "mcp"], "argument_mapping": { - "positional": "listed in input_schema.positional, in order; array values expand to several arguments", - "options": "key -> --key with '_' replaced by '-'; booleans are flags; arrays repeat the flag; input_schema.properties[key].cli lists the accepted spellings", + "positional": "listed in input_schema.positional, in order; array values expand to several arguments; when any of them starts with '-', all of them follow '--' after every option, so none is read as an option", + "options": "key -> --key with '_' replaced by '-'; booleans are flags; arrays repeat the flag; input_schema.properties[key].cli lists the accepted spellings; a value starting with '-' is one token, --long-flag=value (\"-50%\" is text, not an option)", "exceptions": "per tool in mcp.argument_exceptions (key -> flag), e.g. output -> -o, loudness.lufs -> -I, graphics.count_from -> --from", "json": "--json is appended for every tool except look and probe (probe prints JSON by default)", }, diff --git a/scripts/audio.py b/scripts/audio.py index 62cb9c2a..1e1a6f72 100755 --- a/scripts/audio.py +++ b/scripts/audio.py @@ -13,7 +13,7 @@ python3 audio.py clip.mp4 --fade-in 0.5 --fade-out 1 --stereo python3 audio.py talk.mp4 --music bed.mp3 --duck --duck-threshold -30 --duck-release 250 # ducks earlier and recovers faster python3 audio.py band.wav --stereo-widen 0.5 -o wide.wav # wider stereo image (a real stereo source; mono is refused) - python3 audio.py surround.mov --downmix # 5.1 -> stereo with proper centre/LFE weights + python3 audio.py surround.mov --downmix # 5.1 -> stereo, ITU-R BS.775 weights, scaled not to clip python3 audio.py clip.mp4 --replace narration.wav # swap the audio track entirely python3 audio.py interview.mp4 -o interview.wav # extract the audio (no video in the output) python3 audio.py multi.mkv --audio-stream 1 --voice -o lav.m4a # pick the second audio track, clean it, write M4A @@ -29,6 +29,8 @@ from _common import STATE, add_common, dry_run_input_pending, require_tool, apply_common, audio_codec_for, db_to_linear, default_output, die, emit, ffmpeg_base, info, is_audio_output, measured_level_dbfs, probe, run, run_keeping_subtitles, fmt_secs from _common.emit import warn +# --downmix: ITU-R BS.775 fold-down, renormalised by pan's '<' (see main()). +DOWNMIX_PAN = "pan=stereo|FL int: help="widen the stereo image, 0..1 (0 = untouched, 1 = maximum); needs a real stereo source: a mono input is " "refused (duplicating it leaves both channels identical, so there is nothing to widen) and more than two " "channels are refused unless --downmix folds them to stereo first") - fades.add_argument("--downmix", action="store_true", help="downmix 5.1/7.1 to stereo using standard weights") + fades.add_argument("--downmix", action="store_true", help="downmix 5.1/7.1 to stereo with the ITU-R BS.775 weights (centre and surrounds -3 dB, LFE dropped), scaled so the fold-down cannot clip") fades.add_argument("--replace", help="replace the audio with this file (trimmed/padded to the video)") dyn = ap.add_argument_group("dynamics (typed; each flag is one option of ffmpeg's acompressor / alimiter / agate)") dyn.add_argument("--compress", action="store_true", help="compressor (acompressor); order: gate -> compressor -> limiter") @@ -301,7 +303,11 @@ def main() -> int: fx: List[str] = [] if args.downmix: - fx.append("pan=stereo|FL=0.707*FC+FL+0.5*BL+0.5*SL+0.5*LFE|FR=0.707*FC+FR+0.5*BR+0.5*SR+0.5*LFE") + # ITU-R BS.775: centre and surrounds at -3 dB (0.707), LFE dropped. pan's '<' scales each + # output so its gains sum to 1 over the channels the input has (1/2.414 on 5.1, 1/3.121 on + # 7.1), so in-phase channels at full scale cannot clip; '=' summed 2.7x and clipped a loud + # 5.1 while the run said verified. + fx.append(DOWNMIX_PAN) if args.voice: fx.append(VOICE_LEVELS[args.voice]) elif args.denoise: diff --git a/scripts/batch.py b/scripts/batch.py index a4a166a7..581bd1cb 100755 --- a/scripts/batch.py +++ b/scripts/batch.py @@ -33,8 +33,10 @@ from typing import Any, Dict, List, Optional from _common import STATE, add_common, apply_common, child_args, die, emit, info, run_tool, MEDIA_EXT as _MEDIA_EXT -from _common.runner import HW_FORCED_ENV, _OutputLock, add_hw_orchestrator_args, copy_to_sibling, env_hw, stop_children +from _common.runner import (HW_FORCED_ENV, OWNED_ENV, _OutputLock, add_hw_orchestrator_args, copy_to_sibling, env_hw, + owned_identity, stop_children) from _common.emit import absorb_stage_hw, audit, note, shown, steps_encoder_report, warn +from _common.decision import rebase_project_paths HERE = Path(__file__).resolve().parent MEDIA_EXT = {e for e in _MEDIA_EXT if e not in (".png", ".jpg", ".jpeg", ".webp")} # one list (_common); a batch walks media, not stills @@ -117,6 +119,16 @@ def gpu_key() -> str: JOBS_CAP = 8 # beyond this, concurrent encodes contend for the same cores and memory _LOG = threading.local() +# source -> {real path -> owned_identity()} of every file this process delivered for that item (a +# --watch loop's earlier passes): its final output and every side file its steps were allowed to +# write (an .srt beside the video). An existing path that is still exactly that file is the item's +# own, not someone else's, so a later pass of the SAME item replaces it (design-decisions: "A path +# this same run wrote ... is never someone else's file"); the permission is per item and per path, +# and is checked again where the write happens -- by deliver() under the output lock, by a step at +# its own output check (runner.OWNED_ENV). A file edited or replaced since, one an earlier run +# left, or one another item delivered (two items naming one side file) is still refused. +_OWN_OUTPUTS: Dict[str, Dict[str, Dict[str, int]]] = {} +_OWN_LOCK = threading.Lock() # set by batch.py's signal handler: a --jobs worker between two steps must not start the next one _STOP = threading.Event() @@ -155,13 +167,15 @@ def step_error(index: Optional[int], tool: Optional[str], doc: Any) -> Dict[str, return out -def run_step(argv: List[str], per_call: "Optional[float]" = None) -> "tuple[bool, Optional[dict]]": +def run_step(argv: List[str], per_call: "Optional[float]" = None, + owned: "Optional[Path]" = None) -> "tuple[bool, Optional[dict]]": """Returns (ok, doc): doc is the step's own --json result document; a failed step that printed none that parses gets one in the failure shape (error.reason step_output_unparsed, the last stderr line as the message), so its row still says why. {in}/{out} are already resolved by the caller before argv reaches here, so forcing --json on every step (harmless: no step reads another step's stdout, only its output path) costs nothing and lets a caller like process() read back tool-specific fields, e.g. cut.py's - `reencoded`, without adding a bespoke flag per field it might someday want.""" + `reencoded`, without adding a bespoke flag per field it might someday want. `owned`: the + owned_config() file the step reads under runner.OWNED_ENV.""" script = argv[0] if script not in ALLOWED_STEP_SCRIPTS: # main() refuses such a recipe before any item runs; this stays as the last line of defence @@ -173,7 +187,8 @@ def run_step(argv: List[str], per_call: "Optional[float]" = None) -> "tuple[bool "error": {"kind": "interrupted", "message": f"interrupted before {script} started"}} cmd = [str(HERE / script)] + argv[1:] + child_args() + (["--json"] if "--json" not in argv[1:] else []) log(" → " + " ".join(os.path.basename(c) if i < 1 else c for i, c in enumerate(cmd))) - proc = run_tool(cmd, per_call=per_call) + env = dict(os.environ, **{OWNED_ENV: str(owned)}) if owned else None + proc = run_tool(cmd, per_call=per_call, env=env) try: doc = json.loads(proc.stdout) except ValueError: @@ -240,9 +255,69 @@ def row_output(src: Path, recipe: Dict[str, Any], outdir: Path) -> Dict[str, Any return fields -def deliver(src: str, final: Path) -> None: +def own_expected(src: Path, path: "Path | str") -> "Optional[Dict[str, int]]": + """The identity this process left `path` with, if it delivered a file there for `src`.""" + with _OWN_LOCK: + return _OWN_OUTPUTS.get(str(src), {}).get(os.path.realpath(str(path))) + + +def is_own_output(src: Path, final: Path) -> bool: + """Whether `final` is still the file this process delivered there for `src` (_OWN_OUTPUTS).""" + expected = own_expected(src, final) + return expected is not None and owned_identity(str(final)) == expected + + +def record_own(src: Path, committed: "Dict[str, Dict[str, int]]", work: Path) -> None: + """Remember the files this item delivered, each with the identity it had when it was written + (a step's report_committed(), deliver() for the final path), never as it is now: a sidecar + edited while a later step ran no longer matches, so the item's next pass refuses it. Paths in + the item's work directory go with it and are skipped. Replaces what an earlier pass of the item + recorded.""" + base = os.path.realpath(str(work)) + mine: Dict[str, Dict[str, int]] = {} + for p, ident in committed.items(): + real = os.path.realpath(p) + if real == base or real.startswith(base + os.sep) or not isinstance(ident, dict): + continue + mine[real] = ident + with _OWN_LOCK: + _OWN_OUTPUTS[str(src)] = mine + + +def owned_config(src: Path, work: Path, i: int) -> Path: + """The file a step reads under runner.OWNED_ENV: a snapshot of what this process delivered for + `src`, taken right before the step starts (one file per step, since a step reads it once), and + the directory where each of its processes reports the paths it passed the output check for.""" + report = work / "owned-report" + report.mkdir(exist_ok=True) + with _OWN_LOCK: + replaceable = dict(_OWN_OUTPUTS.get(str(src), {})) + cfg = work / f"owned{i}.json" + cfg.write_text(json.dumps({"replaceable": replaceable, "report": str(report)}), encoding="utf-8") + return cfg + + +def owned_reports(work: Path) -> "Dict[str, Dict[str, int]]": + """{path: identity when written} for every file the item's steps reported committing + (owned_config's report directory). A path a step was allowed to write but never reported + writing is not in it.""" + found: Dict[str, Dict[str, int]] = {} + for f in sorted((work / "owned-report").glob("*.json")) if (work / "owned-report").is_dir() else []: + try: + doc = json.loads(f.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + committed = doc.get("committed") if isinstance(doc, dict) else None + if isinstance(committed, dict): + found.update({p: i for p, i in committed.items() if isinstance(p, str) and isinstance(i, dict)}) + return found + + +def deliver(src: str, final: Path, expected: "Optional[Dict[str, int]]" = None) -> "Optional[Dict[str, int]]": """Move an item's last written intermediate to its final path, under the output guard every - tool applies: an existing file is replaced only with --overwrite. A dry run writes nothing.""" + tool applies: an existing file is replaced only with --overwrite, or when it is still exactly + `expected` -- the file this process delivered there on an earlier pass -- checked under the + output lock, so a file another process put there since is refused. A dry run writes nothing.""" if STATE.dry_run: # the refusal the real run would meet, as a step writing {out} plans it (#428 review) if final.exists() and not STATE.overwrite: @@ -257,7 +332,7 @@ def locked(msg: str, hint: str, details: Dict[str, Any]) -> None: # under the output lock every tool's write holds, so the existence check and the move are one # step for another run delivering or writing the same path with _OutputLock(str(final), on_locked=locked): - if final.exists() and not STATE.overwrite: + if final.exists() and not STATE.overwrite and not (expected is not None and owned_identity(str(final)) == expected): raise ItemError({"step": None, "tool": None, "kind": "input", "reason": "output_exists", "message": f"refusing to overwrite existing output {str(final)!r}: pass --overwrite to replace it"}) try: @@ -279,13 +354,25 @@ def locked(msg: str, hint: str, details: Dict[str, Any]) -> None: pass raise ItemError({"step": None, "tool": None, "kind": "output", "message": f"could not deliver {src} to {final}: {exc}"}) + # the delivered file as it was put there, taken under the lock: what a later pass may replace + ident = owned_identity(str(final)) log(f" delivered the last written file to {final}") + return ident def process(src: Path, recipe: Dict[str, Any], outdir: Path, work: Path, deadline: "Optional[float]" = None) -> Dict[str, Any]: final = final_path(src, recipe, outdir) t0 = time.time() + # replacing this run's own earlier output is decided per path and at the write, never here: + # each step gets a snapshot of what this process delivered (runner.OWNED_ENV) and checks every + # existing file it would write against it; deliver() checks the final path under its lock + track = not STATE.dry_run and not STATE.overwrite + + delivered: Dict[str, Dict[str, int]] = {} + + def owned(i: int) -> "Optional[Path]": + return owned_config(src, work, i) if track else None def budget() -> "Optional[float]": """What is left of the BATCH's time limit -- not a fresh one per item. A --timeout is a @@ -306,11 +393,15 @@ def budget() -> "Optional[float]": proj.setdefault("clips", [{}]) while len(proj["clips"]) <= idx: proj["clips"].append({}) + # the copy is written in the item's work directory, and render.py reads a project's + # relative paths against the project file's own directory: they are made absolute + # against the recipe's project, or "bed.wav" beside it was looked for in .work/-xxx/ + proj = rebase_project_paths(proj, Path(str(recipe["project"])).resolve().parent) proj["clips"][idx]["src"] = str(src.resolve()) proj["output"] = str(final.resolve()) pj = work / "project.json" pj.write_text(json.dumps(proj, indent=2), encoding="utf-8") - ok, _doc = run_step(["render.py", str(pj)], budget()) + ok, _doc = run_step(["render.py", str(pj)], budget(), owned=owned(0)) step_docs = [("render", _doc)] failure = None if ok else step_error(0, "render.py", _doc) cut_reencoded: List[bool] = [] @@ -334,7 +425,9 @@ def budget() -> "Optional[float]": # what this one read, and when it is the last, the last file written is delivered (#415) writes = step_writes(step) argv = [str(a).replace("{in}", cur).replace("{out}", out) for a in step] - step_ok, doc = run_step(argv, budget()) + if last and writes and track and is_own_output(src, final): + log(f" {final.name} is this run's own earlier output, unchanged since: it may be replaced") + step_ok, doc = run_step(argv, budget(), owned=owned(i)) step_docs.append((Path(str(argv[0])).stem if argv else "step", doc)) # only cut.py's own doc carries `reencoded` -- true if ANY range this call cut (or its # --segments join) re-encoded, for the reasons in its `reencode_reason` (distinct per @@ -353,9 +446,15 @@ def budget() -> "Optional[float]": # the last step wrote nothing: the item's deliverable is the last file a step wrote, # which is in this item's work directory and would be removed with it (#415) try: - deliver(cur, final) + ident = deliver(cur, final, expected=own_expected(src, final)) + if ident: + delivered[str(final)] = ident except ItemError as exc: ok, failure = False, exc.error + if ok and track: + # what this item delivered, each file as it was when written: the paths its steps reported + # committing (side files included) and the final path deliver() moved into place + record_own(src, {**owned_reports(work), **delivered}, work) result: Dict[str, Any] = {"file": str(src), **row_output(src, recipe, outdir), "ok": ok, "seconds": round(time.time() - t0, 1)} if failure: result["error"] = failure @@ -434,6 +533,8 @@ def main() -> int: apply_common(args) started = time.time() + with _OWN_LOCK: + _OWN_OUTPUTS.clear() folder = Path(args.folder).resolve() # relative 'bdir' used to become bdir/bdir/out once joined with the default outdir if not folder.is_dir(): die(f"not a folder: {folder}") @@ -553,6 +654,35 @@ def _interrupt(signum: int, frame: Any) -> None: except (ValueError, OSError): pass + # --watch: each source's (size, mtime_ns) at the previous poll. A folder files are copied, + # downloaded or recorded into is read while they grow: a file processed mid-copy delivered a + # fraction of itself as ok (verified, for an MKV or TS), and once it settled every later pass + # met that output as someone else's and refused it. A source is processed once it is + # unchanged across one poll interval -- the same size and mtime as at the previous poll, or, + # first seen, last modified at least one interval ago; until then its row is + # skipped: "settling", not cached, and the next pass looks again. + last_seen: Dict[str, "tuple[int, int]"] = {} + + def settle_poll(files: List[Path]) -> "set": + if not args.watch: + return set() + unsettled = set() + now = time.time() + current: Dict[str, "tuple[int, int]"] = {} + for src in files: + try: + st = src.stat() + except OSError: + continue # gone since the scan: file_key() reports it as before + sig = (st.st_size, st.st_mtime_ns) + current[str(src)] = sig + before = last_seen.get(str(src)) + if (before != sig) if before is not None else (now - st.st_mtime < args.watch): + unsettled.add(str(src)) + last_seen.clear() + last_seen.update(current) + return unsettled + def one_pass() -> List[Dict[str, Any]]: files = sorted(p for p in folder.glob(glob) if p.is_file() and p.suffix.lower() in MEDIA_EXT and outdir not in p.parents) # Two different sources can compute the same final path (most often a fixed recipe "ext" @@ -632,7 +762,13 @@ def store(key: str, r: Dict[str, Any]) -> None: pending: List[tuple] = [] cache_miss: Dict[int, Dict[str, Any]] = {} pass_key = rkey + side_inputs_key(recipe) + polled = settle_poll(files) for i, src in enumerate(files): + if str(src) in polled: + info(f"skip (still being written) {src.name}: it changed since the last poll") + slots[i] = {"file": str(src), **row_output(src, recipe, outdir), "ok": False, "verified": False, + "seconds": 0.0, "skipped": "settling"} + continue key = f"{file_key(src)}:{pass_key}" hit = cache.get(key) if isinstance(hit, dict) and not args.force: @@ -842,7 +978,9 @@ def work_one(i: int, src: Path, key: str) -> Dict[str, Any]: timed_out=True) if done != len(results): failed_files = [r["file"] for r in results if not r["ok"]] - die(f"{len(results) - done} of {len(results)} items failed: {', '.join(failed_files[:5])}" + (" ..." if len(failed_files) > 5 else ""), + settling = sum(1 for r in results if r.get("skipped") == "settling") + die(f"{len(results) - done} of {len(results)} items failed: {', '.join(failed_files[:5])}" + (" ..." if len(failed_files) > 5 else "") + + (f" ({settling} of them still being written when the watch stopped, skipped: settling, not processed)" if settling else ""), kind="verification", output=None, dry_run=STATE.dry_run, results=results, processed=done, total=len(results)) # cut.py's own `reencoded`, rolled up across every cut.py step this batch ran: how much of the # folder landed on the fast lossless path (mode "copy") vs re-encoded on at least one range. diff --git a/scripts/broll.py b/scripts/broll.py index 45acbc31..af249590 100755 --- a/scripts/broll.py +++ b/scripts/broll.py @@ -22,7 +22,7 @@ from typing import Any, Dict, List from _common.decision import av_trim, frame_rate, grid_at_or_after, stream_starts -from _common import STATE, add_common, aac_args, apply_common, cfr_args, as_limited, default_output, die, draws_chroma, drawn_chain, emit, ffmpeg_base, info, time_arg, label_chain, matrix_chain, range_chain, probe, run, validate_color, video_args, X264_PRESETS, fmt_secs +from _common import STATE, add_common, aac_args, apply_common, cfr_args, as_limited, default_output, die, draws_chroma, drawn_chain, emit, ffmpeg_base, info, time_arg, label_chain, matrix_chain, overlay_format, range_chain, probe, run, validate_color, video_args, X264_PRESETS, fmt_secs from _common.emit import note, record_fallback from _common.probe import display_size, square_pixels, square_pixels_note @@ -136,7 +136,7 @@ def per(values: List[str], i: int, default: str) -> str: b_sq = (square_pixels(c["meta"]["video"]) + ",") if square_pixels(c["meta"]["video"]) else "" parts.append(f"[{i + 1}:v]trim=start={c['from']:.3f}:duration={c['length']:.3f},setpts=PTS-STARTPTS+{c['at']:.3f}/TB," f"{convert[i + 1]}{b_sq}{geo},setsar=1,fps={fps:g},format={'yuv420p10le' if (meta_a.get('video') or {}).get('bt2020_or_hdr') else 'yuv420p'}[b{i}]") - parts.append(f"{cur}[b{i}]overlay=0:0:eof_action=pass:enable='between(t,{c['at']:.3f},{c['at'] + c['length']:.3f})'[v{i}]") + parts.append(f"{cur}[b{i}]overlay=0:0{overlay_format(meta_a)}:eof_action=pass:enable='between(t,{c['at']:.3f},{c['at'] + c['length']:.3f})'[v{i}]") cur = f"[v{i}]" vout = cur diff --git a/scripts/caption.py b/scripts/caption.py index 24fe3a2e..37c1da10 100755 --- a/scripts/caption.py +++ b/scripts/caption.py @@ -43,9 +43,10 @@ from _platforms import (PLATFORMS, PLATFORM_CHOICES, ASS_SCRIPT_HEIGHT, ass_units, resolve as resolve_platform) -from _ass_overlay import EMOJI_SENTINEL, emoji_placeholder, ass_escape +from _ass_overlay import (EMOJI_SENTINEL, emoji_placeholder, ass_escape, ass_escape_markup, + strip_srt_markup, reinsert_srt_markup, carry_srt_markup, srt_alignment) from _common.emit import refuse # noqa: E402 -from _common import require_filter, emoji_filter_chain, EMOJI_ASSET_HINT, emoji_asset_for, emoji_codepoint_name, emoji_support, resolve_emoji_assets, ADVANCE_EM, LATIN_EM, NO_SPACE_SCRIPTS, _char_em, char_script, text_width_em, emoji_clusters, has_emoji, detect_script, BIDI_SCRIPTS, STATE, brand_states_font, script_font_for_text, signed_time_arg, brand_caption_style, color_hex, drawn_chain, label_chain, layer_base_chain, load_brand, rgb_chain_to_bt709, video_args, add_common, apply_common, emit, aac_args, cfr_args, default_output, die, escape_filter_path, ffmpeg_base, fmt_srt_time, fmt_smpte_time, info, MissingFpsError, parse_time, probe, run, x264_args, X264_PRESETS, read_text_or_die, fmt_secs, refuse_existing_outputs, guard_sidecar +from _common import require_filter, emoji_filter_chain, EMOJI_ASSET_HINT, emoji_asset_for, emoji_codepoint_name, emoji_support, resolve_emoji_assets, ADVANCE_EM, LATIN_EM, NO_SPACE_SCRIPTS, _char_em, char_script, text_width_em, emoji_clusters, has_emoji, detect_script, BIDI_SCRIPTS, STATE, brand_states_font, script_font_for_text, signed_time_arg, brand_caption_style, color_hex, drawn_chain, label_chain, layer_base_chain, load_brand, overlay_format, rgb_chain_to_bt709, video_args, add_common, apply_common, emit, aac_args, cfr_args, default_output, die, escape_filter_path, ffmpeg_base, fmt_srt_time, fmt_smpte_time, info, MissingFpsError, parse_time, probe, run, x264_args, X264_PRESETS, read_text_or_die, fmt_secs, refuse_existing_outputs, guard_sidecar # The line breaker, lifted into _common/text.py in 1.16.0 so graphics.py can use the same rules. # The ASR bridge and the SRT reader/writer live in _common.asr since 1.17 (silence.py --filler # shares them). They stay caption.py's public names -- every caller and test that reached for @@ -277,12 +278,12 @@ def plan_emoji(cues, args, play_w, play_h, brand=None): margin_px = args.margin * play_h / 288.0 line_h = size_px * 1.2 box_px = size_px * scale - align = ALIGN[args.position] out_cues = [] for start, end, text in cues: lines = _cue_lines(text) n = len(lines) new_lines = [] + align = srt_alignment(text) or ALIGN[args.position] # a cue's {\anN} moves its emoji too for i, line in enumerate(lines): if align in (7, 8, 9): y_top = margin_px + i * line_h @@ -290,7 +291,7 @@ def plan_emoji(cues, args, play_w, play_h, brand=None): y_top = play_h / 2.0 - (n * line_h) / 2.0 + i * line_h else: y_top = play_h - margin_px - (n - i) * line_h - line_w = text_width_em(line, scale) * size_px + line_w = text_width_em(strip_srt_markup(line), scale) * size_px if align in (1, 4, 7): x0 = margin_px elif align in (3, 6, 9): @@ -300,7 +301,7 @@ def plan_emoji(cues, args, play_w, play_h, brand=None): # libass lays an RTL line out right-to-left, so the LOGICAL prefix of a cluster # occupies the RIGHT end of the rendered line. Measuring the prefix from the left # edge put the PNG on top of the text, mirrored, on every Arabic/Hebrew cue (1.15.0). - rtl = detect_script(line) in BIDI_SCRIPTS + rtl = detect_script(strip_srt_markup(line)) in BIDI_SCRIPTS rebuilt = "" cursor = 0 for idx, cluster in emoji_clusters(line): @@ -313,6 +314,7 @@ def plan_emoji(cues, args, play_w, play_h, brand=None): rebuilt += line[cursor:idx + len(cluster)] cursor = idx + len(cluster) continue + prefix = strip_srt_markup(prefix) # a tag before the emoji takes no room if rtl: x = x0 + line_w - text_width_em(prefix + cluster, scale) * size_px else: @@ -390,9 +392,12 @@ def layout_cues(cues: List[Tuple[float, float, str]], *, max_em: Optional[float] # greedy wrap would have given (`rebalanced`) and what 1.15's wrap would have # given (`phrase_breaks`). Three wrap_text() calls re-ran the atomiser each time. sliced_atoms: List[int] = [] - lines, greedy, measured = wrap_variants(text, own_em, mode=wrap, lang=lang, + # SRT markup (, , {\an8}) is not text: the wrap measures and breaks + # the words a viewer reads, and the tags go back next to the characters they styled + plain = strip_srt_markup(text) + lines, greedy, measured = wrap_variants(plain, own_em, mode=wrap, lang=lang, slice_overlong=True, sliced=sliced_atoms) - if lines != [l for l in text.split("\n") if l.strip()]: + if lines != [l for l in plain.split("\n") if l.strip()]: stats["wrapped"] += 1 if sliced_atoms: # the escape hatch fired: a word or run with no break point the wrapper may use @@ -407,19 +412,22 @@ def layout_cues(cues: List[Tuple[float, float, str]], *, max_em: Optional[float] stats["rebalanced"] += 1 if wrap != "measured" and lines != measured: stats["phrase_breaks"] += 1 + marked = reinsert_srt_markup(text, lines) if plain != text else lines if len(lines) > max_lines: chunks = [lines[i:i + max_lines] for i in range(0, len(lines), max_lines)] weights = [max(1.0, sum(len(l) for l in c)) for c in chunks] total_w = sum(weights) + pieces = carry_srt_markup(["\n".join(marked[i:i + max_lines]) + for i in range(0, len(marked), max_lines)]) t = start - for chunk, weight in zip(chunks, weights): + for piece, weight in zip(pieces, weights): seg = (end - start) * weight / total_w - staged.append((t, min(end, t + seg), "\n".join(chunk))) + staged.append((t, min(end, t + seg), piece)) staged_sizes.append(own_size) t += seg stats["split"] += len(chunks) - 1 continue - text = "\n".join(lines) + text = "\n".join(marked) staged.append((start, end, text)) staged_sizes.append(own_size) out: List[Tuple[float, float, str]] = [] @@ -572,7 +580,7 @@ def secs(stamp: str) -> float: def cue_text_visible(text: str) -> bool: - return bool((text or "").strip()) + return bool(strip_srt_markup(text or "").strip()) # `` draws nothing def count_visible_cues(cues, duration) -> Tuple[int, int]: @@ -664,6 +672,17 @@ def _karaoke_word_durations(args, video: Optional[str], start: float, end: float return durs +# an override block this file wrote (a converted SRT tag), not the escaped `\{` a user typed +_OVERRIDE_BLOCK_RE = re.compile(r"(? bool: + """Does this space-separated Dialogue token hold something a viewer reads? An emoji + placeholder does not, and neither does a lone converted tag (`{\\i1}` from ` Hello`): + neither gets a share of the cue's time or a highlight of its own.""" + return bool(_OVERRIDE_BLOCK_RE.sub("", token).strip(EMOJI_SENTINEL)) + + def write_ass(cues: List[Tuple[float, float, str]], path: str, args, play_w: int, play_h: int, video: str = None) -> None: """Write a styled ASS file with optional animation and word-by-word highlight.""" def t(sec: float) -> str: @@ -687,6 +706,7 @@ def t(sec: float) -> str: word_active = ass_color(args.highlight_color) word_upcoming = ass_color(args.upcoming_color) word_bord = f"{args.outline * scale * (max(1.0, args.karaoke_scale) / 100.0):.2f}" + style_bord = f"{args.outline * scale:.2f}" word_scale = int(round(args.karaoke_scale)) # --karaoke-style word emits one Dialogue per word carrying the WHOLE cue with explicit \N # breaks (the layout's own fixed line list -- see max_line_em's docstring). WrapStyle: 2 tells @@ -713,7 +733,9 @@ def t(sec: float) -> str: # escapes for the braces, so 1.15 escapes them (ass_escape) rather than deleting them: # a cue that says "use {curly} braces" is read out with its braces, and still cannot open # an override block. Newlines become \N in the same pass. - text = ass_escape(text) + # SRT markup is the exception (#X2): ///, and {\anN} become the + # overrides ffmpeg's SRT reader gives them, so this route draws what the plain one draws + text = ass_escape_markup(text, colour=not args.karaoke) fx = "" if args.animate == "fade": fx = "{\\fad(200,200)}" @@ -740,7 +762,7 @@ def t(sec: float) -> str: # the active word's larger \fscx/\fscy, which is the jump the issue is filed against. line_words = [[w for w in seg.split(" ") if w] for seg in body.split("\\N")] flat = [w for lw in line_words for w in lw] - real_idx = [i for i, w in enumerate(flat) if w.strip(EMOJI_SENTINEL)] + real_idx = [i for i, w in enumerate(flat) if _karaoke_word(w)] if not real_idx: # nothing to highlight (an all-emoji or empty cue) -- one plain event, same as # today's karaoke-off rendering, rather than zero Dialogue lines for this cue @@ -764,15 +786,18 @@ def t(sec: float) -> str: for lw in line_words: styled_line = [] for w in lw: - if not w.strip(EMOJI_SENTINEL): + if not _karaoke_word(w): styled_line.append(w) else: rank = real_idx.index(flat_pos) if rank < j: styled_line.append(f"{{\\c{word_past}}}{w}") elif rank == j: + # put back what the active word changed, not {\r}: a reset would + # also drop the cue's own //colour and a --fit-size \fs styled_line.append( - f"{{\\c{word_active}\\fscx{word_scale}\\fscy{word_scale}\\bord{word_bord}}}{w}{{\\r}}") + f"{{\\c{word_active}\\fscx{word_scale}\\fscy{word_scale}\\bord{word_bord}}}{w}" + f"{{\\fscx100\\fscy100\\bord{style_bord}}}") else: styled_line.append(f"{{\\c{word_upcoming}}}{w}") flat_pos += 1 @@ -789,7 +814,7 @@ def t(sec: float) -> str: if args.karaoke: # split each line into words and give every word an equal share of the cue (\k is in centiseconds) segments = body.split("\\N") - words = [w for seg in segments for w in seg.split(" ") if w and w.strip(EMOJI_SENTINEL)] + words = [w for seg in segments for w in seg.split(" ") if w and _karaoke_word(w)] durs = _karaoke_word_durations(args, video, start, end, len(words)) it = iter(durs) out_segments = [] @@ -799,7 +824,7 @@ def t(sec: float) -> str: # past the reserved gap without spending cue time on a glyph nobody sees (rendered # and confirmed -- libass keeps the full gap inside a karaoke run). out_segments.append(" ".join( - ("{\\kf0}" + w) if not w.strip(EMOJI_SENTINEL) else f"{{\\kf{next(it)}}}{w}" for w in ws)) + ("{\\kf0}" + w) if not _karaoke_word(w) else f"{{\\kf{next(it)}}}{w}" for w in ws)) body = "\\N".join(out_segments) if EMOJI_SENTINEL in body: body = body.replace(EMOJI_SENTINEL, emoji_placeholder(getattr(args, "_emoji_box_px", size))) @@ -1223,7 +1248,7 @@ def fit_the_size(cue_list): # viewer sees and would silently change the SRT this run writes, so the mux path is # left exactly as 1.16 wrote it. return - fit = fit_size(cue_list, **fit_params()) + fit = fit_size([(s, e, strip_srt_markup(t)) for s, e, t in cue_list], **fit_params()) fit_stats["size_used"] = fit["size"] fit_stats["size_source"] = "platform-frame" if planned_frame else "input" fit_stats["shrunk"] = fit["shrunk"] @@ -1774,7 +1799,7 @@ def claim(items) -> None: side_notes.append(square_pixels_note(meta["video"])) cmd = ffmpeg_base() + ["-i", args.input] chains, emoji_inputs = (emoji_filter_chain(emoji_plan or {}, "vsub", "vout", to_yuv=rgb_chain_to_bt709(meta), - base_chain=layer_base_chain(meta)) + base_chain=layer_base_chain(meta), overlay_opts=overlay_format(meta)) if emoji_plan else ([], [])) if chains: for spec in emoji_inputs: diff --git a/scripts/check.py b/scripts/check.py index 053298f1..405ccd7b 100755 --- a/scripts/check.py +++ b/scripts/check.py @@ -525,7 +525,7 @@ def row(name: str, status: str, value: Any, expect: Any, fix: str = "", reason: # comes back at a level nobody checked. ch = a.get("channels") or 0 row("channels", "PASS" if ch in (1, 2) else "WARN", f"{ch}ch", "1 (mono) or 2 (stereo)", - "audio.py --downmix (5.1/7.1 to stereo with the standard weights) or audio.py --mono", + "audio.py --downmix (5.1/7.1 to stereo with the ITU-R BS.775 weights) or audio.py --mono", reason="podcast players downmix 5.1 unpredictably") if args.platform == "podcast": chapters = meta.get("chapters") or [] diff --git a/scripts/color.py b/scripts/color.py index 1dcb8913..3e6b4aeb 100755 --- a/scripts/color.py +++ b/scripts/color.py @@ -38,6 +38,7 @@ from _common.alpha import (add_alpha_args, audio_line, check_alpha_args, decoder_args_for, default_ext, finish_chain, report as alpha_report, source_alpha, tag_args, video_line) from _common.decision import keeps_source_tags +from _common.color import bt2020_sdr_to_bt709_chain, hdr_to_sdr_chain, is_bt2020_sdr # noqa: E402,F401 # the curves of FFmpeg's CPU `tonemap` filter (every version from 5.1 to 9). bt2390 was listed # until 2.24 but exists only in tonemap_opencl/libplacebo, so choosing it always failed. @@ -198,47 +199,6 @@ def lut_legs(meta=None, out_fmt: str = "yuv420p") -> "tuple": f"scale=out_color_matrix={matrix}:out_range=tv,format={out_fmt}") -def hdr_to_sdr_chain(meta: dict, tonemap: str, peak: float, desat: float) -> str: - v = meta["video"] - trc = v.get("color_transfer") or "smpte2084" - prim = v.get("color_primaries") or "bt2020" - space = v.get("color_space") or "bt2020nc" - # zscale needs explicit input tags when the file lacks them - chain: List[str] = [ - f"zscale=tin={trc}:pin={prim}:min={space}:rin={v.get('color_range') or 'tv'}:t=linear:npl={peak:g}", - "format=gbrpf32le", - "zscale=p=bt709", - f"tonemap=tonemap={tonemap}:desat={desat:g}", - "zscale=t=bt709:m=bt709:r=tv", - "format=yuv420p", - ] - return ",".join(chain) - - -def bt2020_sdr_to_bt709_chain(meta: dict) -> str: - """Gamut conversion only, for BT.2020 primaries on an SDR transfer (probe: `bt2020_or_hdr` - true, `hdr_signal` false). Such a file is already display-referred SDR: linearising it with - its own transfer, mapping the primaries to BT.709 and re-applying the BT.709 transfer keeps - white at white and grey at grey. The tone map the HDR chain adds would treat 100 % SDR white - as a fraction of a 1000-nit peak and darken the picture (white Y 235 -> 151, grey 126 -> 90).""" - v = meta["video"] - trc = v.get("color_transfer") or "bt709" - space = v.get("color_space") or "bt2020nc" - return ",".join([ - f"zscale=tin={trc}:pin=bt2020:min={space}:rin={v.get('color_range') or 'tv'}:t=linear", - "format=gbrpf32le", - "zscale=p=bt709", - "zscale=t=bt709:m=bt709:r=tv", - "format=yuv420p", - ]) - - -def is_bt2020_sdr(v: dict) -> bool: - """BT.2020 primaries on a non-HDR transfer and no Dolby Vision: gamut-only --to-sdr.""" - return bool(v.get("bt2020_or_hdr")) and not v.get("hdr_signal") and not v.get("dolby_vision") \ - and v.get("color_transfer") not in ("smpte2084", "arib-std-b67") - - # --retag targets: ffprobe's names (colorspace, primaries, transfer) and the ITU-T H.273 code points # h264_metadata/hevc_metadata write into the VUI (matrix_coefficients, colour_primaries, # transfer_characteristics). @@ -436,7 +396,7 @@ def main() -> int: "(luma 14-241, chroma 5-251) legal range after the 8-bit 4:2:0 conversion, then measure the written " "file's out-of-range share (JSON legalize). --correct with no correction values clamps only") ap.add_argument("--tonemap", choices=TONEMAPS, default="hable", help="tone-mapping curve (default hable)") - ap.add_argument("--peak", type=float, default=1000.0, help="source peak brightness in nits used for PQ (default 1000)") + ap.add_argument("--peak", type=float, default=1000.0, help="the PQ source's peak brightness in nits, the level the tone map takes to SDR white (default 1000; an HLG source peaks at 1000 by definition, so it does not apply there)") ap.add_argument("--desat", type=float, default=0.0, help="tonemap desaturation strength (default 0)") ap.add_argument("--lut-strength", type=float, default=1.0, help="blend LUT result with the original, 0..1 (default 1)") ap.add_argument("--force", action="store_true", help="run --to-sdr even if the file is tagged neither HDR nor BT.2020 (tone-mapped as PQ; a BT.2020 SDR file never needs it)") diff --git a/scripts/cut.py b/scripts/cut.py index 6faf3121..0fa82cce 100755 --- a/scripts/cut.py +++ b/scripts/cut.py @@ -62,7 +62,8 @@ from _common import (beat_grid, snap_points, decode_pcm_mono, rms_envelope, BEAT_MIN_CONFIDENCE) from _common import require_tool from _common.probe import TS_EXTS, signatures_match, stream_signature -from _common.decision import grid_at_or_after +from _common.decision import SEEK_MARGIN, delivered_rate, frame_rate, grid_at_or_after, stream_starts +from _common.timeline import output_timecode_args from _common import warn, record_fallback, refuse_existing_outputs, sibling_temp, source_codec_video_args, STATE, add_common, apply_common, audio_codec_for, emit, aac_args, cfr_args, default_output, die, ffmpeg_base, info, is_audio_output, time_arg, probe, run, X264_PRESETS, keyframes_near, MissingFpsError, concat_list_line, place_output, refuse_output_is_input, fmt_secs, measure_frame_timing, dry_run_input_pending # outputs whose re-encode dropped a subtitle/data stream (reported as dropped_non_av_streams) @@ -352,6 +353,28 @@ def copy_presentation(t: float, key, edit_list: bool, fps) -> dict: return {"start_snapped": abs(k_pts - t) > frame + 1e-6, "stored_preroll_seconds": None} +def first_frame_from(src: str, meta: dict, start: float, reencode: bool, edit_list: bool): + """Seconds from the source picture's first frame to the frame a cut from `start` begins on, + for its start timecode: the first frame at or after `start` for a re-encode and an edit-listed + copy that hides its keyframe's pre-roll, the keyframe for any other copy (it shows the + pre-roll). None when where a copy lands cannot be read, and for an MXF source, whose demuxer + lands a seek on another keyframe than seek_keyframe's read (-ss 1.5: the one at 1.44 s, read as + the one at 0.96 s), so no label would be known to be right.""" + v0, rate = stream_starts(meta)[0], frame_rate(meta) + if reencode: + # a start before the picture's first frame (a video that starts after the file's origin) + # begins on that first frame, not on a grid frame before it + return grid_at_or_after(max(start, v0), v0, rate) - v0 + if "mxf" in str(meta.get("format") or "").split(","): + return None + key = seek_keyframe(src, start) + if key is None: + return None + if edit_list and key[0] <= start + 1e-6: + return None if key[2] is None else key[2] - v0 + return key[0] - v0 + + def av_skew(out_meta: dict, edit_list_remedy: bool = False, expected: float = 0.0): """(seconds audio starts after video, warning note or None); None when either is missing. A lossless cut can start its streams apart -- Core Media HEVC once gave 3.7 s of sound with no @@ -891,6 +914,9 @@ def cut_one(src: str, start: float, end: float, dst: str, reencode: bool, crf: i edit_list = (edit_list_ok or copy_t is not None) and os.path.splitext(dst)[1].lower() in EDIT_LIST_EXTS cmd = (ffmpeg_base() + ["-ss", f"{start:.6f}", "-i", src, "-t", f"{dur if copy_t is None else copy_t:.6f}"] + copy_args(meta, dst) + ([] if edit_list else ["-avoid_negative_ts", "make_zero"]) + [dst]) + if not audio_only and (meta.get("timecode") or {}).get("value"): + cmd[-1:-1] = output_timecode_args(meta, first_frame_from(src, meta, start, reencode, edit_list and copy_t is None), + delivered_rate(meta) if reencode else None) proc = run(cmd, check=False) if proc.returncode != 0: if not reencode: @@ -956,9 +982,6 @@ def cut_one(src: str, start: float, end: float, dst: str, reencode: bool, crf: i # plays as a click: measured on a 440 Hz tone, the step at the join was 14x the tone's own largest # sample-to-sample change. A 20 ms fade is too short to hear as a dip and long enough to remove it. DECLICK_S = 0.02 -# how far before a segment the fallback seeks, so a start inside the B-frame reorder delay before a -# keyframe still decodes from the keyframe BEFORE it -SEEK_MARGIN = 1.0 # codecs whose configuration travels in the packets, so a part with no extradata is not missing any _PARTS: dict = {} @@ -1102,7 +1125,9 @@ def _join_chunk(src: str, segments: List[Tuple[float, float]], dst: str, meta: d # the source's rate, stated: FFmpeg 7.0 encoded the trimmed, concatenated pictures at 25 fps # and dropped frames to fit (98 frames for 117 at 30 fps) rate_out = ["-r", rate_arg] if has_v and rate_arg and rate and "-r" not in codec else [] - cmd += ["-sn", "-dn"] + codec + rate_out + [dst] + # a join's frames after the first range are not the source's next frames: no start timecode + # labels them, so the output carries none (a single range's cut moves it to its first frame) + cmd += ["-sn", "-dn"] + codec + rate_out + ([] if intermediate or not has_v else output_timecode_args(meta, None)) + [dst] run(cmd) return codec @@ -1122,7 +1147,7 @@ def join_from_source(src: str, segments: List[Tuple[float, float]], dst: str, me if len(segments) <= JOIN_CHUNK: _join_chunk(src, segments, dst, meta, crf, preset, has_v, vend=vend, hold=hold) return - ext = os.path.splitext(dst)[1] or ".mp4" + ext = os.path.splitext(dst)[1].lower() or ".mp4" # IMG_0001.MOV is a .mov # Matroska chunks with PCM audio: no edit lists and no AAC priming to carry into the join (MP4 # chunks measured the video 23 ms late against the audio after a copy concat) # the chunks' codec tag (hvc1), which a stream copy out of Matroska does not carry into an MP4 @@ -1163,7 +1188,8 @@ def encode_chunks() -> List[str]: audio = [] if not meta.get("audio") else (aac_args() if has_v else audio_codec_for(dst)) run(ffmpeg_base() + ["-f", "concat", "-safe", "0", "-i", listfile] + (["-c:v", "copy"] + (tag if ext in (".mp4", ".mov", ".m4v") else []) if has_v else ["-vn"]) + audio - + (["-movflags", "+faststart"] if ext in (".mp4", ".mov", ".m4v") and has_v else []) + [dst]) + + (["-movflags", "+faststart"] if ext in (".mp4", ".mov", ".m4v") and has_v else []) + + (output_timecode_args(meta, None) if has_v else []) + [dst]) BEAT_RATE = 22050 # the decode rate the onset pass uses, matching scenes.py --beats @@ -1333,7 +1359,7 @@ def fallback(reason: str, code: str) -> dict: # line can outgrow Windows' limit, and the full re-encode joins in chunks of this size return fallback(f"{len(segments)} segments with sound: the smart join takes at most {JOIN_CHUNK}, the " "full re-encode joins them in chunks", "segments") - ext = os.path.splitext(output)[1] or ".mp4" + ext = os.path.splitext(output)[1].lower() or ".mp4" params = _smartcut.stream_params(args.input) bad = _smartcut.ineligible(params, meta, ext) if bad: @@ -1510,7 +1536,7 @@ def main() -> int: output = args.output or default_output(args.input, "cut") refuse_output_is_input(output, args.input) - ext = os.path.splitext(output)[1] or ".mp4" + ext = os.path.splitext(output)[1].lower() or ".mp4" # an upper-case .MP4 is still faststart # A segment that runs past the end of the video, with a join after it: whatever joins it (the # concat demuxer or the concat filter) starts the next segment after its LONGER stream, so the @@ -1693,6 +1719,8 @@ def sample_timing() -> dict: cmd = ffmpeg_base() + ["-f", "concat", "-safe", "0", "-i", listfile, "-c", "copy"] if ext in (".mp4", ".mov", ".m4v"): cmd += ["-movflags", "+faststart"] + if not is_audio_output(output): + cmd += output_timecode_args(meta, None) # a join: no start timecode (_join_chunk) cmd += [joined] proc = run(cmd, check=False) fail_if_classified(proc) # #415: a full disk is not a reason to re-cut every segment diff --git a/scripts/fit.py b/scripts/fit.py index b316bee1..2ced300e 100755 --- a/scripts/fit.py +++ b/scripts/fit.py @@ -39,6 +39,8 @@ from fractions import Fraction from typing import List +from _common.decision import SEEK_MARGIN, delivered_rate, frame_rate, grid_at_or_after, stream_starts +from _common.timeline import exact_rate, output_timecode_args from _common import video_args, STATE, add_common, apply_common, emit, aac_args, aspect_ratio, cfr_args, default_output, die, ffmpeg_base, info, parse_time, probe, run, run_keeping_subtitles, validate_color, x264_args, pad_filters, add_pad_fill_args, X264_PRESETS, time_arg, fmt_secs, frame_size, display_size, square_pixels, square_pixels_note, note BLUR_DARKEN = 0.15 # how much --fit blur dims the blurred background copy (eq brightness) @@ -136,6 +138,7 @@ def main() -> int: pre_input: List[str] = [] post: List[str] = [] factor = 1.0 + tc_first = None # seconds from the source's first frame to the output's, when a trim moves it # ---- rotate / flip if args.rotate == 90: @@ -175,8 +178,18 @@ def main() -> int: else: if target < src_dur: start = (src_dur - target) / 2 if args.from_center else 0.0 - pre_input += ["-ss", f"{start:.3f}"] - post += ["-t", f"{target:.3f}"] + # seek SEEK_MARGIN early and drop the margin on the output side, as cut.py does: + # an input seek straight to `start` can land on a keyframe whose picture comes + # after it (a B-frame HEVC source) and lose the frames before it, so the first + # picture would not be the one the start timecode below names + m = min(SEEK_MARGIN, round(start, 3)) + pre_input += ["-ss", f"{start - m:.3f}"] + post += (["-ss", f"{m:.3f}"] if m > 0 else []) + ["-t", f"{target:.3f}"] + # the output starts on the first frame at or after the seek (a seek before a + # picture that starts after the file's origin: on its first frame): its start + # timecode is that frame's, not the source's first frame's + v0, rate = stream_starts(meta)[0], frame_rate(meta) + tc_first = grid_at_or_after(max(round(start, 3), v0), v0, rate) - v0 else: info(f"source ({src_dur:.2f}s) is already shorter than {target:.2f}s; trim does nothing") @@ -213,6 +226,15 @@ def main() -> int: vf.append(f"fps={args.fps:g}") elif meta["video"].get("variable_frame_rate_suspected"): info("source looks variable-frame-rate; conforming to constant fps automatically") + # the start timecode is counted at the rate the output is delivered at: a --fps that changes + # the rate relabels it (01:00:00:12 at 25 fps is 01:00:00:24 at 50) even with no trim, which + # is also how a render.py clip cut at its own rate reaches the project's frame.fps + out_rate = args.fps or delivered_rate(meta) + src_fps = meta["video"].get("fps") + rate_changed = bool(out_rate and src_fps and exact_rate(float(out_rate)) != exact_rate(float(src_fps))) + relabel = tc_first is not None or rate_changed + if relabel: + post += output_timecode_args(meta, tc_first or 0.0, out_rate) output = args.output or default_output(args.input, "fit") cmd = ffmpeg_base() + pre_input + ["-i", args.input] @@ -240,7 +262,7 @@ def main() -> int: run(cmd + [output]) dropped_streams = bool(meta.get("subtitle_streams") or meta.get("data_streams")) else: - dropped_streams = run_keeping_subtitles(cmd, output) + dropped_streams = run_keeping_subtitles(cmd, output, relabel_timecode=relabel) result = probe(output, role="output") msg = f"wrote {output} ({fmt_secs(result['duration'])}, {result['video']['width']}x{result['video']['height']})" diff --git a/scripts/graphics.py b/scripts/graphics.py index b4947a2d..ab30fe54 100755 --- a/scripts/graphics.py +++ b/scripts/graphics.py @@ -36,7 +36,7 @@ from _platforms import PLATFORMS, PLATFORM_CHOICES, safe_margins_px, resolve as resolve_platform from _common import (guard_sidecar, require_filter, aac_args, add_common, brand_caption_style, script_font_for_text, apply_common, cfr_args, color_hex, default_output, die, draws_chroma, drawn_chain, emit, escape_drawtext, escape_filter_path, - ffmpeg_base, info, layer_base_chain, load_brand, parse_time, probe, rgb_chain_to_bt709, run, run_keeping_subtitles, video_args, + ffmpeg_base, info, layer_base_chain, load_brand, overlay_format, parse_time, probe, rgb_chain_to_bt709, run, run_keeping_subtitles, video_args, drawtext_boxborderw, drawtext_center_align, X264_PRESETS, time_arg, fmt_secs, STATE, drawtext_text_opts, LANGUAGE_NAMES, needs_shaping, detect_script, BIDI_SCRIPTS, font_family_of_file, font_family_for_script, has_emoji, emoji_clusters, emoji_codepoint_name, char_script, emoji_filter_chain, emoji_asset_for, emoji_support, resolve_emoji_assets, @@ -497,8 +497,9 @@ def add_text(text, drawtext, *, target=None, **el): h = max(3, int(base * 0.008)) fps = meta['video'].get('fps') or 30 fc.append(f"{layer_src(primary, 1.0, W, h)}[pb]") - if draws_chroma([bg], meta) or alpha_codec: - # a layer (overlay composites the alpha too; drawbox leaves it as it was) + if draws_chroma([bg], meta) or alpha_codec or overlay_format(meta): + # a layer (overlay composites the alpha too; drawbox leaves it as it was, and takes a + # 10-bit picture to 8 bits) fc.append(f"{layer_src(bg, 0.5, W, h, 0, (H - h) % 2)}[pbg]") fc.append(f"[0:v][pbg]overlay=x=0:y={(H - h) // 2 * 2}:{en}:eof_action=pass[v1]") else: @@ -712,8 +713,9 @@ def add_text(text, drawtext, *, target=None, **el): # converts neither picture to the other's matrix. The text is drawn in BT.709 (drawn_chain). # On an alpha line every box is a layer: overlay composites the box's alpha over the picture's, # while drawbox paints the colour and leaves the alpha as it was (a box on a transparent canvas - # stayed invisible) - as_layers = bool(alpha_codec) or any(draws_chroma([b[0]], meta) for b in boxes) + # stayed invisible). On a 10-bit picture too: drawbox is 8-bit only on 6.1, so FFmpeg took the + # whole picture to 8 bits around it (overlay_format) + as_layers = bool(alpha_codec) or bool(overlay_format(meta)) or any(draws_chroma([b[0]], meta) for b in boxes) drawboxes = [] for colour, alpha, geo, x, y, w, h, enable, x_expr in boxes: if as_layers: @@ -743,6 +745,9 @@ def add_text(text, drawtext, *, target=None, **el): if fc: fc = ["[0:v]format=yuva444p[ka0]"] + [c.replace("[0:v]", "[ka0]") for c in fc] square = "format=yuva444p" + ("," + square if square else "") + elif overlay_format(meta): + # a 10-bit picture on a 10-bit line is composited at 10 bits (overlay's default is 8-bit) + fc = [c.replace(":eof_action=pass", ":eof_action=pass" + overlay_format(meta)) for c in fc] finish = finish_chain(alpha_codec, args.alpha_mode, meta) if alpha_codec else "" cmd = ffmpeg_base() + (decoder_args_for(meta, args.input) if alpha_codec else []) + ["-i", args.input] @@ -762,7 +767,7 @@ def add_text(text, drawtext, *, target=None, **el): last = "vtxt" eo, emoji_inputs = emoji_filter_chain({"overlays": emoji_overlays}, last, "vfinal", first_input=1, to_yuv=rgb_chain_to_bt709(meta), - base_chain=layer_base_chain(meta)) + base_chain=layer_base_chain(meta), overlay_opts=overlay_format(meta)) for spec in emoji_inputs: cmd += spec asset = spec[-1] diff --git a/scripts/look.py b/scripts/look.py index 73d6e497..d8d94016 100755 --- a/scripts/look.py +++ b/scripts/look.py @@ -23,7 +23,7 @@ from _common import drawtext_fontfile # noqa: E402 from _common import ffmpeg_filters, filter_missing_hint # noqa: E402 from _common.emit import note, record_fallback # noqa: E402 -from _common.color import bt709_label_filter # noqa: E402 +from _common.color import bt709_label_filter, bt2020_sdr_to_bt709_chain, hdr_to_sdr_chain, is_bt2020_sdr # noqa: E402 from _common.probe import display_size, square_pixels, square_pixels_note # noqa: E402 # --guides: broadcast safe areas as outlines, in fractions of the frame (a tile as much as a frame). @@ -177,6 +177,23 @@ def px(x, y): } +def display_conversion(meta: dict, what: str = "source") -> str: + """The filter that shows `meta`'s frames on an SDR BT.709 display, or "" for a picture that is + already one. HDR is tone-mapped so the agent judges representative colours, not raw HLG/PQ.""" + v = meta.get("video") or {} + if not v.get("bt2020_or_hdr"): + return "" + # color.py --to-sdr's own chains at its defaults (hable, a 1000-nit PQ peak), so the sheet + # shows what that conversion gives. Through 3.6.0 this tone map linearised at npl=1000 and + # a sheet's white came out at about Y 151 of 235; a BT.2020 SDR source was tone-mapped + # too, where color.py only converts its gamut. + if is_bt2020_sdr(v): + info(f"BT.2020 SDR {what}: frames are converted to BT.709 for display") + return bt2020_sdr_to_bt709_chain(meta) + info(f"HDR {what}: frames are tone-mapped to SDR for display") + return hdr_to_sdr_chain({"video": dict(v, color_transfer=v.get("color_transfer") or "arib-std-b67")}, "hable") + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("input") @@ -264,13 +281,10 @@ def main() -> int: boxes, guides = guide_boxes(list(dict.fromkeys(names))) safe_filter += "," + ",".join(boxes) tc = safe_filter + tc - # HDR sources: tone-map for the PNG so the agent judges representative colours, not raw HLG/PQ - if meta["video"].get("bt2020_or_hdr"): - v = meta["video"] - tm = (f"zscale=tin={v.get('color_transfer') or 'arib-std-b67'}:pin={v.get('color_primaries') or 'bt2020'}:min={v.get('color_space') or 'bt2020nc'}:rin=tv:t=linear:npl=1000," - "format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable,zscale=t=bt709:m=bt709:r=tv,format=yuv420p,") - tc = "," + tm.rstrip(",") + tc - info("HDR source: frames are tone-mapped to SDR for display") + marks = tc # the boxes and the stamp, without the display conversion (each --compare input has its own) + tm = display_conversion(meta) + if tm: + tc = "," + tm + tc outputs: List[str] = [] # An untagged SDR source is BT.709 (what players and every encoder line here take it as), but the # conversion to the PNG's RGB reads "unknown" as BT.601: C03028 came out B31F28 (#410). Its @@ -293,6 +307,9 @@ def main() -> int: cw, ch = display_size(cmeta.get("video") or {}) if cw and ch and cw / ch < 4 / 3 + 0.01: die(f"--guides 4:3 draws the 4:3 centre cut of a wider frame; the --compare clip is {cw}x{ch}") + # the second picture is converted for display by its own tags: through 3.6.1 the first + # input's tone map ran on both, so an SDR clip beside a PQ one was read as PQ + tm2 = display_conversion(cmeta, "--compare clip") lab2 = bt709_label_filter(cmeta) lab2 = lab2 + "," if lab2 else "" csq = (square_pixels(cmeta.get("video")) + ",") if square_pixels(cmeta.get("video")) else "" @@ -310,15 +327,17 @@ def main() -> int: # the tone map and the --safe / --guides boxes go on each picture as scaled, before the # pad: drawn after it, a picture letterboxed in the box got guides from the padded # height (a 2:1 clip beside a 16:9 one); the timecode stamp is drawn on the box - pre = tc.replace("," + timecode_filter(font_prefix), "") + pre = marks.replace("," + timecode_filter(font_prefix), "") # each picture keeps its own aspect inside a half-width box as tall as the taller of the # two (letterboxed, not stretched): through 3.1.3 scale2ref forced the first to the # second's size, so a 4:3 beside a 16:9 came out squashed (#411) box_h = max(_half_height(half, display_size(meta["video"])), _half_height(half, display_size(cmeta.get("video")))) - fit = (f"scale={half}:{box_h}:force_original_aspect_ratio=decrease{pre}," - f"pad={half}:{box_h}:(ow-iw)/2:(oh-ih)/2:color=0x202020,setsar=1{stamp}") - fc = (f"[0:v]{lab}{sq}{fit}[a];[1:v]{lab2}{csq}{fit}[b];" + def fit(conv): + conv = "," + conv if conv else "" + return (f"scale={half}:{box_h}:force_original_aspect_ratio=decrease{conv}{pre}," + f"pad={half}:{box_h}:(ow-iw)/2:(oh-ih)/2:color=0x202020,setsar=1{stamp}") + fc = (f"[0:v]{lab}{sq}{fit(tm)}[a];[1:v]{lab2}{csq}{fit(tm2)}[b];" f"[a][b]hstack=inputs=2[out]") cmd = ffmpeg_base() + ["-ss", f"{sec:.3f}", *still_input_args(args.input), "-i", args.input, "-ss", f"{sec:.3f}", *still_input_args(args.compare), "-i", args.compare, "-filter_complex", fc, "-map", "[out]", "-frames:v", "1", "-update", "1", out] diff --git a/scripts/multicam.py b/scripts/multicam.py index 3fbcff54..13b84456 100755 --- a/scripts/multicam.py +++ b/scripts/multicam.py @@ -28,12 +28,13 @@ import argparse import json import math +import os import sys from typing import Dict, List, Tuple from _common import guard_sidecar, label_chain, matrix_chain, range_chain, video_args, aac_args, add_common, apply_common, default_output, die, emit, ffmpeg_base, info, time_arg, probe, run, x264_args, X264_PRESETS, fmt_secs, decode_pcm_mono, rms_envelope, STATE from _common.emit import note, record_fallback, warn -from _common.decision import av_trim +from _common.decision import av_trim, rebase_project_paths from _common.probe import display_size, square_pixels, square_pixels_note from sync import measure_offset @@ -335,6 +336,10 @@ def main() -> int: # a replacement: the combined output below is still written as usual. project = {"output": default_output(args.inputs[0], "multicam_project", "mp4"), "clips": project_clips} + # the inputs are named as given, relative to the working directory, and render.py reads a + # project's relative paths against the project file's directory: written elsewhere + # (--write-project out/cut.json), they named files that are not there + project = rebase_project_paths(project, os.getcwd()) if not STATE.dry_run: with open(args.write_project, "w", encoding="utf-8") as fh: fh.write(json.dumps(project, indent=2) + "\n") diff --git a/scripts/overlay.py b/scripts/overlay.py index ffb60cef..8edef436 100755 --- a/scripts/overlay.py +++ b/scripts/overlay.py @@ -31,11 +31,11 @@ import argparse import re import sys -from typing import List, Optional +from typing import Any, Dict, List, Optional from _platforms import PLATFORMS, PLATFORM_CHOICES, safe_margins_px, resolve as resolve_platform from _common import still_input_args # noqa: E402 -from _common import STATE, script_font_for_text, drawtext_clean, drawtext_line_metrics, drawtext_text_opts, drawtext_draws_newline_glyph, is_multiline, needs_shaping, LANGUAGE_NAMES, has_emoji, detect_script, emoji_clusters, load_brand, drawn_chain, label_chain, layer_base_chain, matrix_chain, range_chain, rgb_chain_to_bt709, video_args, add_common, apply_common, emit, aac_args, cfr_args, default_output, die, escape_drawtext, ffmpeg_base, info, parse_time, probe, run, run_keeping_subtitles, validate_color, x264_args, X264_PRESETS, time_arg, fmt_secs +from _common import STATE, script_font_for_text, drawtext_clean, drawtext_line_metrics, drawtext_text_opts, drawtext_draws_newline_glyph, is_multiline, needs_shaping, LANGUAGE_NAMES, has_emoji, detect_script, emoji_clusters, load_brand, drawn_chain, label_chain, layer_base_chain, matrix_chain, overlay_format, range_chain, rgb_chain_to_bt709, video_args, add_common, apply_common, emit, aac_args, cfr_args, default_output, die, escape_drawtext, ffmpeg_base, info, parse_time, probe, run, run_keeping_subtitles, validate_color, x264_args, X264_PRESETS, time_arg, fmt_secs from _common.emit import note, warn from _common.alpha import (add_alpha_args, audio_line, check_alpha_args, decoder_args_for, default_ext, finish_chain, has_alpha_plane, report as alpha_report, source_alpha, tag_args, video_line) @@ -428,7 +428,9 @@ def main() -> int: if end is not None and end > args.fade: chain.append(f"fade=t=out:st={end - args.fade:.3f}:d={args.fade:g}:alpha=1") x, y = position_exprs(args.position, args.margin, text_mode=False, margins=safe_margins) - ov = f"overlay={x}:{y}:format=auto" + # a 10-bit main on a 10-bit line is composited at 10 bits in YUV: format=auto with an RGBA + # still took the whole picture through 8-bit RGB (overlay_format) + ov = f"overlay={x}:{y}" + (overlay_format(meta) or ":format=auto") if enable: ov += f":enable='{enable}'" # -loop 1 turns the still into a timed stream so fade/enable expressions see real timestamps @@ -480,13 +482,18 @@ def main() -> int: elif args.scale: chain.append(f"scale={args.scale}:-2") # an alpha line composites at 4:4:4: the layer is not taken through 4:2:0 on the way - chain.append("format=yuva444p" if alpha_codec else "format=yuva420p") + # a 10-bit layer over a main overlay_format() composites at 10 bits stays 10-bit + deep = bool(overlay_format(meta)) and (pip_meta["video"].get("bit_depth") or 8) > 8 + chain.append("format=yuva444p" if alpha_codec else "format=yuva420p10le" if deep else "format=yuva420p") if args.chromakey: chain.append(f"chromakey={args.chromakey}:{args.chromakey_similarity:g}:{args.chromakey_blend:g}") if args.opacity < 1: - chain.append(f"colorchannelmixer=aa={args.opacity:g}") + # the alpha is scaled in YUV: colorchannelmixer takes RGB only, so FFmpeg converted the + # layer to RGB and overlay=format=auto then composited the whole main picture in RGB, + # which came back BT.601 under the BT.709 tag (#333's defect, on this path) + chain.append(f"lut=y=val:u=val:v=val:a=val*{args.opacity:g}") x, y = position_exprs(args.position, args.margin, text_mode=False, margins=safe_margins) - ov = f"overlay={x}:{y}:format=auto" + ov = f"overlay={x}:{y}" + (overlay_format(meta) or ":format=auto") if enable: ov += f":enable='{enable}'" # a VP9 layer stored with an alpha layer is decoded with libvpx-vp9: FFmpeg's own decoder @@ -501,7 +508,13 @@ def main() -> int: # converted to BT.709 YUV with its alpha plane: the 4:4:4 the conversion ends in had none, # and the layer covered the picture as an opaque rectangle pip = pip.replace("format=yuv444p", "format=yuva444p") - if base or pip: + # an untagged picture is labelled beside a converted one, and beside a tagged one too: FFmpeg + # 8.0's overlay negotiates one colorspace for both inputs, and a BT.709-tagged main under an + # untagged PiP was converted to the PiP's "unknown" (BT.601) matrix, away from the PiP too + # (C03028 at 80/110/192 came back 88/103/192 on 8.0.1). Two untagged pictures agree as they are. + def tagged(m: Dict[str, Any]) -> bool: + return ((m.get("video") or {}).get("color_space") or "unknown") not in ("unknown", "unspecified") + if base or pip or tagged(meta) != tagged(pip_meta): base, pip = base or label_chain(meta, meta), pip or label_chain(pip_meta, meta) # overlay mixes samples: a full-range picture is taken to limited range (#410) base, pip = base + range_chain(meta, meta), pip + range_chain(pip_meta, meta) diff --git a/scripts/redact.py b/scripts/redact.py index 47c35e8c..4367d291 100644 --- a/scripts/redact.py +++ b/scripts/redact.py @@ -27,7 +27,7 @@ import argparse import sys -from _common import add_common, apply_common, aac_args, cfr_args, default_output, die, emit, ffmpeg_base, info, probe, run_keeping_subtitles, video_args, X264_PRESETS, fmt_secs +from _common import add_common, overlay_format, apply_common, aac_args, cfr_args, default_output, die, emit, ffmpeg_base, info, probe, run_keeping_subtitles, video_args, X264_PRESETS, fmt_secs from _common.emit import note from _common.probe import display_size, square_pixels, square_pixels_note @@ -110,7 +110,7 @@ def main() -> int: region = f"{crop},boxblur={radius}:{radius}:{chroma}:{radius}" else: region = f"{crop},scale={max(1, args.width // args.block_size)}:{max(1, args.height // args.block_size)}:flags=neighbor,scale={args.width}:{args.height}:flags=neighbor" - fc = f"[0:v]{square + ',' if square else ''}split=2[base][region];[region]{region}[patched];[base][patched]overlay={args.x}:{args.y}[out]" + fc = f"[0:v]{square + ',' if square else ''}split=2[base][region];[region]{region}[patched];[base][patched]overlay={args.x}:{args.y}{overlay_format(meta)}[out]" cmd = ffmpeg_base() + ["-i", args.input, "-filter_complex", fc, "-map", "[out]"] if has_audio: cmd += ["-map", f"0:a:{args.audio_stream}?"] diff --git a/scripts/render.py b/scripts/render.py index 53c03219..ed02107f 100755 --- a/scripts/render.py +++ b/scripts/render.py @@ -74,9 +74,10 @@ from export import PRESETS, PLATFORM_OF from _platforms import PLATFORMS, caption_defaults, resolve as resolve_platform -from _common import guard_sidecar, STATE, add_common, aspect_ratio, brand_caption_style, load_brand, apply_common, child_args, die, emit, info, probe, require_tool, run, run_tool, place_output, refuse_output_is_input, _check_existing_output, _check_output_path, fingerprint, PLAN_VERSION, ffmpeg_version, X264_PRESETS +from _common import guard_sidecar, STATE, add_common, aspect_ratio, brand_caption_style, load_brand, apply_common, child_args, die, emit, info, option_args, probe, require_tool, run, run_tool, place_output, refuse_output_is_input, _check_existing_output, _check_output_path, fingerprint, PLAN_VERSION, ffmpeg_version, X264_PRESETS from _common.runner import HW_ENV, HW_FORCED_ENV, add_hw_orchestrator_args, child_exit_code, found_exe, kept_clause, kept_deliverables, mark_delivered, remember_written from _common.emit import absorb_stage, audit, record_fallback, record_skip, refuse, warn +from _common.decision import rebase_project_paths import subprocess from _contract import CONTRACT_VERSION from batch import file_key @@ -362,7 +363,7 @@ def render_pack(names: List[str], args) -> int: ("--chapters", args.chapters), ("--image", args.image), ("--intermediate-preset", args.intermediate_preset)): if value: - argv += [flag, str(value)] + argv += option_args(flag, value) info(f"→ pack: {name}") proc = run_tool(argv + child_args() + ["--json"]) try: @@ -1260,6 +1261,12 @@ def main() -> int: emit(args.write_project, template=args.template, stages=[], check=None) return 0 + # every file the project names is read against the project file's directory (a template's, + # the working directory): the same keys batch.py and multicam.py rebase on a copy they write + # elsewhere, including the two rel() below never saw (audiogram.image, overlays[].font_file + # were read against the working directory) + proj = rebase_project_paths(proj, base) + def rel(p: Any) -> str: p = str(p) return p if os.path.isabs(p) else str(base / p) @@ -1407,17 +1414,17 @@ def rel(p: Any) -> str: if needs_cut: argv: List[Any] = [src, "-o", part, "--accurate"] if c.get("in") is not None: - argv += ["--start", c["in"]] + argv += option_args("--start", c["in"]) if c.get("out") is not None: - argv += ["--end", c["out"]] + argv += option_args("--end", c["out"]) clip_snap = dict(snap_spec) clip_snap.update(c.get("snap") or {}) if str(clip_snap.get("to") or "none") == "beats": argv += ["--snap", "beats"] if clip_snap.get("tolerance") is not None: - argv += ["--snap-tolerance", str(clip_snap["tolerance"])] + argv += option_args("--snap-tolerance", clip_snap["tolerance"]) if clip_snap.get("min_confidence") is not None: - argv += ["--min-confidence", str(clip_snap["min_confidence"])] + argv += option_args("--min-confidence", clip_snap["min_confidence"]) if clip_snap.get("source"): argv += ["--snap-source", rel(clip_snap["source"])] sh("cut.py", *argv, stage="clips") @@ -1464,7 +1471,7 @@ def rel(p: Any) -> str: # against the project file, as every other path here and as the output guard # checks it; it used to reach waveform.py as written, against the cwd value = rel(value) - argv += [flag, str(value)] + argv += option_args(flag, value) argv += brand_args sh("waveform.py", *argv, stage="audiogram") current = nxt @@ -1480,11 +1487,11 @@ def rel(p: Any) -> str: current = str(work / f"joined{mid}") argv = list(parts) + ["-o", current, "--transition", trans.get("type", "fade"), "--duration", str(trans.get("duration", 0.5))] if frame.get("width"): - argv += ["--width", str(frame["width"])] + argv += option_args("--width", frame["width"]) if frame.get("height"): - argv += ["--height", str(frame["height"])] + argv += option_args("--height", frame["height"]) if frame.get("fps"): - argv += ["--fps", str(frame["fps"])] + argv += option_args("--fps", frame["fps"]) sh("join.py", *argv, stage="join") stages_done.append("join") if args.stop_after == "join": @@ -1499,7 +1506,7 @@ def rel(p: Any) -> str: for k, flag in (("threshold", "--threshold"), ("min_silence", "--min-silence"), ("margin", "--margin"), ("margin_lead", "--margin-lead"), ("margin_tail", "--margin-tail")): if sil.get(k) is not None: - argv += [flag, str(sil[k])] + argv += option_args(flag, sil[k]) sh("silence.py", *argv, stage="silence") current = nxt stages_done.append("silence") @@ -1536,7 +1543,7 @@ def rel(p: Any) -> str: argv = [current, "-o", nxt] for k, flag in (("duration", "--duration"), ("method", "--method"), ("aspect", "--aspect"), ("fit", "--fit"), ("width", "--width"), ("height", "--height"), ("fps", "--fps"), ("smooth", "--smooth")): if fit.get(k) is not None: - argv += [flag, str(fit[k])] + argv += option_args(flag, fit[k]) sh("fit.py", *argv, stage="fit") current = nxt stages_done.append("fit") @@ -1560,7 +1567,7 @@ def rel(p: Any) -> str: for k, flag in (("font", "--font"), ("size", "--size"), ("color", "--color"), ("position", "--position"), ("margin", "--margin"), ("animate", "--animate"), ("highlight_color", "--highlight-color"), ("outline", "--outline"), ("lang", "--lang"), ("offset", "--offset"), ("max_lines", "--max-lines"), ("min_duration", "--min-duration"), ("fit_size", "--fit-size"), ("min_size", "--min-size"), ("fit_size_scope", "--fit-size-scope")): if cap.get(k) is not None: - argv += [flag, str(cap[k])] + argv += option_args(flag, cap[k]) for k, flag in (("karaoke", "--karaoke"), ("bold", "--bold"), ("box", "--box")): if cap.get(k): argv.append(flag) @@ -1578,11 +1585,11 @@ def rel(p: Any) -> str: nxt = str(work / f"graphics{i:02d}{mid}") if not g.get("template"): die(f"graphics[{i}] needs a template") - argv = [current, "-o", nxt, "--template", g["template"]] + argv = [current, "-o", nxt] + option_args("--template", g["template"]) for k, flag in (("name", "--name"), ("title", "--title"), ("subtitle", "--subtitle"), ("start", "--start"), ("end", "--end"), ("position", "--position"), ("from", "--from"), ("scale", "--scale"), ("primary", "--primary"), ("text_color", "--text-color"), ("lang", "--lang"), ("text", "--text"), ("top", "--top"), ("bottom", "--bottom"), ("duration", "--duration"), ("margin", "--margin"), ("platform", "--platform")): if g.get(k) is not None: - argv += [flag, str(g[k])] + argv += option_args(flag, g[k]) # the entry's own "platform" is the more specific statement than the template's destination sh("graphics.py", *(argv + brand_args + ([] if g.get("platform") else platform_args)), stage="graphics") current = nxt @@ -1601,7 +1608,7 @@ def rel(p: Any) -> str: elif ov.get("image"): argv += ["--image", rel(ov["image"])] elif ov.get("text"): - argv += ["--text", ov["text"]] + argv += option_args("--text", ov["text"]) elif ov.get("video"): # #326: sized and placed here, after fit, against the frame that ships argv += ["--video", rel(ov["video"])] @@ -1611,7 +1618,7 @@ def rel(p: Any) -> str: ("chromakey", "--chromakey"), ("chromakey_similarity", "--chromakey-similarity"), ("chromakey_blend", "--chromakey-blend")): if ov.get(k) is not None: # font_file is a path: the same one project_input_files() guards - argv += [flag, rel(ov[k]) if k == "font_file" and ov[k] else str(ov[k])] + argv += option_args(flag, rel(ov[k]) if k == "font_file" and ov[k] else ov[k]) if ov.get("box"): argv.append("--box") # review 12: the overlay stage was the one stage that never heard which destination this @@ -1647,10 +1654,10 @@ def rel(p: Any) -> str: argv += [flag, rel(au[k])] for k, flag in (("music_volume", "--music-volume"), ("effects_volume", "--effects-volume"), ("fade_in", "--fade-in"), ("fade_out", "--fade-out"), ("music_fade_out", "--music-fade-out"), ("gain", "--gain"), ("duck_amount", "--duck-amount"), ("duck_threshold", "--duck-threshold"), ("duck_attack", "--duck-attack"), ("duck_release", "--duck-release"), ("stereo_widen", "--stereo-widen")): if au.get(k) is not None: - argv += [flag, str(au[k])] + argv += option_args(flag, au[k]) # "voice": true is the medium chain; "voice": "light"|"medium"|"strong" names one if au.get("voice") is not None and au.get("voice") is not False: - argv += ["--voice"] + ([] if au["voice"] is True else [str(au["voice"])]) + argv += ["--voice"] if au["voice"] is True else option_args("--voice", au["voice"]) for k, flag in (("denoise", "--denoise"), ("duck", "--duck"), ("music_loop", "--music-loop"), ("stereo", "--stereo"), ("mono", "--mono"), ("downmix", "--downmix")): if au.get(k): argv.append(flag) @@ -1672,9 +1679,9 @@ def rel(p: Any) -> str: nxt = str(work / f"loudnorm{mid}") argv = [current, "-o", nxt] if ld.get("lufs") is not None: - argv += ["-I", str(ld["lufs"])] + argv += option_args("--lufs", ld["lufs"]) if ld.get("tp") is not None: - argv += ["--tp", str(ld["tp"])] + argv += option_args("--tp", ld["tp"]) sh("loudness.py", *argv, stage="loudness") current = nxt stages_done.append("loudness") @@ -1687,9 +1694,9 @@ def rel(p: Any) -> str: if ex and ex.get("preset"): argv = [current, "--preset", ex["preset"], "-o", output] if ex.get("fit"): - argv += ["--fit", ex["fit"]] + argv += option_args("--fit", ex["fit"]) if ex.get("crf") is not None: - argv += ["--crf", str(ex["crf"])] + argv += option_args("--crf", ex["crf"]) normalize = ex.get("normalize") if normalize is None and ex["preset"] in PLATFORM_OF and not proj.get("loudness"): # eval 8: a reels project without the key rendered fully, failed the loudness check and diff --git a/scripts/silence.py b/scripts/silence.py index a0c6d2c9..3bb27545 100755 --- a/scripts/silence.py +++ b/scripts/silence.py @@ -82,14 +82,17 @@ def silence_positions(silences: List[Tuple[float, float]], words: "list", of the word after (on demos/footage/ai_talk_en.mp4 three word starts sit 0.45 to 0.5 s before the sound does), and a word's midpoint stays on its own side of the pause either way. A silence with no word before it is `inside`.""" + import bisect timed = sorted(words, key=lambda w: w["start"] + w["end"]) + mids = [(w["start"] + w["end"]) / 2 for w in timed] out: List[str] = [] for s, e in silences: if e == float("inf"): e = max(s, duration) if duration is not None else s - ref = (s + e) / 2 - before = [w for w in timed if (w["start"] + w["end"]) / 2 < ref] - out.append("sentence_end" if before and ends_sentence(before[-1]["word"]) else "inside") + # the last word whose midpoint is before the silence's: a binary search, not a scan of the + # whole transcript per silence (an hour of speech has thousands of each) + k = bisect.bisect_left(mids, (s + e) / 2) + out.append("sentence_end" if k and ends_sentence(timed[k - 1]["word"]) else "inside") return out diff --git a/scripts/speedramp.py b/scripts/speedramp.py index d2c54af0..7af931df 100644 --- a/scripts/speedramp.py +++ b/scripts/speedramp.py @@ -19,7 +19,7 @@ import sys from typing import List, Tuple -from _common.decision import av_trim, frame_rate, grid_at_or_after, stream_starts +from _common.decision import av_trim, frame_durations, frame_rate, grid_at_or_after, stream_starts from _common import add_common, apply_common, aac_args, default_output, die, emit, ffmpeg_base, info, probe, run, video_args, X264_PRESETS, fmt_secs, time_arg MAX_SPEED = 20.0 @@ -106,7 +106,11 @@ def main() -> int: die(f"--segment {start:g}-{end:g}:{factor:g} holds no frame: both ends fall on the frame at " f"{bounds[i]:.3f}s; widen it or merge it into a neighbour") vtrim, atrim = av_trim(bounds[i], seg_end, rate) - vparts.append(f"[0:v]{vtrim},setpts=PTS/{factor:.6f}[{vlabel}]") + # the closing fps puts the sped-up frames back on the source's grid with its frame + # duration: concat starts the next segment after the last frame's duration, which setpts + # leaves at the unscaled 1/rate, so every 2x segment pushed the rest of the picture two + # frames late against the sound (and added those frames) + vparts.append(f"[0:v]{vtrim},setpts=PTS/{factor:.6f},{frame_durations(meta, rate)}[{vlabel}]") labels.append(f"[{vlabel}]") if has_audio: aparts.append(f"[0:a]{atrim},{atempo_chain(factor)}[{alabel}]") diff --git a/scripts/sync.py b/scripts/sync.py index 612d92d4..de1c568d 100755 --- a/scripts/sync.py +++ b/scripts/sync.py @@ -35,7 +35,8 @@ from _common.runner import fail_if_classified from _common import video_args, add_common, apply_common, emit, aac_args, audio_codec_for, default_output, die, ffmpeg_base, info, probe, require_tool, run, run_analysis, x264_args, decode_pcm_mono, rms_envelope from _common.emit import record_skip, warn -from _common.decision import stream_starts +from _common.decision import SEEK_MARGIN, frame_rate, grid_at_or_after, stream_starts +from _common.timeline import output_timecode_args SR = 8000 # decode sample rate @@ -385,7 +386,12 @@ def main() -> int: cmd = cmd[:-1] + video_args(probe(args.reference) if args.replace_audio else probe(args.second), args.crf) + [output] run(cmd) else: - if head_trim > 0 and not drift_af: + # A picture is never stream-copied here: a copy starts at the keyframe at or before -ss + # and -avoid_negative_ts make_zero shows the frames from it, so the output started + # early by up to a GOP (a 5 s GOP: nothing trimmed at all) and was reported as synced. + # Even a keyframe on the trim point moved the picture by its B-frame delay (+69 ms). + # The re-encode below starts on the frame itself; a sound-only file is still copied. + if head_trim > 0 and not drift_af and not has_video: cmd = ffmpeg_base() + ["-ss", f"{head_trim:.4f}", "-i", args.second, "-c", "copy"] if os.path.splitext(output)[1].lower() in (".mp4", ".mov", ".m4v"): cmd += ["-movflags", "+faststart"] @@ -396,10 +402,20 @@ def main() -> int: cmd = ffmpeg_base() + ["-ss", f"{head_trim:.4f}", "-i", args.second] + (video_args(probe(args.reference) if args.replace_audio else probe(args.second), args.crf) if has_video else []) + audio_codec_for(output) + [output] run(cmd) else: + # Seek SEEK_MARGIN early and drop the margin on the output side, as cut.py's + # re-encode does: an input seek straight to head_trim can land (the MP4 demuxer + # seeks over raw decode times) on a keyframe whose picture comes after it, and a + # B-frame HEVC file lost the frames before it. The output-side -ss decodes and + # drops the margin from both streams alike; it counts on the output's clock, which + # a drift fix runs faster by drift_ratio (setpts and asetrate alike). cmd = ffmpeg_base() + out_seek: List[str] = [] if head_trim > 0: - cmd += ["-ss", f"{head_trim:.4f}"] - cmd += ["-i", args.second] + m = min(SEEK_MARGIN, head_trim) + cmd += ["-ss", f"{head_trim - m:.6f}"] + if m > 0: + out_seek = ["-ss", f"{m / (drift_ratio if drift_af else 1.0):.6f}"] + cmd += ["-i", args.second] + out_seek af_parts = on_clock + drift_af if delay_ms > 0: af_parts.append(f"adelay={delay_ms}:all=1") @@ -412,6 +428,14 @@ def main() -> int: if vf: cmd += ["-vf", ",".join(vf)] cmd += video_args(probe(args.reference) if args.replace_audio else probe(args.second), args.crf) + # the output's first frame is the second file's frame at head_trim, or (padded) + # one `offset` before its first: its start timecode moves with it + v0, rate = stream_starts(second_meta)[0], frame_rate(second_meta) + # (a trim before a picture that starts after the file's origin starts on its + # first frame; only the padded head counts back before it) + first = (grid_at_or_after(max(round(head_trim, 4), v0), v0, rate) - v0 if head_trim > 0 + else -offset if delay_ms > 0 else 0.0) + cmd += output_timecode_args(second_meta, first) if af_parts: cmd += ["-af", ",".join(af_parts)] cmd += audio_codec_for(output) + [output] diff --git a/site/index.html b/site/index.html index f149fd39..2f3cc4fa 100644 --- a/site/index.html +++ b/site/index.html @@ -23,7 +23,8 @@ - + +