From b20a68d730706b83bc1ee1cc1f282e909a9034e9 Mon Sep 17 00:00:00 2001 From: mitimaicode <309470861+mitimaicode@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:11:43 +0000 Subject: [PATCH 1/2] feat: publish Linux server edition --- .github/workflows/ci.yml | 63 +- .github/workflows/release.yml | 26 +- CONTRIBUTING.md | 3 + INSTALL.md | 97 +- README.md | 75 +- RELEASE_NOTES_v0.5.0-beta.md | 46 + Resources/Info.plist | 4 +- scripts/package_server_release.sh | 32 + server/linux/PROVENANCE.md | 36 + server/linux/README.md | 112 ++ server/linux/VERSION | 1 + server/linux/healthcheck.py | 26 + server/linux/install.sh | 157 +++ server/linux/openclaw-plugin/README.md | 29 + server/linux/openclaw-plugin/index.js | 617 +++++++++++ server/linux/openclaw-plugin/index.test.js | 127 +++ server/linux/openclaw-plugin/job-isolation.js | 59 ++ .../openclaw-plugin/openclaw.plugin.json | 38 + .../linux/openclaw-plugin/package-lock.json | 24 + server/linux/openclaw-plugin/package.json | 17 + .../openclaw-plugin/publication-completion.js | 74 ++ .../openclaw-plugin/source-validation.js | 49 + .../openclaw-plugin/source-validation.test.js | 35 + .../linux/openclaw-plugin/telegram-origin.js | 57 + server/linux/openclaw-plugin/tool-result.js | 7 + .../linux/openclaw-plugin/tool-result.test.js | 11 + server/linux/openclaw-plugin/worker-scope.js | 54 + .../openclaw-plugin/worker-scope.test.js | 53 + server/linux/requirements-stt.txt | 9 + server/linux/requirements-video.txt | 11 + server/linux/stt/server.py | 362 +++++++ server/linux/stt/transcribe.py | 82 ++ server/linux/systemd/mitim-stt.service.in | 19 + server/linux/tests/test_enrich_ollama.py | 136 +++ server/linux/tests/test_release_contract.py | 62 ++ .../linux/tests/test_telegram_publication.py | 207 ++++ server/linux/video/align_qwen.py | 148 +++ server/linux/video/diarize_pyannote.py | 147 +++ server/linux/video/enrich_ollama.py | 621 +++++++++++ server/linux/video/telegram_publication.py | 481 +++++++++ server/linux/video/transcribe_mitim_stt.py | 292 +++++ server/linux/video/video_pipeline.py | 993 ++++++++++++++++++ 42 files changed, 5461 insertions(+), 38 deletions(-) create mode 100644 RELEASE_NOTES_v0.5.0-beta.md create mode 100755 scripts/package_server_release.sh create mode 100644 server/linux/PROVENANCE.md create mode 100644 server/linux/README.md create mode 100644 server/linux/VERSION create mode 100755 server/linux/healthcheck.py create mode 100755 server/linux/install.sh create mode 100644 server/linux/openclaw-plugin/README.md create mode 100644 server/linux/openclaw-plugin/index.js create mode 100644 server/linux/openclaw-plugin/index.test.js create mode 100644 server/linux/openclaw-plugin/job-isolation.js create mode 100644 server/linux/openclaw-plugin/openclaw.plugin.json create mode 100644 server/linux/openclaw-plugin/package-lock.json create mode 100644 server/linux/openclaw-plugin/package.json create mode 100644 server/linux/openclaw-plugin/publication-completion.js create mode 100644 server/linux/openclaw-plugin/source-validation.js create mode 100644 server/linux/openclaw-plugin/source-validation.test.js create mode 100644 server/linux/openclaw-plugin/telegram-origin.js create mode 100644 server/linux/openclaw-plugin/tool-result.js create mode 100644 server/linux/openclaw-plugin/tool-result.test.js create mode 100644 server/linux/openclaw-plugin/worker-scope.js create mode 100644 server/linux/openclaw-plugin/worker-scope.test.js create mode 100644 server/linux/requirements-stt.txt create mode 100644 server/linux/requirements-video.txt create mode 100644 server/linux/stt/server.py create mode 100644 server/linux/stt/transcribe.py create mode 100644 server/linux/systemd/mitim-stt.service.in create mode 100644 server/linux/tests/test_enrich_ollama.py create mode 100644 server/linux/tests/test_release_contract.py create mode 100644 server/linux/tests/test_telegram_publication.py create mode 100644 server/linux/video/align_qwen.py create mode 100644 server/linux/video/diarize_pyannote.py create mode 100644 server/linux/video/enrich_ollama.py create mode 100644 server/linux/video/telegram_publication.py create mode 100644 server/linux/video/transcribe_mitim_stt.py create mode 100644 server/linux/video/video_pipeline.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2c6c07..a40420b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,20 +73,71 @@ jobs: - name: Package pull request test build if: github.event_name == 'pull_request' run: | + version=$(tr -d '[:space:]' < server/linux/VERSION) ditto -c -k --sequesterRsrc --keepParent \ "$RUNNER_TEMP/VoiceSwitch/VoiceSwitch.app" \ - "$RUNNER_TEMP/VoiceSwitch-v0.4.0-beta-test.zip" + "$RUNNER_TEMP/VoiceSwitch-${version}-test.zip" cd "$RUNNER_TEMP" - shasum -a 256 VoiceSwitch-v0.4.0-beta-test.zip \ - > VoiceSwitch-v0.4.0-beta-test.zip.sha256 + shasum -a 256 "VoiceSwitch-${version}-test.zip" \ + > "VoiceSwitch-${version}-test.zip.sha256" - name: Upload pull request test build if: github.event_name == 'pull_request' uses: actions/upload-artifact@v7 with: - name: VoiceSwitch-v0.4.0-beta-macos-arm64-test + name: VoiceSwitch-${{ github.event.pull_request.number }}-macos-arm64-test path: | - ${{ runner.temp }}/VoiceSwitch-v0.4.0-beta-test.zip - ${{ runner.temp }}/VoiceSwitch-v0.4.0-beta-test.zip.sha256 + ${{ runner.temp }}/VoiceSwitch-*-test.zip + ${{ runner.temp }}/VoiceSwitch-*-test.zip.sha256 + if-no-files-found: error + retention-days: 7 + + server: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Validate Linux Server Edition + run: | + bash -n server/linux/install.sh scripts/package_server_release.sh + python3 -m py_compile \ + server/linux/stt/*.py \ + server/linux/video/*.py \ + server/linux/healthcheck.py + PYTHONPATH=server/linux/video \ + python3 -m unittest discover -s server/linux/tests -v + VOICESWITCH_SETUP_VALIDATE_ONLY=1 \ + server/linux/install.sh --core-only --no-start + + - name: Validate optional OpenClaw plugin + working-directory: server/linux/openclaw-plugin + run: | + npm ci --omit=dev --omit=peer --ignore-scripts + node --check index.js + node --test \ + index.test.js \ + source-validation.test.js \ + tool-result.test.js \ + worker-scope.test.js + + - name: Package Linux Server Edition + run: | + chmod +x server/linux/install.sh server/linux/healthcheck.py scripts/package_server_release.sh + ./scripts/package_server_release.sh + archive="dist/VoiceSwitch-Server-$(tr -d '[:space:]' < server/linux/VERSION)-linux-x86_64.tar.gz" + test -f "$archive" + test -f "$archive.sha256" + tar -tzf "$archive" | grep -q '/install.sh$' + + - name: Upload Linux Server test build + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: VoiceSwitch-${{ github.event.pull_request.number }}-linux-x86_64-test + path: | + dist/VoiceSwitch-Server-*.tar.gz + dist/VoiceSwitch-Server-*.tar.gz.sha256 if-no-files-found: error retention-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index afaf979..36fee32 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,11 +19,27 @@ jobs: - name: Validate release source run: | zsh -n scripts/*.sh Resources/install_runtime.sh - python3 -m py_compile worker/asr_worker.py worker/media_worker.py worker/text_worker.py + bash -n server/linux/install.sh scripts/package_server_release.sh + python3 -m py_compile \ + worker/asr_worker.py \ + worker/media_worker.py \ + worker/text_worker.py \ + server/linux/stt/*.py \ + server/linux/video/*.py \ + server/linux/healthcheck.py python3 -m unittest discover -s tests -v + PYTHONPATH=server/linux/video \ + python3 -m unittest discover -s server/linux/tests -v + node --check server/linux/openclaw-plugin/index.js + node --test \ + server/linux/openclaw-plugin/index.test.js \ + server/linux/openclaw-plugin/source-validation.test.js \ + server/linux/openclaw-plugin/tool-result.test.js \ + server/linux/openclaw-plugin/worker-scope.test.js bundle_version=$(/usr/libexec/PlistBuddy -c 'Print :CFBundleShortVersionString' Resources/Info.plist) tag_version="${GITHUB_REF_NAME#v}" test "$bundle_version" = "${tag_version%%-*}" + test "$(tr -d '[:space:]' < server/linux/VERSION)" = "$tag_version" - name: Validate clean GigaAM dependency install run: | @@ -37,9 +53,11 @@ jobs: - name: Build release archive run: | - chmod +x scripts/*.sh Resources/install_runtime.sh + chmod +x scripts/*.sh Resources/install_runtime.sh server/linux/install.sh server/linux/healthcheck.py ./scripts/package_release.sh "${GITHUB_REF_NAME#v}" + ./scripts/package_server_release.sh "${GITHUB_REF_NAME#v}" test -f "dist/VoiceSwitch-${GITHUB_REF_NAME#v}-macos-arm64/VoiceSwitch.app/Contents/Resources/media_worker.py" + test -f "dist/VoiceSwitch-Server-${GITHUB_REF_NAME#v}-linux-x86_64.tar.gz" - name: Publish GitHub release env: @@ -52,7 +70,9 @@ jobs: dist/VoiceSwitch-*.dmg.sha256 dist/VoiceSwitch-*.zip dist/VoiceSwitch-*.zip.sha256 - --title "VoiceSwitch ${GITHUB_REF_NAME}" + dist/VoiceSwitch-Server-*.tar.gz + dist/VoiceSwitch-Server-*.tar.gz.sha256 + --title "VoiceSwitch ${GITHUB_REF_NAME} — Linux Server Edition" ) if [[ -f "$notes_file" ]]; then args+=(--notes-file "$notes_file") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcbc8fa..1593548 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,9 @@ swift build zsh -n scripts/*.sh Resources/install_runtime.sh python3 -m py_compile worker/asr_worker.py worker/media_worker.py worker/text_worker.py scripts/collect_github_stats.py python3 -m unittest discover -s tests -v +bash -n server/linux/install.sh scripts/package_server_release.sh +PYTHONPATH=server/linux/video python3 -m unittest discover -s server/linux/tests -v +(cd server/linux/openclaw-plugin && npm ci --omit=dev --omit=peer --ignore-scripts && node --test index.test.js source-validation.test.js tool-result.test.js worker-scope.test.js) ``` Для изменения распознавания желательно приложить обезличенный набор тестовых diff --git a/INSTALL.md b/INSTALL.md index 6c99db4..3af0a1e 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,8 +1,87 @@ # Установка VoiceSwitch -## Готовый beta-релиз +В релизе есть два независимых пакета: Linux Server Edition и приложение для +Mac. DMG не предназначен для Linux, а серверный архив не содержит SwiftUI GUI. -### 1. Скачивание +## Linux Server Edition + +### Требования Linux + +- Linux x86_64 и NVIDIA GPU с рабочим драйвером; +- Python 3.12, user systemd и системный `ffprobe`; +- место для двух Python-окружений и загружаемых моделей; +- для pyannote diarization — принятие условий модели и локальная авторизация + Hugging Face; +- для summary и анализа кадров — локальный Ollama с `qwen3-vl:8b`. + +### Установка Linux + +1. На странице [GitHub Releases](https://github.com/mitimaicode/VoiceSwitch/releases) + скачайте `VoiceSwitch-Server-0.5.0-beta-linux-x86_64.tar.gz` и соседний + файл `.sha256`. +2. Проверьте архив и распакуйте его: + +```bash +sha256sum -c VoiceSwitch-Server-0.5.0-beta-linux-x86_64.tar.gz.sha256 +tar -xzf VoiceSwitch-Server-0.5.0-beta-linux-x86_64.tar.gz +cd VoiceSwitch-Server-0.5.0-beta-linux-x86_64 +``` + +3. Запустите установщик без `sudo`: + +```bash +./install.sh +``` + +Он создаёт отдельные окружения в `~/.local/share/mitim-stt` и +`~/.local/share/mitim-video`, user service `mitim-stt.service` и команду +`~/.local/bin/voiceswitch-video`. Системный Python и shell-профиль не +изменяются. + +Только resident STT без длинного видеоконвейера: + +```bash +./install.sh --core-only +``` + +Полный контур вместе с OpenClaw TaskFlow plugin: + +```bash +./install.sh --with-openclaw +``` + +### Проверка Linux + +Проверка сервиса: + +```bash +python3 healthcheck.py +``` + +Короткое аудио: + +```bash +~/.local/share/mitim-stt/transcribe.py /absolute/path/to/audio.ogg +``` + +Длинное видео: + +```bash +~/.local/bin/voiceswitch-video /absolute/path/to/video.mp4 --profile standard +``` + +Сервис слушает только `127.0.0.1:18790`. Не публикуйте этот порт наружу: +endpoint принимает путь к локальному файлу и предназначен для доверенных +процессов того же пользователя. + +Полное устройство пакета, профили и закреплённые версии описаны в +[`server/linux/README.md`](server/linux/README.md). + +## Приложение для macOS + +### Готовый beta-релиз + +#### 1. Скачивание Откройте [GitHub Releases](https://github.com/mitimaicode/VoiceSwitch/releases) и скачайте файл с окончанием `macos-arm64.dmg` из самого нового релиза. @@ -15,7 +94,7 @@ DMG — рекомендуемый и самый простой вариант. shasum -a 256 VoiceSwitch-*.dmg ``` -### 2. Копирование в «Программы» +#### 2. Копирование в «Программы» 1. Дважды нажмите скачанный DMG. 2. Перетащите `VoiceSwitch.app` на ярлык **«Программы»** внутри открывшегося @@ -25,7 +104,7 @@ shasum -a 256 VoiceSwitch-*.dmg Не запускайте VoiceSwitch прямо из DMG или папки «Загрузки»: путь приложения важен для разрешения автоматической вставки. -### 3. Первый запуск неподписанной beta +#### 3. Первый запуск неподписанной beta Публичная версия подписана ad-hoc и не нотарифицирована Apple, потому что проект не использует платную подписку Apple Developer. @@ -39,7 +118,7 @@ shasum -a 256 VoiceSwitch-*.dmg не удалось проверить разработчика. В этом случае закройте предупреждение и используйте правый клик → **Открыть**. -### 4. Выбор моделей +#### 4. Выбор моделей Встроенный мастер предлагает три стартовых комплекта: @@ -55,7 +134,7 @@ Python 3.12, Python-зависимости, ffmpeg и только выбран Администраторский пароль не требуется; системный Python и shell-профиль не изменяются. -### 5. Разрешения macOS +#### 5. Разрешения macOS Мастер последовательно попросит: @@ -68,7 +147,7 @@ Python 3.12, Python-зависимости, ffmpeg и только выбран включите переключатель. macOS может запросить Touch ID или пароль владельца Mac. Вернитесь в VoiceSwitch и нажмите **Проверить снова**. -### 6. Первая проверка +#### 6. Первая проверка 1. Откройте любое поле ввода и поставьте в него курсор. 2. Нажмите `fn + Option` — появится красный индикатор записи. @@ -80,7 +159,7 @@ Mac. Вернитесь в VoiceSwitch и нажмите **Проверить с буфере обмена, поэтому при проблеме его можно вставить вручную через `Command + V`. -### 7. Проверка аудио или видео +#### 7. Проверка аудио или видео 1. В меню VoiceSwitch переключите **Диктовка** на **Файл**. 2. Оставьте GigaAM либо выберите установленный Whisper/Qwen. @@ -96,7 +175,7 @@ Application Support. Кадры видео не анализируются. Ме Повторно выберите тот же неизменённый файл, тот же движок и контекст, чтобы продолжить. Apple SpeechAnalyzer в файловом режиме пока недоступен. -### Если загрузка моделей прервалась +#### Если загрузка моделей прервалась Не удаляйте папку Runtime. Нажмите **Продолжить установку** в меню VoiceSwitch: установщик делает до трёх попыток для загружаемых компонентов, diff --git a/README.md b/README.md index 0250ee1..ef636df 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,28 @@ # VoiceSwitch

- VoiceSwitch — локальная диктовка для macOS + VoiceSwitch — локальное распознавание речи

-> Локальная диктовка и расшифровка аудио/видео для macOS. +> Локальная диктовка и расшифровка аудио/видео для macOS и Linux-сервера. [![CI](https://github.com/mitimaicode/VoiceSwitch/actions/workflows/ci.yml/badge.svg)](https://github.com/mitimaicode/VoiceSwitch/actions/workflows/ci.yml) [![Latest release](https://img.shields.io/github/v/release/mitimaicode/VoiceSwitch?include_prereleases)](https://github.com/mitimaicode/VoiceSwitch/releases) [![Downloads](https://img.shields.io/github/downloads/mitimaicode/VoiceSwitch/total)](https://github.com/mitimaicode/VoiceSwitch/releases) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -VoiceSwitch записывает речь по глобальной горячей клавише, распознаёт её -полностью локально и вставляет текст в активное приложение. Отдельный режим -«Файл» расшифровывает аудиодорожку локального аудио или видео, умеет продолжить -прерванную задачу и сохраняет TXT, SRT, VTT и JSON. +В репозитории две редакции. Приложение для macOS записывает речь по глобальной +горячей клавише, распознаёт её локально и вставляет текст в активное приложение. +Linux Server Edition воспроизводит установленный серверный контур для короткой +речи и длинных видео: resident STT, checkpoint/resume, точное выравнивание, +диаризация, визуальный анализ и несколько форматов экспорта. + +| Редакция | Платформа | Пакет релиза | +|---|---|---| +| VoiceSwitch for Mac | Apple Silicon, macOS 14+ | `VoiceSwitch-…-macos-arm64.dmg` | +| VoiceSwitch Server Edition | Linux x86_64, NVIDIA CUDA | `VoiceSwitch-Server-…-linux-x86_64.tar.gz` | + +## Приложение для macOS

@@ -39,12 +47,38 @@ VoiceSwitch записывает речь по глобальной горяче - аудио и текст не отправляются во внешние API; - журнал сравнения помогает выбрать модель под собственную речь. +## Linux Server Edition + +Серверный пакет — это не запуск DMG на Linux. Он содержит отдельный контур, +снятый с реально работающей серверной установки: + +- GigaAM v3 E2E RNNT на CUDA и Whisper Turbo fallback; +- loopback HTTP-сервис `127.0.0.1:18790` под user systemd; +- длинные локальные файлы, Telegram media и YouTube; +- профили `quick`, `standard`, `interview`, `multilingual` и `deep`; +- Qwen3 ForcedAligner, pyannote diarization и локальный Ollama `qwen3-vl:8b`; +- Markdown, JSON, SRT, VTT, QC, manifest и возобновляемые checkpoint; +- необязательный OpenClaw TaskFlow plugin со start/resume/status/cancel. + +Краткая установка: + +```bash +tar -xzf VoiceSwitch-Server-0.5.0-beta-linux-x86_64.tar.gz +cd VoiceSwitch-Server-0.5.0-beta-linux-x86_64 +./install.sh +python3 healthcheck.py +``` + +Подробности, требования и варианты установки находятся в +[`server/linux/README.md`](server/linux/README.md) и [INSTALL.md](INSTALL.md). +Порт `18790` намеренно доступен только локально; публиковать его в сеть нельзя. + > [!IMPORTANT] -> Это beta-релиз для Mac с Apple Silicon. Приложение пока подписано ad-hoc и -> не нотарифицировано Apple. Подробно о различиях и плане выпуска: +> Обе редакции пока beta. Приложение для Mac подписано ad-hoc и не +> нотарифицировано Apple. Подробно о различиях и плане выпуска: > [подпись и нотарификация](docs/SIGNING_AND_NOTARIZATION.md). -## Системные требования +## Системные требования macOS - Mac с Apple Silicon (`M1` или новее); - macOS 14 Sonoma или новее; @@ -52,7 +86,7 @@ VoiceSwitch записывает речь по глобальной горяче - около 2,5 ГБ для рекомендуемой установки с GigaAM или около 10 ГБ для всех моделей; - интернет только во время первоначальной установки моделей. -## Установка +## Установка macOS 1. Откройте [GitHub Releases](https://github.com/mitimaicode/VoiceSwitch/releases), выберите самый новый релиз и скачайте `VoiceSwitch-…-macos-arm64.dmg`. @@ -218,7 +252,8 @@ chmod +x scripts/*.sh Resources/install_runtime.sh Создание компактного release-архива без весов моделей: ```zsh -./scripts/package_release.sh 0.4.0-beta +./scripts/package_release.sh 0.5.0-beta +./scripts/package_server_release.sh 0.5.0-beta ``` Публичный release-скрипт по умолчанию использует ad-hoc подпись. Для стабильной @@ -247,6 +282,10 @@ identity через `VOICESWITCH_CODESIGN_IDENTITY`. - [OpenAI Whisper](https://github.com/openai/whisper) — MIT; - [MLX Whisper](https://github.com/ml-explore/mlx-examples/tree/main/whisper); - [Qwen3-ASR](https://github.com/QwenLM/Qwen3-ASR) — Apache-2.0; +- [Qwen3 ForcedAligner](https://huggingface.co/Qwen/Qwen3-ForcedAligner-0.6B) — + условия модели опубликованы её автором; +- [pyannote.audio](https://github.com/pyannote/pyannote-audio) — MIT, отдельная + модель diarization требует принятия условий на Hugging Face; - [mlx-qwen3-asr](https://github.com/moona3k/mlx-qwen3-asr) — Apache-2.0; - [Qwen3-4B](https://huggingface.co/Qwen/Qwen3-4B-MLX-4bit) — Apache-2.0; - [MLX LM](https://github.com/ml-explore/mlx-lm) — MIT; @@ -260,13 +299,13 @@ VoiceSwitch не связан с авторами перечисленных п

English summary -VoiceSwitch is a local macOS dictation and media-transcription menu-bar app for -Apple Silicon. It switches between GigaAM, Whisper Large V3 Turbo, Qwen3-ASR -1.7B, and Apple SpeechAnalyzer. Press `fn + Option` to dictate, or import a -local audio/video file and export TXT, SRT, VTT, and JSON. Video import -transcribes the audio track only; subtitle timestamps are coarse chunk -boundaries. Audio and transcripts stay on the Mac. Apple SpeechAnalyzer -requires macOS 26 or newer and is currently limited to dictation mode. +VoiceSwitch provides a local macOS dictation app and a separate Linux Server +Edition. The Mac app supports GigaAM, Whisper, Qwen3-ASR and Apple +SpeechAnalyzer. The Linux package reproduces the project's NVIDIA/CUDA server +stack for resident speech recognition and resumable long-video processing, +including optional alignment, diarization, visual analysis and OpenClaw task +orchestration. Media and transcripts stay on the machine unless a user +explicitly uses the YouTube download or Telegram publication integrations. See [INSTALL.md](INSTALL.md) for installation details and use GitHub Issues or Discussions for feedback. diff --git a/RELEASE_NOTES_v0.5.0-beta.md b/RELEASE_NOTES_v0.5.0-beta.md new file mode 100644 index 0000000..ded7f5d --- /dev/null +++ b/RELEASE_NOTES_v0.5.0-beta.md @@ -0,0 +1,46 @@ +# VoiceSwitch v0.5.0-beta — Linux Server Edition + +Этот релиз публикует воспроизводимый пакет той версии локального распознавания +речи и видео, которая фактически работает на Linux-сервере владельца проекта. +macOS-приложение остаётся в релизе, но главное изменение v0.5 — отдельный архив +`VoiceSwitch-Server-0.5.0-beta-linux-x86_64.tar.gz`. + +## Linux Server Edition + +- resident GigaAM v3 E2E RNNT на CUDA; +- Whisper Turbo как явный режим и fallback; +- loopback HTTP endpoint `127.0.0.1:18790`; +- user-systemd service без root; +- длинные локальные файлы, Telegram media и YouTube; +- минутные блоки, checkpoint и resume; +- TXT, Markdown, JSON, SRT и VTT; +- Qwen3 ForcedAligner; +- pyannote diarization; +- локальный визуальный enrichment через Ollama `qwen3-vl:8b`; +- OpenClaw TaskFlow plugin `0.1.6` с start/resume/status/cancel. + +Установленные серверные версии PyTorch, GigaAM, Whisper, Qwen ASR и pyannote +закреплены в requirements. В `PROVENANCE.md` опубликованы SHA-256 исходных +файлов серверного снимка и перечислены санитарные изменения перед публикацией. + +## Установка Linux + +1. Скачайте `VoiceSwitch-Server-0.5.0-beta-linux-x86_64.tar.gz` и файл + `.sha256`. +2. Проверьте SHA-256 и распакуйте архив. +3. Внутри каталога запустите `./install.sh`. + +Требуются Linux x86_64, NVIDIA GPU, Python 3.12, user systemd и `ffprobe`. +Модели и зависимости загружаются локально при установке и первом запуске. + +## Безопасность и ограничения + +- HTTP endpoint доступен только с localhost и не должен публиковаться наружу. +- В архив не входят модели, cache, медиа, расшифровки, токены, cookies, + Telegram chat ID или серверные журналы. +- Diarization требует отдельно принять условия модели pyannote и выполнить + локальную авторизацию Hugging Face. +- Визуальный enrichment требует отдельно установленный Ollama. + +Подробная инструкция находится в `server/linux/README.md` внутри исходников и +релизного архива. diff --git a/Resources/Info.plist b/Resources/Info.plist index 521a576..093ec89 100644 --- a/Resources/Info.plist +++ b/Resources/Info.plist @@ -17,9 +17,9 @@ CFBundlePackageType APPL CFBundleShortVersionString - 0.4.0 + 0.5.0 CFBundleVersion - 13 + 14 LSMinimumSystemVersion 14.0 LSUIElement diff --git a/scripts/package_server_release.sh b/scripts/package_server_release.sh new file mode 100755 index 0000000..9eb96f4 --- /dev/null +++ b/scripts/package_server_release.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VERSION="${1:-$(tr -d '[:space:]' < "$ROOT/server/linux/VERSION")}" +OUTPUT_ROOT="$ROOT/dist" +ARCHIVE_NAME="VoiceSwitch-Server-${VERSION}-linux-x86_64" +TEMP_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/voiceswitch-server-package.XXXXXX")" + +cleanup() { + rm -rf "$TEMP_ROOT" +} +trap cleanup EXIT + +mkdir -p "$OUTPUT_ROOT" "$TEMP_ROOT/$ARCHIVE_NAME" +cp -R "$ROOT/server/linux/." "$TEMP_ROOT/$ARCHIVE_NAME/" +find "$TEMP_ROOT/$ARCHIVE_NAME" -type d -name __pycache__ -prune -exec rm -rf {} + +chmod 0755 \ + "$TEMP_ROOT/$ARCHIVE_NAME/install.sh" \ + "$TEMP_ROOT/$ARCHIVE_NAME/healthcheck.py" + +tar -C "$TEMP_ROOT" -czf "$OUTPUT_ROOT/$ARCHIVE_NAME.tar.gz" "$ARCHIVE_NAME" +( + cd "$OUTPUT_ROOT" + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$ARCHIVE_NAME.tar.gz" > "$ARCHIVE_NAME.tar.gz.sha256" + else + shasum -a 256 "$ARCHIVE_NAME.tar.gz" > "$ARCHIVE_NAME.tar.gz.sha256" + fi +) + +echo "$OUTPUT_ROOT/$ARCHIVE_NAME.tar.gz" diff --git a/server/linux/PROVENANCE.md b/server/linux/PROVENANCE.md new file mode 100644 index 0000000..ff861c1 --- /dev/null +++ b/server/linux/PROVENANCE.md @@ -0,0 +1,36 @@ +# Provenance: installed Linux server snapshot + +Снимок снят 21 сентября 2026 года с фактически установленного серверного +контура. Модели, cache, пользовательские медиа, результаты, токены и журналы в +релиз не входят. + +## Resident STT + +| Файл | SHA-256 установленного источника | +|---|---| +| `stt/server.py` | `2a7ffdde45929a7f3151620c9795d6a990e88e442d78068ccdaddfee734c3c3e` | +| `stt/transcribe.py` | `8340c4452a89c003c18453d9792ca8f61a27af3247af863bd2518ffdba10ce92` | +| systemd unit | `89d98f730650b7927132520527ce191c783481f4f83257fb33ede8ef153b9436` | + +В публичном `transcribe.py` изменён только комментарий с абсолютным домашним +путём. Unit представлен переносимым шаблоном с теми же параметрами запуска. + +## Long-video pipeline + +| Файл | SHA-256 установленного источника | +|---|---| +| `video/video_pipeline.py` | `0c44b359f4453288048575bd442fc49c497113bfca03fb888629df0679893443` | +| `video/align_qwen.py` | `4390900317e7f98c929497ac075a7af01fd72b1d6f506ac9211bef86f3155f49` | +| `video/diarize_pyannote.py` | `c6af00ab0d74051e94ed37af21122118c52ed73cfa02055bb591bc2f75aa5977` | +| `video/enrich_ollama.py` | `8cca0137d2411bfdc3505768f7ad89bb347fb2a1a45e416644af8ee84fa3d139` | + +В Telegram outbox изменено только имя подключения по умолчанию. Реальный chat +ID из внутренних примеров заменён фиктивным. OpenClaw plugin `0.1.6` сохраняет +логику установленной версии, но абсолютные пути заменены путями от домашнего +каталога и переменными окружения. + +## Подтверждение работы + +На момент снимка `mitim-stt.service` был `active (running)` более трёх недель. +Журнал содержал успешные ответы `/transcribe` за 21 сентября 2026 года. +HTTP endpoint намеренно loopback-only. diff --git a/server/linux/README.md b/server/linux/README.md new file mode 100644 index 0000000..b70bda3 --- /dev/null +++ b/server/linux/README.md @@ -0,0 +1,112 @@ +# VoiceSwitch Server Edition for Linux + +Этот пакет воспроизводит локальный контур распознавания, установленный на +сервере владельца VoiceSwitch. Это отдельная Linux-версия, а не запуск +SwiftUI-приложения или DMG на сервере. + +## Что входит + +- resident HTTP STT на `127.0.0.1:18790`; +- GigaAM v3 E2E RNNT на CUDA как основной русский движок; +- Whisper Turbo как явный режим и fallback; +- локальная нормализация аудио через ffmpeg; +- длинные аудио и видео минутными блоками с checkpoint/resume; +- TXT, Markdown, JSON, SRT и VTT; +- профили `quick`, `standard`, `interview`, `multilingual` и `deep`; +- Qwen3 ForcedAligner, pyannote diarization и визуальный enrichment через + локальный Ollama `qwen3-vl:8b`; +- необязательный OpenClaw TaskFlow plugin `0.1.6` с start/resume/status/cancel. + +Исходные медиа, расшифровки и запросы не отправляются во внешние ASR API. +Первоначальная установка загружает Python-пакеты и модели из их официальных +репозиториев. YouTube-режим обращается к YouTube через `yt-dlp`. + +## Требования + +- Linux x86_64; +- NVIDIA GPU и рабочий драйвер; +- Python 3.12 и user systemd; +- `ffprobe` из системного пакета ffmpeg; +- свободное место для двух Python-окружений и моделей; +- для diarization — принятие условий модели pyannote и локальная авторизация + Hugging Face (`HF_TOKEN` или `hf auth login`); +- для визуального enrichment — локальный Ollama и модель `qwen3-vl:8b`. + +Сервис привязан только к loopback. Не публикуйте порт `18790` наружу: endpoint +принимает абсолютный путь к локальному файлу и рассчитан на доверенные процессы +того же пользователя. + +## Установка полного серверного контура + +Распакуйте архив `VoiceSwitch-Server-…-linux-x86_64.tar.gz`, перейдите в его +каталог и выполните: + +```bash +./install.sh +``` + +Установщик не требует root и размещает данные в: + +- `~/.local/share/mitim-stt` — resident STT; +- `~/.local/share/mitim-video` — длинные видео, alignment и diarization; +- `~/.config/systemd/user/mitim-stt.service` — user service; +- `~/.local/bin/voiceswitch-video` — CLI длинного конвейера. + +Только короткая речь без видеоконвейера: + +```bash +./install.sh --core-only +``` + +Добавить установленный на исходном сервере OpenClaw TaskFlow plugin: + +```bash +./install.sh --with-openclaw +``` + +## Проверка + +```bash +python3 healthcheck.py +``` + +Короткое аудио: + +```bash +~/.local/share/mitim-stt/transcribe.py /absolute/path/to/audio.ogg +``` + +Длинное локальное видео: + +```bash +~/.local/bin/voiceswitch-video /absolute/path/to/video.mp4 --profile standard +``` + +YouTube: + +```bash +~/.local/bin/voiceswitch-video 'https://youtu.be/VIDEO_ID' --profile standard +``` + +Результаты по умолчанию сохраняются в +`~/.local/share/mitim-video/pipeline/knowledge/videos/`. + +## Закреплённые версии + +Resident STT повторяет установленное окружение: + +- `torch 2.6.0+cu124`; +- `torchaudio 2.6.0+cu124`; +- `gigaam 0.2.0`; +- `openai-whisper 20250625`; +- `imageio-ffmpeg 0.6.0`. + +Видео-окружение: + +- `torch 2.8.0+cu128`; +- `torchaudio 2.8.0+cu128`; +- `qwen-asr 0.0.6`; +- `pyannote.audio 4.0.7`; +- `yt-dlp 2026.8.19`. + +Полные доказательства происхождения файлов перечислены в `PROVENANCE.md`. diff --git a/server/linux/VERSION b/server/linux/VERSION new file mode 100644 index 0000000..3d67fbf --- /dev/null +++ b/server/linux/VERSION @@ -0,0 +1 @@ +0.5.0-beta diff --git a/server/linux/healthcheck.py b/server/linux/healthcheck.py new file mode 100755 index 0000000..47b4aa2 --- /dev/null +++ b/server/linux/healthcheck.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import sys +import urllib.error +import urllib.request + + +def main() -> int: + url = "http://127.0.0.1:18790/health" + try: + with urllib.request.urlopen(url, timeout=10) as response: + payload = json.loads(response.read().decode("utf-8")) + except (OSError, urllib.error.URLError, json.JSONDecodeError) as error: + print(f"VoiceSwitch Server health check failed: {error}", file=sys.stderr) + return 1 + if not payload.get("ok") or not payload.get("ready"): + print(json.dumps(payload, ensure_ascii=False), file=sys.stderr) + return 1 + print(json.dumps(payload, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/linux/install.sh b/server/linux/install.sh new file mode 100755 index 0000000..ea3ddf6 --- /dev/null +++ b/server/linux/install.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +set -euo pipefail + +SOURCE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STT_ROOT="${VOICESWITCH_STT_ROOT:-${HOME}/.local/share/mitim-stt}" +VIDEO_ROOT="${VOICESWITCH_VIDEO_ROOT:-${HOME}/.local/share/mitim-video}" +CACHE_ROOT="${VOICESWITCH_CACHE_ROOT:-${HOME}/.cache/mitim-stt}" +UNIT_ROOT="${HOME}/.config/systemd/user" +BIN_ROOT="${HOME}/.local/bin" +PYTHON="${VOICESWITCH_PYTHON:-python3.12}" +INSTALL_VIDEO=1 +INSTALL_OPENCLAW=0 +START_SERVICE=1 + +usage() { + cat <<'EOF' +Usage: ./install.sh [--core-only] [--with-openclaw] [--no-start] + + --core-only Install resident GigaAM/Whisper STT without the long-video tools. + --with-openclaw Also install the optional OpenClaw TaskFlow plugin. + --no-start Install files but do not enable or start the user systemd service. + +Environment overrides: + VOICESWITCH_PYTHON, VOICESWITCH_STT_ROOT, VOICESWITCH_VIDEO_ROOT, + VOICESWITCH_CACHE_ROOT, VOICESWITCH_SETUP_VALIDATE_ONLY=1 +EOF +} + +while (($#)); do + case "$1" in + --core-only) INSTALL_VIDEO=0 ;; + --with-openclaw) INSTALL_OPENCLAW=1 ;; + --no-start) START_SERVICE=0 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage >&2; exit 2 ;; + esac + shift +done + +require_command() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Missing required command: $1" >&2 + return 1 + fi +} + +require_command "$PYTHON" +require_command systemctl +if ((INSTALL_VIDEO)); then + if command -v ffprobe >/dev/null 2>&1; then + FFPROBE_SOURCE="$(command -v ffprobe)" + elif [[ -x "$STT_ROOT/bin/ffprobe" ]]; then + FFPROBE_SOURCE="$STT_ROOT/bin/ffprobe" + else + echo "Missing ffprobe. Install the ffmpeg package before the full video setup." >&2 + exit 1 + fi +fi + +if ((INSTALL_OPENCLAW)); then + if ((!INSTALL_VIDEO)); then + echo "--with-openclaw cannot be combined with --core-only" >&2 + exit 2 + fi + require_command node + require_command npm +fi + +case "$(uname -s)-$(uname -m)" in + Linux-x86_64) ;; + *) echo "VoiceSwitch Server supports Linux x86_64 with an NVIDIA GPU." >&2; exit 1 ;; +esac + +if [[ "${VOICESWITCH_SETUP_VALIDATE_ONLY:-0}" == "1" ]]; then + "$PYTHON" -m py_compile \ + "$SOURCE_ROOT/stt/server.py" \ + "$SOURCE_ROOT/stt/transcribe.py" \ + "$SOURCE_ROOT/video/video_pipeline.py" \ + "$SOURCE_ROOT/video/transcribe_mitim_stt.py" \ + "$SOURCE_ROOT/video/align_qwen.py" \ + "$SOURCE_ROOT/video/diarize_pyannote.py" \ + "$SOURCE_ROOT/video/enrich_ollama.py" \ + "$SOURCE_ROOT/video/telegram_publication.py" + echo "VoiceSwitch Server installer validation completed." + exit 0 +fi + +mkdir -p "$STT_ROOT" "$STT_ROOT/work" "$STT_ROOT/bin" "$CACHE_ROOT" "$UNIT_ROOT" "$BIN_ROOT" +"$PYTHON" -m venv "$STT_ROOT/venv" +"$STT_ROOT/venv/bin/python" -m pip install --upgrade pip +"$STT_ROOT/venv/bin/python" -m pip install -r "$SOURCE_ROOT/requirements-stt.txt" +install -m 0644 "$SOURCE_ROOT/stt/server.py" "$STT_ROOT/server.py" +install -m 0755 "$SOURCE_ROOT/stt/transcribe.py" "$STT_ROOT/transcribe.py" +FFMPEG_SOURCE="$("$STT_ROOT/venv/bin/python" -c 'import imageio_ffmpeg; print(imageio_ffmpeg.get_ffmpeg_exe())')" +ln -sfn "$FFMPEG_SOURCE" "$STT_ROOT/bin/ffmpeg" + +if ((INSTALL_VIDEO)); then + if [[ "$FFPROBE_SOURCE" != "$STT_ROOT/bin/ffprobe" ]]; then + ln -sfn "$FFPROBE_SOURCE" "$STT_ROOT/bin/ffprobe" + fi +fi + +STT_ROOT="$STT_ROOT" CACHE_ROOT="$CACHE_ROOT" SOURCE_ROOT="$SOURCE_ROOT" UNIT_ROOT="$UNIT_ROOT" \ + "$PYTHON" - <<'PY' +import os +from pathlib import Path + +template = Path(os.environ["SOURCE_ROOT"]) / "systemd" / "mitim-stt.service.in" +content = template.read_text(encoding="utf-8") +content = content.replace("@STT_ROOT@", os.environ["STT_ROOT"]) +content = content.replace("@CACHE_ROOT@", os.environ["CACHE_ROOT"]) +target = Path(os.environ["UNIT_ROOT"]) / "mitim-stt.service" +target.write_text(content, encoding="utf-8") +PY + +if ((INSTALL_VIDEO)); then + mkdir -p "$VIDEO_ROOT/pipeline" + "$PYTHON" -m venv "$VIDEO_ROOT/venv" + "$VIDEO_ROOT/venv/bin/python" -m pip install --upgrade pip + "$VIDEO_ROOT/venv/bin/python" -m pip install -r "$SOURCE_ROOT/requirements-video.txt" + install -m 0644 "$SOURCE_ROOT"/video/*.py "$VIDEO_ROOT/pipeline/" + cat > "$BIN_ROOT/voiceswitch-video" <`; состояние соседней или более новой транскрибации больше не +может быть принято за состояние текущей. + +Инструмент `video_transcription_resume` продолжает тот же оборванный Flow, +включая `failed` после смерти старого gateway-child процесса, без создания +дубликата. `cancelled` и `succeeded` Flow не возобновляются. + +Успешный терминальный статус дополнительно возвращает `ok=true`. Это не даёт +Codex-провайдеру ошибочно трактовать штатный `status=succeeded` как tool error и +запускать лишние повторные проверки уже завершённой задачи. + +Проверки: + +```text +node --check index.js +node --test index.test.js source-validation.test.js tool-result.test.js worker-scope.test.js +``` diff --git a/server/linux/openclaw-plugin/index.js b/server/linux/openclaw-plugin/index.js new file mode 100644 index 0000000..db89d2e --- /dev/null +++ b/server/linux/openclaw-plugin/index.js @@ -0,0 +1,617 @@ +import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { basename, dirname, join, resolve } from "node:path"; +import { spawn } from "node:child_process"; +import { Type } from "typebox"; +import { defineToolPlugin } from "openclaw/plugin-sdk/tool-plugin"; +import { inspectTelegramPublication } from "./publication-completion.js"; +import { expectsTelegramPublication, resolveTelegramPublicationTarget } from "./telegram-origin.js"; +import { validateVideoSource } from "./source-validation.js"; +import { toolResult } from "./tool-result.js"; +import { flowOutputRoot, newestJobState, sha256, videoStartFingerprint } from "./job-isolation.js"; +import { + isWorkerScopeActive, + stopWorkerScope, + workerScopeArgs, + workerScopeUnitName, +} from "./worker-scope.js"; + +const CONTROLLER_ID = "video-transcription-taskflow"; +const HOME = homedir(); +const PROJECT = process.env.VOICESWITCH_VIDEO_PROJECT + ?? join(HOME, ".local", "share", "mitim-video", "pipeline"); +const VIDEO_PYTHON = process.env.VOICESWITCH_VIDEO_PYTHON + ?? join(HOME, ".local", "share", "mitim-video", "venv", "bin", "python"); +const DEFAULT_FFMPEG = join(HOME, ".local", "share", "mitim-stt", "bin", "ffmpeg"); +const DEFAULT_FFPROBE = join(HOME, ".local", "share", "mitim-stt", "bin", "ffprobe"); +const DEFAULT_OUTPUT_ROOT = join(PROJECT, "artifacts"); +const DEFAULT_PIPELINE = join(PROJECT, "video_pipeline.py"); +const ALIGNER = `${VIDEO_PYTHON} ${join(PROJECT, "align_qwen.py")} --audio \"{audio}\" --transcript \"{transcript}\" --output \"{output}\"`; +const DIARIZER = `${VIDEO_PYTHON} ${join(PROJECT, "diarize_pyannote.py")} --audio \"{audio}\" --transcript \"{transcript}\" --output \"{output}\"`; +const TERMINAL = new Set(["succeeded", "failed", "cancelled", "lost"]); + +const profiles = ["quick", "standard", "interview", "multilingual", "deep"]; +const sourceKinds = ["auto", "youtube", "telegram", "local"]; + +const startParameters = Type.Object({ + idempotencyKey: Type.String({ minLength: 1, maxLength: 200, description: "Stable key reused for retries of the same video job." }), + source: Type.String({ description: "Exact YouTube URL or absolute local MediaPath. Never pass the literal marker media:." }), + profile: Type.Optional(Type.Union(profiles.map((value) => Type.Literal(value)))), + sourceKind: Type.Optional(Type.Union(sourceKinds.map((value) => Type.Literal(value)))), + title: Type.Optional(Type.String()), + force: Type.Optional(Type.Boolean()), + telegramChatId: Type.Optional(Type.String({ description: "Telegram target chat id; normally derived from trusted runtime context." })), + telegramSourceTopicId: Type.Optional(Type.Integer({ minimum: 1, description: "Telegram source topic id; normally derived from trusted runtime context." })), +}); +const flowParameters = Type.Object({ + flowId: Type.Optional(Type.String({ description: "TaskFlow id; latest flow is used when omitted." })), +}); + +const pluginConfigSchema = Type.Object({ + outputRoot: Type.Optional(Type.String()), + pipelinePath: Type.Optional(Type.String()), + pythonPath: Type.Optional(Type.String()), + ffmpegPath: Type.Optional(Type.String()), + ffprobePath: Type.Optional(Type.String()), +}, { additionalProperties: false }); + +function getFlow(runtime, token) { + return token ? runtime.resolve(token) : runtime.findLatest(); +} + +function isAlive(pid) { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function isWorkerAlive(state) { + if (state?.unitName) { + if (isWorkerScopeActive(state.unitName)) return true; + return Date.now() - Number(state.startedAt ?? 0) < 10_000 && isAlive(Number(state.pid)); + } + return isAlive(Number(state?.pid)); +} + +function readJson(path) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +function reserveVideoStart(baseOutputRoot, idempotencyKey, startFingerprint) { + const reservationRoot = join(resolve(baseOutputRoot), ".taskflow-start"); + mkdirSync(reservationRoot, { recursive: true, mode: 0o700 }); + const keyHash = sha256(idempotencyKey); + const reservationPath = join(reservationRoot, `${keyHash}.json`); + const reservation = { + schemaVersion: 1, + keyHash, + startFingerprint, + status: "reserved", + gatewayPid: process.pid, + reservedAt: Date.now(), + }; + let descriptor; + try { + descriptor = openSync(reservationPath, "wx", 0o600); + writeFileSync(descriptor, `${JSON.stringify(reservation)}\n`, "utf8"); + closeSync(descriptor); + return { path: reservationPath, data: reservation, created: true }; + } catch (error) { + if (descriptor !== undefined) closeSync(descriptor); + if (error?.code !== "EEXIST") throw error; + const existing = readJson(reservationPath); + if (!existing || existing.keyHash !== keyHash) { + throw new Error("Video start reservation is corrupted; refusing a duplicate worker."); + } + if (existing.startFingerprint !== startFingerprint) { + throw new Error("Video idempotency key was already used with a different payload."); + } + return { path: reservationPath, data: existing, created: false }; + } +} + +function updateVideoStartReservation(reservationPath, patch) { + const current = readJson(reservationPath); + if (!current) throw new Error("Video start reservation disappeared."); + const temporary = `${reservationPath}.${process.pid}.tmp`; + writeFileSync(temporary, `${JSON.stringify({ ...current, ...patch, updatedAt: Date.now() })}\n`, { encoding: "utf8", mode: 0o600 }); + renameSync(temporary, reservationPath); +} + +function stageFromState(state) { + if (!state || typeof state !== "object") return "processing"; + const stage = String(state.stage ?? state.current_stage ?? state.status ?? "processing"); + return ["complete", "completed", "succeeded"].includes(stage.toLowerCase()) ? "ready" : stage; +} + +function settleStoppedFlow(runtime, flow, nextState, stage) { + if (!["ready", "completed", "complete", "succeeded"].includes(stage.toLowerCase())) { + return runtime.fail({ + flowId: flow.flowId, + expectedRevision: flow.revision, + stateJson: nextState, + blockedSummary: `Video worker stopped at stage: ${stage}`, + }); + } + + const publication = inspectTelegramPublication( + nextState.jobState, + Boolean(nextState.publicationTarget), + ); + const stateJson = { ...nextState, publication }; + if (publication.required && !publication.complete) { + return runtime.setWaiting({ + flowId: flow.flowId, + expectedRevision: flow.revision, + currentStep: "waiting_publication", + stateJson, + waitJson: { + kind: "telegram_publication", + outboxPath: publication.outboxPath, + deliveryStatus: publication.status, + missing: publication.missing, + }, + blockedSummary: "Local processing is complete; Telegram publication is still pending.", + }); + } + return runtime.finish({ + flowId: flow.flowId, + expectedRevision: flow.revision, + stateJson, + }); +} + +function refreshFlow(runtime, flow) { + if (!flow || TERMINAL.has(flow.status)) return flow; + const state = flow.stateJson && typeof flow.stateJson === "object" ? flow.stateJson : {}; + const pid = Number(state.pid); + const job = newestJobState(String(state.outputRoot ?? DEFAULT_OUTPUT_ROOT), Number(state.startedAt ?? 0)); + const stage = stageFromState(job?.data); + const nextState = { ...state, stage, ...(job ? { jobStatePath: job.path, jobState: job.data } : {}) }; + + if (isWorkerAlive(state)) { + const updated = runtime.resume({ + flowId: flow.flowId, + expectedRevision: flow.revision, + status: "running", + currentStep: stage, + stateJson: nextState, + }); + return updated.applied ? updated.flow : updated.current ?? flow; + } + + const result = settleStoppedFlow(runtime, flow, nextState, stage); + return result.applied ? result.flow : result.current ?? flow; +} + +function pipelineArgs({ + source, + profile, + sourceKind, + title, + force, + pipelinePath, + outputRoot, + ffmpegPath, + ffprobePath, + publicationTarget, +}) { + const args = [ + pipelinePath, + source, + "--profile", profile, + "--source-kind", sourceKind ?? "auto", + "--output-root", outputRoot, + "--align-command", ALIGNER, + "--diarize-command", DIARIZER, + "--ffmpeg-path", ffmpegPath, + "--ffprobe-path", ffprobePath, + ]; + if (title) args.push("--title", title); + if (force) args.push("--force"); + if (publicationTarget) { + args.push("--telegram-chat-id", publicationTarget.chatId); + if (publicationTarget.sourceTopicId) { + args.push("--telegram-source-topic-id", String(publicationTarget.sourceTopicId)); + } + } + return args; +} + +function launchWorker({ flowId, pythonPath, args, logPath, ffmpegPath, ffprobePath }) { + const unitName = workerScopeUnitName(flowId); + const logFd = openSync(logPath, "a"); + let child; + try { + child = spawn("/usr/bin/systemd-run", workerScopeArgs(unitName, pythonPath, args), { + detached: true, + stdio: ["ignore", logFd, logFd], + env: { + ...process.env, + PATH: [dirname(ffmpegPath), dirname(ffprobePath), process.env.PATH].filter(Boolean).join(":"), + }, + }); + } finally { + closeSync(logFd); + } + if (!child.pid) throw new Error("Video worker scope did not return a PID."); + return { child, unitName }; +} + +function finishFromExit(runtime, flowId, code, signal) { + let flow = runtime.get(flowId); + if (!flow || TERMINAL.has(flow.status)) return; + const state = flow.stateJson && typeof flow.stateJson === "object" ? flow.stateJson : {}; + const job = newestJobState(String(state.outputRoot ?? DEFAULT_OUTPUT_ROOT), Number(state.startedAt ?? 0)); + const stage = stageFromState(job?.data); + const nextState = { + ...state, + stage, + ...(job ? { jobStatePath: job.path, jobState: job.data } : {}), + exitCode: code, + exitSignal: signal ?? null, + }; + if (code === 0) { + settleStoppedFlow(runtime, flow, nextState, stage); + } else { + runtime.fail({ + flowId, + expectedRevision: flow.revision, + stateJson: nextState, + blockedSummary: `Video worker exited with code ${code ?? "unknown"}`, + }); + } +} + +function createStartTool(api, config, toolContext) { + const runtime = api.runtime.tasks.managedFlows.fromToolContext(toolContext); + return { + name: "video_transcription_start", + label: "Start video transcription", + description: "Start the local GigaAM-first video pipeline and track it as a managed TaskFlow.", + parameters: startParameters, + execute: async (_toolCallId, params) => { + const profile = params.profile ?? "standard"; + // Fail before createManaged/spawn so an invalid attachment cannot be + // reported as "started" and then disappear as a failed background job. + const source = validateVideoSource(params.source); + const idempotencyKey = String(params.idempotencyKey ?? "").trim(); + if (!idempotencyKey || idempotencyKey.length > 200) { + throw new Error("video_transcription_start requires a stable idempotencyKey."); + } + const publicationTarget = resolveTelegramPublicationTarget(toolContext, params); + if (expectsTelegramPublication(toolContext) && !publicationTarget) { + throw new Error("Telegram context detected, but publication chat could not be resolved."); + } + + const pipelinePath = resolve(config.pipelinePath ?? DEFAULT_PIPELINE); + const pythonPath = config.pythonPath ? resolve(config.pythonPath) : "/usr/bin/python3"; + const ffmpegPath = resolve(config.ffmpegPath ?? DEFAULT_FFMPEG); + const ffprobePath = resolve(config.ffprobePath ?? DEFAULT_FFPROBE); + const baseOutputRoot = resolve(config.outputRoot ?? DEFAULT_OUTPUT_ROOT); + if (!existsSync(pipelinePath)) throw new Error(`Pipeline not found: ${pipelinePath}`); + if (!existsSync(ffmpegPath)) throw new Error(`ffmpeg not found: ${ffmpegPath}`); + if (!existsSync(ffprobePath)) throw new Error(`ffprobe not found: ${ffprobePath}`); + mkdirSync(baseOutputRoot, { recursive: true }); + const startFingerprint = videoStartFingerprint(params, source, publicationTarget); + const existing = runtime.list().find((candidate) => { + const state = candidate?.stateJson && typeof candidate.stateJson === "object" ? candidate.stateJson : {}; + return candidate.controllerId === CONTROLLER_ID && state.startIdempotencyKey === idempotencyKey; + }); + if (existing) { + const state = existing.stateJson && typeof existing.stateJson === "object" ? existing.stateJson : {}; + if (state.startFingerprint !== startFingerprint) { + throw new Error("Video idempotency key was already used with a different payload."); + } + return toolResult({ ...refreshFlow(runtime, existing), idempotent: true, duplicateWorkerCreated: false }); + } + const reservation = reserveVideoStart(baseOutputRoot, idempotencyKey, startFingerprint); + if (!reservation.created) { + if (reservation.data.flowId) { + const reservedFlow = runtime.get(reservation.data.flowId); + if (reservedFlow) { + return toolResult({ ...refreshFlow(runtime, reservedFlow), idempotent: true, duplicateWorkerCreated: false }); + } + throw new Error(`Video job already belongs to TaskFlow ${reservation.data.flowId}; refusing a cross-session duplicate.`); + } + throw new Error("A previous video start stopped before its TaskFlow id was recorded; manual reconcile is required and no duplicate was started."); + } + const jobId = sha256(idempotencyKey).slice(0, 24); + const outputRoot = flowOutputRoot(baseOutputRoot, jobId); + mkdirSync(outputRoot, { recursive: true }); + const logRoot = join(outputRoot, "logs"); + mkdirSync(logRoot, { recursive: true }); + + const startedAt = Date.now(); + const flow = runtime.createManaged({ + controllerId: CONTROLLER_ID, + goal: `Transcribe video: ${params.title || basename(source) || source}`, + status: "running", + notifyPolicy: "state_changes", + currentStep: "received", + stateJson: { + source, + sourceHash: sha256(source), + profile, + baseOutputRoot, + outputRoot, + jobId, + startIdempotencyKey: idempotencyKey, + startFingerprint, + startedAt, + publicationTarget, + }, + waitJson: { kind: "local_process" }, + }); + updateVideoStartReservation(reservation.path, { status: "flow_created", flowId: flow.flowId, jobId }); + + const logPath = join(logRoot, `${flow.flowId}.log`); + const args = pipelineArgs({ + source, + profile, + sourceKind: params.sourceKind, + title: params.title, + force: params.force, + pipelinePath, + outputRoot, + ffmpegPath, + ffprobePath, + publicationTarget, + }); + + let launched; + try { + launched = launchWorker({ flowId: flow.flowId, pythonPath, args, logPath, ffmpegPath, ffprobePath }); + } catch (error) { + const current = runtime.get(flow.flowId) ?? flow; + runtime.fail({ + flowId: current.flowId, + expectedRevision: current.revision, + stateJson: { ...current.stateJson, stage: "worker_launch_failed" }, + blockedSummary: `Video worker launch failed: ${error instanceof Error ? error.message : String(error)}`, + }); + updateVideoStartReservation(reservation.path, { status: "failed", flowId: flow.flowId, jobId }); + throw error; + } + const { child, unitName } = launched; + if (!child.pid) { + finishFromExit(runtime, flow.flowId, null, "spawn_failed"); + throw new Error("Video worker did not return a PID."); + } + + const current = runtime.get(flow.flowId); + const stateJson = { + source, + sourceHash: sha256(source), + profile, + baseOutputRoot, + outputRoot, + jobId, + startIdempotencyKey: idempotencyKey, + startFingerprint, + startedAt, + pid: child.pid, + unitName, + logPath, + ffmpegPath, + ffprobePath, + sourceKind: params.sourceKind ?? "auto", + title: params.title ?? null, + force: params.force === true, + publicationTarget, + }; + if (current) { + runtime.resume({ + flowId: flow.flowId, + expectedRevision: current.revision, + status: "running", + currentStep: "received", + stateJson, + }); + } + child.once("exit", (code, signal) => finishFromExit(runtime, flow.flowId, code, signal)); + child.once("error", () => finishFromExit(runtime, flow.flowId, null, "spawn_error")); + child.unref(); + updateVideoStartReservation(reservation.path, { status: "worker_started", flowId: flow.flowId, jobId, unitName }); + + return toolResult({ flowId: flow.flowId, jobId, status: "running", pid: child.pid, logPath, profile, publicationTarget, idempotent: false }); + }, + }; +} + +function createResumeTool(api, config, toolContext) { + const runtime = api.runtime.tasks.managedFlows.fromToolContext(toolContext); + return { + name: "video_transcription_resume", + label: "Resume video transcription", + description: "Resume the same interrupted video TaskFlow without creating a duplicate Flow.", + parameters: flowParameters, + execute: async (_toolCallId, params) => { + let flow = getFlow(runtime, params.flowId); + if (!flow) throw new Error("Video transcription TaskFlow was not found."); + if (TERMINAL.has(flow.status) && flow.status !== "failed") { + throw new Error(`Video transcription TaskFlow cannot be resumed from: ${flow.status}`); + } + const state = flow.stateJson && typeof flow.stateJson === "object" ? flow.stateJson : {}; + if (isWorkerAlive(state)) return toolResult({ ...flow, idempotent: true, workerActive: true }); + + const completedJob = newestJobState( + String(state.outputRoot ?? DEFAULT_OUTPUT_ROOT), + Number(state.startedAt ?? 0), + ); + const completedStage = stageFromState(completedJob?.data); + if (["ready", "completed", "complete", "succeeded"].includes(completedStage.toLowerCase())) { + const settled = settleStoppedFlow(runtime, flow, { + ...state, + stage: completedStage, + jobStatePath: completedJob.path, + jobState: completedJob.data, + }, completedStage); + return toolResult({ ...(settled.applied ? settled.flow : settled.current ?? flow), idempotent: true, workerActive: false }); + } + const source = validateVideoSource(state.source); + const profile = profiles.includes(state.profile) ? state.profile : "standard"; + const pipelinePath = resolve(config.pipelinePath ?? DEFAULT_PIPELINE); + const pythonPath = config.pythonPath ? resolve(config.pythonPath) : "/usr/bin/python3"; + const ffmpegPath = resolve(state.ffmpegPath ?? config.ffmpegPath ?? DEFAULT_FFMPEG); + const ffprobePath = resolve(state.ffprobePath ?? config.ffprobePath ?? DEFAULT_FFPROBE); + const outputRoot = resolve(state.outputRoot ?? config.outputRoot ?? DEFAULT_OUTPUT_ROOT); + const logPath = resolve(state.logPath ?? join(outputRoot, "logs", `${flow.flowId}.log`)); + if (!existsSync(pipelinePath)) throw new Error(`Pipeline not found: ${pipelinePath}`); + if (!existsSync(ffmpegPath)) throw new Error(`ffmpeg not found: ${ffmpegPath}`); + if (!existsSync(ffprobePath)) throw new Error(`ffprobe not found: ${ffprobePath}`); + mkdirSync(dirname(logPath), { recursive: true }); + const startedAt = Date.now(); + const args = pipelineArgs({ + source, + profile, + sourceKind: state.sourceKind, + title: state.title, + force: false, + pipelinePath, + outputRoot, + ffmpegPath, + ffprobePath, + publicationTarget: state.publicationTarget ?? null, + }); + const { child, unitName } = launchWorker({ + flowId: flow.flowId, + pythonPath, + args, + logPath, + ffmpegPath, + ffprobePath, + }); + const resumed = runtime.resume({ + flowId: flow.flowId, + expectedRevision: flow.revision, + status: "running", + currentStep: "resumed_after_gateway_restart", + stateJson: { + ...state, + originalStartedAt: state.originalStartedAt ?? state.startedAt, + startedAt, + pid: child.pid, + unitName, + recoveryCount: Number(state.recoveryCount ?? 0) + 1, + recoveredAt: startedAt, + stage: "resumed_after_gateway_restart", + }, + }); + if (!resumed.applied) { + stopWorkerScope(unitName); + throw new Error(`Video TaskFlow changed during recovery: ${resumed.code ?? "revision conflict"}`); + } + child.once("exit", (code, signal) => finishFromExit(runtime, flow.flowId, code, signal)); + child.once("error", () => finishFromExit(runtime, flow.flowId, null, "spawn_error")); + child.unref(); + return toolResult({ + flowId: flow.flowId, + status: "running", + resumed: true, + duplicateFlowCreated: false, + pid: child.pid, + unitName, + }); + }, + }; +} + +function createStatusTool(api, toolContext) { + const runtime = api.runtime.tasks.managedFlows.fromToolContext(toolContext); + return { + name: "video_transcription_status", + label: "Video transcription status", + description: "Refresh and return the latest state of a managed video transcription TaskFlow.", + parameters: flowParameters, + execute: async (_toolCallId, params) => { + const flow = getFlow(runtime, params.flowId); + if (!flow) throw new Error("Video transcription TaskFlow was not found."); + return toolResult(refreshFlow(runtime, flow)); + }, + }; +} + +function createCancelTool(api, toolContext) { + const runtime = api.runtime.tasks.managedFlows.fromToolContext(toolContext); + return { + name: "video_transcription_cancel", + label: "Cancel video transcription", + description: "Terminate the local worker and cancel its managed video transcription TaskFlow.", + parameters: flowParameters, + execute: async (_toolCallId, params) => { + const flow = getFlow(runtime, params.flowId); + if (!flow) throw new Error("Video transcription TaskFlow was not found."); + const state = flow.stateJson && typeof flow.stateJson === "object" ? flow.stateJson : {}; + const pid = Number(state.pid); + if (state.unitName) { + stopWorkerScope(state.unitName); + } else if (isAlive(pid)) { + try { + process.kill(-pid, "SIGTERM"); + } catch { + process.kill(pid, "SIGTERM"); + } + } + const requested = runtime.requestCancel({ flowId: flow.flowId, expectedRevision: flow.revision }); + const current = requested.applied ? requested.flow : requested.current ?? flow; + const cfg = toolContext.getRuntimeConfig?.() ?? toolContext.runtimeConfig ?? toolContext.config; + if (!cfg) return toolResult(current); + const cancelled = await runtime.cancel({ flowId: flow.flowId, cfg }); + return toolResult(cancelled); + }, + }; +} + +export default defineToolPlugin({ + id: CONTROLLER_ID, + name: "Video Transcription TaskFlow", + description: "Managed background orchestration for the local GigaAM-first video pipeline.", + configSchema: pluginConfigSchema, + tools: (tool) => [ + tool({ + name: "video_transcription_start", + label: "Start video transcription", + description: "Start the local GigaAM-first video pipeline and track it as a managed TaskFlow.", + parameters: startParameters, + factory: ({ api, config, toolContext }) => ( + toolContext?.sessionKey ? createStartTool(api, config, toolContext) : null + ), + }), + tool({ + name: "video_transcription_resume", + label: "Resume video transcription", + description: "Resume the same interrupted video TaskFlow without creating a duplicate Flow.", + parameters: flowParameters, + factory: ({ api, config, toolContext }) => ( + toolContext?.sessionKey ? createResumeTool(api, config, toolContext) : null + ), + }), + tool({ + name: "video_transcription_status", + label: "Video transcription status", + description: "Refresh and return a managed video transcription TaskFlow.", + parameters: flowParameters, + factory: ({ api, toolContext }) => ( + toolContext?.sessionKey ? createStatusTool(api, toolContext) : null + ), + }), + tool({ + name: "video_transcription_cancel", + label: "Cancel video transcription", + description: "Cancel a managed video transcription TaskFlow.", + parameters: flowParameters, + factory: ({ api, toolContext }) => ( + toolContext?.sessionKey ? createCancelTool(api, toolContext) : null + ), + }), + ], +}); diff --git a/server/linux/openclaw-plugin/index.test.js b/server/linux/openclaw-plugin/index.test.js new file mode 100644 index 0000000..e002d82 --- /dev/null +++ b/server/linux/openclaw-plugin/index.test.js @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { flowOutputRoot, newestJobState, videoStartFingerprint } from "./job-isolation.js"; + +import { inspectTelegramPublication } from "./publication-completion.js"; +import { + expectsTelegramPublication, + resolveTelegramPublicationTarget, + telegramOriginFromDestination, + telegramOriginFromSessionKey, +} from "./telegram-origin.js"; + +test("uses the OpenClaw 2026.9 managed TaskFlow runtime", () => { + const source = readFileSync(new URL("./index.js", import.meta.url), "utf8"); + assert.match(source, /api\.runtime\.tasks\.managedFlows\.fromToolContext/); + assert.doesNotMatch(source, /api\.runtime\.tasks\.flow\./); +}); + +test("parses Telegram topic session keys", () => { + assert.deepEqual( + telegramOriginFromSessionKey("agent:main:telegram:group:-1001234567890:topic:8"), + { chatId: "-1001234567890", sourceTopicId: 8 }, + ); +}); + +test("parses marked Telegram destinations", () => { + assert.deepEqual( + telegramOriginFromDestination("telegram:-1001234567890:topic:8"), + { chatId: "-1001234567890", sourceTopicId: 8 }, + ); +}); + +test("trusted delivery context becomes the publication target", () => { + const target = resolveTelegramPublicationTarget({ + deliveryContext: { channel: "telegram", to: "-1001234567890", threadId: "8" }, + }); + assert.deepEqual(target, { chatId: "-1001234567890", sourceTopicId: 8 }); +}); + +test("session key is a fallback when delivery context is absent", () => { + const target = resolveTelegramPublicationTarget({ + messageChannel: "telegram", + sessionKey: "agent:main:telegram:group:-1001234567890:topic:8", + }); + assert.deepEqual(target, { chatId: "-1001234567890", sourceTopicId: 8 }); + assert.equal(expectsTelegramPublication({ messageChannel: "telegram" }), true); +}); + +test("non-Telegram calls do not gain a publication target", () => { + assert.equal(resolveTelegramPublicationTarget({ messageChannel: "webchat" }), null); + assert.equal(expectsTelegramPublication({ messageChannel: "webchat" }), false); +}); + +test("parallel video jobs only inspect their own output directory", (t) => { + const root = mkdtempSync(join(tmpdir(), "video-isolation-test-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const firstRoot = flowOutputRoot(root, "a".repeat(24)); + const secondRoot = flowOutputRoot(root, "b".repeat(24)); + mkdirSync(join(firstRoot, "result"), { recursive: true }); + mkdirSync(join(secondRoot, "result"), { recursive: true }); + writeFileSync(join(firstRoot, "result", "job-state.json"), JSON.stringify({ stage: "complete", owner: "first" })); + writeFileSync(join(secondRoot, "result", "job-state.json"), JSON.stringify({ stage: "complete", owner: "second" })); + assert.equal(newestJobState(firstRoot, 0).data.owner, "first"); + assert.equal(newestJobState(secondRoot, 0).data.owner, "second"); +}); + +test("video start fingerprint is stable and payload-sensitive", () => { + const params = { profile: "standard", sourceKind: "youtube", title: "One" }; + const target = { chatId: "-1001", sourceTopicId: 7 }; + const first = videoStartFingerprint(params, "https://youtu.be/example", target); + assert.equal(first, videoStartFingerprint(params, "https://youtu.be/example", target)); + assert.notEqual(first, videoStartFingerprint({ ...params, profile: "deep" }, "https://youtu.be/example", target)); +}); + +function publicationFixture(t, delivery) { + const root = mkdtempSync(join(tmpdir(), "video-publication-test-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const outbox = join(root, "telegram-publication.json"); + writeFileSync(outbox, JSON.stringify({ delivery }), "utf8"); + return { stages: { publication: { status: "ready", outbox } } }; +} + +test("pending Telegram delivery cannot complete the flow", (t) => { + const job = publicationFixture(t, { + status: "pending_topic", + topic_id: null, + messages: {}, + completed_at: null, + }); + const result = inspectTelegramPublication(job); + assert.equal(result.complete, false); + assert.ok(result.missing.includes("topic_id")); + assert.ok(result.missing.includes("delivery.status")); +}); + +test("completed status alone is insufficient without message ids", (t) => { + const job = publicationFixture(t, { + status: "completed", + topic_id: 777, + messages: { status: 10, summary: 11 }, + completed_at: "2026-08-30T07:00:00+00:00", + }); + const result = inspectTelegramPublication(job); + assert.equal(result.complete, false); + assert.deepEqual(result.missing, ["message:summary_document", "message:transcript_document"]); +}); + +test("flow completion requires topic and every publication message", (t) => { + const job = publicationFixture(t, { + status: "completed", + topic_id: 777, + messages: { + status: 10, + summary: 11, + summary_document: 12, + transcript_document: 13, + }, + completed_at: "2026-08-30T07:00:00+00:00", + }); + const result = inspectTelegramPublication(job); + assert.equal(result.complete, true); + assert.deepEqual(result.missing, []); +}); diff --git a/server/linux/openclaw-plugin/job-isolation.js b/server/linux/openclaw-plugin/job-isolation.js new file mode 100644 index 0000000..80e4839 --- /dev/null +++ b/server/linux/openclaw-plugin/job-isolation.js @@ -0,0 +1,59 @@ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join, resolve } from "node:path"; + +export function sha256(value) { + return createHash("sha256").update(String(value)).digest("hex"); +} + +export function videoStartFingerprint(params, source, publicationTarget) { + return sha256(JSON.stringify({ + sourceHash: sha256(source), + profile: params.profile ?? "standard", + sourceKind: params.sourceKind ?? "auto", + title: params.title ?? null, + force: params.force === true, + publicationTarget: publicationTarget ?? null, + })); +} + +export function flowOutputRoot(baseOutputRoot, jobId) { + if (!/^[a-f0-9]{24}$/.test(jobId)) throw new Error("Unsupported video job id."); + return join(resolve(baseOutputRoot), "jobs", jobId); +} + +function readJson(path) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +export function newestJobState(root, startedAt) { + if (!existsSync(root)) return null; + let newest = null; + const queue = [{ path: root, depth: 0 }]; + while (queue.length) { + const current = queue.shift(); + if (current.depth > 5) continue; + let entries; + try { + entries = readdirSync(current.path, { withFileTypes: true }); + } catch { + continue; + } + for (const entry of entries) { + const path = join(current.path, entry.name); + if (entry.isDirectory()) { + queue.push({ path, depth: current.depth + 1 }); + } else if (entry.name === "job-state.json") { + const modified = statSync(path).mtimeMs; + if (modified >= startedAt - 5000 && (!newest || modified > newest.modified)) { + newest = { path, modified, data: readJson(path) }; + } + } + } + } + return newest; +} diff --git a/server/linux/openclaw-plugin/openclaw.plugin.json b/server/linux/openclaw-plugin/openclaw.plugin.json new file mode 100644 index 0000000..bd46491 --- /dev/null +++ b/server/linux/openclaw-plugin/openclaw.plugin.json @@ -0,0 +1,38 @@ +{ + "id": "video-transcription-taskflow", + "name": "Video Transcription TaskFlow", + "description": "Managed background orchestration for the local GigaAM-first video pipeline.", + "version": "0.1.6", + "configSchema": { + "type": "object", + "properties": { + "outputRoot": { + "type": "string" + }, + "pipelinePath": { + "type": "string" + }, + "pythonPath": { + "type": "string" + }, + "ffmpegPath": { + "type": "string" + }, + "ffprobePath": { + "type": "string" + } + }, + "additionalProperties": false + }, + "activation": { + "onStartup": true + }, + "contracts": { + "tools": [ + "video_transcription_start", + "video_transcription_resume", + "video_transcription_status", + "video_transcription_cancel" + ] + } +} diff --git a/server/linux/openclaw-plugin/package-lock.json b/server/linux/openclaw-plugin/package-lock.json new file mode 100644 index 0000000..e9b278c --- /dev/null +++ b/server/linux/openclaw-plugin/package-lock.json @@ -0,0 +1,24 @@ +{ + "name": "openclaw-plugin-video-transcription-taskflow", + "version": "0.1.6", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "openclaw-plugin-video-transcription-taskflow", + "version": "0.1.6", + "dependencies": { + "typebox": "1.3.3" + }, + "peerDependencies": { + "openclaw": ">=2026.7.1" + } + }, + "node_modules/typebox": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.3.tgz", + "integrity": "sha512-URXGUE31PJDQC+PtRMJeLdF4kmmOdFoVPikPCtV2oOIhUpNpppEdIz7W8bH8cFYPYHdDpaRvqwdegMTmHliudg==", + "license": "MIT" + } + } +} diff --git a/server/linux/openclaw-plugin/package.json b/server/linux/openclaw-plugin/package.json new file mode 100644 index 0000000..71cefb4 --- /dev/null +++ b/server/linux/openclaw-plugin/package.json @@ -0,0 +1,17 @@ +{ + "name": "openclaw-plugin-video-transcription-taskflow", + "version": "0.1.6", + "type": "module", + "private": true, + "peerDependencies": { + "openclaw": ">=2026.7.1" + }, + "dependencies": { + "typebox": "1.3.3" + }, + "openclaw": { + "extensions": [ + "./index.js" + ] + } +} diff --git a/server/linux/openclaw-plugin/publication-completion.js b/server/linux/openclaw-plugin/publication-completion.js new file mode 100644 index 0000000..05a0ac5 --- /dev/null +++ b/server/linux/openclaw-plugin/publication-completion.js @@ -0,0 +1,74 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const REQUIRED_PUBLICATION_MESSAGES = ["status", "summary", "summary_document", "transcript_document"]; + +function readJson(path) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +function positiveInteger(value) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0; +} + +function publicationPathFromJob(job) { + const stages = job?.stages && typeof job.stages === "object" ? job.stages : {}; + const publication = stages.publication && typeof stages.publication === "object" + ? stages.publication + : {}; + return typeof publication.outbox === "string" && publication.outbox.trim() + ? resolve(publication.outbox) + : null; +} + +export function inspectTelegramPublication(job, publicationExpected = true) { + if (!publicationExpected) { + return { required: false, complete: true, status: "skipped", missing: [] }; + } + const outboxPath = publicationPathFromJob(job); + if (!outboxPath) { + return { + required: true, + complete: false, + status: "missing_outbox", + outboxPath: null, + missing: ["outbox"], + }; + } + const plan = readJson(outboxPath); + if (!plan) { + return { + required: true, + complete: false, + status: "unreadable_outbox", + outboxPath, + missing: ["outbox"], + }; + } + const delivery = plan.delivery && typeof plan.delivery === "object" ? plan.delivery : {}; + const messages = delivery.messages && typeof delivery.messages === "object" ? delivery.messages : {}; + const missing = []; + if (!positiveInteger(delivery.topic_id)) missing.push("topic_id"); + for (const kind of REQUIRED_PUBLICATION_MESSAGES) { + if (!positiveInteger(messages[kind])) missing.push(`message:${kind}`); + } + if (typeof delivery.completed_at !== "string" || !delivery.completed_at.trim()) { + missing.push("completed_at"); + } + if (delivery.status !== "completed") missing.push("delivery.status"); + return { + required: true, + complete: missing.length === 0, + status: String(delivery.status ?? "missing"), + outboxPath, + topicId: positiveInteger(delivery.topic_id) ? Number(delivery.topic_id) : null, + messages, + completedAt: delivery.completed_at ?? null, + missing, + }; +} diff --git a/server/linux/openclaw-plugin/source-validation.js b/server/linux/openclaw-plugin/source-validation.js new file mode 100644 index 0000000..9968172 --- /dev/null +++ b/server/linux/openclaw-plugin/source-validation.js @@ -0,0 +1,49 @@ +import { existsSync, statSync } from "node:fs"; +import { isAbsolute, resolve } from "node:path"; + +const YOUTUBE_HOSTS = new Set(["youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be", "www.youtu.be"]); + +function isYoutubeUrl(value) { + try { + const parsed = new URL(value); + return ["http:", "https:"].includes(parsed.protocol) && YOUTUBE_HOSTS.has(parsed.hostname.toLowerCase()); + } catch { + return false; + } +} + +/** + * Validate before creating a managed flow. In particular, `media:` is an + * inbound-media marker, not a filesystem path; accepting it creates a flow + * that can only fail after the user has already been told it started. + */ +export function validateVideoSource(input) { + const source = typeof input === "string" ? input.trim() : ""; + if (!source) { + throw new Error("Источник видео пуст. Передай точный MediaPath или YouTube-ссылку."); + } + if (/^media(?::|$)/i.test(source)) { + throw new Error("Получен маркер `media:`, а не путь к файлу. Передай точный MediaPath из входящего вложения."); + } + if (/^https?:\/\//i.test(source)) { + if (!isYoutubeUrl(source)) { + throw new Error("Поддерживаются только YouTube-ссылки; Instagram/Reels нужно скачать и передать как видеофайл."); + } + return source; + } + + const path = isAbsolute(source) ? source : resolve(source); + if (!existsSync(path)) { + throw new Error(`Файл видео не найден: ${path}. Обработка не запускалась.`); + } + let stats; + try { + stats = statSync(path); + } catch { + throw new Error(`Невозможно прочитать файл видео: ${path}. Обработка не запускалась.`); + } + if (!stats.isFile()) { + throw new Error(`Источник не является файлом: ${path}. Обработка не запускалась.`); + } + return path; +} diff --git a/server/linux/openclaw-plugin/source-validation.test.js b/server/linux/openclaw-plugin/source-validation.test.js new file mode 100644 index 0000000..fdec04d --- /dev/null +++ b/server/linux/openclaw-plugin/source-validation.test.js @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { validateVideoSource } from "./source-validation.js"; + +test("rejects media marker before a flow can be created", () => { + assert.throws( + () => validateVideoSource("media:"), + /маркер `media:`/, + ); +}); + +test("rejects a missing local file", () => { + assert.throws( + () => validateVideoSource(join(tmpdir(), "video-does-not-exist.mp4")), + /Файл видео не найден/, + ); +}); + +test("returns an existing regular file as an absolute path", () => { + const root = mkdtempSync(join(tmpdir(), "video-source-")); + const file = join(root, "clip.mp4"); + writeFileSync(file, "test"); + assert.equal(validateVideoSource(file), file); +}); + +test("accepts YouTube URLs and rejects unsupported web sources", () => { + assert.equal(validateVideoSource("https://youtu.be/abc123"), "https://youtu.be/abc123"); + assert.throws( + () => validateVideoSource("https://www.instagram.com/reel/abc123/"), + /только YouTube-ссылки/, + ); +}); diff --git a/server/linux/openclaw-plugin/telegram-origin.js b/server/linux/openclaw-plugin/telegram-origin.js new file mode 100644 index 0000000..8a3d98c --- /dev/null +++ b/server/linux/openclaw-plugin/telegram-origin.js @@ -0,0 +1,57 @@ +function positiveInteger(value) { + if (value === null || value === undefined || value === "") return null; + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : null; +} + +export function telegramOriginFromSessionKey(sessionKey) { + if (typeof sessionKey !== "string" || !sessionKey.trim()) return null; + const parts = sessionKey.split(":"); + const channelIndex = parts.indexOf("telegram"); + if (channelIndex < 0) return null; + const kind = parts[channelIndex + 1]; + const chatId = parts[channelIndex + 2]; + if (!chatId || !["direct", "group"].includes(kind)) return null; + const topicIndex = parts.indexOf("topic", channelIndex + 3); + return { + chatId, + sourceTopicId: topicIndex >= 0 ? positiveInteger(parts[topicIndex + 1]) : null, + }; +} + +export function telegramOriginFromDestination(value) { + if (value === null || value === undefined) return null; + const rendered = String(value).trim(); + if (!rendered) return null; + const match = rendered.match(/^(?:telegram:)?(-?\d+)(?::topic:(\d+))?$/); + if (!match) return null; + return { + chatId: match[1], + sourceTopicId: positiveInteger(match[2]), + }; +} + +export function expectsTelegramPublication(toolContext = {}) { + const deliveryChannel = String(toolContext.deliveryContext?.channel ?? "").toLowerCase(); + const messageChannel = String(toolContext.messageChannel ?? "").toLowerCase(); + return deliveryChannel === "telegram" + || messageChannel === "telegram" + || telegramOriginFromSessionKey(toolContext.sessionKey) !== null; +} + +export function resolveTelegramPublicationTarget(toolContext = {}, params = {}) { + const explicit = telegramOriginFromDestination(params.telegramChatId); + const delivery = String(toolContext.deliveryContext?.channel ?? "").toLowerCase() === "telegram" + ? telegramOriginFromDestination(toolContext.deliveryContext?.to) + : null; + const session = telegramOriginFromSessionKey(toolContext.sessionKey); + const chatId = explicit?.chatId ?? delivery?.chatId ?? session?.chatId; + if (!chatId) return null; + const sourceTopicId = positiveInteger(params.telegramSourceTopicId) + ?? positiveInteger(toolContext.deliveryContext?.threadId) + ?? explicit?.sourceTopicId + ?? delivery?.sourceTopicId + ?? session?.sourceTopicId + ?? null; + return { chatId: String(chatId), sourceTopicId }; +} diff --git a/server/linux/openclaw-plugin/tool-result.js b/server/linux/openclaw-plugin/tool-result.js new file mode 100644 index 0000000..9b849cc --- /dev/null +++ b/server/linux/openclaw-plugin/tool-result.js @@ -0,0 +1,7 @@ +export function toolResult(details) { + const payload = { ...details, ok: true }; + return { + content: [{ type: "text", text: JSON.stringify(payload, null, 2) }], + details: payload, + }; +} diff --git a/server/linux/openclaw-plugin/tool-result.test.js b/server/linux/openclaw-plugin/tool-result.test.js new file mode 100644 index 0000000..065cfcd --- /dev/null +++ b/server/linux/openclaw-plugin/tool-result.test.js @@ -0,0 +1,11 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { toolResult } from "./tool-result.js"; + +test("successful terminal TaskFlow status is an affirmative tool result", () => { + const result = toolResult({ flowId: "flow-1", status: "succeeded" }); + assert.equal(result.details.ok, true); + assert.equal(result.details.status, "succeeded"); + assert.equal(JSON.parse(result.content[0].text).ok, true); +}); diff --git a/server/linux/openclaw-plugin/worker-scope.js b/server/linux/openclaw-plugin/worker-scope.js new file mode 100644 index 0000000..11b5cf8 --- /dev/null +++ b/server/linux/openclaw-plugin/worker-scope.js @@ -0,0 +1,54 @@ +import { spawnSync } from "node:child_process"; + +const FLOW_ID_PATTERN = /^[a-zA-Z0-9-]{1,80}$/; + +export function workerScopeUnitName(flowId) { + if (typeof flowId !== "string" || !FLOW_ID_PATTERN.test(flowId)) { + throw new Error("Unsupported video TaskFlow id for a systemd scope."); + } + return `openclaw-video-${flowId}.scope`; +} + +export function workerScopeArgs(unitName, command, args = []) { + if (typeof unitName !== "string" || !unitName.endsWith(".scope")) { + throw new Error("Video worker unit must be a systemd scope."); + } + if (typeof command !== "string" || !command.startsWith("/")) { + throw new Error("Video worker command must be an absolute path."); + } + return [ + "--user", + "--scope", + "--quiet", + `--unit=${unitName}`, + "--property=CPUWeight=50", + "--property=CPUQuota=300%", + "--property=IOWeight=50", + "--property=MemoryHigh=3G", + "--property=MemoryMax=4G", + "--property=TasksMax=256", + "--", + "/usr/bin/nice", + "-n", + "10", + command, + ...args, + ]; +} + +export function isWorkerScopeActive(unitName, run = spawnSync) { + if (typeof unitName !== "string" || !unitName.endsWith(".scope")) return false; + const result = run("/usr/bin/systemctl", ["--user", "is-active", "--quiet", unitName], { + stdio: "ignore", + timeout: 5000, + }); + return result.status === 0; +} +export function stopWorkerScope(unitName, run = spawnSync) { + if (typeof unitName !== "string" || !unitName.endsWith(".scope")) return false; + const result = run("/usr/bin/systemctl", ["--user", "stop", unitName], { + stdio: "ignore", + timeout: 10000, + }); + return result.status === 0; +} diff --git a/server/linux/openclaw-plugin/worker-scope.test.js b/server/linux/openclaw-plugin/worker-scope.test.js new file mode 100644 index 0000000..2c7459d --- /dev/null +++ b/server/linux/openclaw-plugin/worker-scope.test.js @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + isWorkerScopeActive, + stopWorkerScope, + workerScopeArgs, + workerScopeUnitName, +} from "./worker-scope.js"; + +test("video worker scope is owner-flow specific and resource friendly", () => { + const unit = workerScopeUnitName("4e090d8d-5f12-4b32-aa42-985608f3c755"); + assert.equal(unit, "openclaw-video-4e090d8d-5f12-4b32-aa42-985608f3c755.scope"); + assert.deepEqual(workerScopeArgs(unit, "/usr/bin/python3", ["/opt/pipeline.py"]), [ + "--user", + "--scope", + "--quiet", + `--unit=${unit}`, + "--property=CPUWeight=50", + "--property=CPUQuota=300%", + "--property=IOWeight=50", + "--property=MemoryHigh=3G", + "--property=MemoryMax=4G", + "--property=TasksMax=256", + "--", + "/usr/bin/nice", + "-n", + "10", + "/usr/bin/python3", + "/opt/pipeline.py", + ]); +}); + +test("scope probes and stops use exact non-shell systemctl arguments", () => { + const calls = []; + const run = (command, args, options) => { + calls.push({ command, args, options }); + return { status: 0 }; + }; + const unit = workerScopeUnitName("flow-1"); + assert.equal(isWorkerScopeActive(unit, run), true); + assert.equal(stopWorkerScope(unit, run), true); + assert.deepEqual(calls.map((call) => call.args), [ + ["--user", "is-active", "--quiet", unit], + ["--user", "stop", unit], + ]); +}); + +test("unsafe flow ids and commands are rejected", () => { + assert.throws(() => workerScopeUnitName("flow/../../escape"), /Unsupported/); + assert.throws(() => workerScopeArgs("worker.service", "/usr/bin/python3"), /scope/); + assert.throws(() => workerScopeArgs("worker.scope", "python3"), /absolute/); +}); diff --git a/server/linux/requirements-stt.txt b/server/linux/requirements-stt.txt new file mode 100644 index 0000000..0059dd2 --- /dev/null +++ b/server/linux/requirements-stt.txt @@ -0,0 +1,9 @@ +--extra-index-url https://download.pytorch.org/whl/cu124 +torch==2.6.0+cu124 +torchaudio==2.6.0+cu124 +gigaam==0.2.0 +openai-whisper==20250625 +imageio-ffmpeg==0.6.0 +numpy==2.5.2 +soundfile==0.14.0 +huggingface-hub==0.36.2 diff --git a/server/linux/requirements-video.txt b/server/linux/requirements-video.txt new file mode 100644 index 0000000..048de1a --- /dev/null +++ b/server/linux/requirements-video.txt @@ -0,0 +1,11 @@ +--extra-index-url https://download.pytorch.org/whl/cu128 +torch==2.8.0+cu128 +torchaudio==2.8.0+cu128 +qwen-asr==0.0.6 +pyannote.audio==4.0.7 +soundfile==0.14.0 +huggingface-hub==0.36.2 +numpy==2.5.2 +transformers==4.57.6 +accelerate==1.12.0 +yt-dlp==2026.8.19 diff --git a/server/linux/stt/server.py b/server/linux/stt/server.py new file mode 100644 index 0000000..03643fc --- /dev/null +++ b/server/linux/stt/server.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +"""Resident local speech-to-text service for OpenClaw. + +GigaAM v3 E2E RNNT is the primary Russian recognizer. Whisper Turbo is +loaded only when explicitly requested or when the primary recognizer fails. +The HTTP endpoint is loopback-only; OpenClaw calls it through transcribe.py. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import stat +import subprocess +import tempfile +import threading +import time +import traceback +import wave +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +GIGAAM_MODEL = "v3_e2e_rnnt" +MAX_INPUT_BYTES = 50 * 1024 * 1024 +MAX_DURATION_SECONDS = 30 * 60 +REQUEST_TIMEOUT_SECONDS = 180 + + +def _duration(path: Path) -> float: + with wave.open(str(path), "rb") as audio: + return audio.getnframes() / float(audio.getframerate()) + + +def _split_wav_for_gigaam( + source: Path, + *, + maximum_seconds: float = 22.0, +) -> tuple[list[Path], tempfile.TemporaryDirectory[str] | None]: + """Split long PCM WAV files near a low-energy point. + + GigaAM's short-form transcribe API is limited to roughly 25 seconds. The + VoiceSwitch worker uses the same strategy, with a little safety margin. + """ + + with wave.open(str(source), "rb") as audio: + parameters = audio.getparams() + frame_rate = audio.getframerate() + channels = audio.getnchannels() + sample_width = audio.getsampwidth() + frame_count = audio.getnframes() + frames = audio.readframes(frame_count) + + if frame_count / float(frame_rate) <= 24.0: + return [source], None + if channels != 1 or sample_width != 2: + raise ValueError("Ожидается mono PCM WAV 16-bit после нормализации.") + + import numpy as np + + samples = np.frombuffer(frames, dtype=" maximum: + hard_end = min(total, start + maximum) + search_start = max(start + minimum, hard_end - search_span) + best_end = hard_end + best_energy = float("inf") + for candidate in range(search_start, hard_end, analysis_window): + segment = samples[candidate : min(candidate + analysis_window, hard_end)] + if segment.size == 0: + continue + energy = float(np.mean(np.abs(segment.astype(np.float32)))) + if energy < best_energy: + best_energy = energy + best_end = candidate + max(1, segment.size // 2) + boundaries.append(best_end) + start = best_end + boundaries.append(total) + + temporary = tempfile.TemporaryDirectory(prefix="mitim-stt-gigaam-") + chunks: list[Path] = [] + for index, (left, right) in enumerate(zip(boundaries, boundaries[1:])): + output = Path(temporary.name) / f"chunk-{index:03d}.wav" + with wave.open(str(output), "wb") as chunk: + chunk.setnchannels(1) + chunk.setsampwidth(2) + chunk.setframerate(frame_rate) + chunk.writeframes(samples[left:right].astype(" None: + self.cache_root = cache_root + self.work_root = work_root + self.work_root.mkdir(parents=True, exist_ok=True) + self._gpu_lock = threading.Lock() + self._whisper_lock = threading.Lock() + self.gigaam: Any = None + self.whisper: Any = None + self.device = "cpu" + + os.environ.setdefault("HF_HOME", str(cache_root / "huggingface")) + os.environ.setdefault("TORCH_HOME", str(cache_root / "torch")) + os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") + + # GigaAM's Hugging Face modeling file invokes ``ffmpeg`` by name. + # imageio-ffmpeg supplies a pinned Linux binary inside the venv, so + # expose that binary to both our normalizer and the model code. + import imageio_ffmpeg + + ffmpeg_executable = Path(imageio_ffmpeg.get_ffmpeg_exe()) + # GigaAM invokes the executable by the literal name ``ffmpeg``. + # imageio-ffmpeg ships a pinned binary with a versioned filename, so + # expose it through a private service-local symlink rather than + # changing the system installation. + ffmpeg_bin = self.work_root.parent / "bin" + ffmpeg_bin.mkdir(parents=True, exist_ok=True) + ffmpeg_link = ffmpeg_bin / "ffmpeg" + if not ffmpeg_link.exists(): + ffmpeg_link.symlink_to(ffmpeg_executable) + os.environ["PATH"] = f"{ffmpeg_bin}{os.pathsep}{os.environ.get('PATH', '')}" + + import torch + + if not torch.cuda.is_available(): + raise RuntimeError("CUDA недоступна: GPU-режим mitim-stt не активирован.") + self.device = "cuda" + + import gigaam + + print("Loading GigaAM v3 E2E RNNT on CUDA", flush=True) + self.gigaam = gigaam.load_model( + GIGAAM_MODEL, + device=self.device, + fp16_encoder=True, + download_root=str(cache_root / "gigaam"), + ) + print("GigaAM ready", flush=True) + + def _normalize(self, source: Path, temporary: Path) -> Path: + import imageio_ffmpeg + + output = temporary / "normalized.wav" + ffmpeg = imageio_ffmpeg.get_ffmpeg_exe() + command = [ + ffmpeg, + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(source), + "-vn", + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "pcm_s16le", + str(output), + ] + completed = subprocess.run( + command, + capture_output=True, + text=True, + timeout=90, + check=False, + ) + if completed.returncode != 0 or not output.exists(): + detail = (completed.stderr or "ffmpeg не создал WAV").strip() + raise RuntimeError(f"Не удалось подготовить аудио: {detail[-500:]}") + return output + + def _gigaam_transcribe(self, audio: Path) -> str: + chunks, temporary = _split_wav_for_gigaam(audio) + try: + texts: list[str] = [] + for chunk in chunks: + result = self.gigaam.transcribe(str(chunk)) + text = getattr(result, "text", str(result)).strip() + if text: + texts.append(text) + return " ".join(texts).strip() + finally: + if temporary is not None: + temporary.cleanup() + + def _load_whisper(self) -> Any: + if self.whisper is not None: + return self.whisper + with self._whisper_lock: + if self.whisper is None: + import whisper + + print("Loading Whisper Turbo fallback on CUDA", flush=True) + self.whisper = whisper.load_model( + "turbo", + device=self.device, + download_root=str(self.cache_root / "whisper"), + ) + print("Whisper fallback ready", flush=True) + return self.whisper + + def _whisper_transcribe(self, audio: Path) -> tuple[str, str | None]: + model = self._load_whisper() + result = model.transcribe( + str(audio), + task="transcribe", + language=None, + temperature=0.0, + condition_on_previous_text=False, + word_timestamps=False, + fp16=True, + verbose=False, + ) + return str(result.get("text", "")).strip(), result.get("language") + + def transcribe(self, source: Path, engine: str = "auto") -> dict[str, Any]: + try: + source = source.resolve(strict=True) + metadata = source.stat() + except FileNotFoundError as error: + raise ValueError(f"Аудиофайл не найден: {source}") from error + if not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"Аудиофайл должен быть обычным файлом: {source}") + if metadata.st_size < 1024: + raise ValueError("Аудиофайл слишком мал.") + if metadata.st_size > MAX_INPUT_BYTES: + raise ValueError("Аудиофайл больше 50 МБ.") + + started = time.perf_counter() + with self._gpu_lock: + with tempfile.TemporaryDirectory(prefix="mitim-stt-request-", dir=self.work_root) as work: + normalized = self._normalize(source, Path(work)) + duration = _duration(normalized) + if duration <= 0 or duration > MAX_DURATION_SECONDS: + raise ValueError("Длительность аудио должна быть от 0 до 30 минут.") + + if engine == "whisper": + text, language = self._whisper_transcribe(normalized) + used_engine = "whisper" + else: + try: + text = self._gigaam_transcribe(normalized) + language = "ru" + used_engine = "gigaam" + except Exception: + if engine == "gigaam": + raise + traceback.print_exc() + text, language = self._whisper_transcribe(normalized) + used_engine = "whisper-fallback" + + if engine == "auto" and not text: + text, language = self._whisper_transcribe(normalized) + used_engine = "whisper-fallback-empty" + + return { + "text": text, + "language": language, + "engine": used_engine, + "duration": duration, + "latency": time.perf_counter() - started, + } + + +class RequestHandler(BaseHTTPRequestHandler): + service: "STTHTTPServer" + + def log_message(self, format: str, *args: Any) -> None: + print(f"{self.address_string()} - {format % args}", flush=True) + + def _json(self, status: int, payload: dict[str, Any]) -> None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self) -> None: + if self.path != "/health": + self._json(404, {"error": "not found"}) + return + self._json( + 200, + {"ok": True, "ready": True, "device": self.service.engine.device}, + ) + + def do_POST(self) -> None: + if self.path != "/transcribe": + self._json(404, {"error": "not found"}) + return + try: + content_length = int(self.headers.get("Content-Length", "0")) + if content_length <= 0 or content_length > 64 * 1024: + raise ValueError("Некорректный размер JSON-запроса.") + request = json.loads(self.rfile.read(content_length)) + source = Path(str(request.get("path", ""))).expanduser() + if not source.is_absolute(): + raise ValueError("Путь к аудио должен быть абсолютным.") + engine = str(request.get("engine", "auto")) + if engine not in {"auto", "gigaam", "whisper"}: + raise ValueError("engine должен быть auto, gigaam или whisper.") + if not self.service.admission.acquire(blocking=False): + self._json(429, {"error": "STT queue is full"}) + return + try: + result = self.service.engine.transcribe(source, engine) + finally: + self.service.admission.release() + self._json(200, result) + except Exception as error: + traceback.print_exc() + self._json(400, {"error": str(error)}) + + +class STTHTTPServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, address: tuple[str, int], engine: SpeechEngine) -> None: + super().__init__(address, RequestHandler) + self.engine = engine + self.admission = threading.BoundedSemaphore(2) + RequestHandler.service = self + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--cache", type=Path, required=True) + parser.add_argument("--work", type=Path, required=True) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=18790) + args = parser.parse_args() + + args.cache.mkdir(parents=True, exist_ok=True) + args.work.mkdir(parents=True, exist_ok=True) + service = SpeechEngine(args.cache, args.work) + server = STTHTTPServer((args.host, args.port), service) + print(f"mitim-stt listening on http://{args.host}:{args.port}", flush=True) + try: + server.serve_forever(poll_interval=0.5) + except KeyboardInterrupt: + pass + finally: + server.server_close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/linux/stt/transcribe.py b/server/linux/stt/transcribe.py new file mode 100644 index 0000000..d4f39a7 --- /dev/null +++ b/server/linux/stt/transcribe.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""OpenClaw CLI adapter: print only the transcript to stdout.""" + +from __future__ import annotations + +import json +import os +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + + +AUDIO_SUFFIXES = {".oga", ".ogg", ".opus", ".mp3", ".m4a", ".wav", ".webm"} + + +def resolve_audio_path(raw: str) -> Path: + """Resolve OpenClaw's path and tolerate its empty-path fallback.""" + candidate = Path(os.path.abspath(os.path.expanduser(raw))) + if candidate.is_file(): + return candidate + + # Some Telegram preflight paths arrive as an empty CLI argument. In that + # case abspath() becomes the gateway working directory. Limit + # the recovery search to OpenClaw's own inbound media directories and only + # accept files written very recently. + if candidate.is_dir(): + roots = [ + Path.home() / ".openclaw" / "media" / "inbound", + Path.home() / ".openclaw" / "workspace" / "media" / "inbound", + ] + deadline = time.time() - 300 + recent = [ + path + for root in roots + if root.is_dir() + for path in root.rglob("*") + if path.is_file() + and path.suffix.lower() in AUDIO_SUFFIXES + and path.stat().st_mtime >= deadline + ] + if recent: + return max(recent, key=lambda path: path.stat().st_mtime) + return candidate + + +def main() -> int: + if len(sys.argv) != 2: + print("usage: mitim-stt-transcribe AUDIO_PATH", file=sys.stderr) + return 2 + + request = urllib.request.Request( + os.environ.get("MITIM_STT_URL", "http://127.0.0.1:18790/transcribe"), + data=json.dumps( + { + "path": str(resolve_audio_path(sys.argv[1])), + "engine": os.environ.get("MITIM_STT_ENGINE", "auto"), + } + ).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=180) as response: + payload = json.loads(response.read().decode("utf-8")) + except (OSError, urllib.error.URLError, json.JSONDecodeError) as error: + print(f"mitim-stt request failed: {error}", file=sys.stderr) + return 1 + + if payload.get("error"): + print(str(payload["error"]), file=sys.stderr) + return 1 + text = str(payload.get("text", "")).strip() + if text: + sys.stdout.write(text) + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/linux/systemd/mitim-stt.service.in b/server/linux/systemd/mitim-stt.service.in new file mode 100644 index 0000000..dd64b94 --- /dev/null +++ b/server/linux/systemd/mitim-stt.service.in @@ -0,0 +1,19 @@ +[Unit] +Description=VoiceSwitch resident local speech recognition +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +WorkingDirectory=@STT_ROOT@ +ExecStart=@STT_ROOT@/venv/bin/python @STT_ROOT@/server.py --cache @CACHE_ROOT@ --work @STT_ROOT@/work --host 127.0.0.1 --port 18790 +Restart=on-failure +RestartSec=5 +TimeoutStartSec=900 +Environment=CUDA_VISIBLE_DEVICES=0 +Environment=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True +Environment=HF_HOME=@CACHE_ROOT@/huggingface +Environment=TORCH_HOME=@CACHE_ROOT@/torch + +[Install] +WantedBy=default.target diff --git a/server/linux/tests/test_enrich_ollama.py b/server/linux/tests/test_enrich_ollama.py new file mode 100644 index 0000000..8c4d49d --- /dev/null +++ b/server/linux/tests/test_enrich_ollama.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +import json +import unittest +from unittest.mock import patch + +from enrich_ollama import ( + OllamaClient, + SUMMARY_SCHEMA, + generate_topic_title, + normalize_topic_title, + reduce_summaries, + summarize_chunk, +) + + +class FakeResponse: + def __init__(self, content: str) -> None: + self.payload = json.dumps({"message": {"content": content}}, ensure_ascii=False).encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def read(self) -> bytes: + return self.payload + + +class BrokenClient: + model = "broken-model" + + def chat(self, prompt, *, schema, images=None): + raise ValueError("невалидный JSON") + + +class EnrichOllamaTests(unittest.TestCase): + def test_client_retries_invalid_json(self) -> None: + responses = iter( + [ + FakeResponse("это не JSON"), + FakeResponse( + json.dumps( + { + "chapter_title": "Тема", + "overview": "Обзор", + "key_points": [], + "decisions": [], + "action_items": [], + "risks": [], + }, + ensure_ascii=False, + ) + ), + ] + ) + calls = [] + + def fake_urlopen(request, timeout): + calls.append(json.loads(request.data.decode("utf-8"))) + return next(responses) + + client = OllamaClient(url="http://ollama.test", model="test", json_attempts=2) + with patch("enrich_ollama.urlopen", side_effect=fake_urlopen): + result = client.chat("Сделай саммари", schema=SUMMARY_SCHEMA) + + self.assertEqual(result["chapter_title"], "Тема") + self.assertEqual(len(calls), 2) + self.assertEqual(len(calls[1]["messages"]), 3) + + def test_chunk_uses_deterministic_fallback(self) -> None: + result = summarize_chunk( + { + "start": 0, + "end": 5, + "text": "[0:00] Первая мысль.\n[0:03] Вторая мысль.", + }, + client=BrokenClient(), + ) + + self.assertTrue(result["_fallback"]) + self.assertIn("Первая мысль", result["overview"]) + self.assertEqual(result["key_points"], ["Первая мысль.", "Вторая мысль."]) + + def test_reduce_uses_deterministic_fallback(self) -> None: + items = [ + { + "start": 0, + "end": 5, + "overview": "Первая часть.", + "key_points": ["Тезис 1"], + "decisions": [], + "action_items": [], + "risks": [], + }, + { + "start": 5, + "end": 10, + "overview": "Вторая часть.", + "key_points": ["Тезис 2"], + "decisions": [], + "action_items": [], + "risks": [], + }, + ] + + result = reduce_summaries(items, client=BrokenClient()) + + self.assertTrue(result["_fallback"]) + self.assertEqual(result["key_points"], ["Тезис 1", "Тезис 2"]) + + def test_topic_title_is_short_and_semantic(self) -> None: + title = normalize_topic_title("🎬 Проверка сайта к первому сентября 2026") + self.assertEqual(title, "Проверка сайта к первому сентября 2026") + self.assertLessEqual(len(title.split()), 8) + self.assertLessEqual(len(title), 56) + + def test_topic_title_rejects_generic_and_file_names(self) -> None: + with self.assertRaises(ValueError): + normalize_topic_title("Видео") + with self.assertRaises(ValueError): + normalize_topic_title("camera-upload-1234.mp4") + + def test_topic_title_has_deterministic_summary_fallback(self) -> None: + title = generate_topic_title( + {"overview": "Подробная проверка готовности сайта.", "key_points": []}, + [{"chapter_title": "Проверка сайта к первому сентября"}], + source_title="camera-upload-1234.mp4", + client=BrokenClient(), + ) + self.assertEqual(title, "Проверка сайта к первому сентября") + + +if __name__ == "__main__": + unittest.main() diff --git a/server/linux/tests/test_release_contract.py b/server/linux/tests/test_release_contract.py new file mode 100644 index 0000000..35a6287 --- /dev/null +++ b/server/linux/tests/test_release_contract.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +class LinuxServerReleaseContractTests(unittest.TestCase): + def test_installed_versions_are_pinned(self) -> None: + stt = (ROOT / "requirements-stt.txt").read_text(encoding="utf-8") + video = (ROOT / "requirements-video.txt").read_text(encoding="utf-8") + for expected in ( + "torch==2.6.0+cu124", + "torchaudio==2.6.0+cu124", + "gigaam==0.2.0", + "openai-whisper==20250625", + ): + self.assertIn(expected, stt) + for expected in ( + "torch==2.8.0+cu128", + "torchaudio==2.8.0+cu128", + "qwen-asr==0.0.6", + "pyannote.audio==4.0.7", + "transformers==4.57.6", + "accelerate==1.12.0", + ): + self.assertIn(expected, video) + + def test_service_is_loopback_only(self) -> None: + unit = (ROOT / "systemd" / "mitim-stt.service.in").read_text( + encoding="utf-8" + ) + self.assertIn("--host 127.0.0.1 --port 18790", unit) + + def test_private_server_values_are_not_published(self) -> None: + forbidden = ( + "/home/" + "ai", + "-100" + "4497951319", + "bot:" + "mitim_" + "openclaw", + ) + for path in ROOT.rglob("*"): + if not path.is_file() or path.suffix in {".pyc"}: + continue + text = path.read_text(encoding="utf-8", errors="ignore") + for value in forbidden: + self.assertNotIn(value, text, f"{value!r} leaked through {path}") + + def test_pipeline_and_plugin_versions_match_snapshot(self) -> None: + pipeline = (ROOT / "video" / "video_pipeline.py").read_text( + encoding="utf-8" + ) + plugin = (ROOT / "openclaw-plugin" / "openclaw.plugin.json").read_text( + encoding="utf-8" + ) + self.assertIn('PIPELINE_VERSION = "mitim-video-pipeline-v3"', pipeline) + self.assertIn('"version": "0.1.6"', plugin) + + +if __name__ == "__main__": + unittest.main() diff --git a/server/linux/tests/test_telegram_publication.py b/server/linux/tests/test_telegram_publication.py new file mode 100644 index 0000000..3c63135 --- /dev/null +++ b/server/linux/tests/test_telegram_publication.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from telegram_publication import ( + SAFE_TEXT_LIMIT, + action_descriptors, + build_publication_plan, + record_message, + record_topic, +) + + +def write_json(path: Path, value: dict) -> None: + path.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8") + + +class TelegramPublicationTests(unittest.TestCase): + def make_artifacts(self, root: Path, *, youtube: bool = False, long_overview: bool = False) -> Path: + source = ( + {"kind": "youtube", "canonical_url": "https://youtu.be/abc123", "original": "https://youtu.be/abc123"} + if youtube + else {"kind": "telegram", "original": "/private/incoming/video.mp4", "sha256": "a" * 64} + ) + write_json( + root / "transcript.json", + { + "asset_id": "asset-1", + "title": "Тестовое видео", + "duration": 125, + "profile": "deep", + "asr": {"engines_used": ["gigaam"]}, + "segments": [], + }, + ) + write_json( + root / "manifest.json", + { + "asset_id": "asset-1", + "title": "Тестовое видео", + "profile": "deep", + "duration": 125, + "qc_status": "ok", + "vision_status": "complete", + "summary_status": "complete", + }, + ) + write_json( + root / "summary.json", + { + "title": "Практическая проверка сайта к празднику", + "overview": "я" * 5000 if long_overview else "Краткое содержание.", + "key_points": ["Первый тезис"], + "chapters": [{"start": 65, "title": "Главный фрагмент"}], + "risks": ["Проверить выводы"], + }, + ) + write_json(root / "metadata.json", {"channel": "Тестовый канал"}) + write_json(root / "source-info.json", source) + write_json(root / "qc.json", {"status": "ok"}) + write_json( + root / "job-state.json", + { + "profile": "deep", + "stages": { + "alignment": {"status": "complete"}, + "diarization": {"status": "complete"}, + "visuals": {"status": "complete"}, + "summary": {"status": "complete"}, + }, + }, + ) + (root / "summary.md").write_text("summary", encoding="utf-8") + (root / "transcript.md").write_text("transcript", encoding="utf-8") + return root + + def test_build_is_idempotent_and_does_not_publish_local_path(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.make_artifacts(Path(directory)) + first = build_publication_plan(root, chat_id="-1001234567890", source_topic_id=7) + second = build_publication_plan(root, chat_id="-1001234567890", source_topic_id=7) + self.assertEqual(first["topic"]["name"], "🎬 Практическая проверка сайта к празднику") + self.assertLessEqual(len(first["topic"]["name"]), 60) + self.assertNotIn("/private/incoming", first["content"]["summary"]) + self.assertEqual(first["topic"]["idempotency_key"], second["topic"]["idempotency_key"]) + self.assertEqual(2, len(first["attachments"])) + + def test_youtube_timestamp_and_message_limit(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.make_artifacts(Path(directory), youtube=True) + plan = build_publication_plan(root, chat_id=-1001234567890) + self.assertIn("https://youtu.be/abc123?t=65", plan["content"]["summary"]) + write_json( + root / "summary.json", + {"title": "Практическая проверка сайта к празднику", "overview": "я" * 5000}, + ) + plan = build_publication_plan(root, chat_id=-1001234567890) + self.assertLessEqual(len(plan["content"]["summary"]), SAFE_TEXT_LIMIT) + self.assertIn("Текст сокращён", plan["content"]["summary"]) + + def test_internal_stage_error_is_not_published(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.make_artifacts(Path(directory)) + write_json( + root / "job-state.json", + { + "profile": "deep", + "stages": { + "visuals": { + "status": "partial", + "error": 'Traceback: File "/home/private/enrich.py" RuntimeError: Ollama failed', + } + }, + }, + ) + plan = build_publication_plan(root, chat_id=-1001234567890) + summary = plan["content"]["summary"] + self.assertIn("OCR/vision: partial", summary) + self.assertNotIn("Traceback", summary) + self.assertNotIn("/home/private", summary) + + def test_idempotency_keys_follow_payload_content(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.make_artifacts(Path(directory)) + first_plan = build_publication_plan(root, chat_id=-1001234567890) + first = {item["kind"]: item["idempotency_key"] for item in action_descriptors(first_plan, topic_id=777)} + + write_json( + root / "summary.json", + {"title": "Практическая проверка сайта к празднику", "overview": "Новая версия."}, + ) + (root / "summary.md").write_text("updated summary", encoding="utf-8") + second_plan = build_publication_plan(root, chat_id=-1001234567890) + second = { + item["kind"]: item["idempotency_key"] + for item in action_descriptors(second_plan, topic_id=777) + } + + self.assertEqual(first["status"], second["status"]) + self.assertEqual(first["transcript_document"], second["transcript_document"]) + self.assertNotEqual(first["summary"], second["summary"]) + self.assertNotEqual(first["summary_document"], second["summary_document"]) + + def test_actions_resume_without_duplicates(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.make_artifacts(Path(directory)) + plan_path = root / "telegram-publication.json" + plan = build_publication_plan(root, chat_id=-1001234567890) + topic_action = action_descriptors(plan) + self.assertEqual("create_forum_topic", topic_action[0]["operation"]) + self.assertTrue(topic_action[0]["owner_direct_approval"]) + record_topic(plan_path, 777) + plan = json.loads(plan_path.read_text(encoding="utf-8")) + actions = action_descriptors(plan) + self.assertEqual({"status", "summary", "summary_document", "transcript_document"}, {a["kind"] for a in actions}) + self.assertTrue(all(action["owner_direct_approval"] for action in actions)) + for index, action in enumerate(actions, start=100): + record_message(plan_path, action["kind"], index) + plan = json.loads(plan_path.read_text(encoding="utf-8")) + self.assertEqual("completed", plan["delivery"]["status"]) + self.assertEqual([], action_descriptors(plan)) + + def test_required_publication_documents_must_exist(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.make_artifacts(Path(directory)) + (root / "summary.md").unlink() + with self.assertRaisesRegex(FileNotFoundError, "обязательный файл публикации"): + build_publication_plan(root, chat_id=-1001234567890) + + def test_topic_name_must_be_semantic_and_come_from_summary(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.make_artifacts(Path(directory)) + write_json(root / "summary.json", {"title": "video-1234.mp4", "overview": "Содержание."}) + with self.assertRaisesRegex(ValueError, "именем файла"): + build_publication_plan(root, chat_id=-1001234567890) + + write_json(root / "summary.json", {"overview": "Содержание без заголовка."}) + with self.assertRaisesRegex(ValueError, "отсутствует смысловое название"): + build_publication_plan(root, chat_id=-1001234567890) + + def test_legacy_complete_delivery_is_migrated(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = self.make_artifacts(Path(directory)) + plan_path = root / "telegram-publication.json" + build_publication_plan(root, chat_id=-1001234567890) + plan = json.loads(plan_path.read_text(encoding="utf-8")) + plan["delivery"] = { + "status": "complete", + "topic_id": 777, + "messages": { + "status": 10, + "summary": 11, + "summary_document": 12, + "transcript_document": 13, + }, + "completed_at": "2026-08-30T07:00:00+00:00", + } + write_json(plan_path, plan) + rebuilt = build_publication_plan(root, chat_id=-1001234567890) + self.assertEqual("completed", rebuilt["delivery"]["status"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/server/linux/video/align_qwen.py b/server/linux/video/align_qwen.py new file mode 100644 index 0000000..1cadec4 --- /dev/null +++ b/server/linux/video/align_qwen.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Add word timestamps to an existing transcript with Qwen3 ForcedAligner.""" + +from __future__ import annotations + +import argparse +import json +import math +from pathlib import Path +from typing import Any + +import soundfile as sf +import torch +from qwen_asr import Qwen3ForcedAligner + + +DEFAULT_MODEL = "Qwen/Qwen3-ForcedAligner-0.6B" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--audio", required=True, type=Path) + parser.add_argument("--transcript", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--model", default=DEFAULT_MODEL) + parser.add_argument("--language", default="Russian") + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--max-seconds", type=float, default=280.0) + return parser.parse_args() + + +def read_transcript(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload, list): + return {"segments": payload}, payload + if not isinstance(payload, dict) or not isinstance(payload.get("segments"), list): + raise ValueError("Transcript must be a JSON object with a segments array") + return payload, payload["segments"] + + +def split_segment(segment: dict[str, Any], max_seconds: float) -> list[dict[str, Any]]: + start = float(segment.get("start", 0.0)) + end = float(segment.get("end", start)) + text = str(segment.get("text", "")).strip() + duration = max(0.0, end - start) + if not text or duration <= 0.0: + return [] + count = max(1, math.ceil(duration / max_seconds)) + if count == 1: + return [{"start": start, "end": end, "text": text}] + + words = text.split() + units: list[dict[str, Any]] = [] + for index in range(count): + left = math.floor(len(words) * index / count) + right = math.floor(len(words) * (index + 1) / count) + unit_text = " ".join(words[left:right]).strip() + unit_start = start + duration * index / count + unit_end = start + duration * (index + 1) / count + if unit_text: + units.append({"start": unit_start, "end": unit_end, "text": unit_text}) + return units + + +def read_audio_slice(handle: sf.SoundFile, start: float, end: float) -> Any: + sample_rate = int(handle.samplerate) + first = max(0, round(start * sample_rate)) + frames = max(1, round((end - start) * sample_rate)) + handle.seek(min(first, len(handle))) + audio = handle.read(frames=frames, dtype="float32", always_2d=True) + if audio.shape[1] > 1: + audio = audio.mean(axis=1) + else: + audio = audio[:, 0] + return audio, sample_rate + + +def main() -> int: + args = parse_args() + payload, segments = read_transcript(args.transcript) + if not args.audio.is_file(): + raise FileNotFoundError(args.audio) + + use_cuda = args.device.startswith("cuda") and torch.cuda.is_available() + device = args.device if use_cuda else "cpu" + dtype = torch.float16 if use_cuda else torch.float32 + aligner = Qwen3ForcedAligner.from_pretrained( + args.model, + device_map=device, + dtype=dtype, + attn_implementation="sdpa", + ) + + aligned_segments: list[dict[str, Any]] = [] + all_words: list[dict[str, Any]] = [] + errors: list[dict[str, Any]] = [] + + with sf.SoundFile(str(args.audio)) as audio_file: + for index, original in enumerate(segments): + segment = dict(original) + segment_words: list[dict[str, Any]] = [] + try: + for unit in split_segment(segment, args.max_seconds): + audio = read_audio_slice(audio_file, unit["start"], unit["end"]) + result = aligner.align( + audio=audio, + text=unit["text"], + language=args.language, + )[0] + for item in result: + word = { + "text": item.text, + "start": round(unit["start"] + float(item.start_time), 3), + "end": round(unit["start"] + float(item.end_time), 3), + } + segment_words.append(word) + all_words.append(word) + except Exception as exc: # Keep the canonical GigaAM text on local failures. + errors.append({"segment": index, "error": str(exc)}) + + if segment_words: + segment["start"] = segment_words[0]["start"] + segment["end"] = segment_words[-1]["end"] + segment["words"] = segment_words + segment["alignment_engine"] = "qwen3-forced-aligner" + aligned_segments.append(segment) + + result_payload = dict(payload) + result_payload["segments"] = aligned_segments + result_payload["words"] = all_words + result_payload["alignment"] = { + "engine": "qwen3-forced-aligner", + "model": args.model, + "language": args.language, + "device": device, + "aligned_segments": len(segments) - len(errors), + "failed_segments": errors, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result_payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/linux/video/diarize_pyannote.py b/server/linux/video/diarize_pyannote.py new file mode 100644 index 0000000..5d25ee9 --- /dev/null +++ b/server/linux/video/diarize_pyannote.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Assign pyannote speaker labels to transcript segments and words.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any + +import torch +import soundfile as sf +from huggingface_hub import get_token +from pyannote.audio import Pipeline + + +DEFAULT_MODEL = "pyannote/speaker-diarization-community-1" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--audio", required=True, type=Path) + parser.add_argument("--transcript", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--model", default=DEFAULT_MODEL) + parser.add_argument("--device", default="cuda") + parser.add_argument("--num-speakers", type=int) + parser.add_argument("--min-speakers", type=int) + parser.add_argument("--max-speakers", type=int) + return parser.parse_args() + + +def load_transcript(path: Path) -> tuple[dict[str, Any], list[dict[str, Any]]]: + payload = json.loads(path.read_text(encoding="utf-8")) + if isinstance(payload, list): + return {"segments": payload}, payload + if not isinstance(payload, dict) or not isinstance(payload.get("segments"), list): + raise ValueError("Transcript must be a JSON object with a segments array") + return payload, payload["segments"] + + +def best_speaker(start: float, end: float, turns: list[dict[str, Any]]) -> str | None: + best_label: str | None = None + best_overlap = 0.0 + midpoint = (start + end) / 2.0 + nearest_distance = float("inf") + for turn in turns: + overlap = max(0.0, min(end, turn["end"]) - max(start, turn["start"])) + if overlap > best_overlap: + best_overlap = overlap + best_label = turn["speaker"] + if best_overlap == 0.0: + turn_midpoint = (turn["start"] + turn["end"]) / 2.0 + distance = abs(midpoint - turn_midpoint) + if distance < nearest_distance: + nearest_distance = distance + best_label = turn["speaker"] + return best_label + + +def main() -> int: + args = parse_args() + token = ( + os.environ.get("HF_TOKEN") + or os.environ.get("HUGGINGFACE_HUB_TOKEN") + or get_token() + ) + if not token: + raise RuntimeError( + "Hugging Face authentication is required after accepting the " + "pyannote community-1 model terms" + ) + + payload, segments = load_transcript(args.transcript) + if not args.audio.is_file(): + raise FileNotFoundError(args.audio) + + pipeline = Pipeline.from_pretrained(args.model, token=token) + device = torch.device(args.device if torch.cuda.is_available() else "cpu") + pipeline.to(device) + call_args = { + key: value + for key, value in { + "num_speakers": args.num_speakers, + "min_speakers": args.min_speakers, + "max_speakers": args.max_speakers, + }.items() + if value is not None + } + samples, sample_rate = sf.read( + args.audio, + dtype="float32", + always_2d=True, + ) + audio = { + "waveform": torch.from_numpy(samples.T.copy()), + "sample_rate": int(sample_rate), + } + diarization_output = pipeline(audio, **call_args) + annotation = getattr(diarization_output, "speaker_diarization", diarization_output) + turns = [ + { + "start": round(float(turn.start), 3), + "end": round(float(turn.end), 3), + "speaker": str(speaker), + } + for turn, _, speaker in annotation.itertracks(yield_label=True) + ] + + diarized_segments: list[dict[str, Any]] = [] + for original in segments: + segment = dict(original) + start = float(segment.get("start", 0.0)) + end = float(segment.get("end", start)) + segment["speaker"] = best_speaker(start, end, turns) + words = segment.get("words") + if isinstance(words, list): + labeled_words = [] + for original_word in words: + word = dict(original_word) + word_start = float(word.get("start", start)) + word_end = float(word.get("end", word_start)) + word["speaker"] = best_speaker(word_start, word_end, turns) + labeled_words.append(word) + segment["words"] = labeled_words + diarized_segments.append(segment) + + result_payload = dict(payload) + result_payload["segments"] = diarized_segments + result_payload["speaker_turns"] = turns + result_payload["speakers"] = sorted({turn["speaker"] for turn in turns}) + result_payload["diarization"] = { + "engine": "pyannote.audio", + "model": args.model, + "device": str(device), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps(result_payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/linux/video/enrich_ollama.py b/server/linux/video/enrich_ollama.py new file mode 100644 index 0000000..9c93429 --- /dev/null +++ b/server/linux/video/enrich_ollama.py @@ -0,0 +1,621 @@ +#!/usr/bin/env python3 +"""Local OCR/vision and transcript summarization through Ollama. + +The helper is deliberately dependency-free: frames and transcripts stay on the +server and are sent only to the local Ollama HTTP endpoint. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + + +DEFAULT_MODEL = "qwen3-vl:8b" +DEFAULT_OLLAMA_URL = "http://127.0.0.1:11434" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def atomic_write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def atomic_write_text(path: Path, value: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(value, encoding="utf-8") + temporary.replace(path) + + +def read_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"Ожидался JSON-объект: {path}") + return value + + +def human_timestamp(seconds: float) -> str: + total = max(0, int(seconds)) + hours, remainder = divmod(total, 3600) + minutes, secs = divmod(remainder, 60) + return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}" + + +def parse_json_object(value: str) -> dict[str, Any]: + text = value.strip() + if text.startswith("```"): + text = re.sub(r"^```(?:json)?\s*", "", text, flags=re.IGNORECASE) + text = re.sub(r"\s*```$", "", text) + try: + parsed = json.loads(text) + except json.JSONDecodeError: + start = text.find("{") + end = text.rfind("}") + if start < 0 or end <= start: + raise ValueError("Ollama не вернул JSON-объект") + parsed = json.loads(text[start : end + 1]) + if not isinstance(parsed, dict): + raise ValueError("Ollama вернул JSON другого типа") + return parsed + + +class OllamaClient: + def __init__(self, *, url: str, model: str, timeout: int = 900, json_attempts: int = 3) -> None: + self.endpoint = url.rstrip("/") + "/api/chat" + self.model = model + self.timeout = timeout + self.json_attempts = max(1, json_attempts) + + def chat( + self, + prompt: str, + *, + schema: dict[str, Any], + images: list[Path] | None = None, + ) -> dict[str, Any]: + message: dict[str, Any] = {"role": "user", "content": prompt} + if images: + message["images"] = [base64.b64encode(path.read_bytes()).decode("ascii") for path in images] + messages = [message] + last_error: Exception | None = None + for attempt in range(1, self.json_attempts + 1): + payload = { + "model": self.model, + "messages": messages, + "stream": False, + "think": False, + "format": schema, + "keep_alive": "10m", + "options": {"temperature": 0, "num_ctx": 8192}, + } + request = Request( + self.endpoint, + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urlopen(request, timeout=self.timeout) as response: + result = json.loads(response.read().decode("utf-8")) + except HTTPError as error: + detail = error.read().decode("utf-8", errors="replace")[-1200:] + raise RuntimeError(f"Ollama HTTP {error.code}: {detail}") from error + except URLError as error: + raise RuntimeError(f"Локальный Ollama недоступен: {error.reason}") from error + message_result = result.get("message", {}) + content = message_result.get("content") + if not isinstance(content, str) or not content.strip(): + content = message_result.get("thinking") + if not isinstance(content, str) or not content.strip(): + last_error = RuntimeError("Ollama вернул пустой ответ") + content = "" + else: + try: + return parse_json_object(content) + except (ValueError, json.JSONDecodeError) as error: + last_error = error + if attempt < self.json_attempts: + messages.extend( + [ + {"role": "assistant", "content": content}, + { + "role": "user", + "content": ( + "Предыдущий ответ не удалось разобрать как JSON. " + "Повтори ответ строго как один валидный JSON-объект по заданной схеме, " + "без пояснений и Markdown." + ), + }, + ] + ) + raise ValueError(f"Ollama не вернул валидный JSON за {self.json_attempts} попытки") from last_error + + +VISION_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "ocr_text": {"type": "string"}, + "description": {"type": "string"}, + "visible_actions": {"type": "array", "items": {"type": "string"}}, + "visual_type": { + "type": "string", + "enum": ["slide", "interface", "document", "camera", "diagram", "other"], + }, + "confidence": {"type": "number", "minimum": 0, "maximum": 1}, + "warnings": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["ocr_text", "description", "visible_actions", "visual_type", "confidence", "warnings"], + "additionalProperties": False, +} + + +SUMMARY_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "chapter_title": {"type": "string"}, + "overview": {"type": "string"}, + "key_points": {"type": "array", "items": {"type": "string"}}, + "decisions": {"type": "array", "items": {"type": "string"}}, + "action_items": {"type": "array", "items": {"type": "string"}}, + "risks": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["chapter_title", "overview", "key_points", "decisions", "action_items", "risks"], + "additionalProperties": False, +} + +TOPIC_TITLE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": {"title": {"type": "string"}}, + "required": ["title"], + "additionalProperties": False, +} + +GENERIC_TOPIC_TITLES = { + "видео", + "тестовое видео", + "фрагмент транскрипта", + "итоговый конспект", +} + + +def normalize_topic_title(value: Any) -> str: + """Return a short semantic topic title without an emoji or technical ids.""" + rendered = re.sub(r"\s+", " ", str(value or "")).strip(" \t\r\n🎬:—–-.,;!?«»\"'") + if not rendered or rendered.casefold() in GENERIC_TOPIC_TITLES: + raise ValueError("Название топика не отражает содержание видео") + if re.search(r"\.(?:mp4|mov|mkv|webm|avi|mp3|wav|m4a)$", rendered, flags=re.IGNORECASE): + raise ValueError("Название топика похоже на имя файла") + if re.fullmatch(r"[0-9a-f]{12,64}", rendered, flags=re.IGNORECASE) or re.fullmatch( + r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", rendered, flags=re.IGNORECASE + ): + raise ValueError("Название топика похоже на технический идентификатор") + + words = re.findall(r"[0-9A-Za-zА-Яа-яЁё]+(?:[-–][0-9A-Za-zА-Яа-яЁё]+)*", rendered) + if len(words) < 3: + raise ValueError("Название топика должно содержать не менее трёх слов") + words = words[:8] + while len(words) >= 3 and len(" ".join(words)) > 56: + words.pop() + if len(words) < 3: + raise ValueError("Название топика нельзя сократить до 56 символов без потери смысла") + return " ".join(words) + + +def generate_topic_title( + reduced: dict[str, Any], + partials: list[dict[str, Any]], + *, + source_title: Any, + client: OllamaClient, +) -> str: + """Generate the Telegram topic title from the completed transcript summary.""" + evidence = { + "overview": str(reduced.get("overview") or "").strip(), + "key_points": unique_strings(list(reduced.get("key_points") or []), limit=8), + "chapters": unique_strings( + [str(item.get("chapter_title") or "") for item in partials], limit=8 + ), + } + prompt = ( + "Сформулируй короткое фактическое название темы видео по готовому конспекту. " + "Верни только JSON. Поле title: 3–8 слов, не более 56 символов, без эмодзи, " + "кавычек, имени файла и технических идентификаторов. Название должно отражать " + "главную тему или практический результат, не быть кликбейтом.\n\n" + + json.dumps(evidence, ensure_ascii=False) + ) + try: + result = client.chat(prompt, schema=TOPIC_TITLE_SCHEMA) + return normalize_topic_title(result.get("title")) + except (RuntimeError, ValueError): + candidates = [ + *(item.get("chapter_title") for item in partials), + *(reduced.get("key_points") or []), + reduced.get("overview"), + source_title, + ] + for candidate in candidates: + try: + return normalize_topic_title(candidate) + except ValueError: + continue + raise ValueError("Не удалось сформировать смысловое название топика из транскрипта") + + +def analyze_visual_timeline( + timeline_path: Path, + *, + client: OllamaClient, + force: bool, +) -> dict[str, Any]: + timeline = read_json(timeline_path) + frames = timeline.get("frames") + if not isinstance(frames, list): + raise ValueError("В visual-timeline.json отсутствует список frames") + for index, frame in enumerate(frames): + if not isinstance(frame, dict): + raise ValueError(f"Некорректная запись кадра #{index + 1}") + if frame.get("status") == "complete" and not force: + continue + image_path = Path(str(frame.get("path", ""))).expanduser() + if not image_path.is_absolute(): + image_path = image_path.resolve() if image_path.is_file() else (timeline_path.parent / image_path).resolve() + if not image_path.is_file(): + raise FileNotFoundError(f"Не найден кадр: {image_path}") + timestamp = float(frame.get("timestamp") or 0) + prompt = ( + "Проанализируй один кадр видео. Верни только JSON по заданной схеме. " + "Дословно перепиши весь уверенно читаемый русский и английский текст в ocr_text. " + "В ocr_text запрещено копировать слова этой инструкции: записывай только текст, " + "физически видимый на изображении; если текста нет, верни пустую строку. " + "Кратко опиши только видимое содержимое и действия. Не угадывай скрытый контекст, " + "имена людей и события вне кадра. Неразборчивое не выдумывай, а укажи в warnings. " + f"Таймкод кадра: {human_timestamp(timestamp)}." + ) + try: + result = client.chat(prompt, schema=VISION_SCHEMA, images=[image_path]) + frame.update( + { + "ocr": str(result.get("ocr_text") or "").strip() or None, + "description": str(result.get("description") or "").strip(), + "actions": [str(item).strip() for item in result.get("visible_actions", []) if str(item).strip()], + "visual_type": result.get("visual_type", "other"), + "confidence": float(result.get("confidence") or 0), + "warnings": [str(item).strip() for item in result.get("warnings", []) if str(item).strip()], + "model": client.model, + "status": "complete", + "analyzed_at": utc_now(), + } + ) + except Exception as error: + frame.update({"status": "failed", "error": str(error), "analyzed_at": utc_now()}) + timeline.update({"status": "failed", "model": client.model, "updated_at": utc_now()}) + atomic_write_json(timeline_path, timeline) + raise + timeline.update({"status": "running", "model": client.model, "updated_at": utc_now()}) + atomic_write_json(timeline_path, timeline) + timeline.update( + { + "status": "complete", + "model": client.model, + "frame_count": len(frames), + "analyzed_frame_count": sum(frame.get("status") == "complete" for frame in frames), + "completed_at": utc_now(), + } + ) + atomic_write_json(timeline_path, timeline) + return timeline + + +def transcript_chunks(segments: list[dict[str, Any]], *, max_chars: int = 8500) -> list[dict[str, Any]]: + chunks: list[dict[str, Any]] = [] + current: list[str] = [] + current_start = 0.0 + current_end = 0.0 + current_size = 0 + for segment in segments: + text = str(segment.get("text") or "").strip() + if not text: + continue + start = float(segment.get("start") or 0) + end = float(segment.get("end") or start) + line = f"[{human_timestamp(start)}] {text}" + if current and current_size + len(line) + 1 > max_chars: + chunks.append({"start": current_start, "end": current_end, "text": "\n".join(current)}) + current = [] + current_size = 0 + if not current: + current_start = start + current.append(line) + current_end = end + current_size += len(line) + 1 + if current: + chunks.append({"start": current_start, "end": current_end, "text": "\n".join(current)}) + return chunks + + +def summarize_chunk(chunk: dict[str, Any], *, client: OllamaClient) -> dict[str, Any]: + prompt = ( + "Сделай фактический конспект фрагмента транскрипта видео. Верни только JSON по схеме. " + "Не добавляй знания извне и не исправляй утверждения автора молча. Пустые разделы оставляй " + "пустыми массивами. chapter_title должен коротко отражать тему фрагмента.\n\n" + f"Диапазон: {human_timestamp(float(chunk['start']))}–{human_timestamp(float(chunk['end']))}\n" + f"Транскрипт:\n{chunk['text']}" + ) + try: + result = client.chat(prompt, schema=SUMMARY_SCHEMA) + except (RuntimeError, ValueError) as error: + lines = [ + re.sub(r"^\[[^\]]+\]\s*", "", line).strip() + for line in str(chunk.get("text") or "").splitlines() + ] + lines = [line for line in lines if line] + overview = " ".join(lines).strip() + if len(overview) > 1600: + overview = overview[:1599].rstrip() + "…" + result = { + "chapter_title": (lines[0][:120] if lines else "Фрагмент транскрипта"), + "overview": overview or "Нет распознанного текста.", + "key_points": lines[:8], + "decisions": [], + "action_items": [], + "risks": [], + "_fallback": True, + "_fallback_reason": str(error), + } + return {"start": chunk["start"], "end": chunk["end"], **result} + + +def unique_strings(values: list[Any], *, limit: int = 30) -> list[str]: + result: list[str] = [] + seen: set[str] = set() + for value in values: + text = str(value).strip() + key = text.casefold() + if not text or key in seen: + continue + seen.add(key) + result.append(text) + if len(result) >= limit: + break + return result + + +def merge_summary_items(items: list[dict[str, Any]], *, fallback_reason: str | None = None) -> dict[str, Any]: + merged = { + "start": min(float(item.get("start") or 0) for item in items), + "end": max(float(item.get("end") or 0) for item in items), + "chapter_title": "Итоговый конспект", + "overview": " ".join(str(item.get("overview") or "") for item in items).strip(), + "key_points": unique_strings([point for item in items for point in item.get("key_points", [])]), + "decisions": unique_strings([point for item in items for point in item.get("decisions", [])]), + "action_items": unique_strings([point for item in items for point in item.get("action_items", [])]), + "risks": unique_strings([point for item in items for point in item.get("risks", [])]), + } + if fallback_reason: + merged["_fallback"] = True + merged["_fallback_reason"] = fallback_reason + return merged + + +def reduce_summaries(items: list[dict[str, Any]], *, client: OllamaClient) -> dict[str, Any]: + if len(items) == 1: + return items[0] + current = items + while len(current) > 1: + groups: list[list[dict[str, Any]]] = [] + group: list[dict[str, Any]] = [] + size = 0 + for item in current: + serialized = json.dumps(item, ensure_ascii=False) + if group and size + len(serialized) > 8000: + groups.append(group) + group = [] + size = 0 + group.append(item) + size += len(serialized) + if group: + groups.append(group) + next_items: list[dict[str, Any]] = [] + for batch in groups: + start = min(float(item.get("start") or 0) for item in batch) + end = max(float(item.get("end") or start) for item in batch) + prompt = ( + "Объедини промежуточные конспекты одного видео в единый фактический конспект. " + "Верни только JSON по схеме, устрани повторы, не добавляй знания извне.\n\n" + + json.dumps(batch, ensure_ascii=False) + ) + try: + reduced = client.chat(prompt, schema=SUMMARY_SCHEMA) + except (RuntimeError, ValueError) as error: + return merge_summary_items(current, fallback_reason=str(error)) + next_items.append({"start": start, "end": end, **reduced}) + if len(next_items) >= len(current) and len(current) > 1: + return merge_summary_items(current) + current = next_items + return current[0] + + +def render_summary_markdown(summary: dict[str, Any]) -> str: + title = str(summary.get("title") or summary.get("asset_id") or "Видео") + lines = [f"# {title}", "", "## Кратко", "", str(summary.get("overview") or "Нет данных."), ""] + + def section(name: str, values: list[Any]) -> None: + cleaned = unique_strings(values) + if not cleaned: + return + lines.extend([f"## {name}", ""]) + lines.extend(f"- {value}" for value in cleaned) + lines.append("") + + section("Ключевые тезисы", list(summary.get("key_points") or [])) + chapters = summary.get("chapters") or [] + if chapters: + lines.extend(["## Содержание по времени", ""]) + for chapter in chapters: + start = human_timestamp(float(chapter.get("start") or 0)) + end = human_timestamp(float(chapter.get("end") or 0)) + heading = str(chapter.get("title") or "Фрагмент") + overview = str(chapter.get("overview") or "").strip() + lines.extend([f"### {start}–{end} — {heading}", "", overview, ""]) + visual_findings = summary.get("visual_findings") or [] + if visual_findings: + lines.extend(["## Визуальные наблюдения", ""]) + for finding in visual_findings: + timestamp = human_timestamp(float(finding.get("timestamp") or 0)) + description = str(finding.get("description") or "").strip() + ocr = str(finding.get("ocr") or "").strip() + line = f"- **{timestamp}:** {description}" + if ocr: + line += f" Текст на экране: «{ocr}»" + lines.append(line) + lines.append("") + section("Решения", list(summary.get("decisions") or [])) + section("Дальнейшие действия", list(summary.get("action_items") or [])) + section("Предупреждения", list(summary.get("warnings") or [])) + section("Риски и оговорки", list(summary.get("risks") or [])) + lines.extend( + [ + "---", + "", + f"_Сформировано локально моделью `{summary.get('model')}`; источник текста — локальная ASR._", + "", + ] + ) + return "\n".join(lines) + + +def generate_summary( + transcript_path: Path, + *, + visual_timeline_path: Path | None, + output_json: Path, + output_markdown: Path, + client: OllamaClient, + force: bool, +) -> dict[str, Any]: + if output_json.exists() and output_markdown.exists() and not force: + return read_json(output_json) + transcript = read_json(transcript_path) + segments = transcript.get("segments") + if not isinstance(segments, list): + raise ValueError("В transcript.json отсутствует список segments") + chunks = transcript_chunks(segments) + if not chunks: + raise ValueError("Невозможно создать саммари: транскрипт пуст") + partials = [summarize_chunk(chunk, client=client) for chunk in chunks] + reduced = reduce_summaries(partials, client=client) + fallback_reasons = unique_strings( + [ + str(item.get("_fallback_reason") or "") + for item in [*partials, reduced] + if item.get("_fallback") + ], + limit=10, + ) + visual_findings: list[dict[str, Any]] = [] + if visual_timeline_path and visual_timeline_path.exists(): + timeline = read_json(visual_timeline_path) + for frame in timeline.get("frames", []): + if not isinstance(frame, dict) or frame.get("status") != "complete": + continue + if not frame.get("description") and not frame.get("ocr"): + continue + visual_findings.append( + { + "timestamp": float(frame.get("timestamp") or 0), + "description": str(frame.get("description") or "").strip(), + "ocr": str(frame.get("ocr") or "").strip() or None, + "actions": frame.get("actions") or [], + } + ) + source_title = transcript.get("title") + topic_title = generate_topic_title( + reduced, + partials, + source_title=source_title, + client=client, + ) + summary = { + "asset_id": transcript.get("asset_id"), + "title": topic_title, + "source_title": source_title, + "status": "complete", + "model": client.model, + "generation_mode": "hybrid_fallback" if fallback_reasons else "model", + "source": "local_transcript_and_visual_timeline", + "generated_at": utc_now(), + "overview": str(reduced.get("overview") or "").strip(), + "key_points": unique_strings(list(reduced.get("key_points") or [])), + "chapters": [ + { + "start": float(item.get("start") or 0), + "end": float(item.get("end") or 0), + "title": str(item.get("chapter_title") or "Фрагмент").strip(), + "overview": str(item.get("overview") or "").strip(), + } + for item in partials + ], + "visual_findings": visual_findings, + "decisions": unique_strings(list(reduced.get("decisions") or [])), + "action_items": unique_strings(list(reduced.get("action_items") or [])), + "warnings": [ + "Часть саммари сформирована детерминированным fallback из локального транскрипта: " + reason + for reason in fallback_reasons + ], + "risks": unique_strings(list(reduced.get("risks") or [])), + } + atomic_write_json(output_json, summary) + atomic_write_text(output_markdown, render_summary_markdown(summary)) + return summary + + +def main() -> int: + parser = argparse.ArgumentParser(description="Local OCR/vision and summaries through Ollama") + parser.add_argument("--ollama-url", default=DEFAULT_OLLAMA_URL) + parser.add_argument("--model", default=DEFAULT_MODEL) + parser.add_argument("--timeout", type=int, default=900) + subparsers = parser.add_subparsers(dest="command", required=True) + + visuals = subparsers.add_parser("visuals") + visuals.add_argument("--timeline", type=Path, required=True) + visuals.add_argument("--force", action="store_true") + + summary = subparsers.add_parser("summary") + summary.add_argument("--transcript", type=Path, required=True) + summary.add_argument("--visual-timeline", type=Path) + summary.add_argument("--output-json", type=Path, required=True) + summary.add_argument("--output-markdown", type=Path, required=True) + summary.add_argument("--force", action="store_true") + + args = parser.parse_args() + client = OllamaClient(url=args.ollama_url, model=args.model, timeout=args.timeout) + if args.command == "visuals": + result = analyze_visual_timeline(args.timeline, client=client, force=args.force) + else: + result = generate_summary( + args.transcript, + visual_timeline_path=args.visual_timeline, + output_json=args.output_json, + output_markdown=args.output_markdown, + client=client, + force=args.force, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/linux/video/telegram_publication.py b/server/linux/video/telegram_publication.py new file mode 100644 index 0000000..e6afd3d --- /dev/null +++ b/server/linux/video/telegram_publication.py @@ -0,0 +1,481 @@ +#!/usr/bin/env python3 +"""Build a safe, resumable Telegram publication outbox for one video. + +This module never contacts Telegram. It renders exact action descriptors for +Telegram Suite. A fresh video from Ivan is the direct command for the exact +low-risk publication package, so descriptors request owner-direct execution +without a duplicate confirmation prompt. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse + + +SCHEMA_VERSION = 1 +SAFE_TEXT_LIMIT = 3900 +DEFAULT_CONNECTION = "bot:voiceswitch" + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def read_json(path: Path, *, required: bool = True) -> dict[str, Any]: + if not path.exists(): + if required: + raise FileNotFoundError(f"Не найден обязательный артефакт: {path}") + return {} + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"Ожидался JSON-объект: {path}") + return value + + +def atomic_write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def normalize_space(value: Any) -> str: + return re.sub(r"\s+", " ", str(value or "")).strip() + + +def stable_key(*parts: Any) -> str: + source = "\x1f".join(str(part) for part in parts) + digest = hashlib.sha256(source.encode("utf-8")).hexdigest()[:20] + return f"video-{digest}" + + +def file_digest(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def parse_chat_id(value: str | int) -> str | int: + rendered = str(value).strip() + if re.fullmatch(r"-?\d+", rendered): + return int(rendered) + if not rendered: + raise ValueError("Telegram chat id не может быть пустым") + return rendered + + +def make_topic_name(title: str) -> str: + cleaned = normalize_space(title).strip(" 🎬:—–-.,;!?«»\"'").replace("/", "／") + words = re.findall(r"[0-9A-Za-zА-Яа-яЁё]+(?:[-–][0-9A-Za-zА-Яа-яЁё]+)*", cleaned) + generic = {"видео", "тестовое видео", "фрагмент транскрипта", "итоговый конспект"} + if re.search(r"\.(?:mp4|mov|mkv|webm|avi|mp3|wav|m4a)$", cleaned, flags=re.IGNORECASE): + raise ValueError("Название топика не должно быть именем файла") + if re.fullmatch(r"[0-9a-f]{12,64}", cleaned, flags=re.IGNORECASE) or re.fullmatch( + r"[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}", cleaned, flags=re.IGNORECASE + ): + raise ValueError("Название топика не должно быть техническим идентификатором") + if cleaned.casefold() in generic or len(words) < 3: + raise ValueError("Название топика должно содержательно описывать видео и включать 3–8 слов") + words = words[:8] + cleaned = " ".join(words) + prefix = "🎬 " + available = 60 - len(prefix) + while len(words) >= 3 and len(cleaned) > available: + words.pop() + cleaned = " ".join(words) + if len(words) < 3: + raise ValueError("Название топика нельзя сократить до 60 символов без потери смысла") + return prefix + cleaned + + +def human_duration(value: Any) -> str: + try: + total = max(0, int(round(float(value)))) + except (TypeError, ValueError): + return "не определена" + hours, remainder = divmod(total, 3600) + minutes, seconds = divmod(remainder, 60) + return f"{hours}:{minutes:02d}:{seconds:02d}" if hours else f"{minutes}:{seconds:02d}" + + +def timestamp(value: Any) -> str: + try: + total = max(0, int(float(value))) + except (TypeError, ValueError): + total = 0 + hours, remainder = divmod(total, 3600) + minutes, seconds = divmod(remainder, 60) + return f"{hours}:{minutes:02d}:{seconds:02d}" if hours else f"{minutes}:{seconds:02d}" + + +def timestamp_url(url: str | None, seconds: Any) -> str | None: + if not url: + return None + parsed = urlparse(url) + allowed = {"youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be", "www.youtu.be"} + if parsed.hostname not in allowed: + return None + query = dict(parse_qsl(parsed.query, keep_blank_values=True)) + query["t"] = str(max(0, int(float(seconds or 0)))) + return urlunparse(parsed._replace(query=urlencode(query))) + + +def source_label(source: dict[str, Any]) -> str: + if source.get("canonical_url"): + return str(source["canonical_url"]) + digest = normalize_space(source.get("sha256"))[:12] + label = "Telegram-вложение" if source.get("kind") == "telegram" else "Локальный файл" + return f"{label} · SHA-256 {digest}…" if digest else label + + +def author_label(metadata: dict[str, Any]) -> str | None: + for key in ("uploader", "channel", "creator", "artist", "author"): + value = normalize_space(metadata.get(key)) + if value: + return value + return None + + +def engine_label(transcript: dict[str, Any], manifest: dict[str, Any]) -> str: + asr = transcript.get("asr") if isinstance(transcript.get("asr"), dict) else {} + engines = asr.get("engines_used") + if isinstance(engines, list) and engines: + return ", ".join(normalize_space(item) for item in engines if normalize_space(item)) + return normalize_space(asr.get("requested_engine") or manifest.get("engine") or "не определён") + + +def stage_warnings(job: dict[str, Any]) -> list[str]: + result: list[str] = [] + stages = job.get("stages") if isinstance(job.get("stages"), dict) else {} + labels = { + "alignment": "точное выравнивание", + "diarization": "диаризация", + "visuals": "OCR/vision", + "summary": "саммари", + } + for stage, label in labels.items(): + value = stages.get(stage) if isinstance(stages.get(stage), dict) else {} + status = normalize_space(value.get("status")) + if status in {"skipped", "partial", "failed"}: + reason = normalize_space(value.get("reason") or value.get("error")) + internal_markers = ("Traceback", "eback (most recent call", 'File "', "/home/", "RuntimeError:") + if any(marker in reason for marker in internal_markers): + reason = "" + elif len(reason) > 180: + reason = reason[:179].rstrip() + "…" + result.append(f"{label}: {status}" + (f" ({reason})" if reason else "")) + return result + + +def bounded_text(value: str, *, limit: int = SAFE_TEXT_LIMIT) -> str: + if len(value) <= limit: + return value + marker = "\n\n…Текст сокращён; полный материал приложен файлами." + return value[: limit - len(marker)].rstrip() + marker + + +def build_summary_text( + *, + transcript: dict[str, Any], + summary: dict[str, Any], + manifest: dict[str, Any], + metadata: dict[str, Any], + source: dict[str, Any], + qc: dict[str, Any], + job: dict[str, Any], +) -> str: + title = normalize_space(summary.get("title") or transcript.get("title") or manifest.get("title") or "Видео") + profile = normalize_space(manifest.get("profile") or transcript.get("profile") or job.get("profile") or "standard") + duration = manifest.get("duration", transcript.get("duration")) + lines = [ + f"🎬 {title}", + "", + f"Источник: {source_label(source)}", + f"Длительность: {human_duration(duration)}", + ] + author = author_label(metadata) + if author: + lines.append(f"Автор/канал: {author}") + lines.extend( + [ + f"Профиль: {profile}", + f"Локальная ASR: {engine_label(transcript, manifest)}", + f"QC: {normalize_space(qc.get('status') or manifest.get('qc_status') or 'не определён')}", + ] + ) + + overview = normalize_space(summary.get("overview")) + if overview: + lines.extend(["", "Кратко", overview]) + key_points = [normalize_space(item) for item in summary.get("key_points", []) if normalize_space(item)] + if key_points: + lines.extend(["", "Ключевые тезисы"]) + lines.extend(f"• {item}" for item in key_points[:6]) + chapters = [item for item in summary.get("chapters", []) if isinstance(item, dict)] + if chapters: + lines.extend(["", "Ключевые фрагменты"]) + base_url = source.get("canonical_url") + for chapter in chapters[:6]: + start = chapter.get("start", 0) + chapter_title = normalize_space(chapter.get("title") or chapter.get("chapter_title") or "Фрагмент") + link = timestamp_url(str(base_url), start) if base_url else None + lines.append(f"• {timestamp(start)} — {chapter_title}" + (f" — {link}" if link else "")) + risks = [normalize_space(item) for item in summary.get("risks", []) if normalize_space(item)] + if risks: + lines.extend(["", "Риски и оговорки"]) + lines.extend(f"• {item}" for item in risks[:4]) + warnings = stage_warnings(job) + if warnings: + lines.extend(["", "Ограничения обработки"]) + lines.extend(f"• {item}" for item in warnings) + lines.extend(["", "Полные саммари и транскрипт приложены файлами."]) + return bounded_text("\n".join(lines)) + + +def build_status_text(manifest: dict[str, Any], qc: dict[str, Any]) -> str: + return "\n".join( + [ + "📦 Локальная обработка готова", + f"Профиль: {normalize_space(manifest.get('profile') or 'standard')}", + f"QC: {normalize_space(qc.get('status') or manifest.get('qc_status') or 'не определён')}", + f"OCR/vision: {normalize_space(manifest.get('vision_status') or 'не применялся')}", + f"Саммари: {normalize_space(manifest.get('summary_status') or 'не создано')}", + ] + ) + + +def build_publication_plan( + artifact_dir: Path, + *, + chat_id: str | int, + source_topic_id: int | None = None, + connection: str = DEFAULT_CONNECTION, + output: Path | None = None, +) -> dict[str, Any]: + artifact_dir = artifact_dir.expanduser().resolve(strict=True) + transcript = read_json(artifact_dir / "transcript.json") + manifest = read_json(artifact_dir / "manifest.json") + summary = read_json(artifact_dir / "summary.json", required=False) + metadata = read_json(artifact_dir / "metadata.json", required=False) + source = read_json(artifact_dir / "source-info.json") + qc = read_json(artifact_dir / "qc.json") + job = read_json(artifact_dir / "job-state.json") + asset_id = normalize_space(manifest.get("asset_id") or transcript.get("asset_id")) + if not asset_id: + raise ValueError("В артефактах отсутствует asset_id") + title = normalize_space(summary.get("title")) + if not title: + raise ValueError("В summary.json отсутствует смысловое название для Telegram-топика") + target_chat = parse_chat_id(chat_id) + output = output or artifact_dir / "telegram-publication.json" + + previous: dict[str, Any] = {} + if output.exists(): + previous = read_json(output) + old_target = previous.get("target") if isinstance(previous.get("target"), dict) else {} + if previous.get("asset_id") != asset_id or old_target.get("chat_id") != target_chat: + raise ValueError("Существующий outbox относится к другому asset или Telegram-чату") + + attachments: list[dict[str, Any]] = [] + for kind, filename, caption in ( + ("summary_document", "summary.md", "Содержательное саммари"), + ("transcript_document", "transcript.md", "Полный локальный транскрипт"), + ): + path = artifact_dir / filename + if not path.is_file(): + raise FileNotFoundError(f"Не найден обязательный файл публикации: {path}") + attachments.append({"kind": kind, "path": str(path), "caption": caption}) + + now = utc_now() + delivery = previous.get("delivery") or { + "status": "pending_topic", + "topic_id": None, + "messages": {}, + "completed_at": None, + } + if delivery.get("status") == "complete": + delivery["status"] = "completed" + plan = { + "schema_version": SCHEMA_VERSION, + "asset_id": asset_id, + "artifact_dir": str(artifact_dir), + "created_at": previous.get("created_at") or now, + "updated_at": now, + "target": {"connection": connection, "chat_id": target_chat, "source_topic_id": source_topic_id}, + "topic": { + "name": make_topic_name(title), + "idempotency_key": stable_key(asset_id, target_chat, "topic", SCHEMA_VERSION), + }, + "content": { + "status_ready": build_status_text(manifest, qc), + "summary": build_summary_text( + transcript=transcript, + summary=summary, + manifest=manifest, + metadata=metadata, + source=source, + qc=qc, + job=job, + ), + }, + "attachments": attachments, + "delivery": delivery, + } + atomic_write_json(output, plan) + return plan + + +def action_descriptors(plan: dict[str, Any], *, topic_id: int | None = None) -> list[dict[str, Any]]: + target = plan["target"] + delivery = plan.get("delivery") if isinstance(plan.get("delivery"), dict) else {} + messages = delivery.get("messages") if isinstance(delivery.get("messages"), dict) else {} + resolved_topic = topic_id or delivery.get("topic_id") + connection = target["connection"] + chat_id = target["chat_id"] + asset_id = plan["asset_id"] + + if not resolved_topic: + return [ + { + "kind": "topic", + "connection": connection, + "operation": "create_forum_topic", + "params": {"chat_id": chat_id, "name": plan["topic"]["name"]}, + "idempotency_key": plan["topic"]["idempotency_key"], + "owner_direct_approval": True, + } + ] + + resolved_topic = int(resolved_topic) + common = {"chat_id": chat_id, "message_thread_id": resolved_topic} + result: list[dict[str, Any]] = [] + for kind, text_key in (("status", "status_ready"), ("summary", "summary")): + if kind not in messages: + text = plan["content"][text_key] + result.append( + { + "kind": kind, + "connection": connection, + "operation": "send_message", + "params": {**common, "text": text}, + "idempotency_key": stable_key( + asset_id, chat_id, resolved_topic, kind, text, SCHEMA_VERSION + ), + "owner_direct_approval": True, + } + ) + for attachment in plan.get("attachments", []): + kind = attachment["kind"] + if kind in messages: + continue + result.append( + { + "kind": kind, + "connection": connection, + "operation": "send_media", + "params": { + **common, + "media_type": "document", + "file": {"$file": attachment["path"]}, + "caption": attachment["caption"], + }, + "idempotency_key": stable_key( + asset_id, + chat_id, + resolved_topic, + kind, + file_digest(attachment["path"]), + SCHEMA_VERSION, + ), + "owner_direct_approval": True, + } + ) + return result + + +def record_topic(plan_path: Path, topic_id: int) -> dict[str, Any]: + if topic_id < 1: + raise ValueError("topic_id должен быть положительным") + plan = read_json(plan_path) + delivery = plan.setdefault("delivery", {}) + existing = delivery.get("topic_id") + if existing and int(existing) != topic_id: + raise ValueError(f"Outbox уже привязан к другому topic_id: {existing}") + delivery["topic_id"] = topic_id + delivery["status"] = "pending_messages" + plan["updated_at"] = utc_now() + atomic_write_json(plan_path, plan) + return plan + + +def record_message(plan_path: Path, kind: str, message_id: int) -> dict[str, Any]: + if message_id < 1: + raise ValueError("message_id должен быть положительным") + plan = read_json(plan_path) + delivery = plan.setdefault("delivery", {}) + messages = delivery.setdefault("messages", {}) + existing = messages.get(kind) + if existing and int(existing) != message_id: + raise ValueError(f"Для {kind} уже записан другой message_id: {existing}") + messages[kind] = message_id + expected = {"status", "summary", *(item["kind"] for item in plan.get("attachments", []))} + if expected.issubset(messages): + delivery["status"] = "completed" + delivery["completed_at"] = delivery.get("completed_at") or utc_now() + else: + delivery["status"] = "pending_messages" + plan["updated_at"] = utc_now() + atomic_write_json(plan_path, plan) + return plan + + +def main() -> int: + parser = argparse.ArgumentParser(description="Telegram topic publication outbox for video artifacts") + subparsers = parser.add_subparsers(dest="command", required=True) + build = subparsers.add_parser("build") + build.add_argument("artifact_dir", type=Path) + build.add_argument("--chat-id", required=True) + build.add_argument("--source-topic-id", type=int) + build.add_argument("--connection", default=DEFAULT_CONNECTION) + build.add_argument("--output", type=Path) + actions = subparsers.add_parser("actions") + actions.add_argument("plan", type=Path) + actions.add_argument("--topic-id", type=int) + bind = subparsers.add_parser("record-topic") + bind.add_argument("plan", type=Path) + bind.add_argument("topic_id", type=int) + record = subparsers.add_parser("record-message") + record.add_argument("plan", type=Path) + record.add_argument("kind") + record.add_argument("message_id", type=int) + + args = parser.parse_args() + if args.command == "build": + result = build_publication_plan( + args.artifact_dir, + chat_id=args.chat_id, + source_topic_id=args.source_topic_id, + connection=args.connection, + output=args.output, + ) + elif args.command == "actions": + result = action_descriptors(read_json(args.plan), topic_id=args.topic_id) + elif args.command == "record-topic": + result = record_topic(args.plan, args.topic_id) + else: + result = record_message(args.plan, args.kind, args.message_id) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/linux/video/transcribe_mitim_stt.py b/server/linux/video/transcribe_mitim_stt.py new file mode 100644 index 0000000..eb2efc3 --- /dev/null +++ b/server/linux/video/transcribe_mitim_stt.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Transcribe long WAV files through the local loopback-only mitim-stt service. + +The resident service accepts files up to 30 minutes and returns one text block. +This adapter cuts long recordings into timestamped pieces, adds a small overlap, +deduplicates exact words at joins, and checkpoints every completed request. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +import urllib.error +import urllib.request +import wave +from pathlib import Path +from typing import Any + + +WORD_RE = re.compile(r"[\wёЁ-]+", re.UNICODE) + + +def resolve_binary(name: str, configured: Path | None) -> str: + resolved = str(configured.expanduser().resolve()) if configured else shutil.which(name) + if not resolved: + raise RuntimeError(f"{name} не найден") + path = Path(resolved) + if not path.is_file() or not os.access(path, os.X_OK): + raise RuntimeError(f"{name} недоступен для запуска: {path}") + return resolved + + +def media_duration(path: Path, ffprobe: str) -> float: + if ffprobe: + completed = subprocess.run( + [ + ffprobe, + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + str(path), + ], + check=False, + capture_output=True, + text=True, + timeout=60, + ) + if completed.returncode == 0: + try: + duration = float(completed.stdout.strip()) + if duration > 0: + return duration + except ValueError: + pass + with wave.open(str(path), "rb") as audio: + return audio.getnframes() / float(audio.getframerate()) + + +def post_transcribe(endpoint: str, audio: Path, engine: str) -> dict[str, Any]: + body = json.dumps({"path": str(audio.resolve()), "engine": engine}).encode("utf-8") + request = urllib.request.Request( + endpoint, + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + result: Any = None + for attempt in range(4): + try: + with urllib.request.urlopen(request, timeout=3600) as response: + result = json.load(response) + break + except urllib.error.HTTPError as error: + detail = error.read().decode("utf-8", errors="replace") + if error.code not in {429, 500, 502, 503, 504} or attempt == 3: + raise RuntimeError(f"mitim-stt HTTP {error.code}: {detail}") from error + except urllib.error.URLError as error: + if attempt == 3: + raise RuntimeError(f"mitim-stt недоступен: {error}") from error + time.sleep(2**attempt) + if not isinstance(result, dict) or "text" not in result: + raise RuntimeError(f"Некорректный ответ mitim-stt: {result!r}") + return result + + +def normalized_words(text: str) -> list[str]: + return [word.casefold() for word in WORD_RE.findall(text)] + + +def remove_overlap(previous: str, current: str, maximum_words: int = 24) -> str: + """Remove an exact 3+ word suffix/prefix repeated by overlapped audio.""" + previous_words = normalized_words(previous) + current_matches = list(WORD_RE.finditer(current)) + current_words = [match.group(0).casefold() for match in current_matches] + limit = min(maximum_words, len(previous_words), len(current_words)) + for count in range(limit, 2, -1): + if previous_words[-count:] == current_words[:count]: + return current[current_matches[count - 1].end() :].lstrip(" ,.;:—–-\t") + return current.strip() + + +def quality_flags(text: str, duration: float, language: Any, engine: Any) -> list[str]: + flags: list[str] = [] + words = normalized_words(text) + if not words: + return ["empty_text"] + if duration >= 20 and len(words) / duration < 0.03: + flags.append("low_text_density") + if len(words) >= 12: + most_common = max(words.count(word) for word in set(words)) + if most_common / len(words) >= 0.45: + flags.append("high_token_repetition") + letters = [character for character in text if character.isalpha()] + cyrillic = [character for character in letters if "а" <= character.casefold() <= "я" or character.casefold() == "ё"] + if str(language) == "ru" and str(engine).startswith("gigaam") and len(letters) >= 20: + if len(cyrillic) / len(letters) < 0.2: + flags.append("unexpected_script_for_russian") + return flags + + +def extract_chunk(ffmpeg: str, source: Path, output: Path, start: float, duration: float) -> None: + command = [ + ffmpeg, + "-y", + "-hide_banner", + "-loglevel", + "error", + "-ss", + f"{start:.3f}", + "-t", + f"{duration:.3f}", + "-i", + str(source), + "-vn", + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "pcm_s16le", + str(output), + ] + subprocess.run(command, check=True, timeout=180) + + +def atomic_write_json(path: Path, value: Any) -> None: + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8") + temporary.replace(path) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Long-form adapter for local mitim-stt") + parser.add_argument("--audio", required=True, type=Path) + parser.add_argument("--endpoint", default="http://127.0.0.1:18790/transcribe") + parser.add_argument("--engine", choices=("auto", "gigaam", "whisper"), default="auto") + parser.add_argument("--chunk-seconds", type=float, default=60.0) + parser.add_argument("--overlap-seconds", type=float, default=1.5) + parser.add_argument("--checkpoint", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--ffmpeg-path", type=Path) + parser.add_argument("--ffprobe-path", type=Path) + args = parser.parse_args() + + source = args.audio.resolve(strict=True) + ffmpeg = resolve_binary("ffmpeg", args.ffmpeg_path) + ffprobe = resolve_binary("ffprobe", args.ffprobe_path) + if args.chunk_seconds < 30 or args.chunk_seconds > 600: + raise ValueError("chunk-seconds должен быть от 30 до 600") + if args.overlap_seconds < 0 or args.overlap_seconds >= args.chunk_seconds / 4: + raise ValueError("Некорректный overlap-seconds") + + total_duration = media_duration(source, ffprobe) + checkpoint = args.checkpoint or source.with_suffix(".mitim-stt-checkpoint.json") + fingerprint = { + "path": str(source), + "size": source.stat().st_size, + "mtime_ns": source.stat().st_mtime_ns, + "duration": round(total_duration, 3), + "chunk_seconds": args.chunk_seconds, + "overlap_seconds": args.overlap_seconds, + "endpoint": args.endpoint, + "requested_engine": args.engine, + "ffmpeg_path": ffmpeg, + "ffprobe_path": ffprobe, + } + + state: dict[str, Any] = {"fingerprint": fingerprint, "segments": []} + if checkpoint.exists(): + candidate = json.loads(checkpoint.read_text(encoding="utf-8")) + if candidate.get("fingerprint") == fingerprint and isinstance(candidate.get("segments"), list): + state = candidate + + segment_count = max(1, math.ceil(total_duration / args.chunk_seconds)) + done = {int(segment["index"]): segment for segment in state["segments"]} + started_all = time.perf_counter() + + for index in range(segment_count): + if index in done: + print(f"[{index + 1}/{segment_count}] checkpoint", file=sys.stderr, flush=True) + continue + + nominal_start = index * args.chunk_seconds + nominal_end = min(total_duration, (index + 1) * args.chunk_seconds) + actual_start = max(0.0, nominal_start - (args.overlap_seconds if index else 0.0)) + actual_duration = nominal_end - actual_start + print( + f"[{index + 1}/{segment_count}] {nominal_start:.1f}-{nominal_end:.1f}s -> mitim-stt", + file=sys.stderr, + flush=True, + ) + with tempfile.TemporaryDirectory(prefix="youtube-mitim-stt-") as temporary: + chunk = Path(temporary) / f"chunk-{index:04d}.wav" + extract_chunk(ffmpeg, source, chunk, actual_start, actual_duration) + result = post_transcribe(args.endpoint, chunk, args.engine) + + previous_text = "" + if index > 0 and index - 1 in done: + previous_text = str(done[index - 1].get("text", "")) + text = remove_overlap(previous_text, str(result.get("text", "")).strip()) + segment = { + "index": index, + "start": round(nominal_start, 3), + "end": round(nominal_end, 3), + "text": text, + "confidence": None, + "source": f"mitim-stt:{result.get('engine', args.engine)}", + "engine": result.get("engine", args.engine), + "language": result.get("language"), + "latency": result.get("latency"), + "quality_flags": quality_flags( + text, + nominal_end - nominal_start, + result.get("language"), + result.get("engine", args.engine), + ), + } + done[index] = segment + state["segments"] = [done[key] for key in sorted(done)] + atomic_write_json(checkpoint, state) + + segments = [done[key] for key in sorted(done)] + for segment in segments: + segment.setdefault( + "quality_flags", + quality_flags( + str(segment.get("text", "")), + float(segment.get("end", 0)) - float(segment.get("start", 0)), + segment.get("language"), + segment.get("engine"), + ), + ) + payload = { + "segments": segments, + "metadata": { + "adapter": "mitim-stt-long-form-v2", + "requested_engine": args.engine, + "engines_used": sorted({str(segment.get("engine")) for segment in segments}), + "duration": round(total_duration, 3), + "segment_count": len(segments), + "empty_segment_count": sum(not str(segment.get("text", "")).strip() for segment in segments), + "flagged_segment_count": sum(bool(segment.get("quality_flags")) for segment in segments), + "coverage": { + "start": 0.0, + "end": round(max((float(segment.get("end", 0)) for segment in segments), default=0.0), 3), + "expected_end": round(total_duration, 3), + }, + "elapsed": round(time.perf_counter() - started_all, 3), + "checkpoint": str(checkpoint), + }, + } + rendered = json.dumps(payload, ensure_ascii=False, indent=2) + if args.output: + args.output.write_text(rendered + "\n", encoding="utf-8") + else: + print(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/server/linux/video/video_pipeline.py b/server/linux/video/video_pipeline.py new file mode 100644 index 0000000..8c46ab0 --- /dev/null +++ b/server/linux/video/video_pipeline.py @@ -0,0 +1,993 @@ +#!/usr/bin/env python3 +"""Local-first video transcription pipeline built around resident mitim-stt. + +Supported sources: +- YouTube URLs through yt-dlp +- local audio/video files, including media downloaded from Telegram + +GigaAM is the canonical Russian ASR. YouTube captions are preserved only as +auxiliary evidence. Optional external commands can add forced alignment and +speaker diarization without changing the stable voice-message endpoint. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from telegram_publication import DEFAULT_CONNECTION as DEFAULT_TELEGRAM_CONNECTION +from telegram_publication import build_publication_plan + + +PIPELINE_VERSION = "mitim-video-pipeline-v3" +PROJECT_DIR = Path(__file__).resolve().parent +DEFAULT_OUTPUT_ROOT = PROJECT_DIR / "knowledge" / "videos" +DEFAULT_STT_ADAPTER = PROJECT_DIR / "transcribe_mitim_stt.py" +DEFAULT_OLLAMA_ENRICHER = PROJECT_DIR / "enrich_ollama.py" +YOUTUBE_HOSTS = {"youtube.com", "www.youtube.com", "m.youtube.com", "youtu.be", "www.youtu.be"} +PROFILES = { + "quick": {"alignment": False, "diarization": False, "visuals": False, "frame_interval": 0}, + "standard": {"alignment": True, "diarization": False, "visuals": True, "frame_interval": 300}, + "interview": {"alignment": True, "diarization": True, "visuals": True, "frame_interval": 300}, + "multilingual": {"alignment": True, "diarization": False, "visuals": True, "frame_interval": 300}, + "deep": {"alignment": True, "diarization": True, "visuals": True, "frame_interval": 60}, +} + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def atomic_write_json(path: Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def atomic_write_text(path: Path, value: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(value, encoding="utf-8") + temporary.replace(path) + + +def run_cmd(command: list[str], *, timeout: int = 3600) -> str: + completed = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + timeout=timeout, + ) + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout or "command failed").strip() + raise RuntimeError(f"Команда завершилась с кодом {completed.returncode}: {detail[-1200:]}") + return completed.stdout.strip() + + +def require_binary(name: str, configured: Path | str | None = None) -> str: + resolved = str(Path(configured).expanduser().resolve()) if configured else shutil.which(name) + if not resolved: + raise RuntimeError(f"Не найдена обязательная команда: {name}") + path = Path(resolved) + if not path.is_file() or not os.access(path, os.X_OK): + raise RuntimeError(f"Команда {name} недоступна для запуска: {path}") + return resolved + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def safe_name(value: str, fallback: str) -> str: + cleaned = re.sub(r"[^0-9A-Za-zА-Яа-яЁё._-]+", "-", value).strip("-._") + return (cleaned[:96] or fallback).strip("-._") or fallback + + +def seconds_to_timestamp(seconds: float, *, srt: bool = False) -> str: + milliseconds = max(0, int(round(seconds * 1000))) + hours, remainder = divmod(milliseconds, 3_600_000) + minutes, remainder = divmod(remainder, 60_000) + secs, millis = divmod(remainder, 1000) + separator = "," if srt else "." + return f"{hours:02d}:{minutes:02d}:{secs:02d}{separator}{millis:03d}" + + +def human_timestamp(seconds: float) -> str: + total = max(0, int(seconds)) + hours, remainder = divmod(total, 3600) + minutes, secs = divmod(remainder, 60) + return f"{hours}:{minutes:02d}:{secs:02d}" if hours else f"{minutes}:{secs:02d}" + + +def is_youtube_url(value: str) -> bool: + parsed = urlparse(value) + return parsed.scheme in {"http", "https"} and parsed.hostname in YOUTUBE_HOSTS + + +def probe_media(path: Path, *, ffprobe: str) -> dict[str, Any]: + data = json.loads( + run_cmd([ffprobe, "-v", "error", "-show_format", "-show_streams", "-of", "json", str(path)], timeout=120) + ) + duration = 0.0 + try: + duration = float(data.get("format", {}).get("duration") or 0) + except (TypeError, ValueError): + pass + if duration <= 0: + for stream in data.get("streams", []): + try: + duration = max(duration, float(stream.get("duration") or 0)) + except (TypeError, ValueError): + continue + return { + "duration": round(duration, 3), + "format": data.get("format", {}), + "streams": data.get("streams", []), + "has_video": any(stream.get("codec_type") == "video" for stream in data.get("streams", [])), + "has_audio": any(stream.get("codec_type") == "audio" for stream in data.get("streams", [])), + } + + +def prepare_stage_audio(media: Path, output: Path, *, ffmpeg: str, force: bool) -> Path: + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists() and not force: + return output + run_cmd( + [ + ffmpeg, + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(media), + "-vn", + "-ac", + "1", + "-ar", + "16000", + "-c:a", + "pcm_s16le", + str(output), + ], + timeout=7200, + ) + return output + + +def choose_youtube_caption(metadata: dict[str, Any], languages: list[str]) -> tuple[str, str] | None: + for source, bucket_name in (("youtube_manual", "subtitles"), ("youtube_auto", "automatic_captions")): + bucket = metadata.get(bucket_name, {}) + if not isinstance(bucket, dict): + continue + for language in languages: + if bucket.get(language): + return source, language + for language, tracks in bucket.items(): + if tracks: + return source, language + return None + + +def download_youtube_caption( + url: str, + destination: Path, + source: str, + language: str, + *, + ffmpeg: str, +) -> Path: + destination.mkdir(parents=True, exist_ok=True) + command = [ + require_binary("yt-dlp"), + "--no-warnings", + "--no-playlist", + "--skip-download", + "--sub-lang", + language, + "--sub-format", + "vtt", + "--convert-subs", + "vtt", + "--ffmpeg-location", + ffmpeg, + "--output", + str(destination / "captions.%(ext)s"), + "--write-subs" if source == "youtube_manual" else "--write-auto-subs", + url, + ] + run_cmd(command) + candidates = sorted(destination.glob("*.vtt")) + if not candidates: + raise RuntimeError("yt-dlp не создал файл субтитров") + return candidates[0] + + +def download_youtube_media(url: str, work_dir: Path, include_video: bool, *, ffmpeg: str) -> Path: + work_dir.mkdir(parents=True, exist_ok=True) + yt_dlp = require_binary("yt-dlp") + output = str(work_dir / "source.%(ext)s") + if include_video: + command = [ + yt_dlp, + "--no-warnings", + "--no-playlist", + "-f", + "bv*[height<=720]+ba/b[height<=720]/b", + "--merge-output-format", + "mp4", + "--ffmpeg-location", + ffmpeg, + "--output", + output, + url, + ] + else: + command = [ + yt_dlp, + "--no-warnings", + "--no-playlist", + "-x", + "--audio-format", + "wav", + "--audio-quality", + "0", + "--ffmpeg-location", + ffmpeg, + "--postprocessor-args", + "ffmpeg:-ac 1 -ar 16000", + "--output", + output, + url, + ] + run_cmd(command, timeout=7200) + candidates = sorted( + path for path in work_dir.glob("source.*") if path.is_file() and path.suffix not in {".part", ".ytdl"} + ) + if not candidates: + raise RuntimeError("Не удалось найти скачанный медиафайл") + return candidates[0] + + +class JobState: + def __init__(self, path: Path, *, asset_id: str, source: str, profile: str) -> None: + self.path = path + if path.exists(): + self.value = json.loads(path.read_text(encoding="utf-8")) + self.value.update( + { + "pipeline": PIPELINE_VERSION, + "asset_id": asset_id, + "source": source, + "profile": profile, + "status": "running", + "current_stage": "received", + "stages": {}, + } + ) + for stale_key in ("error", "failed_at", "completed_at"): + self.value.pop(stale_key, None) + else: + self.value = { + "pipeline": PIPELINE_VERSION, + "asset_id": asset_id, + "source": source, + "profile": profile, + "status": "running", + "current_stage": "received", + "stages": {}, + "created_at": utc_now(), + } + self.save() + + def stage(self, name: str, status: str, **details: Any) -> None: + self.value["current_stage"] = name + self.value["status"] = "failed" if status == "failed" else "running" + self.value.setdefault("stages", {})[name] = {"status": status, "updated_at": utc_now(), **details} + self.save() + + def finish(self) -> None: + self.value["status"] = "complete" + self.value["current_stage"] = "complete" + self.value["completed_at"] = utc_now() + self.save() + + def fail(self, error: Exception) -> None: + self.value["status"] = "failed" + self.value["error"] = str(error) + self.value["failed_at"] = utc_now() + self.save() + + def save(self) -> None: + self.value["updated_at"] = utc_now() + atomic_write_json(self.path, self.value) + + +def run_transcription( + media: Path, + output_dir: Path, + *, + adapter: Path, + engine: str, + chunk_seconds: float, + overlap_seconds: float, + ffmpeg: str, + ffprobe: str, + force: bool, +) -> dict[str, Any]: + raw_name = "transcript.raw.gigaam.json" if engine in {"auto", "gigaam"} else "transcript.raw.whisper.json" + raw_path = output_dir / raw_name + if raw_path.exists() and not force: + return json.loads(raw_path.read_text(encoding="utf-8")) + checkpoint = output_dir / "checkpoints" / "mitim-stt.json" + checkpoint.parent.mkdir(parents=True, exist_ok=True) + command = [ + sys.executable, + str(adapter), + "--audio", + str(media), + "--engine", + engine, + "--chunk-seconds", + str(chunk_seconds), + "--overlap-seconds", + str(overlap_seconds), + "--ffmpeg-path", + ffmpeg, + "--ffprobe-path", + ffprobe, + "--checkpoint", + str(checkpoint), + "--output", + str(raw_path), + ] + run_cmd(command, timeout=24 * 3600) + return json.loads(raw_path.read_text(encoding="utf-8")) + + +def canonical_segments(raw: dict[str, Any]) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for index, source in enumerate(raw.get("segments", [])): + start = round(float(source.get("start", 0)), 3) + end = round(float(source.get("end", start)), 3) + result.append( + { + "id": index, + "start": start, + "end": max(start, end), + "text": str(source.get("text", "")).strip(), + "speaker": source.get("speaker"), + "words": source.get("words", []), + "confidence": source.get("confidence"), + "source": source.get("source", "mitim-stt"), + "engine": source.get("engine"), + "language": source.get("language"), + "quality_flags": list(source.get("quality_flags", [])), + } + ) + return result + + +def run_stage_command( + template: str, + *, + media: Path, + audio: Path | None = None, + transcript: Path, + output: Path, +) -> dict[str, Any]: + command = shlex.split( + template.format( + audio=str(audio or media), + media=str(media), + transcript=str(transcript), + output=str(output), + ) + ) + run_cmd(command, timeout=24 * 3600) + if not output.exists(): + raise RuntimeError(f"Этап не создал ожидаемый файл: {output}") + payload = json.loads(output.read_text(encoding="utf-8")) + if not isinstance(payload, dict) or not isinstance(payload.get("segments"), list): + raise RuntimeError(f"Некорректный JSON этапа: {output}") + return payload + + +def build_qc(segments: list[dict[str, Any]], duration: float) -> dict[str, Any]: + gaps: list[dict[str, float]] = [] + overlaps: list[dict[str, float]] = [] + previous_end = 0.0 + for segment in segments: + start = float(segment["start"]) + end = float(segment["end"]) + if start > previous_end + 0.25: + gaps.append({"start": round(previous_end, 3), "end": round(start, 3)}) + if start < previous_end - 0.25: + overlaps.append({"start": round(start, 3), "previous_end": round(previous_end, 3)}) + previous_end = max(previous_end, end) + flagged = [ + {"id": segment["id"], "start": segment["start"], "flags": segment["quality_flags"]} + for segment in segments + if segment["quality_flags"] + ] + return { + "status": "review" if flagged or gaps else "ok", + "duration": duration, + "segment_count": len(segments), + "nonempty_segment_count": sum(bool(segment["text"]) for segment in segments), + "coverage_end": max((segment["end"] for segment in segments), default=0.0), + "gaps": gaps, + "overlaps": overlaps, + "flagged_segments": flagged, + "generated_at": utc_now(), + } + + +def export_markdown( + path: Path, + *, + asset_id: str, + title: str, + source_url: str | None, + duration: float, + profile: str, + segments: list[dict[str, Any]], +) -> None: + lines = [ + "---", + f"asset_id: {asset_id}", + f"title: {json.dumps(title, ensure_ascii=False)}", + f"duration: {duration}", + f"pipeline: {PIPELINE_VERSION}", + f"profile: {profile}", + "canonical_source: local_asr", + f"created_at: {utc_now()}", + "---", + "", + f"# {title}", + "", + "## Полный локальный транскрипт", + "", + ] + for segment in segments: + if not segment["text"]: + continue + start = float(segment["start"]) + label = human_timestamp(start) + if source_url and is_youtube_url(source_url): + heading = f"### [{label}]({source_url}{'&' if '?' in source_url else '?'}t={int(start)})" + else: + heading = f"### {label}" + lines.extend([heading, "", segment["text"], ""]) + if segment["quality_flags"]: + lines.extend([f"_QC: {', '.join(segment['quality_flags'])}_", ""]) + atomic_write_text(path, "\n".join(lines).strip() + "\n") + + +def export_srt(path: Path, segments: list[dict[str, Any]]) -> None: + blocks: list[str] = [] + cue = 0 + for segment in segments: + if not segment["text"]: + continue + cue += 1 + blocks.append( + f"{cue}\n{seconds_to_timestamp(segment['start'], srt=True)} --> " + f"{seconds_to_timestamp(segment['end'], srt=True)}\n{segment['text']}" + ) + atomic_write_text(path, "\n\n".join(blocks) + ("\n" if blocks else "")) + + +def export_vtt(path: Path, segments: list[dict[str, Any]]) -> None: + blocks = ["WEBVTT"] + for segment in segments: + if not segment["text"]: + continue + blocks.append( + f"{seconds_to_timestamp(segment['start'])} --> {seconds_to_timestamp(segment['end'])}\n{segment['text']}" + ) + atomic_write_text(path, "\n\n".join(blocks) + "\n") + + +def extract_frames( + media: Path, + output_dir: Path, + *, + duration: float, + interval: int, + ffmpeg: str, + force: bool, +) -> list[dict[str, Any]]: + if interval <= 0: + return [] + output_dir.mkdir(parents=True, exist_ok=True) + existing = sorted(output_dir.glob("frame-*.jpg")) + if not existing or force: + run_cmd( + [ + ffmpeg, + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(media), + "-vf", + f"fps=1/{interval},scale=1280:-2", + "-q:v", + "3", + str(output_dir / "frame-%05d.jpg"), + ], + timeout=7200, + ) + existing = sorted(output_dir.glob("frame-*.jpg")) + if not existing and duration > 0: + first_frame = output_dir / "frame-00001.jpg" + run_cmd( + [ + ffmpeg, + "-y", + "-hide_banner", + "-loglevel", + "error", + "-i", + str(media), + "-frames:v", + "1", + "-vf", + "scale=1280:-2", + "-q:v", + "3", + str(first_frame), + ], + timeout=7200, + ) + existing = sorted(output_dir.glob("frame-*.jpg")) + return [ + { + "timestamp": round(min(index * interval, duration), 3), + "path": str(path), + "ocr": None, + "description": None, + "status": "frame_extracted", + } + for index, path in enumerate(existing) + ] + + +def reusable_visual_timeline(path: Path, *, force: bool) -> dict[str, Any] | None: + if force or not path.exists(): + return None + try: + timeline = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + frames = timeline.get("frames") + if ( + timeline.get("status") != "complete" + or not isinstance(frames, list) + or not frames + or any(not isinstance(frame, dict) or frame.get("status") != "complete" for frame in frames) + ): + return None + return timeline + + +def main() -> int: + parser = argparse.ArgumentParser(description="Local-first video transcription around GigaAM/mitim-stt") + parser.add_argument("source", help="YouTube URL or local audio/video path") + parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT_ROOT) + parser.add_argument("--profile", choices=tuple(PROFILES), default="standard") + parser.add_argument("--engine", choices=("auto", "gigaam", "whisper"), default="gigaam") + parser.add_argument("--chunk-seconds", type=float, default=60.0) + parser.add_argument("--overlap-seconds", type=float, default=1.5) + parser.add_argument("--langs", default="ru,en,en-US,en-GB") + parser.add_argument("--title") + parser.add_argument("--source-kind", choices=("auto", "youtube", "telegram", "local"), default="auto") + parser.add_argument("--stt-adapter", type=Path, default=DEFAULT_STT_ADAPTER) + parser.add_argument("--ollama-enricher", type=Path, default=DEFAULT_OLLAMA_ENRICHER) + parser.add_argument("--ollama-url", default="http://127.0.0.1:11434") + parser.add_argument("--vision-model", default="qwen3-vl:8b") + parser.add_argument("--align-command", help="Command template using {audio}, {transcript}, {output}") + parser.add_argument("--diarize-command", help="Command template using {audio}, {transcript}, {output}") + parser.add_argument("--ffmpeg-path", type=Path, help="Absolute path to ffmpeg for detached workers") + parser.add_argument("--ffprobe-path", type=Path, help="Absolute path to ffprobe for detached workers") + parser.add_argument("--skip-vision-analysis", action="store_true") + parser.add_argument("--skip-summary", action="store_true") + parser.add_argument("--telegram-chat-id", help="Build a Telegram Suite publication outbox for this chat") + parser.add_argument("--telegram-source-topic-id", type=int, help="Topic where the source was received") + parser.add_argument("--telegram-connection", default=DEFAULT_TELEGRAM_CONNECTION) + parser.add_argument("--force", action="store_true") + args = parser.parse_args() + + if args.telegram_source_topic_id is not None and not args.telegram_chat_id: + parser.error("--telegram-source-topic-id требует --telegram-chat-id") + + ffmpeg = require_binary("ffmpeg", args.ffmpeg_path) + ffprobe = require_binary("ffprobe", args.ffprobe_path) + source_is_youtube = is_youtube_url(args.source) + if urlparse(args.source).scheme in {"http", "https"} and not source_is_youtube: + raise ValueError("В первой версии URL поддерживаются только для YouTube; остальные источники передавайте файлом") + + profile = PROFILES[args.profile] + metadata: dict[str, Any] + source_info: dict[str, Any] + output_dir: Path + work_dir: Path + + if source_is_youtube: + yt_dlp = require_binary("yt-dlp") + metadata = json.loads(run_cmd([yt_dlp, "--dump-single-json", "--no-playlist", "--no-warnings", args.source])) + asset_id = str(metadata.get("id") or safe_name(args.source, "youtube")) + output_dir = args.output_root / asset_id + work_dir = output_dir / "_work" + title = args.title or str(metadata.get("title") or asset_id) + source_info = { + "kind": "youtube", + "original": args.source, + "canonical_url": metadata.get("webpage_url") or args.source, + "video_id": asset_id, + "received_at": utc_now(), + } + else: + local_source = Path(args.source).expanduser().resolve(strict=True) + digest = file_sha256(local_source) + asset_id = safe_name(local_source.stem, digest[:16]) + "-" + digest[:12] + output_dir = args.output_root / asset_id + work_dir = output_dir / "_work" + title = args.title or local_source.stem + metadata = {"title": title, "id": asset_id} + source_info = { + "kind": "telegram" if args.source_kind == "telegram" else "local", + "original": str(local_source), + "sha256": digest, + "size": local_source.stat().st_size, + "received_at": utc_now(), + } + + output_dir.mkdir(parents=True, exist_ok=True) + job = JobState(output_dir / "job-state.json", asset_id=asset_id, source=args.source, profile=args.profile) + + try: + job.stage("metadata", "running") + atomic_write_json(output_dir / "metadata.json", metadata) + atomic_write_json(output_dir / "source-info.json", source_info) + job.stage("metadata", "complete") + + if source_is_youtube: + job.stage("media", "running") + media = download_youtube_media(args.source, work_dir, bool(profile["visuals"]), ffmpeg=ffmpeg) + caption = choose_youtube_caption(metadata, [item.strip() for item in args.langs.split(",") if item.strip()]) + if caption: + caption_source, caption_language = caption + try: + caption_path = download_youtube_caption( + args.source, + work_dir / "captions", + caption_source, + caption_language, + ffmpeg=ffmpeg, + ) + shutil.copy2(caption_path, output_dir / "subtitles.raw.vtt") + atomic_write_json( + output_dir / "subtitles.source.json", + {"source": caption_source, "language": caption_language, "canonical": False}, + ) + except Exception as error: + atomic_write_json(output_dir / "subtitles.source.json", {"status": "failed", "error": str(error)}) + job.stage("media", "complete", path=str(media)) + else: + media = Path(args.source).expanduser().resolve(strict=True) + job.stage("media", "complete", path=str(media)) + + media_probe = probe_media(media, ffprobe=ffprobe) + if not media_probe["has_audio"]: + raise RuntimeError("В источнике не найден аудиопоток") + atomic_write_json(output_dir / "media-info.json", media_probe) + + effective_engine = "whisper" if args.profile == "multilingual" and args.engine == "gigaam" else args.engine + job.stage("transcription", "running", engine=effective_engine) + raw = run_transcription( + media, + output_dir, + adapter=args.stt_adapter, + engine=effective_engine, + chunk_seconds=args.chunk_seconds, + overlap_seconds=args.overlap_seconds, + ffmpeg=ffmpeg, + ffprobe=ffprobe, + force=args.force, + ) + segments = canonical_segments(raw) + transcript_path = output_dir / "transcript.json" + transcript = { + "asset_id": asset_id, + "title": title, + "source": source_info, + "duration": media_probe["duration"], + "language": "ru" if effective_engine == "gigaam" else "auto", + "canonical_source": "local_asr", + "pipeline": PIPELINE_VERSION, + "profile": args.profile, + "asr": raw.get("metadata", {}), + "created_at": utc_now(), + "segments": segments, + } + atomic_write_json(transcript_path, transcript) + job.stage("transcription", "complete", segment_count=len(segments)) + + stage_audio = media + needs_stage_audio = (profile["alignment"] and bool(args.align_command)) or ( + profile["diarization"] and bool(args.diarize_command) + ) + if needs_stage_audio: + stage_audio = prepare_stage_audio( + media, + work_dir / "stage-audio-16k-mono.wav", + ffmpeg=ffmpeg, + force=args.force, + ) + + if profile["alignment"]: + if args.align_command: + job.stage("alignment", "running") + aligned_path = output_dir / "transcript.aligned.json" + aligned = run_stage_command( + args.align_command, + media=media, + audio=stage_audio, + transcript=transcript_path, + output=aligned_path, + ) + segments = canonical_segments(aligned) + transcript["segments"] = segments + transcript["alignment"] = aligned.get("metadata", {"status": "complete"}) + atomic_write_json(transcript_path, transcript) + job.stage("alignment", "complete") + else: + job.stage("alignment", "skipped", reason="align_command_not_configured", precision="chunk") + else: + job.stage("alignment", "skipped", reason="profile") + + if profile["diarization"]: + if args.diarize_command: + job.stage("diarization", "running") + diarized_path = output_dir / "transcript.diarized.json" + diarized = run_stage_command( + args.diarize_command, + media=media, + audio=stage_audio, + transcript=transcript_path, + output=diarized_path, + ) + segments = canonical_segments(diarized) + transcript["segments"] = segments + transcript["diarization"] = diarized.get( + "diarization", + diarized.get("metadata", {"status": "complete"}), + ) + if isinstance(diarized.get("speakers"), list): + transcript["speakers"] = diarized["speakers"] + if isinstance(diarized.get("speaker_turns"), list): + transcript["speaker_turns"] = diarized["speaker_turns"] + atomic_write_json(transcript_path, transcript) + job.stage("diarization", "complete") + else: + job.stage("diarization", "skipped", reason="diarize_command_not_configured") + else: + job.stage("diarization", "skipped", reason="profile") + + visual_timeline_path = output_dir / "visual-timeline.json" + if profile["visuals"] and media_probe["has_video"]: + existing_timeline = reusable_visual_timeline(visual_timeline_path, force=args.force) + if existing_timeline: + existing_frames = existing_timeline["frames"] + job.stage( + "visuals", + "complete", + frame_count=len(existing_frames), + analyzed_frame_count=len(existing_frames), + ocr="complete", + model=existing_timeline.get("model") or args.vision_model, + resumed=True, + ) + else: + job.stage("visuals", "running") + frames = extract_frames( + media, + output_dir / "frames", + duration=media_probe["duration"], + interval=int(profile["frame_interval"]), + ffmpeg=ffmpeg, + force=args.force, + ) + atomic_write_json( + visual_timeline_path, + { + "interval_seconds": profile["frame_interval"], + "frames": frames, + "status": "frames_extracted", + }, + ) + if frames and not args.skip_vision_analysis: + enrich_command = [ + sys.executable, + str(args.ollama_enricher), + "--ollama-url", + args.ollama_url, + "--model", + args.vision_model, + "visuals", + "--timeline", + str(visual_timeline_path), + ] + if args.force: + enrich_command.append("--force") + try: + run_cmd(enrich_command, timeout=7200) + job.stage( + "visuals", + "complete", + frame_count=len(frames), + analyzed_frame_count=len(frames), + ocr="complete", + model=args.vision_model, + ) + except Exception as error: + if args.profile == "deep": + raise + job.stage( + "visuals", + "partial", + frame_count=len(frames), + ocr="failed", + model=args.vision_model, + error=str(error), + ) + else: + job.stage( + "visuals", + "complete", + frame_count=len(frames), + analyzed_frame_count=0, + ocr="skipped" if args.skip_vision_analysis else "not_applicable", + ) + else: + job.stage("visuals", "skipped", reason="profile_or_no_video_stream") + + if args.skip_summary: + job.stage("summary", "skipped", reason="command_line") + else: + job.stage("summary", "running", model=args.vision_model) + summary_command = [ + sys.executable, + str(args.ollama_enricher), + "--ollama-url", + args.ollama_url, + "--model", + args.vision_model, + "summary", + "--transcript", + str(transcript_path), + "--output-json", + str(output_dir / "summary.json"), + "--output-markdown", + str(output_dir / "summary.md"), + ] + if visual_timeline_path.exists(): + summary_command.extend(["--visual-timeline", str(visual_timeline_path)]) + if args.force: + summary_command.append("--force") + run_cmd(summary_command, timeout=7200) + summary_result = json.loads((output_dir / "summary.json").read_text(encoding="utf-8")) + job.stage( + "summary", + "complete", + model=args.vision_model, + chapter_count=len(summary_result.get("chapters", [])), + visual_finding_count=len(summary_result.get("visual_findings", [])), + ) + + job.stage("exports", "running") + qc = build_qc(segments, media_probe["duration"]) + atomic_write_json(output_dir / "qc.json", qc) + export_markdown( + output_dir / "transcript.md", + asset_id=asset_id, + title=title, + source_url=source_info.get("canonical_url"), + duration=media_probe["duration"], + profile=args.profile, + segments=segments, + ) + export_srt(output_dir / "transcript.srt", segments) + export_vtt(output_dir / "transcript.vtt", segments) + artifacts = [ + "metadata.json", + "source-info.json", + "media-info.json", + "transcript.json", + "transcript.md", + "transcript.srt", + "transcript.vtt", + "qc.json", + "job-state.json", + ] + if (output_dir / "visual-timeline.json").exists(): + artifacts.append("visual-timeline.json") + if (output_dir / "frames").exists(): + artifacts.append("frames/") + if (output_dir / "summary.json").exists(): + artifacts.append("summary.json") + if (output_dir / "summary.md").exists(): + artifacts.append("summary.md") + manifest = { + "asset_id": asset_id, + "title": title, + "status": "complete", + "pipeline": PIPELINE_VERSION, + "profile": args.profile, + "canonical_source": "local_asr", + "engine": effective_engine, + "duration": media_probe["duration"], + "segment_count": len(segments), + "qc_status": qc["status"], + "vision_status": job.value.get("stages", {}).get("visuals", {}).get("status"), + "summary_status": job.value.get("stages", {}).get("summary", {}).get("status"), + "publication_status": "pending" if args.telegram_chat_id else "skipped", + "vision_model": args.vision_model, + "artifacts": artifacts, + "completed_at": utc_now(), + } + atomic_write_json(output_dir / "manifest.json", manifest) + job.stage("exports", "complete") + + if args.telegram_chat_id: + job.stage("publication", "running", connection=args.telegram_connection) + publication_path = output_dir / "telegram-publication.json" + publication = build_publication_plan( + output_dir, + chat_id=args.telegram_chat_id, + source_topic_id=args.telegram_source_topic_id, + connection=args.telegram_connection, + output=publication_path, + ) + artifacts.append("telegram-publication.json") + manifest["artifacts"] = artifacts + manifest["publication_status"] = "ready" + atomic_write_json(output_dir / "manifest.json", manifest) + job.stage( + "publication", + "ready", + outbox=str(publication_path), + topic_name=publication["topic"]["name"], + ) + else: + job.stage("publication", "skipped", reason="telegram_chat_not_configured") + job.finish() + print(json.dumps({"output_dir": str(output_dir), "manifest": manifest}, ensure_ascii=False, indent=2)) + return 0 + except Exception as error: + job.fail(error) + raise + + +if __name__ == "__main__": + raise SystemExit(main()) From af52fa18a9d18f4f56c7a77b4ed5d43eb2f67d83 Mon Sep 17 00:00:00 2001 From: mitimaicode <309470861+mitimaicode@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:14:33 +0000 Subject: [PATCH 2/2] fix: install OpenClaw plugin without peer tree --- .github/workflows/ci.yml | 2 +- CONTRIBUTING.md | 2 +- server/linux/install.sh | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a40420b..d23bc59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,7 +114,7 @@ jobs: - name: Validate optional OpenClaw plugin working-directory: server/linux/openclaw-plugin run: | - npm ci --omit=dev --omit=peer --ignore-scripts + npm ci --omit=dev --omit=peer --ignore-scripts --legacy-peer-deps node --check index.js node --test \ index.test.js \ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1593548..f7c7ac8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ python3 -m py_compile worker/asr_worker.py worker/media_worker.py worker/text_wo python3 -m unittest discover -s tests -v bash -n server/linux/install.sh scripts/package_server_release.sh PYTHONPATH=server/linux/video python3 -m unittest discover -s server/linux/tests -v -(cd server/linux/openclaw-plugin && npm ci --omit=dev --omit=peer --ignore-scripts && node --test index.test.js source-validation.test.js tool-result.test.js worker-scope.test.js) +(cd server/linux/openclaw-plugin && npm ci --omit=dev --omit=peer --ignore-scripts --legacy-peer-deps && node --test index.test.js source-validation.test.js tool-result.test.js worker-scope.test.js) ``` Для изменения распознавания желательно приложить обезличенный набор тестовых diff --git a/server/linux/install.sh b/server/linux/install.sh index ea3ddf6..ca9d03c 100755 --- a/server/linux/install.sh +++ b/server/linux/install.sh @@ -139,7 +139,11 @@ if ((INSTALL_OPENCLAW)); then PLUGIN_ROOT="${HOME}/.openclaw/workspace/plugins/video-transcription-taskflow" mkdir -p "$PLUGIN_ROOT" cp -R "$SOURCE_ROOT/openclaw-plugin/." "$PLUGIN_ROOT/" - npm --prefix "$PLUGIN_ROOT" install --omit=dev --omit=peer --ignore-scripts + npm --prefix "$PLUGIN_ROOT" install \ + --omit=dev \ + --omit=peer \ + --ignore-scripts \ + --legacy-peer-deps fi systemctl --user daemon-reload