From d2773f2ee2f055174cc26c343007c4b253602983 Mon Sep 17 00:00:00 2001 From: Thales <> Date: Thu, 25 Jun 2026 16:37:46 +0100 Subject: [PATCH 1/3] feat: export as MP4 (karaoke video) for MP4 uploads and YouTube (#219) Add an MP4 export that muxes the current mixer state (e.g. vocals muted) with the source video, producing a karaoke-style video. Backend: - Preserve a silent video.mp4 from .mp4 uploads (stream-copy, no re-encode). - YouTube jobs do a best-effort video-only download (H.264/avc1, <=720p) to video.mp4, decoupled from the audio source so failures degrade to audio-only. New STEMDECK_VIDEO_MAX_HEIGHT config. - GET /api/jobs/{id}/video.mp4 streams a fragmented MP4: the amix audio graph encoded as AAC, video stream-copied. - has_video flag on Job, surfaced in state and persisted to metadata. Frontend: - MP4 added as a fourth export format (WAV/MP3/FLAC/MP4), shown only for jobs with a preserved video track. In MP4 mode, Export Mix produces the karaoke video and the audio-only Stems/Region rows are hidden. SoundCloud and plain audio uploads are audio-only (no MP4 option). --- README.md | 1 + app/api/stems.py | 100 +++++++++++++++++++++++++++++----- app/core/config.py | 3 + app/core/models.py | 4 ++ app/pipeline/download.py | 73 ++++++++++++++++++++++++- app/pipeline/runner.py | 43 ++++++++++++++- static/css/daw.css | 10 ++++ static/index.html | 1 + static/js/catalog.js | 3 +- static/js/job.js | 2 + static/js/main.js | 43 ++++++++++++--- static/js/player.js | 28 +++++++++- tests/test_pipeline_runner.py | 63 ++++++++++++++++++++- tests/test_stems_api.py | 64 ++++++++++++++++++++++ 14 files changed, 411 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 904c54f8..657935db 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,7 @@ Stems land in `./jobs/` on the host. Demucs weights are cached in a named volume | PATCH | `/api/jobs/{id}/sections` | Save waveform section markers for a job | | GET | `/api/jobs/{id}/stems/{name}.wav` | Stream a single stem WAV file | | GET | `/api/jobs/{id}/stems/{name}.mp3` | Transcode and stream a stem as MP3 | +| GET | `/api/jobs/{id}/video.mp4` | Mux the current mix with the source video (MP4 upload or YouTube) into a karaoke MP4 | | DELETE | `/api/jobs/{id}` | Remove job dir from disk (terminal jobs only) | --- diff --git a/app/api/stems.py b/app/api/stems.py index 964d108d..66badf0e 100644 --- a/app/api/stems.py +++ b/app/api/stems.py @@ -59,6 +59,27 @@ def _validate_stem_path(job_id: str, name: str): return path +def _parse_lane_gains(stems: str, gains: str) -> tuple[list[str], list[float]]: + """Parse and validate parallel comma-separated lane names and linear gains. + Shared by the audio mixdown and the karaoke-video mux. Raises HTTPException + on malformed input, unknown lanes, or out-of-range gains.""" + names = [s for s in stems.split(",") if s] + raw_gains = [g for g in gains.split(",") if g] + if not names or len(names) != len(raw_gains): + raise HTTPException( + status_code=422, detail="stems and gains must be non-empty and equal length" + ) + try: + parsed_gains = [float(g) for g in raw_gains] + except ValueError: + raise HTTPException(status_code=422, detail="gains must be numbers") from None + if any(g < 0 or g > _MIXDOWN_MAX_GAIN for g in parsed_gains): + raise HTTPException(status_code=422, detail="gain out of range") + if not set(names) <= _MIXDOWN_NAMES: + raise HTTPException(status_code=422, detail="unknown stem requested") + return names, parsed_gains + + async def _stream_ffmpeg(cmd: list[str]): """Yield ffmpeg stdout in 64 KB chunks; kill process on client disconnect.""" proc = await asyncio.create_subprocess_exec( @@ -198,20 +219,7 @@ async def get_mixdown( if ext not in ("wav", "mp3", "flac"): raise HTTPException(status_code=404, detail="not found") - names = [s for s in stems.split(",") if s] - raw_gains = [g for g in gains.split(",") if g] - if not names or len(names) != len(raw_gains): - raise HTTPException( - status_code=422, detail="stems and gains must be non-empty and equal length" - ) - try: - parsed_gains = [float(g) for g in raw_gains] - except ValueError: - raise HTTPException(status_code=422, detail="gains must be numbers") from None - if any(g < 0 or g > _MIXDOWN_MAX_GAIN for g in parsed_gains): - raise HTTPException(status_code=422, detail="gain out of range") - if not set(names) <= _MIXDOWN_NAMES: - raise HTTPException(status_code=422, detail="unknown stem requested") + names, parsed_gains = _parse_lane_gains(stems, gains) if (start is None) != (end is None) or (start is not None and start >= end): raise HTTPException( status_code=422, @@ -255,6 +263,70 @@ def _safe_title(title: str | None) -> str: return safe or "stems" +@router.get("/jobs/{job_id}/video.mp4", response_model=None) +async def get_video_mixdown( + job_id: str, + stems: str = Query(..., description="Comma-separated lane names to sum"), + gains: str = Query(..., description="Comma-separated linear gains, parallel to stems"), +) -> StreamingResponse: + """Mux a fresh audio mixdown of the current mixer state with the job's preserved + video into a karaoke MP4 (issue #219). Mirrors get_mixdown's audio graph (encoded + as AAC) and stream-copies video.mp4 -- the silent video kept from an .mp4 upload + or the real video stream downloaded for a YouTube job. 404 when the job has no + video (SoundCloud / plain audio uploads). + + Streamed as fragmented MP4 (frag_keyframe+empty_moov) since the output pipe is + not seekable -- +faststart would require a seekable file. The full song is + exported; no region trim, to avoid A/V drift from stream-copy seeking.""" + if not JOB_ID_RE.match(job_id): + raise HTTPException(status_code=404, detail="job not found") + job = registry_get(job_id) + if job is None or job.status != "done": + raise HTTPException(status_code=404, detail="job not ready") + + video_path = (JOBS_DIR / job_id / "video.mp4").resolve() + if not video_path.is_file() or not video_path.is_relative_to(JOBS_DIR.resolve()): + raise HTTPException(status_code=404, detail="no video track for this job") + + names, parsed_gains = _parse_lane_gains(stems, gains) + # Validates job_id (404), job done (404), and path traversal (404) per stem. + paths = [_validate_stem_path(job_id, name) for name in names] + + cmd: list[str] = [ffmpeg_executable(), "-nostdin", "-loglevel", "error"] + for p in paths: + cmd += ["-i", str(p)] + cmd += ["-i", str(video_path)] + video_idx = len(paths) + # Per-lane gain then amix (normalize=0 keeps levels faithful). A single audible + # lane skips amix (a 1-input amix is a no-op), matching get_mixdown. + filters = [f"[{i}:a]volume={g:.6f}[a{i}]" for i, g in enumerate(parsed_gains)] + n = len(paths) + if n > 1: + labels = "".join(f"[a{i}]" for i in range(n)) + filters.append(f"{labels}amix=inputs={n}:normalize=0[mix]") + out_label = "[mix]" + else: + out_label = "[a0]" + cmd += [ + "-filter_complex", ";".join(filters), + "-map", out_label, + "-map", f"{video_idx}:v", + "-c:v", "copy", + "-c:a", "aac", "-b:a", "192k", + "-shortest", + "-movflags", "frag_keyframe+empty_moov", + "-f", "mp4", + "pipe:1", + ] + + filename = f"{_safe_title(job.title)}_karaoke.mp4" + return StreamingResponse( + _stream_ffmpeg(cmd), + media_type="video/mp4", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + def _build_stems_zip(sources: list[tuple[str, Path]], fmt: str, dest: Path) -> None: """Blocking: write the stems into a ZIP. WAV files are stored as-is; MP3 and FLAC are transcoded per stem via ffmpeg. ZIP_STORED throughout - audio doesn't diff --git a/app/core/config.py b/app/core/config.py index 1e2481b5..a685ec4e 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -74,6 +74,9 @@ def _detect_device() -> str: TIMEOUT_FFMPEG = _env_int("STEMDECK_TIMEOUT_FFMPEG", 300) TIMEOUT_ANALYZE = _env_int("STEMDECK_TIMEOUT_ANALYZE", 120) TIMEOUT_DEMUCS_STALL = _env_int("STEMDECK_TIMEOUT_DEMUCS_STALL", 1800) +# Max height for the karaoke-MP4 video stream pulled from YouTube (issue #219). +# Capped to keep downloads reasonable; 1080p of a full song is large. +VIDEO_MAX_HEIGHT = max(144, _env_int("STEMDECK_VIDEO_MAX_HEIGHT", 720)) def ffmpeg_executable() -> str: diff --git a/app/core/models.py b/app/core/models.py index a99f5207..b2a76fe3 100644 --- a/app/core/models.py +++ b/app/core/models.py @@ -52,6 +52,9 @@ class Job: selected_stems: list[str] = field(default_factory=list) mix_url: str | None = None # populated when a strict subset was selected source_url: str | None = None # original URL or "local:" for file uploads + # True when a silent video track (video.mp4) was preserved from an .mp4 + # upload, enabling the "Export Mix (with video)" karaoke export. + has_video: bool = False error: str | None = None # Set by POST /api/jobs/{id}/cancel; consumed by pipeline stages. # Not surfaced via to_state() -- it's internal control state. @@ -84,6 +87,7 @@ def to_state(self) -> dict[str, Any]: "selected_stems": self.selected_stems, "mix_url": self.mix_url, "source_url": self.source_url, + "has_video": self.has_video, "error": self.error, "created_at": self.created_at, } diff --git a/app/pipeline/download.py b/app/pipeline/download.py index 2c759d4c..4224a056 100644 --- a/app/pipeline/download.py +++ b/app/pipeline/download.py @@ -8,7 +8,7 @@ from yt_dlp import YoutubeDL -from app.core.config import MAX_DURATION_SEC +from app.core.config import FFMPEG_DIR, MAX_DURATION_SEC, VIDEO_MAX_HEIGHT from app.core.models import Job, JobCancelled, _set logger = logging.getLogger("stemdeck.download") @@ -148,6 +148,68 @@ def normalize_youtube_url(url: str) -> str: return url +def _download_video_track(job: Job, url: str, job_dir: Path) -> None: + """Best-effort: download a video-only H.264/MP4 stream to video.mp4 for the + karaoke-MP4 export (issue #219). The audio source is downloaded separately as + usual; this is a second, additive fetch so the audio pipeline is untouched. + + Video-only MP4 needs no ffmpeg merge, so this can't break an audio-only job: + any failure (no progressive MP4 video, network error, unsupported codec) is + logged and swallowed, leaving has_video False. A cancel mid-download raises + JobCancelled, which the runner treats like any other cancellation. + + Capped at VIDEO_MAX_HEIGHT to keep downloads reasonable -- a full song at + 1080p is large, and karaoke playback doesn't need it.""" + + def vhook(d: dict) -> None: + if job.cancel_requested: + raise JobCancelled() + if d.get("status") == "downloading": + total = d.get("total_bytes") or d.get("total_bytes_estimate") + if total: + p = float(d.get("downloaded_bytes", 0)) / float(total) + _set(job, stage=f"Fetching video {int(p * 100)}%") + + # Prefer H.264 (avc1) so the exported MP4 plays everywhere -- YouTube also + # serves AV1/VP9 in mp4 containers, which many players (Safari/iOS, older + # devices) can't decode. Fall back to any <=cap mp4 only if no avc1 exists. + ydl_opts = { + "format": ( + f"bestvideo[height<={VIDEO_MAX_HEIGHT}][vcodec^=avc1]" + f"/bestvideo[height<={VIDEO_MAX_HEIGHT}][ext=mp4]" + ), + "outtmpl": str(job_dir / "video.%(ext)s"), + "quiet": True, + "noprogress": True, + "noplaylist": True, + "allowed_extractors": _ALLOWED_EXTRACTORS, + "progress_hooks": [vhook], + } + # Point yt-dlp at the bundled ffmpeg in case a DASH stream needs remuxing; + # in portable builds ffmpeg is not on PATH. + if FFMPEG_DIR.is_dir(): + ydl_opts["ffmpeg_location"] = str(FFMPEG_DIR) + + _set(job, stage="Fetching video...") + try: + with YoutubeDL(ydl_opts) as ydl: + ydl.extract_info(url, download=True) + except JobCancelled: + raise + except Exception as exc: + if job.cancel_requested: + raise JobCancelled() from exc + logger.warning("[%s] video track unavailable (audio-only): %s", job.id, exc) + + video = job_dir / "video.mp4" + if video.is_file() and video.stat().st_size > 0: + job.has_video = True + else: + # Drop any partial/non-mp4 leftover so the export endpoint sees nothing. + for f in job_dir.glob("video.*"): + f.unlink(missing_ok=True) + + def download(job: Job, url: str, job_dir: Path) -> Path: url = normalize_youtube_url(url) logger.info("[%s] download starting: %s", job.id, url) @@ -177,6 +239,10 @@ def hook(d: dict) -> None: elif d.get("status") == "finished": _set(job, progress=1.0, stage="Download complete") + # YouTube jobs additionally fetch the real video stream (below) for the + # karaoke-MP4 export (issue #219). SoundCloud is audio-only and excluded. + is_youtube = url.startswith("https://www.youtube.com/") + # No postprocessors -- Demucs reads the raw audio container (webm/m4a/opus/...) # directly via torchaudio + ffmpeg. Skipping the WAV transcode saves the slowest # part of the download pipeline and a lot of disk. @@ -229,6 +295,11 @@ def hook(d: dict) -> None: deduped = [t for t in raw_tags if not (t in seen or seen.add(t))] # type: ignore[func-returns-value] _set(job, tags=deduped[:8] or None) + # Best-effort: fetch the real video stream for the karaoke-MP4 export. + # Non-fatal -- on any failure the job proceeds audio-only. + if is_youtube: + _download_video_track(job, url, job_dir) + candidates = sorted(job_dir.glob("source.*")) if not candidates: raise RuntimeError("yt-dlp finished but no source file was produced") diff --git a/app/pipeline/runner.py b/app/pipeline/runner.py index 34f36ac9..688c2a44 100644 --- a/app/pipeline/runner.py +++ b/app/pipeline/runner.py @@ -42,13 +42,51 @@ def _check_cancel(job: Job) -> None: raise JobCancelled() +def _extract_video_track(job: Job, source: Path, job_dir: Path) -> None: + """For an .mp4 upload, preserve a silent video-only track at + video.mp4 so the studio can later mux it with a custom stem mix + into a karaoke video (issue #219). Stream-copies the video (no + re-encode) -- fast and lossless. + + Best-effort: an .mp4 with no video stream (audio-only container) + fails harmlessly and leaves has_video false.""" + from app.core.config import ffmpeg_executable + + dest = job_dir / "video.mp4" + cmd = [ + ffmpeg_executable(), + "-nostdin", + "-loglevel", + "error", + "-i", + str(source), + "-an", # drop audio -- the mix is added at export time + "-c:v", + "copy", + "-movflags", + "+faststart", + "-y", + str(dest), + ] + result = subprocess.run(cmd, capture_output=True, timeout=TIMEOUT_FFMPEG) + if result.returncode != 0 or not dest.is_file() or dest.stat().st_size == 0: + dest.unlink(missing_ok=True) + logger.info( + "no video track preserved for job %s (source has no video stream?)", job.id + ) + return + job.has_video = True + + def _prepare_local_source(job: Job, source: Path, job_dir: Path) -> Path: """Transcode any local upload to 16-bit 44.1 kHz stereo WAV before handing it to Demucs. Normalises MP3 and non-standard WAV formats (24-bit, 32-bit float, high sample rate, multi-channel) that Demucs would otherwise process silently and output as silence. - Deletes the original source file after a successful transcode.""" + For .mp4 uploads, first preserves a silent video.mp4 for later + karaoke-video export. Deletes the original source file after a + successful transcode.""" from app.core.config import ffmpeg_executable dest = job_dir / "source.wav" @@ -56,6 +94,8 @@ def _prepare_local_source(job: Job, source: Path, job_dir: Path) -> Path: return source _set(job, stage="Preparing audio...") + if source.suffix.lower() == ".mp4": + _extract_video_track(job, source, job_dir) cmd = [ ffmpeg_executable(), "-nostdin", @@ -146,6 +186,7 @@ def _write_metadata(job: Job, job_dir: Path) -> None: "tempo_stability": job.tempo_stability, "stem_presence": job.stem_presence, "tags": job.tags, + "has_video": job.has_video, } try: (job_dir / "metadata.json").write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8") diff --git a/static/css/daw.css b/static/css/daw.css index b83c2893..e6998f43 100644 --- a/static/css/daw.css +++ b/static/css/daw.css @@ -1996,6 +1996,16 @@ input, textarea { font-family: inherit; } .export-item:hover:not([aria-disabled="true"]) { background: var(--panel-3); } .export-item[aria-disabled="true"] { opacity: 0.4; pointer-events: none; } +/* MP4 is offered as an export format only when the job has a preserved video + track (mp4 upload or YouTube). player.js toggles .has-video on the wrap. */ +.export-fmt-video { display: none; } +#footer-export-wrap.has-video .export-fmt-video { display: block; flex: 1; } + +/* In MP4 mode only "Export Mix" applies — the audio-only Stems/Region rows + are hidden. main.js toggles .fmt-mp4 on the panel. */ +#t-export-panel.fmt-mp4 #t-export-stems, +#t-export-panel.fmt-mp4 #t-export-region { display: none; } + /* ── About dialog ── */ .about-backdrop { position: fixed; inset: 0; z-index: 100; diff --git a/static/index.html b/static/index.html index 4c237898..ddacfbae 100644 --- a/static/index.html +++ b/static/index.html @@ -603,6 +603,7 @@ +