feat: export as MP4 (karaoke video) for MP4 uploads and YouTube

README.md

@@ -309,6 +309,7 @@ Stems land in `./jobs/` on the host. Demucs weights are cached in a named volume
 | PATCH | `/api/jobs/{id}/sections` | Save waveform section markers for a job |
 | GET | `/api/jobs/{id}/stems/{name}.wav` | Stream a single stem WAV file |
 | GET | `/api/jobs/{id}/stems/{name}.mp3` | Transcode and stream a stem as MP3 |
+| GET | `/api/jobs/{id}/video.mp4` | Mux the current mix with the source video (MP4 upload or YouTube) into a karaoke MP4 |
 | DELETE | `/api/jobs/{id}` | Remove job dir from disk (terminal jobs only) |
 
 ---

app/api/stems.py

@@ -59,6 +59,27 @@ def _validate_stem_path(job_id: str, name: str):
     return path
 
 
+def _parse_lane_gains(stems: str, gains: str) -> tuple[list[str], list[float]]:
+    """Parse and validate parallel comma-separated lane names and linear gains.
+    Shared by the audio mixdown and the karaoke-video mux. Raises HTTPException
+    on malformed input, unknown lanes, or out-of-range gains."""
+    names = [s for s in stems.split(",") if s]
+    raw_gains = [g for g in gains.split(",") if g]
+    if not names or len(names) != len(raw_gains):
+        raise HTTPException(
+            status_code=422, detail="stems and gains must be non-empty and equal length"
+        )
+    try:
+        parsed_gains = [float(g) for g in raw_gains]
+    except ValueError:
+        raise HTTPException(status_code=422, detail="gains must be numbers") from None
+    if any(g < 0 or g > _MIXDOWN_MAX_GAIN for g in parsed_gains):
+        raise HTTPException(status_code=422, detail="gain out of range")
+    if not set(names) <= _MIXDOWN_NAMES:
+        raise HTTPException(status_code=422, detail="unknown stem requested")
+    return names, parsed_gains
+
+
 async def _stream_ffmpeg(cmd: list[str]):
     """Yield ffmpeg stdout in 64 KB chunks; kill process on client disconnect."""
     proc = await asyncio.create_subprocess_exec(
@@ -198,20 +219,7 @@ async def get_mixdown(
     if ext not in ("wav", "mp3", "flac"):
         raise HTTPException(status_code=404, detail="not found")
 
-    names = [s for s in stems.split(",") if s]
-    raw_gains = [g for g in gains.split(",") if g]
-    if not names or len(names) != len(raw_gains):
-        raise HTTPException(
-            status_code=422, detail="stems and gains must be non-empty and equal length"
-        )
-    try:
-        parsed_gains = [float(g) for g in raw_gains]
-    except ValueError:
-        raise HTTPException(status_code=422, detail="gains must be numbers") from None
-    if any(g < 0 or g > _MIXDOWN_MAX_GAIN for g in parsed_gains):
-        raise HTTPException(status_code=422, detail="gain out of range")
-    if not set(names) <= _MIXDOWN_NAMES:
-        raise HTTPException(status_code=422, detail="unknown stem requested")
+    names, parsed_gains = _parse_lane_gains(stems, gains)
     if (start is None) != (end is None) or (start is not None and start >= end):
         raise HTTPException(
             status_code=422,
@@ -255,6 +263,79 @@ def _safe_title(title: str | None) -> str:
     return safe or "stems"
 
 
+@router.get("/jobs/{job_id}/video.mp4", response_model=None)
+async def get_video_mixdown(
+    job_id: str,
+    stems: str = Query(..., description="Comma-separated lane names to sum"),
+    gains: str = Query(..., description="Comma-separated linear gains, parallel to stems"),
+) -> StreamingResponse:
+    """Mux a fresh audio mixdown of the current mixer state with the job's preserved
+    video into a karaoke MP4 (issue #219). Mirrors get_mixdown's audio graph (encoded
+    as AAC) and stream-copies video.mp4 -- the silent video kept from an .mp4 upload
+    or the real video stream downloaded for a YouTube job. 404 when the job has no
+    video (SoundCloud / plain audio uploads).
+
+    Streamed as fragmented MP4 (frag_keyframe+empty_moov) since the output pipe is
+    not seekable -- +faststart would require a seekable file. The full song is
+    exported; no region trim, to avoid A/V drift from stream-copy seeking."""
+    if not JOB_ID_RE.match(job_id):
+        raise HTTPException(status_code=404, detail="job not found")
+    job = registry_get(job_id)
+    if job is None or job.status != "done":
+        raise HTTPException(status_code=404, detail="job not ready")
+
+    video_path = (JOBS_DIR / job_id / "video.mp4").resolve()
+    if not video_path.is_file() or not video_path.is_relative_to(JOBS_DIR.resolve()):
+        raise HTTPException(status_code=404, detail="no video track for this job")
+
+    names, parsed_gains = _parse_lane_gains(stems, gains)
+    # Validates job_id (404), job done (404), and path traversal (404) per stem.
+    paths = [_validate_stem_path(job_id, name) for name in names]
+
+    cmd: list[str] = [ffmpeg_executable(), "-nostdin", "-loglevel", "error"]
+    for p in paths:
+        cmd += ["-i", str(p)]
+    cmd += ["-i", str(video_path)]
+    video_idx = len(paths)
+    # Per-lane gain then amix (normalize=0 keeps levels faithful). A single audible
+    # lane skips amix (a 1-input amix is a no-op), matching get_mixdown.
+    filters = [f"[{i}:a]volume={g:.6f}[a{i}]" for i, g in enumerate(parsed_gains)]
+    n = len(paths)
+    if n > 1:
+        labels = "".join(f"[a{i}]" for i in range(n))
+        filters.append(f"{labels}amix=inputs={n}:normalize=0[mix]")
+        out_label = "[mix]"
+    else:
+        out_label = "[a0]"
+    cmd += [
+        "-filter_complex",
+        ";".join(filters),
+        "-map",
+        out_label,
+        "-map",
+        f"{video_idx}:v",
+        "-c:v",
+        "copy",
+        "-c:a",
+        "aac",
+        "-b:a",
+        "192k",
+        "-shortest",
+        "-movflags",
+        "frag_keyframe+empty_moov",
+        "-f",
+        "mp4",
+        "pipe:1",
+    ]
+
+    filename = f"{_safe_title(job.title)}_karaoke.mp4"
+    return StreamingResponse(
+        _stream_ffmpeg(cmd),
+        media_type="video/mp4",
+        headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+    )
+
+
 def _build_stems_zip(sources: list[tuple[str, Path]], fmt: str, dest: Path) -> None:
     """Blocking: write the stems into a ZIP. WAV files are stored as-is; MP3 and
     FLAC are transcoded per stem via ffmpeg. ZIP_STORED throughout - audio doesn't

app/core/config.py

@@ -74,6 +74,9 @@ def _detect_device() -> str:
 TIMEOUT_FFMPEG = _env_int("STEMDECK_TIMEOUT_FFMPEG", 300)
 TIMEOUT_ANALYZE = _env_int("STEMDECK_TIMEOUT_ANALYZE", 120)
 TIMEOUT_DEMUCS_STALL = _env_int("STEMDECK_TIMEOUT_DEMUCS_STALL", 1800)
+# Max height for the karaoke-MP4 video stream pulled from YouTube (issue #219).
+# Capped to keep downloads reasonable; 1080p of a full song is large.
+VIDEO_MAX_HEIGHT = max(144, _env_int("STEMDECK_VIDEO_MAX_HEIGHT", 720))
 
 
 def ffmpeg_executable() -> str:

app/core/models.py

@@ -52,6 +52,9 @@ class Job:
     selected_stems: list[str] = field(default_factory=list)
     mix_url: str | None = None  # populated when a strict subset was selected
     source_url: str | None = None  # original URL or "local:<filename>" for file uploads
+    # True when a silent video track (video.mp4) was preserved from an .mp4
+    # upload, enabling the "Export Mix (with video)" karaoke export.
+    has_video: bool = False
     error: str | None = None
     # Set by POST /api/jobs/{id}/cancel; consumed by pipeline stages.
     # Not surfaced via to_state() -- it's internal control state.
@@ -84,6 +87,7 @@ def to_state(self) -> dict[str, Any]:
             "selected_stems": self.selected_stems,
             "mix_url": self.mix_url,
             "source_url": self.source_url,
+            "has_video": self.has_video,
             "error": self.error,
             "created_at": self.created_at,
         }

app/pipeline/download.py

@@ -8,7 +8,7 @@
 
 from yt_dlp import YoutubeDL
 
-from app.core.config import MAX_DURATION_SEC
+from app.core.config import FFMPEG_DIR, MAX_DURATION_SEC, VIDEO_MAX_HEIGHT
 from app.core.models import Job, JobCancelled, _set
 
 logger = logging.getLogger("stemdeck.download")
@@ -148,6 +148,68 @@ def normalize_youtube_url(url: str) -> str:
     return url
 
 
+def _download_video_track(job: Job, url: str, job_dir: Path) -> None:
+    """Best-effort: download a video-only H.264/MP4 stream to video.mp4 for the
+    karaoke-MP4 export (issue #219). The audio source is downloaded separately as
+    usual; this is a second, additive fetch so the audio pipeline is untouched.
+
+    Video-only MP4 needs no ffmpeg merge, so this can't break an audio-only job:
+    any failure (no progressive MP4 video, network error, unsupported codec) is
+    logged and swallowed, leaving has_video False. A cancel mid-download raises
+    JobCancelled, which the runner treats like any other cancellation.
+
+    Capped at VIDEO_MAX_HEIGHT to keep downloads reasonable -- a full song at
+    1080p is large, and karaoke playback doesn't need it."""
+
+    def vhook(d: dict) -> None:
+        if job.cancel_requested:
+            raise JobCancelled()
+        if d.get("status") == "downloading":
+            total = d.get("total_bytes") or d.get("total_bytes_estimate")
+            if total:
+                p = float(d.get("downloaded_bytes", 0)) / float(total)
+                _set(job, stage=f"Fetching video {int(p * 100)}%")
+
+    # Prefer H.264 (avc1) so the exported MP4 plays everywhere -- YouTube also
+    # serves AV1/VP9 in mp4 containers, which many players (Safari/iOS, older
+    # devices) can't decode. Fall back to any <=cap mp4 only if no avc1 exists.
+    ydl_opts = {
+        "format": (
+            f"bestvideo[height<={VIDEO_MAX_HEIGHT}][vcodec^=avc1]"
+            f"/bestvideo[height<={VIDEO_MAX_HEIGHT}][ext=mp4]"
+        ),
+        "outtmpl": str(job_dir / "video.%(ext)s"),
+        "quiet": True,
+        "noprogress": True,
+        "noplaylist": True,
+        "allowed_extractors": _ALLOWED_EXTRACTORS,
+        "progress_hooks": [vhook],
+    }
+    # Point yt-dlp at the bundled ffmpeg in case a DASH stream needs remuxing;
+    # in portable builds ffmpeg is not on PATH.
+    if FFMPEG_DIR.is_dir():
+        ydl_opts["ffmpeg_location"] = str(FFMPEG_DIR)
+
+    _set(job, stage="Fetching video...")
+    try:
+        with YoutubeDL(ydl_opts) as ydl:
+            ydl.extract_info(url, download=True)
+    except JobCancelled:
+        raise
+    except Exception as exc:
+        if job.cancel_requested:
+            raise JobCancelled() from exc
+        logger.warning("[%s] video track unavailable (audio-only): %s", job.id, exc)
+
+    video = job_dir / "video.mp4"
+    if video.is_file() and video.stat().st_size > 0:
+        job.has_video = True
+    else:
+        # Drop any partial/non-mp4 leftover so the export endpoint sees nothing.
+        for f in job_dir.glob("video.*"):
+            f.unlink(missing_ok=True)
+
+
 def download(job: Job, url: str, job_dir: Path) -> Path:
     url = normalize_youtube_url(url)
     logger.info("[%s] download starting: %s", job.id, url)
@@ -177,6 +239,10 @@ def hook(d: dict) -> None:
         elif d.get("status") == "finished":
             _set(job, progress=1.0, stage="Download complete")
 
+    # YouTube jobs additionally fetch the real video stream (below) for the
+    # karaoke-MP4 export (issue #219). SoundCloud is audio-only and excluded.
+    is_youtube = url.startswith("https://www.youtube.com/")
+
     # No postprocessors -- Demucs reads the raw audio container (webm/m4a/opus/...)
     # directly via torchaudio + ffmpeg. Skipping the WAV transcode saves the slowest
     # part of the download pipeline and a lot of disk.
@@ -229,6 +295,11 @@ def hook(d: dict) -> None:
     deduped = [t for t in raw_tags if not (t in seen or seen.add(t))]  # type: ignore[func-returns-value]
     _set(job, tags=deduped[:8] or None)
 
+    # Best-effort: fetch the real video stream for the karaoke-MP4 export.
+    # Non-fatal -- on any failure the job proceeds audio-only.
+    if is_youtube:
+        _download_video_track(job, url, job_dir)
+
     candidates = sorted(job_dir.glob("source.*"))
     if not candidates:
         raise RuntimeError("yt-dlp finished but no source file was produced")

app/pipeline/runner.py

@@ -42,20 +42,58 @@ def _check_cancel(job: Job) -> None:
         raise JobCancelled()
 
 
+def _extract_video_track(job: Job, source: Path, job_dir: Path) -> None:
+    """For an .mp4 upload, preserve a silent video-only track at
+    video.mp4 so the studio can later mux it with a custom stem mix
+    into a karaoke video (issue #219). Stream-copies the video (no
+    re-encode) -- fast and lossless.
+
+    Best-effort: an .mp4 with no video stream (audio-only container)
+    fails harmlessly and leaves has_video false."""
+    from app.core.config import ffmpeg_executable
+
+    dest = job_dir / "video.mp4"
+    cmd = [
+        ffmpeg_executable(),
+        "-nostdin",
+        "-loglevel",
+        "error",
+        "-i",
+        str(source),
+        "-an",  # drop audio -- the mix is added at export time
+        "-c:v",
+        "copy",
+        "-movflags",
+        "+faststart",
+        "-y",
+        str(dest),
+    ]
+    result = subprocess.run(cmd, capture_output=True, timeout=TIMEOUT_FFMPEG)
+    if result.returncode != 0 or not dest.is_file() or dest.stat().st_size == 0:
+        dest.unlink(missing_ok=True)
+        logger.info("no video track preserved for job %s (source has no video stream?)", job.id)
+        return
+    job.has_video = True
+
+
 def _prepare_local_source(job: Job, source: Path, job_dir: Path) -> Path:
     """Transcode any local upload to 16-bit 44.1 kHz stereo WAV before
     handing it to Demucs. Normalises MP3 and non-standard WAV formats
     (24-bit, 32-bit float, high sample rate, multi-channel) that Demucs
     would otherwise process silently and output as silence.
 
-    Deletes the original source file after a successful transcode."""
+    For .mp4 uploads, first preserves a silent video.mp4 for later
+    karaoke-video export. Deletes the original source file after a
+    successful transcode."""
     from app.core.config import ffmpeg_executable
 
     dest = job_dir / "source.wav"
     if source.resolve() == dest.resolve():
         return source
 
     _set(job, stage="Preparing audio...")
+    if source.suffix.lower() == ".mp4":
+        _extract_video_track(job, source, job_dir)
     cmd = [
         ffmpeg_executable(),
         "-nostdin",
@@ -146,6 +184,7 @@ def _write_metadata(job: Job, job_dir: Path) -> None:
         "tempo_stability": job.tempo_stability,
         "stem_presence": job.stem_presence,
         "tags": job.tags,
+        "has_video": job.has_video,
     }
     try:
         (job_dir / "metadata.json").write_text(json.dumps(meta, indent=2) + "\n", encoding="utf-8")