From 9ce88f91ab811d496387255fc1d446bbeb7e7c47 Mon Sep 17 00:00:00 2001 From: Anai-Guo Date: Mon, 31 Aug 2026 12:13:06 -0700 Subject: [PATCH] fix(sft): pass use_audio_in_video and unpack load_video in SFT media loaders `load_video(video, use_audio_in_video, **kwargs)` takes `use_audio_in_video` as a required parameter and returns a 4-tuple `(video, video_meta, audio, audio_meta)`. Both SFT media loaders called it as `load_video(p)`, which raises `TypeError: load_video() missing 1 required positional argument: 'use_audio_in_video'` and, once that is fixed, would still store the whole 4-tuple where downstream expects a `torch.Tensor`. Follow the canonical usage in `relax/utils/multimodal/process.py`, which unpacks the tuple and takes the video tensor, and use `use_audio_in_video=False` to match the default every other definition in the repo already uses. --- relax/engine/sft/dataset/multimodal.py | 2 +- relax/engine/sft/predict/loop.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/relax/engine/sft/dataset/multimodal.py b/relax/engine/sft/dataset/multimodal.py index 7a91e28d8..46b992e48 100644 --- a/relax/engine/sft/dataset/multimodal.py +++ b/relax/engine/sft/dataset/multimodal.py @@ -37,7 +37,7 @@ def _fetch_media(sample: CanonicalSample, rendered_text: str) -> tuple[dict[str, if sample.images: mm_inputs["images"] = [load_image(p) for p in sample.images] if sample.videos: - mm_inputs["videos"] = [load_video(p) for p in sample.videos] + mm_inputs["videos"] = [load_video(p, use_audio_in_video=False)[0] for p in sample.videos] if sample.audios: mm_inputs["audios"] = [load_audio(p) for p in sample.audios] return mm_inputs, rendered_text diff --git a/relax/engine/sft/predict/loop.py b/relax/engine/sft/predict/loop.py index d481f1ced..e4f2f9746 100644 --- a/relax/engine/sft/predict/loop.py +++ b/relax/engine/sft/predict/loop.py @@ -47,7 +47,7 @@ def _build_multimodal_inputs(sample: CanonicalSample) -> dict | None: return { "images": [load_image(p) for p in (sample.images or [])], - "videos": [load_video(p) for p in (sample.videos or [])], + "videos": [load_video(p, use_audio_in_video=False)[0] for p in (sample.videos or [])], "audio": [load_audio(p) for p in (sample.audios or [])], }