From c278561261dee14174352ee61cadf372d701baf0 Mon Sep 17 00:00:00 2001 From: danni <11447317+dannigt@users.noreply.github.com> Date: Tue, 17 Mar 2026 16:43:30 +0100 Subject: [PATCH 1/4] Added qwen3omni --- baselines/main.py | 6 ++ baselines/models/mllm/qwen3_omni.py | 117 ++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 baselines/models/mllm/qwen3_omni.py diff --git a/baselines/main.py b/baselines/main.py index 56d73ff..2302c73 100644 --- a/baselines/main.py +++ b/baselines/main.py @@ -31,6 +31,8 @@ from models.mllm.ola import load_model as load_ola from models.mllm.qwen_omni import generate as generate_qwen_omni from models.mllm.qwen_omni import load_model as load_qwen_omni +from models.mllm.qwen3_omni import generate as generate_qwen3_omni +from models.mllm.qwen3_omni import load_model as load_qwen3_omni # import speech models from models.speech.desta import generate as generate_desta @@ -141,6 +143,9 @@ def load_model(model_name): elif model_name == "qwen_omni": model = load_qwen_omni() generate_func = generate_qwen_omni + elif model_name == "qwen3_omni": + model = load_qwen3_omni() + generate_func = generate_qwen3_omni elif model_name == "ming_lite_omni": model = load_ming_lite_omni() generate_func = generate_ming_lite_omni @@ -228,6 +233,7 @@ def main(in_data_folder, out_folder, model, lang, track, modality, prompt): "ming_lite_omni", "ola", "qwen_omni", + "qwen3_omni", "gpt_oss", "gemini", ] diff --git a/baselines/models/mllm/qwen3_omni.py b/baselines/models/mllm/qwen3_omni.py new file mode 100644 index 0000000..c730946 --- /dev/null +++ b/baselines/models/mllm/qwen3_omni.py @@ -0,0 +1,117 @@ +# Copyright 2025 FBK, KIT + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +from utils import read_txt_file + + +def load_model(): + from transformers import Qwen3OmniMoeForConditionalGeneration, Qwen3OmniMoeProcessor + + model = Qwen3OmniMoeForConditionalGeneration.from_pretrained( + "Qwen/Qwen3-Omni-30B-A3B-Instruct", + torch_dtype="auto", + device_map="auto", + attn_implementation="flash_attention_2", + ) + processor = Qwen3OmniMoeProcessor.from_pretrained("Qwen/Qwen3-Omni-30B-A3B-Instruct") + + return model, processor + + +def generate(model_processor, prompt, example_path, modality): + from qwen_omni_utils import process_mm_info + + model, processor = model_processor + + if modality == "mllm": + USE_AUDIO_IN_VIDEO = True + user_conv_content = [ + {"type": "video", "video": example_path}, + {"type": "text", "text": prompt}, + ] + + elif modality == "video": + USE_AUDIO_IN_VIDEO = False + user_conv_content = [ + {"type": "video", "video": example_path}, + {"type": "text", "text": prompt}, + ] + + elif modality == "audio": + USE_AUDIO_IN_VIDEO = False + user_conv_content = [ + {"type": "audio", "audio": example_path}, + {"type": "text", "text": prompt}, + ] + + elif modality == "text": + USE_AUDIO_IN_VIDEO = False + example = read_txt_file(example_path) + user_conv_content = [ + {"type": "text", "text": f"{example}\n{prompt}\n"}, + ] + + system_conv = { + "role": "system", + "content": [ + { + "type": "text", + "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, " + "capable of perceiving auditory and visual inputs, as well as generating " + "text and speech. Only return the answer requested. Do not include any " + "explanation or introductions.", + } + ], + } + + user_conv = { + "role": "user", + "content": user_conv_content, + } + + conversation = [system_conv, user_conv] + + # Preparation for inference + text = processor.apply_chat_template( + conversation, add_generation_prompt=True, tokenize=False + ) + audios, images, videos = process_mm_info( + conversation, use_audio_in_video=USE_AUDIO_IN_VIDEO + ) + inputs = processor( + text=text, + audio=audios, + images=images, + videos=videos, + return_tensors="pt", + padding=True, + use_audio_in_video=USE_AUDIO_IN_VIDEO, + ) + inputs = inputs.to(model.device).to(model.dtype) + + # Inference: Generation of the output text + generate_output = model.generate( + **inputs, + use_audio_in_video=USE_AUDIO_IN_VIDEO, + return_audio=False, + max_new_tokens=4096, + ) + text_ids = generate_output[0] if isinstance(generate_output, (tuple, list)) else generate_output + text = processor.batch_decode( + text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False + ) + + # postprocess + response = text[-1].split("\nassistant")[-1].strip() + return response From 408c93dbd6ec0a586d07b57e4de6f1eab8d7017c Mon Sep 17 00:00:00 2001 From: danni <11447317+dannigt@users.noreply.github.com> Date: Tue, 17 Mar 2026 16:54:12 +0100 Subject: [PATCH 2/4] added documenta and made output handling more structured --- baselines/models/mllm/qwen3_omni.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/baselines/models/mllm/qwen3_omni.py b/baselines/models/mllm/qwen3_omni.py index c730946..67a8482 100644 --- a/baselines/models/mllm/qwen3_omni.py +++ b/baselines/models/mllm/qwen3_omni.py @@ -101,13 +101,15 @@ def generate(model_processor, prompt, example_path, modality): inputs = inputs.to(model.device).to(model.dtype) # Inference: Generation of the output text - generate_output = model.generate( + # Qwen3-Omni always returns a (text_ids, audio) tuple; with return_audio=False the + # second element is None. thinker_max_new_tokens/thinker_do_sample are Qwen2.5-Omni + # specific parameters and are not supported by Qwen3-Omni. + text_ids, _ = model.generate( **inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO, return_audio=False, max_new_tokens=4096, ) - text_ids = generate_output[0] if isinstance(generate_output, (tuple, list)) else generate_output text = processor.batch_decode( text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False ) From 6094e6044e9c09502288865de229b16004da2108 Mon Sep 17 00:00:00 2001 From: dannigt <11447317+dannigt@users.noreply.github.com> Date: Wed, 18 Mar 2026 15:46:23 +0100 Subject: [PATCH 3/4] Update baselines/models/mllm/qwen3_omni.py Co-authored-by: sarapapi <57095209+sarapapi@users.noreply.github.com> --- baselines/models/mllm/qwen3_omni.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/baselines/models/mllm/qwen3_omni.py b/baselines/models/mllm/qwen3_omni.py index 67a8482..443e0d4 100644 --- a/baselines/models/mllm/qwen3_omni.py +++ b/baselines/models/mllm/qwen3_omni.py @@ -1,4 +1,4 @@ -# Copyright 2025 FBK, KIT +# Copyright 2026 FBK, KIT # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 090be68e26365a6163fe81af22c8f6fa60f340cf Mon Sep 17 00:00:00 2001 From: dannigt <11447317+dannigt@users.noreply.github.com> Date: Wed, 18 Mar 2026 15:46:41 +0100 Subject: [PATCH 4/4] Update baselines/models/mllm/qwen3_omni.py Co-authored-by: sarapapi <57095209+sarapapi@users.noreply.github.com> --- baselines/models/mllm/qwen3_omni.py | 1 - 1 file changed, 1 deletion(-) diff --git a/baselines/models/mllm/qwen3_omni.py b/baselines/models/mllm/qwen3_omni.py index 443e0d4..a5e0ec1 100644 --- a/baselines/models/mllm/qwen3_omni.py +++ b/baselines/models/mllm/qwen3_omni.py @@ -100,7 +100,6 @@ def generate(model_processor, prompt, example_path, modality): ) inputs = inputs.to(model.device).to(model.dtype) - # Inference: Generation of the output text # Qwen3-Omni always returns a (text_ids, audio) tuple; with return_audio=False the # second element is None. thinker_max_new_tokens/thinker_do_sample are Qwen2.5-Omni # specific parameters and are not supported by Qwen3-Omni.