From 5f5b5dcd05826d1c0c87f92f424cb7adfbef408f Mon Sep 17 00:00:00 2001 From: spapi Date: Mon, 13 Jul 2026 15:29:58 +0200 Subject: [PATCH 1/7] Add try-except given the new version-less naming of the MCIF xml files --- baselines/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/baselines/utils.py b/baselines/utils.py index 072b972..ebfa523 100644 --- a/baselines/utils.py +++ b/baselines/utils.py @@ -69,9 +69,11 @@ def read_from_xml(folder_path, lang, track, modality, prompt, version=__benchmar if modality == "text" and track == "short": raise ValueError("Text-to-text is not available in the short track.") - xml_path = f"{folder_path}/MCIF{version}.IF.{track}.{lang}.src.{prompt}prompts.xml" + try: + tree = ET.parse(f"{folder_path}/MCIF.IF.{track}.{lang}.src.{prompt}prompts.xml") + except FileNotFoundError: + tree = ET.parse(f"{folder_path}/MCIF{version}.IF.{track}.{lang}.src.{prompt}prompts.xml") - tree = ET.parse(xml_path) root = tree.getroot() # List to hold the tuples From 33afa6773a35b8ec5a660d517624e1a6dd9f5ec3 Mon Sep 17 00:00:00 2001 From: spapi Date: Mon, 13 Jul 2026 16:41:58 +0200 Subject: [PATCH 2/7] Load MCIF data directly from HF. --- baselines/utils.py | 55 +++++++++++++++++++++++++--------------------- pyproject.toml | 3 ++- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/baselines/utils.py b/baselines/utils.py index ebfa523..12893cf 100644 --- a/baselines/utils.py +++ b/baselines/utils.py @@ -21,6 +21,8 @@ from mcif import __benchmark_version__ from mcif.io import OutputSample, write_output +from datasets import load_dataset + TASK_ATTRIB = ["track", "text_lang"] @@ -69,40 +71,43 @@ def read_from_xml(folder_path, lang, track, modality, prompt, version=__benchmar if modality == "text" and track == "short": raise ValueError("Text-to-text is not available in the short track.") - try: - tree = ET.parse(f"{folder_path}/MCIF.IF.{track}.{lang}.src.{prompt}prompts.xml") - except FileNotFoundError: - tree = ET.parse(f"{folder_path}/MCIF{version}.IF.{track}.{lang}.src.{prompt}prompts.xml") + mcif_dataset = load_dataset("FBK-MT/MCIF", f"{track}_{prompt}prompt")["test"] + + prompt_key = f"prompt_{lang}" - root = tree.getroot() + data = [] - # List to hold the tuples - data_list = [] + for sample in mcif_dataset: + instruction = sample.get(prompt_key) - # Iterate over each 'sample' element in the XML - for sample in root.findall(".//sample"): - # Extract the sample id, instruction, and audio path - sample_id = sample.get("id") - instruction = sample.find("instruction").text - if modality != "mllm": - node = sample.find(f"{modality}_path") - if node is None: + # Skip samples without the requested prompt language + if instruction is None: + continue + + if modality == "audio": + if sample["audio"] is None: continue - example_path = ( - f"{folder_path}/{track.upper()}_{modality.upper()}S/{node.text}" - ) + example_path = os.path.join(folder_path, sample["audio"]) + elif modality == "mllm": - node = sample.find("video_path") - if node is None: + if sample["video"] is None: continue - example_path = f"{folder_path}/{track.upper()}_VIDEOS/{node.text}" + example_path = os.path.join(folder_path, sample["video"]) + else: - raise NotImplementedError(f"No example path found for modality {modality}") + raise NotImplementedError( + f"Unsupported modality: {modality}" + ) - # Append the tuple to the list - data_list.append((sample_id, instruction, example_path)) + data.append( + ( + sample["id"], + instruction, + example_path, + ) + ) - return data_list + return data def write_to_xml(outputs, lang, track, output_file): diff --git a/pyproject.toml b/pyproject.toml index a443d9f..0a155df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,8 @@ dependencies = [ "jiwer==3.0.5", "bert_score==0.3.13", "unbabel-comet==2.2.4", - "whisper_normalizer==0.0.10" + "whisper_normalizer==0.0.10", + "datasets" ] dynamic = ["version"] From e153a6f347560db33a5a65f6bc29f4513f602863 Mon Sep 17 00:00:00 2001 From: spapi Date: Mon, 13 Jul 2026 16:44:24 +0200 Subject: [PATCH 3/7] Revert unnecessary change --- baselines/utils.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/baselines/utils.py b/baselines/utils.py index 12893cf..d2546e4 100644 --- a/baselines/utils.py +++ b/baselines/utils.py @@ -99,13 +99,8 @@ def read_from_xml(folder_path, lang, track, modality, prompt, version=__benchmar f"Unsupported modality: {modality}" ) - data.append( - ( - sample["id"], - instruction, - example_path, - ) - ) + # Append the tuple to the list + data.append((sample["id"], instruction, example_path)) return data From 3ba2ef97c3ea229c47214f068c4b8bb3fa958a07 Mon Sep 17 00:00:00 2001 From: spapi Date: Mon, 13 Jul 2026 16:50:20 +0200 Subject: [PATCH 4/7] Remove unused imports --- baselines/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/baselines/utils.py b/baselines/utils.py index d2546e4..0fd5796 100644 --- a/baselines/utils.py +++ b/baselines/utils.py @@ -16,7 +16,6 @@ import logging import os import sys -import xml.etree.ElementTree as ET from mcif import __benchmark_version__ from mcif.io import OutputSample, write_output From 87a99229cd31ccd25c0e33c2ff506478455e2ec1 Mon Sep 17 00:00:00 2001 From: spapi Date: Mon, 13 Jul 2026 20:38:25 +0200 Subject: [PATCH 5/7] Address comment --- baselines/models/mllm/qwen3_omni.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/baselines/models/mllm/qwen3_omni.py b/baselines/models/mllm/qwen3_omni.py index 082656a..7f35653 100644 --- a/baselines/models/mllm/qwen3_omni.py +++ b/baselines/models/mllm/qwen3_omni.py @@ -104,12 +104,18 @@ def generate(model_processor, prompt, example_path, modality): # Qwen3-Omni always returns a (text_ids, audio) tuple; with return_audio=False the # second element is None. thinker_max_new_tokens/thinker_do_sample are Qwen2.5-Omni # specific parameters and are not supported by Qwen3-Omni. - text_ids, _ = model.generate( + result = model.generate( **inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO, return_audio=False, max_new_tokens=4096, ) + + if isinstance(result, tuple): + text_ids = result[0] + else: + text_ids = result + text = processor.batch_decode( text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False ) From becf5f6d8d99d73524707a7b636af2e6bb840b41 Mon Sep 17 00:00:00 2001 From: spapi Date: Mon, 13 Jul 2026 20:39:05 +0200 Subject: [PATCH 6/7] Revert "Address comment" This reverts commit 87a99229cd31ccd25c0e33c2ff506478455e2ec1. --- baselines/models/mllm/qwen3_omni.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/baselines/models/mllm/qwen3_omni.py b/baselines/models/mllm/qwen3_omni.py index 7f35653..082656a 100644 --- a/baselines/models/mllm/qwen3_omni.py +++ b/baselines/models/mllm/qwen3_omni.py @@ -104,18 +104,12 @@ def generate(model_processor, prompt, example_path, modality): # Qwen3-Omni always returns a (text_ids, audio) tuple; with return_audio=False the # second element is None. thinker_max_new_tokens/thinker_do_sample are Qwen2.5-Omni # specific parameters and are not supported by Qwen3-Omni. - result = model.generate( + text_ids, _ = model.generate( **inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO, return_audio=False, max_new_tokens=4096, ) - - if isinstance(result, tuple): - text_ids = result[0] - else: - text_ids = result - text = processor.batch_decode( text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False ) From 7278edb4b3ba48c20238f39ae36142947205d15a Mon Sep 17 00:00:00 2001 From: spapi Date: Mon, 13 Jul 2026 20:39:14 +0200 Subject: [PATCH 7/7] Address comment --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0a155df..a443d9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,8 +16,7 @@ dependencies = [ "jiwer==3.0.5", "bert_score==0.3.13", "unbabel-comet==2.2.4", - "whisper_normalizer==0.0.10", - "datasets" + "whisper_normalizer==0.0.10" ] dynamic = ["version"]