Skip to content

报告和代码 #244

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vansonquach69-cmyk wants to merge 1 commit into
base: main
Choose a base branch
from
Open

报告和代码 #244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 29 additions & 18 deletions openseek/competition/LongContext-ICL-Annotation/src/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@

# from method import build_prompt, select_examples, annotate

from method import build_prompt, select_examples
from method import build_prompt, select_examples, select_examples_M04, select_examples_M05, select_examples_M06, select_examples_M07, select_examples_M08, select_examples_M12, select_examples_M13, select_examples_M19, select_examples_M20, select_examples_M09, select_examples_M10, select_examples_M11, build_prompt_cot, build_prompt_by_task_type

from method import annotate_nvidia as annotate # For Nvidia GPU
# from method import annotate_ascend as annotate # For Huawei Ascend
# from method import annotate_nvidia as annotate # For Nvidia GPU
from method import annotate_ascend as annotate # For Huawei Ascend

TASK_FILES = {
1: './data/openseek-1_closest_integers.json',
2: './data/openseek-2_count_nouns_verbs.json',
3: './data/openseek-3_collatz_conjecture.json',
4: './data/openseek-4_conala_concat_strings.json',
5: './data/openseek-5_semeval_2018_task1_tweet_sadness_detection.json',
6: './data/openseek-6_mnli_same_genre_classification.json',
7: './data/openseek-7_jeopardy_answer_generation_all.json',
8: '../data/openseek-8_kernel_generation.json',
1: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-1_closest_integers.json',
2: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-2_count_nouns_verbs.json',
3: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-3_collatz_conjecture.json',
4: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-4_conala_concat_strings.json',
5: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-5_semeval_2018_task1_tweet_sadness_detection.json',
6: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-6_mnli_same_genre_classification.json',
7: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-7_jeopardy_answer_generation_all.json',
8: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-8_kernel_generation.json',
}

def parser_args():
Expand All @@ -30,7 +30,7 @@ def parser_args():
default='../outputs/',
help='Prefix path to save the evaluation logs.')
parser.add_argument('--tokenizer_path', type=str,
default='/share/project/wuhaiming/spaces/data_agent/OpenSeek-main/openseek/competition/LongContext-ICL-Annotation/src/Qwen3-4B')
default='/root/Qwen3-4B')
args = parser.parse_args()
return args

Expand All @@ -48,7 +48,7 @@ def evaluate(task_id:int,

task_name = task_dict['task_name']
task_description = task_dict['Definition'][0]
icl_examples = task_dict['examples'][:100]
icl_examples = task_dict['examples'][:50]
test_samples = task_dict['test_samples']

version = 1
Expand All @@ -62,26 +62,37 @@ def evaluate(task_id:int,
pass

examples_str = None
for test_sample in tqdm(test_samples, desc=f'Evaluation on Task {task_id}: {task_name}'):
# 从环境变量获取要使用的select_examples函数名称
select_method = os.environ.get('SELECT_METHOD', 'select_examples_M06')

# 获取对应的函数
select_func = globals().get(select_method, select_examples_M06)

for sample_idx, test_sample in enumerate(tqdm(test_samples, desc=f'Evaluation on Task {task_id}: {task_name}')):
test_record = dict()

test_sample_id = test_sample['id']
test_record['test_sample_id'] = test_sample_id


text2annotate = test_sample['input']
prompt = build_prompt(task_description, text2annotate)

# M03优化：使用任务分型Prompt路由系统
# 根据任务类型自动选择最合适的prompt策略
prompt = build_prompt_by_task_type(task_id, task_description, text2annotate)

if examples_str is None:
examples_str = select_examples(icl_examples, task_description, text2annotate)
# 使用动态选择的select_examples函数
examples_str = select_func(icl_examples, task_description, text2annotate, task_id, sample_idx)
Comment on lines 84 to +86

@gemini-code-assist gemini-code-assist Bot May 27, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

🔴 逻辑漏洞：缓存 examples_str 会破坏动态检索和混合上下文长度策略

在当前的实现中，examples_str 在循环外部被初始化为 None，并在第一次迭代（sample_idx == 0）中被赋值后缓存。这会导致两个严重问题：

  1. 动态检索失效：像 select_examples_M05、select_examples_M19 和 select_examples_M20 这样的高级选择方法依赖于当前样本的 text2annotate 来计算相似度并检索最相关的 few-shot 示例。缓存 examples_str 意味着所有后续的测试样本都将重复使用为第一个样本检索的示例，从而完全使动态检索失效。
  2. 混合上下文长度策略失效：混合上下文长度策略（例如在 select_examples 中）使用 sample_index < 50 来为前 50 个样本分配 30k 的上下文预算，并将其余样本限制为 8k 以提高效率。由于 select_func 仅在 sample_idx = 0 时被调用一次，30k 预算的 prompt 被缓存并重复用于所有后续样本，完全绕过了 8k 优化，严重降低了评估性能。

建议：移除 if examples_str is None: 检查，以便为每个样本动态评估 select_func。

        # 使用动态选择的select_examples函数
        examples_str = select_func(icl_examples, task_description, text2annotate, task_id, sample_idx)

input_prompt = prompt.replace("[[EXAMPLES]]\n\n", examples_str+'\n\n')

# tokenized_input = qwen_tokenizer(input_prompt, return_tensors="pt")
# if tokenized_input['input_ids'].shape[1] > max_input_length:
# test_record['prediction'] = None
# else:
# prediction = annotate(input_prompt)
# prediction = annotate(input_prompt, task_id)
# test_record['prediction'] = prediction
prediction = annotate(input_prompt)
prediction = annotate(input_prompt, task_id)
test_record['prediction'] = prediction
with open(output_file, 'a') as f:
f.write(json.dumps(test_record)+'\n')
Expand Down
Loading