From 0e91b1298566099e2511b5373d38bfe57c804a6e Mon Sep 17 00:00:00 2001 From: jtthie Date: Fri, 29 May 2026 10:31:28 +0800 Subject: [PATCH] Add files via upload --- .../LongContext-ICL-Annotation/src/main.py | 41 +-- .../LongContext-ICL-Annotation/src/method.py | 244 ++++++++++++++---- .../src/requirements.txt | 17 ++ ...2\345\220\215\346\230\237\347\201\253.pdf" | Bin 0 -> 1678295 bytes 4 files changed, 242 insertions(+), 60 deletions(-) create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/requirements.txt create mode 100644 "openseek/competition/LongContext-ICL-Annotation/src/\346\212\200\346\234\257\346\212\245\345\221\212-\346\234\252\345\220\215\346\230\237\347\201\253.pdf" diff --git a/openseek/competition/LongContext-ICL-Annotation/src/main.py b/openseek/competition/LongContext-ICL-Annotation/src/main.py index c2949795..42197674 100644 --- a/openseek/competition/LongContext-ICL-Annotation/src/main.py +++ b/openseek/competition/LongContext-ICL-Annotation/src/main.py @@ -4,20 +4,20 @@ # from method import build_prompt, select_examples, annotate -from method import build_prompt, select_examples +from method import build_prompt, select_examples, build_prompt_cot -from method import annotate_nvidia as annotate # For Nvidia GPU -# from method import annotate_ascend as annotate # For Huawei Ascend +# from method import annotate_nvidia as annotate # For Nvidia GPU +from method import annotate_ascend as annotate # For Huawei Ascend TASK_FILES = { - 1: './data/openseek-1_closest_integers.json', - 2: './data/openseek-2_count_nouns_verbs.json', - 3: './data/openseek-3_collatz_conjecture.json', - 4: './data/openseek-4_conala_concat_strings.json', - 5: './data/openseek-5_semeval_2018_task1_tweet_sadness_detection.json', - 6: './data/openseek-6_mnli_same_genre_classification.json', - 7: './data/openseek-7_jeopardy_answer_generation_all.json', - 8: '../data/openseek-8_kernel_generation.json', + 1: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-1_closest_integers.json', + 2: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-2_count_nouns_verbs.json', + 3: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-3_collatz_conjecture.json', + 4: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-4_conala_concat_strings.json', + 5: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-5_semeval_2018_task1_tweet_sadness_detection.json', + 6: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-6_mnli_same_genre_classification.json', + 7: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-7_jeopardy_answer_generation_all.json', + 8: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-8_kernel_generation.json', } def parser_args(): @@ -30,7 +30,7 @@ def parser_args(): default='../outputs/', help='Prefix path to save the evaluation logs.') parser.add_argument('--tokenizer_path', type=str, - default='/share/project/wuhaiming/spaces/data_agent/OpenSeek-main/openseek/competition/LongContext-ICL-Annotation/src/Qwen3-4B') + default='/root/Qwen3-4B') args = parser.parse_args() return args @@ -48,7 +48,7 @@ def evaluate(task_id:int, task_name = task_dict['task_name'] task_description = task_dict['Definition'][0] - icl_examples = task_dict['examples'][:100] + icl_examples = task_dict['examples'][:50] test_samples = task_dict['test_samples'] version = 1 @@ -70,7 +70,16 @@ def evaluate(task_id:int, text2annotate = test_sample['input'] - prompt = build_prompt(task_description, text2annotate) + + # Use CoT prompt for Task 3 and 4, standard prompt for others (Account 3 strategy) + # Task 3: Collatz conjecture (math reasoning) - CoT helps + # Task 4: String concatenation - CoT significantly helped in Account 2 (+29.2%) + # Task 8: Kernel generation - CoT was harmful in Account 2 (6.0% -> 0.6%) + if task_id in [3, 4]: + prompt = build_prompt_cot(task_description, text2annotate, task_id) + else: + prompt = build_prompt(task_description, text2annotate) + if examples_str is None: examples_str = select_examples(icl_examples, task_description, text2annotate) input_prompt = prompt.replace("[[EXAMPLES]]\n\n", examples_str+'\n\n') @@ -79,9 +88,9 @@ def evaluate(task_id:int, # if tokenized_input['input_ids'].shape[1] > max_input_length: # test_record['prediction'] = None # else: - # prediction = annotate(input_prompt) + # prediction = annotate(input_prompt, task_id) # test_record['prediction'] = prediction - prediction = annotate(input_prompt) + prediction = annotate(input_prompt, task_id) test_record['prediction'] = prediction with open(output_file, 'a') as f: f.write(json.dumps(test_record)+'\n') diff --git a/openseek/competition/LongContext-ICL-Annotation/src/method.py b/openseek/competition/LongContext-ICL-Annotation/src/method.py index 386daf22..516485d8 100644 --- a/openseek/competition/LongContext-ICL-Annotation/src/method.py +++ b/openseek/competition/LongContext-ICL-Annotation/src/method.py @@ -85,6 +85,97 @@ def build_prompt(task_description: str, text2annotate: str) -> str: ) return prompt +def build_prompt_cot(task_description: str, text2annotate: str, task_id: int) -> str: + """ + Build a Chain-of-Thought (CoT) prompt for complex reasoning tasks (Task 3, 8). + This encourages the model to show step-by-step reasoning before final answer. + """ + if task_id == 3: + # Task 3: Collatz Conjecture - Mathematical Reasoning + prompt = ( + "### Role Definition\n" + "You are a mathematical reasoning expert specializing in the Collatz conjecture. " + "You excel at systematic step-by-step mathematical reasoning and verification.\n\n" + + "### Core Task\n" + f"{task_description}\n\n" + + "### Critical Reasoning Guidelines\n" + "1. **Step-by-Step Reasoning**: For each input number, you MUST show your complete reasoning process:\n" + " - Step 1: Identify the current number\n" + " - Step 2: Apply the Collatz rule (if even: n/2; if odd: 3n+1)\n" + " - Step 3: Calculate the next number\n" + " - Step 4: Continue until reaching 1\n" + " - Step 5: Determine the closest integer to 1\n\n" + + "2. **Verification**: Always verify your calculations:\n" + " - Check if the rule was applied correctly\n" + " - Confirm the sequence reaches 1\n" + " - Double-check the final answer\n\n" + + "3. **Output Format**: Your response must follow this structure:\n" + " **Reasoning Process:**\n" + " [Show your step-by-step calculations here]\n\n" + " **Final Answer:** \n\n" + + "### Examples (Must Be Fully Followed)\n" + "[[EXAMPLES]]\n\n" + + "### Text to Annotate\n" + f"{text2annotate}\n\n" + + "### Final Requirement Summary\n" + "1. Show your complete step-by-step reasoning process.\n" + "2. Verify each calculation step.\n" + "3. Final answer MUST be wrapped in