diff --git a/openseek/competition/LongContext-ICL-Annotation/src/main.py b/openseek/competition/LongContext-ICL-Annotation/src/main.py
index c2949795..42197674 100644
--- a/openseek/competition/LongContext-ICL-Annotation/src/main.py
+++ b/openseek/competition/LongContext-ICL-Annotation/src/main.py
@@ -4,20 +4,20 @@
# from method import build_prompt, select_examples, annotate
-from method import build_prompt, select_examples
+from method import build_prompt, select_examples, build_prompt_cot
-from method import annotate_nvidia as annotate # For Nvidia GPU
-# from method import annotate_ascend as annotate # For Huawei Ascend
+# from method import annotate_nvidia as annotate # For Nvidia GPU
+from method import annotate_ascend as annotate # For Huawei Ascend
TASK_FILES = {
- 1: './data/openseek-1_closest_integers.json',
- 2: './data/openseek-2_count_nouns_verbs.json',
- 3: './data/openseek-3_collatz_conjecture.json',
- 4: './data/openseek-4_conala_concat_strings.json',
- 5: './data/openseek-5_semeval_2018_task1_tweet_sadness_detection.json',
- 6: './data/openseek-6_mnli_same_genre_classification.json',
- 7: './data/openseek-7_jeopardy_answer_generation_all.json',
- 8: '../data/openseek-8_kernel_generation.json',
+ 1: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-1_closest_integers.json',
+ 2: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-2_count_nouns_verbs.json',
+ 3: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-3_collatz_conjecture.json',
+ 4: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-4_conala_concat_strings.json',
+ 5: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-5_semeval_2018_task1_tweet_sadness_detection.json',
+ 6: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-6_mnli_same_genre_classification.json',
+ 7: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-7_jeopardy_answer_generation_all.json',
+ 8: '/root/OpenSeek/openseek/competition/LongContext-ICL-Annotation/data/openseek-8_kernel_generation.json',
}
def parser_args():
@@ -30,7 +30,7 @@ def parser_args():
default='../outputs/',
help='Prefix path to save the evaluation logs.')
parser.add_argument('--tokenizer_path', type=str,
- default='/share/project/wuhaiming/spaces/data_agent/OpenSeek-main/openseek/competition/LongContext-ICL-Annotation/src/Qwen3-4B')
+ default='/root/Qwen3-4B')
args = parser.parse_args()
return args
@@ -48,7 +48,7 @@ def evaluate(task_id:int,
task_name = task_dict['task_name']
task_description = task_dict['Definition'][0]
- icl_examples = task_dict['examples'][:100]
+ icl_examples = task_dict['examples'][:50]
test_samples = task_dict['test_samples']
version = 1
@@ -70,7 +70,16 @@ def evaluate(task_id:int,
text2annotate = test_sample['input']
- prompt = build_prompt(task_description, text2annotate)
+
+ # Use CoT prompt for Task 3 and 4, standard prompt for others (Account 3 strategy)
+ # Task 3: Collatz conjecture (math reasoning) - CoT helps
+ # Task 4: String concatenation - CoT significantly helped in Account 2 (+29.2%)
+ # Task 8: Kernel generation - CoT was harmful in Account 2 (6.0% -> 0.6%)
+ if task_id in [3, 4]:
+ prompt = build_prompt_cot(task_description, text2annotate, task_id)
+ else:
+ prompt = build_prompt(task_description, text2annotate)
+
if examples_str is None:
examples_str = select_examples(icl_examples, task_description, text2annotate)
input_prompt = prompt.replace("[[EXAMPLES]]\n\n", examples_str+'\n\n')
@@ -79,9 +88,9 @@ def evaluate(task_id:int,
# if tokenized_input['input_ids'].shape[1] > max_input_length:
# test_record['prediction'] = None
# else:
- # prediction = annotate(input_prompt)
+ # prediction = annotate(input_prompt, task_id)
# test_record['prediction'] = prediction
- prediction = annotate(input_prompt)
+ prediction = annotate(input_prompt, task_id)
test_record['prediction'] = prediction
with open(output_file, 'a') as f:
f.write(json.dumps(test_record)+'\n')
diff --git a/openseek/competition/LongContext-ICL-Annotation/src/method.py b/openseek/competition/LongContext-ICL-Annotation/src/method.py
index 386daf22..516485d8 100644
--- a/openseek/competition/LongContext-ICL-Annotation/src/method.py
+++ b/openseek/competition/LongContext-ICL-Annotation/src/method.py
@@ -85,6 +85,97 @@ def build_prompt(task_description: str, text2annotate: str) -> str:
)
return prompt
+def build_prompt_cot(task_description: str, text2annotate: str, task_id: int) -> str:
+ """
+ Build a Chain-of-Thought (CoT) prompt for complex reasoning tasks (Task 3, 8).
+ This encourages the model to show step-by-step reasoning before final answer.
+ """
+ if task_id == 3:
+ # Task 3: Collatz Conjecture - Mathematical Reasoning
+ prompt = (
+ "### Role Definition\n"
+ "You are a mathematical reasoning expert specializing in the Collatz conjecture. "
+ "You excel at systematic step-by-step mathematical reasoning and verification.\n\n"
+
+ "### Core Task\n"
+ f"{task_description}\n\n"
+
+ "### Critical Reasoning Guidelines\n"
+ "1. **Step-by-Step Reasoning**: For each input number, you MUST show your complete reasoning process:\n"
+ " - Step 1: Identify the current number\n"
+ " - Step 2: Apply the Collatz rule (if even: n/2; if odd: 3n+1)\n"
+ " - Step 3: Calculate the next number\n"
+ " - Step 4: Continue until reaching 1\n"
+ " - Step 5: Determine the closest integer to 1\n\n"
+
+ "2. **Verification**: Always verify your calculations:\n"
+ " - Check if the rule was applied correctly\n"
+ " - Confirm the sequence reaches 1\n"
+ " - Double-check the final answer\n\n"
+
+ "3. **Output Format**: Your response must follow this structure:\n"
+ " **Reasoning Process:**\n"
+ " [Show your step-by-step calculations here]\n\n"
+ " **Final Answer:** \n\n"
+
+ "### Examples (Must Be Fully Followed)\n"
+ "[[EXAMPLES]]\n\n"
+
+ "### Text to Annotate\n"
+ f"{text2annotate}\n\n"
+
+ "### Final Requirement Summary\n"
+ "1. Show your complete step-by-step reasoning process.\n"
+ "2. Verify each calculation step.\n"
+ "3. Final answer MUST be wrapped in