diff --git a/openseek/competition/LongContext-ICL-Annotation/src/README.md b/openseek/competition/LongContext-ICL-Annotation/src/README.md
new file mode 100644
index 00000000..dab1216c
--- /dev/null
+++ b/openseek/competition/LongContext-ICL-Annotation/src/README.md
@@ -0,0 +1,80 @@
+## 目录结构
+
+- `src/`:源码目录,包含推理入口、任务策略、客户端配置和依赖清单
+- `data/`:官方数据集
+- `submission_results.zip`:已测试好的提交结果压缩包
+- `README.md`:使用说明
+
+## 运行环境
+
+建议使用 Python 3.10 及以上版本。
+
+安装依赖:
+
+```bash
+pip install -r src/requirements.txt
+```
+
+当前最小依赖仅包含:
+
+- `requests`
+
+## 模型与合规说明
+
+本方案遵循赛事合规要求,核心逻辑均基于官方指定的 `Qwen3-4B` 模型进行设计与优化,未接入任何外部模型或非公开数据集。
+
+在模型底层支撑方面,本方案依托于官方提供的 `FlagScale` 推理平台(或等效的 `FlagScale + Qwen3-4B` 算力环境)。模型的加载及服务端推理由 `FlagScale` 框架提供核心支持。
+
+为保证工程复现的简洁性与跨平台一致性,本项目通过标准化 OpenAI 接口与 `FlagScale` 推理服务通信。代码专注于 ICL 标注策略、动态上下文组织及结果自修复逻辑的实现,模型的底层生命周期与服务化部署建议参考 `FlagScale` 官方文档与环境预设。
+
+## 配置说明
+
+模型服务接口配置位于 `src/llm_config.yaml`。
+
+默认配置如下:
+
+```yaml
+api:
+ base_url: "http://localhost:30000/v1"
+ model: "Qwen3-4B"
+ api_key: ""
+```
+
+如果服务启用了鉴权,可通过 `api_key` 进行配置。
+
+为确保方案成功复现,请在运行前确认已准备好提供以下标准化接口的 `FlagScale` 推理环境:
+
+- `POST /completions`
+- `POST /chat/completions`
+- `GET /models`
+
+## 快速开始
+
+1. **环境准备**:
+ ```bash
+ pip install -r src/requirements.txt
+ ```
+
+2. **配置服务**:
+ 在 `src/llm_config.yaml` 中配置推理服务的 `base_url`;如果服务要求鉴权,同时填写 `api_key`。
+
+3. **执行推理**:
+ ```bash
+ cd src
+ # 默认运行 Task 1-8 全量任务
+ python main.py
+
+ # 单任务运行 (例如 Task 8)
+ python main.py --task_id 8
+
+ # 冒烟测试 (每任务仅运行前 5 条)
+ python main.py --limit 5
+ ```
+
+## 输出产物
+
+本目录中已包含测试完成后的提交文件 `submission_results.zip`,可直接作为赛事平台提交结果使用。
+
+如在复现环境中重新执行推理,程序会在 `submission_results/` 目录下生成:
+- **JSONL 文件**:`openseek-1-v1.jsonl` 至 `openseek-8-v1.jsonl`
+- **打包文件**:`result.zip`(由当前运行自动生成,包含本次推理得到的全部任务结果)
diff --git a/openseek/competition/LongContext-ICL-Annotation/src/api_test.py b/openseek/competition/LongContext-ICL-Annotation/src/api_test.py
deleted file mode 100644
index 3d309933..00000000
--- a/openseek/competition/LongContext-ICL-Annotation/src/api_test.py
+++ /dev/null
@@ -1,21 +0,0 @@
-import requests
-
-url = "http://0.0.0.0:2026/v1/completions"
-prompts = [
- "Hello, FlagScale + vLLM!",
- "Translate 'Hello World' to Chinese.",
- "Write a short poem about autumn."
- # '用中文写一首短诗,诗句开头用包裹起来'
-]
-
-for prompt in prompts:
- data = {
- "model": "../Qwen3-4B",
- "prompt": prompt,
- "max_tokens": 1000
- }
- resp = requests.post(url, json=data)
- print(f"Prompt: {prompt}")
- print("Response:", resp.json(), "\n")
-
- print("*"*50)
diff --git a/openseek/competition/LongContext-ICL-Annotation/src/llm_client.py b/openseek/competition/LongContext-ICL-Annotation/src/llm_client.py
new file mode 100644
index 00000000..55b9190c
--- /dev/null
+++ b/openseek/competition/LongContext-ICL-Annotation/src/llm_client.py
@@ -0,0 +1,108 @@
+import requests
+from pathlib import Path
+from typing import List, Union, Optional
+
+CONFIG_PATH = Path(__file__).resolve().parent / "llm_config.yaml"
+
+
+def load_config(config_path: Path) -> dict[str, str]:
+ base_url = "http://localhost:8000/v1"
+ model = "Qwen3-4B"
+ api_key = "EMPTY"
+
+ for raw_line in config_path.read_text(encoding="utf-8").splitlines():
+ line = raw_line.strip()
+ if not line or line.startswith("#") or line == "api:":
+ continue
+ if line.startswith("base_url:"):
+ base_url = line.split(":", 1)[1].strip().strip('"').strip("'")
+ elif line.startswith("model:"):
+ model = line.split(":", 1)[1].strip().strip('"').strip("'")
+ elif line.startswith("api_key:"):
+ api_key = line.split(":", 1)[1].strip().strip('"').strip("'")
+
+ return {
+ "base_url": base_url,
+ "model": model,
+ "api_key": api_key,
+ }
+
+
+config = load_config(CONFIG_PATH)
+API_URL = config["base_url"]
+MODEL_NAME = config["model"]
+API_KEY = config["api_key"]
+
+class LLMClient:
+ def __init__(self):
+ self.api_url = API_URL
+ self.model_name = MODEL_NAME
+ self.api_key = API_KEY
+ self.headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
+
+ def post_chat_completion(
+ self,
+ messages: List[dict],
+ max_tokens: int = 5000,
+ temperature: float = 0.0,
+ stop: Optional[List[str]] = None,
+ timeout: int = 600
+ ) -> str:
+ data = {
+ "model": self.model_name,
+ "messages": messages,
+ "max_tokens": max_tokens,
+ "temperature": temperature,
+ }
+ if stop: data["stop"] = stop
+ try:
+ url = f"{self.api_url}/chat/completions"
+ response = requests.post(url, json=data, headers=self.headers, timeout=timeout)
+ response.raise_for_status()
+ res_json = response.json()
+ return str(res_json["choices"][0].get("message", {}).get("content", "")) if res_json.get("choices") else ""
+ except Exception as e:
+ print(f"LLM Chat API Error: {e}")
+ raise
+
+ def post_completion(
+ self,
+ prompt: Union[str, List[str]],
+ max_tokens: int = 5000,
+ temperature: float = 0.0,
+ stop: Optional[List[str]] = None,
+ timeout: int = 600
+ ) -> str:
+ data = {
+ "model": self.model_name,
+ "prompt": prompt,
+ "max_tokens": max_tokens,
+ "temperature": temperature,
+ }
+ if stop: data["stop"] = stop
+ try:
+ url = f"{self.api_url}/completions"
+ response = requests.post(url, json=data, headers=self.headers, timeout=timeout)
+ response.raise_for_status()
+ res_json = response.json()
+ return str(res_json["choices"][0].get("text", "")) if res_json.get("choices") else ""
+ except Exception as e:
+ print(f"LLM API Error: {e}")
+ raise
+
+ def is_available(self) -> bool:
+ try:
+ response = requests.get(f"{self.api_url}/models", headers=self.headers, timeout=5)
+ return response.status_code == 200
+ except Exception: return False
+
+client = LLMClient()
+
+def post_completion(prompt, **kwargs) -> str:
+ return client.post_completion(prompt, **kwargs)
+
+def post_chat(messages: List[dict], **kwargs) -> str:
+ return client.post_chat_completion(messages, **kwargs)
+
+def is_completion_server_available() -> bool:
+ return client.is_available()
diff --git a/openseek/competition/LongContext-ICL-Annotation/src/llm_config.yaml b/openseek/competition/LongContext-ICL-Annotation/src/llm_config.yaml
index 94e8a2cd..24f751a9 100644
--- a/openseek/competition/LongContext-ICL-Annotation/src/llm_config.yaml
+++ b/openseek/competition/LongContext-ICL-Annotation/src/llm_config.yaml
@@ -1,30 +1,4 @@
-serve:
-- serve_id: vllm_model
- engine: vllm
- engine_args:
- model: ../Qwen3-4B
- host: 0.0.0.0
- uvicorn_log_level: warning
- port: 2026
- gpu_memory_utilization: 0.9
- trust_remote_code: true
- no_enable_prefix_caching: true
-
-experiment:
- exp_name: qwen3_4b
- exp_dir: outputs/${experiment.exp_name}
- task:
- type: serve
- runner:
- hostfile: null
- deploy:
- use_fs_serve: false
- envs:
- CUDA_VISIBLE_DEVICES: 0
- CUDA_DEVICE_MAX_CONNECTIONS: 1
-
-action: run
-
-hydra:
- run:
- dir: ${experiment.exp_dir}/hydra
\ No newline at end of file
+api:
+ base_url: "http://localhost:30000/v1"
+ model: "Qwen3-4B"
+ api_key: "EMPTY"
diff --git a/openseek/competition/LongContext-ICL-Annotation/src/main.py b/openseek/competition/LongContext-ICL-Annotation/src/main.py
index c2949795..a590aadf 100644
--- a/openseek/competition/LongContext-ICL-Annotation/src/main.py
+++ b/openseek/competition/LongContext-ICL-Annotation/src/main.py
@@ -1,92 +1,157 @@
-import json, os, argparse
-from tqdm import tqdm, trange
-from transformers import AutoTokenizer
+import argparse
+import json
+from pathlib import Path
+from functools import partial
-# from method import build_prompt, select_examples, annotate
+# 全局设置 print 默认 flush=True,解决日志缓存问题
+print = partial(print, flush=True)
-from method import build_prompt, select_examples
+from method import (
+ get_strategy,
+ normalize_examples,
+ select_examples,
+)
+from llm_client import is_completion_server_available
+from strategy_verified import VerifiedProgramStrategy
+from submission_utils import (
+ ensure_directory,
+ get_output_file,
+ save_jsonl,
+ zip_submission_files,
+)
-from method import annotate_nvidia as annotate # For Nvidia GPU
-# from method import annotate_ascend as annotate # For Huawei Ascend
+SRC_DIR = Path(__file__).resolve().parent
+ROOT_DIR = SRC_DIR.parent
+DEFAULT_OUTPUT_DIR = ROOT_DIR / 'submission_results'
+DEFAULT_ZIP_PATH = DEFAULT_OUTPUT_DIR / 'result.zip'
+DEFAULT_DATA_DIR = ROOT_DIR / 'data'
TASK_FILES = {
- 1: './data/openseek-1_closest_integers.json',
- 2: './data/openseek-2_count_nouns_verbs.json',
- 3: './data/openseek-3_collatz_conjecture.json',
- 4: './data/openseek-4_conala_concat_strings.json',
- 5: './data/openseek-5_semeval_2018_task1_tweet_sadness_detection.json',
- 6: './data/openseek-6_mnli_same_genre_classification.json',
- 7: './data/openseek-7_jeopardy_answer_generation_all.json',
- 8: '../data/openseek-8_kernel_generation.json',
+ 1: DEFAULT_DATA_DIR / 'openseek-1_closest_integers.json',
+ 2: DEFAULT_DATA_DIR / 'openseek-2_count_nouns_verbs.json',
+ 3: DEFAULT_DATA_DIR / 'openseek-3_collatz_conjecture.json',
+ 4: DEFAULT_DATA_DIR / 'openseek-4_conala_concat_strings.json',
+ 5: DEFAULT_DATA_DIR / 'openseek-5_semeval_2018_task1_tweet_sadness_detection.json',
+ 6: DEFAULT_DATA_DIR / 'openseek-6_mnli_same_genre_classification.json',
+ 7: DEFAULT_DATA_DIR / 'openseek-7_jeopardy_answer_generation_all.json',
+ 8: DEFAULT_DATA_DIR / 'openseek-8_kernel_generation.json',
}
def parser_args():
parser = argparse.ArgumentParser()
- parser.add_argument('--task_id', type=int, required=True,
- help='Task ID to evaluate, should be in [1, 7].')
- parser.add_argument('--max_input_length', type=int, default=10_000,
- help='Maximum input length for the model.')
- parser.add_argument('--log_path_prefix', type=str,
- default='../outputs/',
- help='Prefix path to save the evaluation logs.')
- parser.add_argument('--tokenizer_path', type=str,
- default='/share/project/wuhaiming/spaces/data_agent/OpenSeek-main/openseek/competition/LongContext-ICL-Annotation/src/Qwen3-4B')
+ parser.add_argument('--task_id', type=int, nargs='*',
+ help='Task IDs to evaluate. If omitted, runs all tasks 1-8.')
+ parser.add_argument('--limit', type=int, default=None,
+ help='Limit the number of samples for each selected task. Useful for quick testing.')
+ parser.add_argument('--output_dir', type=str,
+ default=str(DEFAULT_OUTPUT_DIR),
+ help='Directory used to store submission jsonl files.')
+ parser.add_argument('--zip_path', type=str,
+ default=str(DEFAULT_ZIP_PATH),
+ help='Zip file path for direct competition submission.')
args = parser.parse_args()
return args
-def evaluate(task_id:int,
- qwen_tokenizer:AutoTokenizer,
- max_input_length:int=128_000,
- log_path_prefix:str='./outputs/'
- )->float:
- assert task_id in [i for i in range(1, 9)],\
- f"task_id should be in [1, 8], but got {task_id}."
-
+
+def resolve_task_ids(task_ids: list[int] | None) -> list[int]:
+ if not task_ids:
+ return list(TASK_FILES.keys())
+
+ invalid_task_ids = [task_id for task_id in task_ids if task_id not in TASK_FILES]
+ if invalid_task_ids:
+ raise ValueError(f"task_id should be in [1, 8], but got {invalid_task_ids}.")
+
+ return list(dict.fromkeys(task_ids))
+
+
+def evaluate_task(task_id: int,
+ output_dir: Path = DEFAULT_OUTPUT_DIR,
+ limit: int | None = None,
+ ):
+ assert task_id in TASK_FILES, f"task_id should be in [1, 8], but got {task_id}."
+
task_file = TASK_FILES[task_id]
- with open(task_file, 'r') as f:
+ with open(task_file, 'r', encoding='utf-8') as f:
task_dict = json.load(f)
-
- task_name = task_dict['task_name']
+
task_description = task_dict['Definition'][0]
- icl_examples = task_dict['examples'][:100]
test_samples = task_dict['test_samples']
-
- version = 1
- output_file = f'{log_path_prefix}openseek-{task_id}-v{version}.jsonl'
- output_path = os.path.dirname(output_file)
- os.makedirs(output_path, exist_ok=True)
- while os.path.exists(output_file):
- version += 1
- output_file = f'{log_path_prefix}openseek-{task_id}-v{version}.jsonl'
- with open(output_file, 'w') as f:
- pass
-
- examples_str = None
- for test_sample in tqdm(test_samples, desc=f'Evaluation on Task {task_id}: {task_name}'):
- test_record = dict()
-
+ if limit is not None:
+ test_samples = test_samples[:limit]
+
+ output_file = get_output_file(task_id, output_dir)
+ ensure_directory(output_dir)
+
+ records: list[dict] = []
+ total_samples = len(test_samples)
+
+ if not is_completion_server_available():
+ print(f'Model service is unavailable. Please check your config.')
+ for test_sample in test_samples:
+ records.append({'test_sample_id': test_sample['id'], 'prediction': None})
+ save_jsonl(records, output_file)
+ return output_file, 0, 0
+
+ raw_examples = task_dict['examples']
+ all_normalized_examples = normalize_examples(raw_examples)
+ prompt_examples = select_examples(raw_examples, example_count=3)
+
+ strategy = get_strategy(task_id)
+ print(f"Starting Task {task_id} using {strategy.__class__.__name__}...")
+
+ if hasattr(strategy, '_select_relevant_examples'):
+ current_context_examples = all_normalized_examples
+ else:
+ current_context_examples = prompt_examples
+
+ solution_code = None
+ if isinstance(strategy, VerifiedProgramStrategy):
+ solution_code = strategy.prepare_solution(
+ task_description=task_description,
+ prompt_examples=prompt_examples,
+ )
+ if not solution_code:
+ print(f"Failed to generate code for Task {task_id}, will retry execution for each sample if needed.")
+
+ for idx, test_sample in enumerate(test_samples, 1):
test_sample_id = test_sample['id']
- test_record['test_sample_id'] = test_sample_id
-
-
- text2annotate = test_sample['input']
- prompt = build_prompt(task_description, text2annotate)
- if examples_str is None:
- examples_str = select_examples(icl_examples, task_description, text2annotate)
- input_prompt = prompt.replace("[[EXAMPLES]]\n\n", examples_str+'\n\n')
-
- # tokenized_input = qwen_tokenizer(input_prompt, return_tensors="pt")
- # if tokenized_input['input_ids'].shape[1] > max_input_length:
- # test_record['prediction'] = None
- # else:
- # prediction = annotate(input_prompt)
- # test_record['prediction'] = prediction
- prediction = annotate(input_prompt)
- test_record['prediction'] = prediction
- with open(output_file, 'a') as f:
- f.write(json.dumps(test_record)+'\n')
+ input_text = test_sample['input']
+
+ try:
+ if isinstance(strategy, VerifiedProgramStrategy) and solution_code:
+ prediction = strategy.execute(solution_code, input_text)
+ else:
+ prediction = strategy.predict(
+ task_id=task_id,
+ task_description=task_description,
+ prompt_examples=current_context_examples,
+ input_text=input_text
+ )
+ except Exception as exc:
+ print(f"Execution failed for sample {test_sample_id}: {exc}")
+ prediction = None
+
+ records.append({'test_sample_id': test_sample_id, 'prediction': prediction})
+ print(f"[{idx}/{total_samples}] ID: {test_sample_id} | Prediction generated.")
+ save_jsonl(records, output_file)
+
+ return output_file, len(records), total_samples
+
if __name__ == '__main__':
args = parser_args()
- qwen_tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_path)
- evaluate(args.task_id, qwen_tokenizer, args.max_input_length, args.log_path_prefix)
\ No newline at end of file
+ output_dir = Path(args.output_dir)
+ zip_path = Path(args.zip_path)
+ task_ids = resolve_task_ids(args.task_id)
+
+ task_results: list[tuple[int, Path, int, int]] = []
+ for task_id in task_ids:
+ output_file, processed, total = evaluate_task(task_id, output_dir, args.limit)
+ task_results.append((task_id, output_file, processed, total))
+
+ zip_submission_files(output_dir, zip_path, DEFAULT_DATA_DIR)
+
+ for task_id, output_file, processed, total in task_results:
+ print(f'Task {task_id} completed. Processed {processed}/{total} samples.')
+ print(f'output_file: {output_file}')
+ print(f'zip_file: {zip_path}')
\ No newline at end of file
diff --git a/openseek/competition/LongContext-ICL-Annotation/src/method.py b/openseek/competition/LongContext-ICL-Annotation/src/method.py
index 386daf22..c6f728f4 100644
--- a/openseek/competition/LongContext-ICL-Annotation/src/method.py
+++ b/openseek/competition/LongContext-ICL-Annotation/src/method.py
@@ -1,277 +1,59 @@
-
-import re
-from collections import Counter
-from transformers import AutoTokenizer
-
-""" Here is an example of implementation of Long-Context Data Annotation. """
-
-def build_prompt____(task_description: str, text2annotate: str) -> str:
- """
- Build a high-precision English prompt for long-context data annotation (optimized for Qwen3-4B).
- Core requirement: Final answer MUST be wrapped in