From 5bf370c9564adb2c5c906c069374377676c36e97 Mon Sep 17 00:00:00 2001 From: cg-zhou Date: Mon, 25 May 2026 20:22:29 +0800 Subject: [PATCH 1/2] feat(competition): add submission for LongContext-ICL-Annotation - [cg-zhou] Signed-off-by: cg-zhou --- .../LongContext-ICL-Annotation/src/README.md | 80 +++ .../src/api_test.py | 21 - .../src/llm_client.py | 108 +++ .../src/llm_config.yaml | 34 +- .../LongContext-ICL-Annotation/src/main.py | 209 ++++-- .../LongContext-ICL-Annotation/src/method.py | 336 ++------- .../src/requirements.txt | 1 + .../src/strategy_base.py | 7 + .../src/strategy_task2.py | 198 ++++++ .../src/strategy_task5.py | 122 ++++ .../src/strategy_task6.py | 255 +++++++ .../src/strategy_task7.py | 167 +++++ .../src/strategy_task8.py | 639 ++++++++++++++++++ .../src/strategy_verified.py | 132 ++++ .../src/submission_utils.py | 40 ++ ...4\257\346\212\245\345\221\212-cg-zhou.pdf" | Bin 0 -> 425802 bytes 16 files changed, 1949 insertions(+), 400 deletions(-) create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/README.md delete mode 100644 openseek/competition/LongContext-ICL-Annotation/src/api_test.py create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/llm_client.py create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/requirements.txt create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/strategy_base.py create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/strategy_task2.py create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/strategy_task5.py create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/strategy_task6.py create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/strategy_task7.py create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/strategy_task8.py create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/strategy_verified.py create mode 100644 openseek/competition/LongContext-ICL-Annotation/src/submission_utils.py create mode 100644 "openseek/competition/LongContext-ICL-Annotation/\346\212\200\346\234\257\346\212\245\345\221\212-cg-zhou.pdf" diff --git a/openseek/competition/LongContext-ICL-Annotation/src/README.md b/openseek/competition/LongContext-ICL-Annotation/src/README.md new file mode 100644 index 00000000..dab1216c --- /dev/null +++ b/openseek/competition/LongContext-ICL-Annotation/src/README.md @@ -0,0 +1,80 @@ +## 目录结构 + +- `src/`：源码目录，包含推理入口、任务策略、客户端配置和依赖清单 +- `data/`：官方数据集 +- `submission_results.zip`：已测试好的提交结果压缩包 +- `README.md`：使用说明 + +## 运行环境 + +建议使用 Python 3.10 及以上版本。 + +安装依赖： + +```bash +pip install -r src/requirements.txt +``` + +当前最小依赖仅包含： + +- `requests` + +## 模型与合规说明 + +本方案遵循赛事合规要求，核心逻辑均基于官方指定的 `Qwen3-4B` 模型进行设计与优化，未接入任何外部模型或非公开数据集。 + +在模型底层支撑方面，本方案依托于官方提供的 `FlagScale` 推理平台（或等效的 `FlagScale + Qwen3-4B` 算力环境）。模型的加载及服务端推理由 `FlagScale` 框架提供核心支持。 + +为保证工程复现的简洁性与跨平台一致性，本项目通过标准化 OpenAI 接口与 `FlagScale` 推理服务通信。代码专注于 ICL 标注策略、动态上下文组织及结果自修复逻辑的实现，模型的底层生命周期与服务化部署建议参考 `FlagScale` 官方文档与环境预设。 + +## 配置说明 + +模型服务接口配置位于 `src/llm_config.yaml`。 + +默认配置如下： + +```yaml +api: + base_url: "http://localhost:30000/v1" + model: "Qwen3-4B" + api_key: "" +``` + +如果服务启用了鉴权，可通过 `api_key` 进行配置。 + +为确保方案成功复现，请在运行前确认已准备好提供以下标准化接口的 `FlagScale` 推理环境： + +- `POST /completions` +- `POST /chat/completions` +- `GET /models` + +## 快速开始 + +1. **环境准备**： + ```bash + pip install -r src/requirements.txt + ``` + +2. **配置服务**： + 在 `src/llm_config.yaml` 中配置推理服务的 `base_url`；如果服务要求鉴权，同时填写 `api_key`。 + +3. **执行推理**： + ```bash + cd src + # 默认运行 Task 1-8 全量任务 + python main.py + + # 单任务运行 (例如 Task 8) + python main.py --task_id 8 + + # 冒烟测试 (每任务仅运行前 5 条) + python main.py --limit 5 + ``` + +## 输出产物 + +本目录中已包含测试完成后的提交文件 `submission_results.zip`，可直接作为赛事平台提交结果使用。 + +如在复现环境中重新执行推理，程序会在 `submission_results/` 目录下生成： +- **JSONL 文件**：`openseek-1-v1.jsonl` 至 `openseek-8-v1.jsonl` +- **打包文件**：`result.zip`（由当前运行自动生成，包含本次推理得到的全部任务结果） diff --git a/openseek/competition/LongContext-ICL-Annotation/src/api_test.py b/openseek/competition/LongContext-ICL-Annotation/src/api_test.py deleted file mode 100644 index 3d309933..00000000 --- a/openseek/competition/LongContext-ICL-Annotation/src/api_test.py +++ /dev/null @@ -1,21 +0,0 @@ -import requests - -url = "http://0.0.0.0:2026/v1/completions" -prompts = [ - "Hello, FlagScale + vLLM!", - "Translate 'Hello World' to Chinese.", - "Write a short poem about autumn." - # '用中文写一首短诗，诗句开头用包裹起来' -] - -for prompt in prompts: - data = { - "model": "../Qwen3-4B", - "prompt": prompt, - "max_tokens": 1000 - } - resp = requests.post(url, json=data) - print(f"Prompt: {prompt}") - print("Response:", resp.json(), "\n") - - print("*"*50) diff --git a/openseek/competition/LongContext-ICL-Annotation/src/llm_client.py b/openseek/competition/LongContext-ICL-Annotation/src/llm_client.py new file mode 100644 index 00000000..0215b993 --- /dev/null +++ b/openseek/competition/LongContext-ICL-Annotation/src/llm_client.py @@ -0,0 +1,108 @@ +import requests +from pathlib import Path +from typing import List, Union, Optional + +CONFIG_PATH = Path(__file__).resolve().parent / "llm_config.yaml" + + +def load_config(config_path: Path) -> dict[str, str]: + base_url = "http://localhost:8000/v1" + model = "Qwen3-4B" + api_key = "EMPTY" + + for raw_line in config_path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#") or line == "api:": + continue + if line.startswith("base_url:"): + base_url = line.split(":", 1)[1].strip().strip('"').strip("'") + elif line.startswith("model:"): + model = line.split(":", 1)[1].strip().strip('"').strip("'") + elif line.startswith("api_key:"): + api_key = line.split(":", 1)[1].strip().strip('"').strip("'") + + return { + "base_url": base_url, + "model": model, + "api_key": api_key, + } + + +config = load_config(CONFIG_PATH) +API_URL = config["base_url"] +MODEL_NAME = config["model"] +API_KEY = config["api_key"] + +class LLMClient: + def __init__(self): + self.api_url = API_URL + self.model_name = MODEL_NAME + self.api_key = API_KEY + self.headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {} + + def post_chat_completion( + self, + messages: List[dict], + max_tokens: int = 5000, + temperature: float = 0.0, + stop: Optional[List[str]] = None, + timeout: int = 600 + ) -> str: + data = { + "model": self.model_name, + "messages": messages, + "max_tokens": max_tokens, + "temperature": temperature, + } + if stop: data["stop"] = stop + try: + url = f"{self.api_url}/chat/completions" + response = requests.post(url, json=data, headers=self.headers, timeout=timeout) + response.raise_for_status() + res_json = response.json() + return str(res_json["choices"][0].get("message", {}).get("content", "")) if "choices" in res_json else "" + except Exception as e: + print(f"LLM Chat API Error: {e}") + raise + + def post_completion( + self, + prompt: Union[str, List[str]], + max_tokens: int = 5000, + temperature: float = 0.0, + stop: Optional[List[str]] = None, + timeout: int = 600 + ) -> str: + data = { + "model": self.model_name, + "prompt": prompt, + "max_tokens": max_tokens, + "temperature": temperature, + } + if stop: data["stop"] = stop + try: + url = f"{self.api_url}/completions" + response = requests.post(url, json=data, headers=self.headers, timeout=timeout) + response.raise_for_status() + res_json = response.json() + return str(res_json["choices"][0].get("text", "")) if "choices" in res_json else "" + except Exception as e: + print(f"LLM API Error: {e}") + raise + + def is_available(self) -> bool: + try: + response = requests.get(f"{self.api_url}/models", headers=self.headers, timeout=5) + return response.status_code == 200 + except: return False + +client = LLMClient() + +def post_completion(prompt, **kwargs) -> str: + return client.post_completion(prompt, **kwargs) + +def post_chat(messages: List[dict], **kwargs) -> str: + return client.post_chat_completion(messages, **kwargs) + +def is_completion_server_available() -> bool: + return client.is_available() diff --git a/openseek/competition/LongContext-ICL-Annotation/src/llm_config.yaml b/openseek/competition/LongContext-ICL-Annotation/src/llm_config.yaml index 94e8a2cd..24f751a9 100644 --- a/openseek/competition/LongContext-ICL-Annotation/src/llm_config.yaml +++ b/openseek/competition/LongContext-ICL-Annotation/src/llm_config.yaml @@ -1,30 +1,4 @@ -serve: -- serve_id: vllm_model - engine: vllm - engine_args: - model: ../Qwen3-4B - host: 0.0.0.0 - uvicorn_log_level: warning - port: 2026 - gpu_memory_utilization: 0.9 - trust_remote_code: true - no_enable_prefix_caching: true - -experiment: - exp_name: qwen3_4b - exp_dir: outputs/${experiment.exp_name} - task: - type: serve - runner: - hostfile: null - deploy: - use_fs_serve: false - envs: - CUDA_VISIBLE_DEVICES: 0 - CUDA_DEVICE_MAX_CONNECTIONS: 1 - -action: run - -hydra: - run: - dir: ${experiment.exp_dir}/hydra \ No newline at end of file +api: + base_url: "http://localhost:30000/v1" + model: "Qwen3-4B" + api_key: "EMPTY" diff --git a/openseek/competition/LongContext-ICL-Annotation/src/main.py b/openseek/competition/LongContext-ICL-Annotation/src/main.py index c2949795..a590aadf 100644 --- a/openseek/competition/LongContext-ICL-Annotation/src/main.py +++ b/openseek/competition/LongContext-ICL-Annotation/src/main.py @@ -1,92 +1,157 @@ -import json, os, argparse -from tqdm import tqdm, trange -from transformers import AutoTokenizer +import argparse +import json +from pathlib import Path +from functools import partial -# from method import build_prompt, select_examples, annotate +# 全局设置 print 默认 flush=True，解决日志缓存问题 +print = partial(print, flush=True) -from method import build_prompt, select_examples +from method import ( + get_strategy, + normalize_examples, + select_examples, +) +from llm_client import is_completion_server_available +from strategy_verified import VerifiedProgramStrategy +from submission_utils import ( + ensure_directory, + get_output_file, + save_jsonl, + zip_submission_files, +) -from method import annotate_nvidia as annotate # For Nvidia GPU -# from method import annotate_ascend as annotate # For Huawei Ascend +SRC_DIR = Path(__file__).resolve().parent +ROOT_DIR = SRC_DIR.parent +DEFAULT_OUTPUT_DIR = ROOT_DIR / 'submission_results' +DEFAULT_ZIP_PATH = DEFAULT_OUTPUT_DIR / 'result.zip' +DEFAULT_DATA_DIR = ROOT_DIR / 'data' TASK_FILES = { - 1: './data/openseek-1_closest_integers.json', - 2: './data/openseek-2_count_nouns_verbs.json', - 3: './data/openseek-3_collatz_conjecture.json', - 4: './data/openseek-4_conala_concat_strings.json', - 5: './data/openseek-5_semeval_2018_task1_tweet_sadness_detection.json', - 6: './data/openseek-6_mnli_same_genre_classification.json', - 7: './data/openseek-7_jeopardy_answer_generation_all.json', - 8: '../data/openseek-8_kernel_generation.json', + 1: DEFAULT_DATA_DIR / 'openseek-1_closest_integers.json', + 2: DEFAULT_DATA_DIR / 'openseek-2_count_nouns_verbs.json', + 3: DEFAULT_DATA_DIR / 'openseek-3_collatz_conjecture.json', + 4: DEFAULT_DATA_DIR / 'openseek-4_conala_concat_strings.json', + 5: DEFAULT_DATA_DIR / 'openseek-5_semeval_2018_task1_tweet_sadness_detection.json', + 6: DEFAULT_DATA_DIR / 'openseek-6_mnli_same_genre_classification.json', + 7: DEFAULT_DATA_DIR / 'openseek-7_jeopardy_answer_generation_all.json', + 8: DEFAULT_DATA_DIR / 'openseek-8_kernel_generation.json', } def parser_args(): parser = argparse.ArgumentParser() - parser.add_argument('--task_id', type=int, required=True, - help='Task ID to evaluate, should be in [1, 7].') - parser.add_argument('--max_input_length', type=int, default=10_000, - help='Maximum input length for the model.') - parser.add_argument('--log_path_prefix', type=str, - default='../outputs/', - help='Prefix path to save the evaluation logs.') - parser.add_argument('--tokenizer_path', type=str, - default='/share/project/wuhaiming/spaces/data_agent/OpenSeek-main/openseek/competition/LongContext-ICL-Annotation/src/Qwen3-4B') + parser.add_argument('--task_id', type=int, nargs='*', + help='Task IDs to evaluate. If omitted, runs all tasks 1-8.') + parser.add_argument('--limit', type=int, default=None, + help='Limit the number of samples for each selected task. Useful for quick testing.') + parser.add_argument('--output_dir', type=str, + default=str(DEFAULT_OUTPUT_DIR), + help='Directory used to store submission jsonl files.') + parser.add_argument('--zip_path', type=str, + default=str(DEFAULT_ZIP_PATH), + help='Zip file path for direct competition submission.') args = parser.parse_args() return args -def evaluate(task_id:int, - qwen_tokenizer:AutoTokenizer, - max_input_length:int=128_000, - log_path_prefix:str='./outputs/' - )->float: - assert task_id in [i for i in range(1, 9)],\ - f"task_id should be in [1, 8], but got {task_id}." - + +def resolve_task_ids(task_ids: list[int] | None) -> list[int]: + if not task_ids: + return list(TASK_FILES.keys()) + + invalid_task_ids = [task_id for task_id in task_ids if task_id not in TASK_FILES] + if invalid_task_ids: + raise ValueError(f"task_id should be in [1, 8], but got {invalid_task_ids}.") + + return list(dict.fromkeys(task_ids)) + + +def evaluate_task(task_id: int, + output_dir: Path = DEFAULT_OUTPUT_DIR, + limit: int | None = None, + ): + assert task_id in TASK_FILES, f"task_id should be in [1, 8], but got {task_id}." + task_file = TASK_FILES[task_id] - with open(task_file, 'r') as f: + with open(task_file, 'r', encoding='utf-8') as f: task_dict = json.load(f) - - task_name = task_dict['task_name'] + task_description = task_dict['Definition'][0] - icl_examples = task_dict['examples'][:100] test_samples = task_dict['test_samples'] - - version = 1 - output_file = f'{log_path_prefix}openseek-{task_id}-v{version}.jsonl' - output_path = os.path.dirname(output_file) - os.makedirs(output_path, exist_ok=True) - while os.path.exists(output_file): - version += 1 - output_file = f'{log_path_prefix}openseek-{task_id}-v{version}.jsonl' - with open(output_file, 'w') as f: - pass - - examples_str = None - for test_sample in tqdm(test_samples, desc=f'Evaluation on Task {task_id}: {task_name}'): - test_record = dict() - + if limit is not None: + test_samples = test_samples[:limit] + + output_file = get_output_file(task_id, output_dir) + ensure_directory(output_dir) + + records: list[dict] = [] + total_samples = len(test_samples) + + if not is_completion_server_available(): + print(f'Model service is unavailable. Please check your config.') + for test_sample in test_samples: + records.append({'test_sample_id': test_sample['id'], 'prediction': None}) + save_jsonl(records, output_file) + return output_file, 0, 0 + + raw_examples = task_dict['examples'] + all_normalized_examples = normalize_examples(raw_examples) + prompt_examples = select_examples(raw_examples, example_count=3) + + strategy = get_strategy(task_id) + print(f"Starting Task {task_id} using {strategy.__class__.__name__}...") + + if hasattr(strategy, '_select_relevant_examples'): + current_context_examples = all_normalized_examples + else: + current_context_examples = prompt_examples + + solution_code = None + if isinstance(strategy, VerifiedProgramStrategy): + solution_code = strategy.prepare_solution( + task_description=task_description, + prompt_examples=prompt_examples, + ) + if not solution_code: + print(f"Failed to generate code for Task {task_id}, will retry execution for each sample if needed.") + + for idx, test_sample in enumerate(test_samples, 1): test_sample_id = test_sample['id'] - test_record['test_sample_id'] = test_sample_id - - - text2annotate = test_sample['input'] - prompt = build_prompt(task_description, text2annotate) - if examples_str is None: - examples_str = select_examples(icl_examples, task_description, text2annotate) - input_prompt = prompt.replace("[[EXAMPLES]]\n\n", examples_str+'\n\n') - - # tokenized_input = qwen_tokenizer(input_prompt, return_tensors="pt") - # if tokenized_input['input_ids'].shape[1] > max_input_length: - # test_record['prediction'] = None - # else: - # prediction = annotate(input_prompt) - # test_record['prediction'] = prediction - prediction = annotate(input_prompt) - test_record['prediction'] = prediction - with open(output_file, 'a') as f: - f.write(json.dumps(test_record)+'\n') + input_text = test_sample['input'] + + try: + if isinstance(strategy, VerifiedProgramStrategy) and solution_code: + prediction = strategy.execute(solution_code, input_text) + else: + prediction = strategy.predict( + task_id=task_id, + task_description=task_description, + prompt_examples=current_context_examples, + input_text=input_text + ) + except Exception as exc: + print(f"Execution failed for sample {test_sample_id}: {exc}") + prediction = None + + records.append({'test_sample_id': test_sample_id, 'prediction': prediction}) + print(f"[{idx}/{total_samples}] ID: {test_sample_id} | Prediction generated.") + save_jsonl(records, output_file) + + return output_file, len(records), total_samples + if __name__ == '__main__': args = parser_args() - qwen_tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_path) - evaluate(args.task_id, qwen_tokenizer, args.max_input_length, args.log_path_prefix) \ No newline at end of file + output_dir = Path(args.output_dir) + zip_path = Path(args.zip_path) + task_ids = resolve_task_ids(args.task_id) + + task_results: list[tuple[int, Path, int, int]] = [] + for task_id in task_ids: + output_file, processed, total = evaluate_task(task_id, output_dir, args.limit) + task_results.append((task_id, output_file, processed, total)) + + zip_submission_files(output_dir, zip_path, DEFAULT_DATA_DIR) + + for task_id, output_file, processed, total in task_results: + print(f'Task {task_id} completed. Processed {processed}/{total} samples.') + print(f'output_file: {output_file}') + print(f'zip_file: {zip_path}') \ No newline at end of file diff --git a/openseek/competition/LongContext-ICL-Annotation/src/method.py b/openseek/competition/LongContext-ICL-Annotation/src/method.py index 386daf22..c6f728f4 100644 --- a/openseek/competition/LongContext-ICL-Annotation/src/method.py +++ b/openseek/competition/LongContext-ICL-Annotation/src/method.py @@ -1,277 +1,59 @@ - -import re -from collections import Counter -from transformers import AutoTokenizer - -""" Here is an example of implementation of Long-Context Data Annotation. """ - -def build_prompt____(task_description: str, text2annotate: str) -> str: - """ - Build a high-precision English prompt for long-context data annotation (optimized for Qwen3-4B). - Core requirement: Final answer MUST be wrapped in