From 1fa49ee7af4fd83c7f1f3a9316f26d21419245fe Mon Sep 17 00:00:00 2001 From: Yoshio Date: Mon, 13 Jul 2026 10:12:57 +0700 Subject: [PATCH 1/2] fix(cpu-thread): optimize cpu processing for local --- agent/classifier.py | 10 +++++----- engines/local_slm.py | 24 +++++++++++++++++++++++- engines/remote_llm.py | 40 ++++++++++++++++++++++++---------------- 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/agent/classifier.py b/agent/classifier.py index 75dc308..1737cf2 100644 --- a/agent/classifier.py +++ b/agent/classifier.py @@ -6,9 +6,9 @@ """ import logging -import os +from typing import Any -from engines.local_slm import LocalSLMEngine +from engines.local_slm import ClassifierEngine logger = logging.getLogger(__name__) @@ -61,10 +61,10 @@ _grammar = None -def _get_grammar(): +def _get_grammar() -> Any: global _grammar if _grammar is None: - from llama_cpp import LlamaGrammar + from llama_cpp import LlamaGrammar # type: ignore _grammar = LlamaGrammar.from_string(_GRAMMAR_STR) return _grammar @@ -78,7 +78,7 @@ def classify(prompt: str) -> str: logger.debug("Classifier: long context (%d chars) → %s", len(prompt), ROUTE_API_LONG) return ROUTE_API_LONG - engine = LocalSLMEngine.get_instance() + engine = ClassifierEngine.get_instance() raw = engine.generate( prompt=f"Task:\n{prompt}\n\nLabel:", system_prompt=_SYSTEM_PROMPT, diff --git a/engines/local_slm.py b/engines/local_slm.py index f4cfd0b..4c6c0b3 100644 --- a/engines/local_slm.py +++ b/engines/local_slm.py @@ -23,7 +23,9 @@ def get_instance(cls) -> "LocalSLMEngine": def __init__(self, model_path: str) -> None: n_ctx = int(os.environ.get("LOCAL_N_CTX", "2048")) - n_threads = int(os.environ.get("LOCAL_N_THREADS", "2")) + # Dynamic thread scaling: fully utilize AMD compute instance + default_threads = str(max(2, (os.cpu_count() or 4) - 1)) + n_threads = int(os.environ.get("LOCAL_N_THREADS", default_threads)) # Default n_gpu_layers to -1 (automatic metal) on Mac/Jupyter, but 0 in production # We can dynamically set this via environment variables. n_gpu_layers = int(os.environ.get("LOCAL_N_GPU_LAYERS", "0")) @@ -75,3 +77,23 @@ def generate(self, prompt: str, system_prompt: str = "", max_tokens: int = 250, return "__ESCALATE__" return text + + +class ClassifierEngine(LocalSLMEngine): + """ + Separate singleton to maintain an isolated KV cache for the Classifier. + Because llama.cpp uses mmap, weights (2.1GB) are shared in memory with LocalSLMEngine. + This entirely eliminates the KV cache thrashing and lock contention between classification and generation. + """ + + _instance: Optional["ClassifierEngine"] = None + _init_lock = threading.Lock() + + @classmethod + def get_instance(cls) -> "ClassifierEngine": + if cls._instance is None: + with cls._init_lock: + if cls._instance is None: + model_path = os.environ.get("LOCAL_MODEL_PATH", "models/qwen2.5-3b-instruct-q4_k_m.gguf") + cls._instance = cls(model_path=model_path) + return cls._instance diff --git a/engines/remote_llm.py b/engines/remote_llm.py index ec7591b..694a1b8 100644 --- a/engines/remote_llm.py +++ b/engines/remote_llm.py @@ -103,6 +103,7 @@ def select_remote_model(category: str) -> str: # --------------------------------------------------------------------------- class RemoteLLMEngine: def __init__(self) -> None: + self._semaphore: asyncio.Semaphore | None = None self.api_key = os.environ.get("FIREWORKS_API_KEY", "") raw_url = os.environ.get("FIREWORKS_BASE_URL", "https://api.fireworks.ai/inference/v1") @@ -184,19 +185,26 @@ async def generate( } timeout = aiohttp.ClientTimeout(total=20, connect=5) - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.post(endpoint, headers=headers, json=payload) as response: - if response.status != 200: - text = await response.text() - logger.error("Fireworks API error (status %d): %s", response.status, text) - response.raise_for_status() - - data = await response.json() - try: - if is_chat: - return str(data["choices"][0]["message"]["content"]).strip() - else: - return str(data["choices"][0]["text"]).strip() - except KeyError as e: - logger.error("Response structure: %s", data) - raise e + # Use a class-level or module-level semaphore to limit concurrency + # Initialize it lazily in an async context since asyncio.Semaphore needs an active event loop + if self._semaphore is None: + self._semaphore = asyncio.Semaphore(15) + + sem: asyncio.Semaphore = self._semaphore + async with sem: + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post(endpoint, headers=headers, json=payload) as response: + if response.status != 200: + text = await response.text() + logger.error("Fireworks API error (status %d): %s", response.status, text) + response.raise_for_status() + + data = await response.json() + try: + if is_chat: + return str(data["choices"][0]["message"]["content"]).strip() + else: + return str(data["choices"][0]["text"]).strip() + except KeyError as e: + logger.error("Response structure: %s", data) + raise e From 18f249af4aaeaf4f638a53c256257f5368401d2d Mon Sep 17 00:00:00 2001 From: Yoshio Date: Mon, 13 Jul 2026 10:32:49 +0700 Subject: [PATCH 2/2] fix(tests): add conftest.py to mock engines in CI --- .../classifier_eval_results.csv | 801 ------------------ output/classifier_eval_results.csv | 801 ------------------ output/math_analysis/math_benchmark.csv | 21 - output/math_benchmark.csv | 21 - output/practice_results.json | 34 - output/results.json | 0 .../prompt_benchmark.csv | 41 - .../prompt_benchmark_summary.md | 23 - tests/conftest.py | 48 ++ 9 files changed, 48 insertions(+), 1742 deletions(-) delete mode 100644 output/classifier_analysis/classifier_eval_results.csv delete mode 100644 output/classifier_eval_results.csv delete mode 100644 output/math_analysis/math_benchmark.csv delete mode 100644 output/math_benchmark.csv delete mode 100644 output/practice_results.json delete mode 100644 output/results.json delete mode 100644 output/sentiment_summarization_analysis/prompt_benchmark.csv delete mode 100644 output/sentiment_summarization_analysis/prompt_benchmark_summary.md create mode 100644 tests/conftest.py diff --git a/output/classifier_analysis/classifier_eval_results.csv b/output/classifier_analysis/classifier_eval_results.csv deleted file mode 100644 index 4c007de..0000000 --- a/output/classifier_analysis/classifier_eval_results.csv +++ /dev/null @@ -1,801 +0,0 @@ -task_id,category,difficulty,prompt_length,expected_route,predicted_route,is_correct -codegen-easy-033,Code Generation,Easy,205,API_CODE,API_CODE,TRUE -summary-medium-075,Summarization,Medium,408,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-074,Code Debugging,Medium,380,API_CODE,API_CODE,TRUE -ner-hard-091,NER,Hard,192,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-048,Logical Reasoning,Medium,166,API_LOGIC,API_LOGIC,TRUE -debug-easy-032,Code Debugging,Easy,399,API_CODE,API_CODE,TRUE -summary-medium-059,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-050,Logical Reasoning,Medium,119,API_LOGIC,API_LOGIC,TRUE -ner-medium-051,NER,Medium,181,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-074,Code Generation,Medium,147,API_CODE,API_CODE,TRUE -ner-hard-096,NER,Hard,233,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-045,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -codegen-hard-089,Code Generation,Hard,199,API_CODE,API_CODE,TRUE -summary-medium-038,Summarization,Medium,360,LOCAL_GENERAL,LOCAL_SENTIMENT,FALSE -factual-easy-025,Factual,Easy,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-092,Code Generation,Hard,200,API_CODE,API_CODE,TRUE -math-ast-hard-078,Math (AST),Hard,36,API_MATH,API_MATH,TRUE -summary-hard-079,Summarization,Hard,1357,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-041,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -math-ast-easy-006,Math (AST),Easy,49,API_MATH,API_MATH,TRUE -summary-hard-099,Summarization,Hard,1207,LOCAL_GENERAL,API_LOGIC,FALSE -factual-hard-089,Factual,Hard,131,LOCAL_GENERAL,API_MATH,FALSE -logic-easy-016,Logical Reasoning,Easy,185,API_LOGIC,API_LOGIC,TRUE -ner-easy-022,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE -sentiment-easy-001,Sentiment,Easy,133,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-easy-002,Sentiment,Easy,156,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-hard-090,NER,Hard,248,LOCAL_NER,LOCAL_NER,TRUE -sentiment-easy-032,Sentiment,Easy,193,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-040,Sentiment,Medium,135,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-hard-077,Summarization,Hard,1342,LOCAL_GENERAL,API_LOGIC,FALSE -factual-medium-065,Factual,Medium,107,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-easy-035,Code Generation,Easy,200,API_CODE,API_CODE,TRUE -codegen-easy-031,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -sentiment-easy-030,Sentiment,Easy,114,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-045,Code Debugging,Medium,433,API_CODE,API_CODE,TRUE -logic-easy-013,Logical Reasoning,Easy,100,API_LOGIC,API_LOGIC,TRUE -debug-easy-011,Code Debugging,Easy,176,API_CODE,API_CODE,TRUE -sentiment-medium-037,Sentiment,Medium,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-032,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE -logic-hard-080,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-022,Sentiment,Easy,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-medium-045,NER,Medium,181,LOCAL_NER,LOCAL_NER,TRUE -ner-hard-100,NER,Hard,156,LOCAL_NER,LOCAL_NER,TRUE -codegen-easy-032,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -summary-easy-030,Summarization,Easy,147,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-081,Sentiment,Hard,106,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-003,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-ast-hard-092,Math (AST),Hard,42,API_MATH,API_MATH,TRUE -ner-easy-002,NER,Easy,203,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-017,Summarization,Easy,130,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-079,Factual,Hard,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-013,Sentiment,Easy,114,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-medium-075,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -summary-hard-084,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE -codegen-medium-037,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -summary-hard-092,Summarization,Hard,1317,LOCAL_GENERAL,LOCAL_NER,FALSE -logic-medium-057,Logical Reasoning,Medium,200,API_LOGIC,API_LOGIC,TRUE -codegen-medium-051,Code Generation,Medium,191,API_CODE,LOCAL_NER,FALSE -summary-easy-002,Summarization,Easy,141,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-medium-051,Math (Word),Medium,151,API_MATH,API_MATH,TRUE -debug-medium-039,Code Debugging,Medium,224,API_CODE,API_CODE,TRUE -factual-medium-037,Factual,Medium,94,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-003,Logical Reasoning,Easy,140,API_LOGIC,API_MATH,FALSE -math-ast-hard-086,Math (AST),Hard,36,API_MATH,API_MATH,TRUE -math-word-medium-073,Math (Word),Medium,152,API_MATH,API_MATH,TRUE -math-word-hard-081,Math (Word),Hard,215,API_MATH,API_MATH,TRUE -factual-hard-093,Factual,Hard,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-010,Factual,Easy,108,LOCAL_GENERAL,API_MATH,FALSE -summary-medium-058,Summarization,Medium,404,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-051,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-064,NER,Medium,248,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-075,Factual,Medium,104,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-067,Logical Reasoning,Medium,166,API_LOGIC,API_LOGIC,TRUE -factual-easy-014,Factual,Easy,76,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-medium-039,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -math-word-medium-075,Math (Word),Medium,152,API_MATH,API_MATH,TRUE -codegen-medium-070,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -debug-medium-040,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE -ner-easy-003,NER,Easy,175,LOCAL_NER,LOCAL_NER,TRUE -ner-medium-052,NER,Medium,217,LOCAL_NER,LOCAL_NER,TRUE -sentiment-hard-078,Sentiment,Hard,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-066,Sentiment,Medium,176,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-060,Summarization,Medium,412,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-076,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE -summary-easy-021,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-029,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-033,Sentiment,Easy,160,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-083,Code Generation,Hard,185,API_CODE,API_CODE,TRUE -ner-easy-035,NER,Easy,202,LOCAL_NER,LOCAL_NER,TRUE -summary-medium-057,Summarization,Medium,328,LOCAL_GENERAL,API_MATH,FALSE -codegen-easy-004,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -debug-easy-008,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -sentiment-easy-014,Sentiment,Easy,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-medium-041,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -logic-hard-090,Logical Reasoning,Hard,185,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-048,Sentiment,Medium,114,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-easy-009,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-055,Summarization,Medium,360,LOCAL_GENERAL,LOCAL_SENTIMENT,FALSE -factual-easy-029,Factual,Easy,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-medium-048,Code Generation,Medium,191,API_CODE,LOCAL_NER,FALSE -logic-medium-047,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -summary-hard-098,Summarization,Hard,1357,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-hard-095,NER,Hard,217,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-024,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE -summary-easy-026,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-011,Factual,Easy,90,LOCAL_GENERAL,API_MATH,FALSE -debug-hard-088,Code Debugging,Hard,433,API_CODE,API_CODE,TRUE -summary-hard-086,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE -ner-easy-023,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE -sentiment-medium-065,Sentiment,Medium,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-006,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE -math-ast-medium-074,Math (AST),Medium,42,API_MATH,API_MATH,TRUE -ner-easy-005,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-038,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -logic-medium-054,Logical Reasoning,Medium,100,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-063,Sentiment,Medium,176,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-027,Logical Reasoning,Easy,100,API_LOGIC,API_LOGIC,TRUE -factual-medium-059,Factual,Medium,89,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-084,Code Debugging,Hard,211,API_CODE,API_CODE,TRUE -ner-hard-078,NER,Hard,248,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-073,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -factual-easy-034,Factual,Easy,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-038,NER,Medium,192,LOCAL_NER,LOCAL_NER,TRUE -factual-easy-024,Factual,Easy,88,LOCAL_GENERAL,API_MATH,FALSE -debug-easy-023,Code Debugging,Easy,274,API_CODE,API_CODE,TRUE -math-word-easy-011,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -factual-easy-018,Factual,Easy,86,LOCAL_GENERAL,API_CODE,FALSE -math-word-hard-099,Math (Word),Hard,103,API_MATH,API_MATH,TRUE -logic-hard-088,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE -logic-easy-008,Logical Reasoning,Easy,140,API_LOGIC,API_MATH,FALSE -ner-hard-082,NER,Hard,162,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-091,Code Debugging,Hard,380,API_CODE,API_CODE,TRUE -debug-hard-078,Code Debugging,Hard,274,API_CODE,API_CODE,TRUE -math-word-medium-063,Math (Word),Medium,143,API_MATH,API_MATH,TRUE -factual-medium-043,Factual,Medium,88,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-easy-027,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -sentiment-medium-064,Sentiment,Medium,177,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-word-medium-049,Math (Word),Medium,153,API_MATH,API_MATH,TRUE -debug-hard-083,Code Debugging,Hard,230,API_CODE,API_CODE,TRUE -codegen-easy-025,Code Generation,Easy,205,API_CODE,API_CODE,TRUE -debug-medium-051,Code Debugging,Medium,176,API_CODE,API_CODE,TRUE -codegen-easy-002,Code Generation,Easy,147,API_CODE,API_CODE,TRUE -ner-easy-032,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE -ner-hard-081,NER,Hard,175,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-012,Summarization,Easy,118,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-easy-015,NER,Easy,233,LOCAL_NER,LOCAL_NER,TRUE -math-ast-medium-038,Math (AST),Medium,43,API_MATH,API_MATH,TRUE -debug-easy-005,Code Debugging,Easy,211,API_CODE,API_CODE,TRUE -summary-medium-070,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_NER,FALSE -codegen-easy-026,Code Generation,Easy,191,API_CODE,LOCAL_NER,FALSE -ner-easy-018,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-090,Code Debugging,Hard,213,API_CODE,API_CODE,TRUE -debug-easy-021,Code Debugging,Easy,289,API_CODE,API_CODE,TRUE -debug-easy-015,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE -sentiment-medium-042,Sentiment,Medium,180,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-easy-006,Factual,Easy,131,LOCAL_GENERAL,API_MATH,FALSE -math-ast-medium-050,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -debug-medium-062,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE -sentiment-medium-073,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-hard-087,Summarization,Hard,1369,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-036,Code Debugging,Medium,211,API_CODE,API_CODE,TRUE -codegen-easy-021,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -math-ast-easy-020,Math (AST),Easy,44,API_MATH,API_MATH,TRUE -ner-easy-010,NER,Easy,211,LOCAL_NER,LOCAL_NER,TRUE -factual-hard-085,Factual,Hard,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-097,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE -logic-medium-068,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE -factual-easy-017,Factual,Easy,118,LOCAL_GENERAL,API_MATH,FALSE -debug-easy-003,Code Debugging,Easy,274,API_CODE,API_CODE,TRUE -factual-medium-040,Factual,Medium,73,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-038,Sentiment,Medium,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-007,Summarization,Easy,129,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-082,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE -sentiment-hard-094,Sentiment,Hard,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-028,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE -math-word-medium-039,Math (Word),Medium,137,API_MATH,API_MATH,TRUE -logic-hard-096,Logical Reasoning,Hard,212,API_LOGIC,API_LOGIC,TRUE -factual-hard-086,Factual,Hard,108,LOCAL_GENERAL,API_MATH,FALSE -debug-hard-079,Code Debugging,Hard,211,API_CODE,API_CODE,TRUE -logic-hard-079,Logical Reasoning,Hard,200,API_LOGIC,API_LOGIC,TRUE -codegen-easy-013,Code Generation,Easy,186,API_CODE,API_CODE,TRUE -codegen-medium-050,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -logic-medium-065,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -math-word-hard-089,Math (Word),Hard,134,API_MATH,API_MATH,TRUE -logic-easy-007,Logical Reasoning,Easy,212,API_LOGIC,API_LOGIC,TRUE -ner-hard-084,NER,Hard,203,LOCAL_NER,LOCAL_NER,TRUE -math-word-easy-025,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -ner-medium-053,NER,Medium,211,LOCAL_NER,LOCAL_NER,TRUE -codegen-easy-003,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -logic-medium-062,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -ner-hard-077,NER,Hard,203,LOCAL_NER,LOCAL_NER,TRUE -summary-hard-094,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE -sentiment-medium-057,Sentiment,Medium,147,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-004,NER,Easy,233,LOCAL_NER,LOCAL_NER,TRUE -math-word-hard-091,Math (Word),Hard,150,API_MATH,API_MATH,TRUE -factual-medium-041,Factual,Medium,73,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-easy-034,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-medium-041,Math (Word),Medium,153,API_MATH,API_MATH,TRUE -summary-hard-076,Summarization,Hard,1342,LOCAL_GENERAL,API_LOGIC,FALSE -math-ast-easy-030,Math (AST),Easy,42,API_MATH,API_MATH,TRUE -math-word-hard-085,Math (Word),Hard,141,API_MATH,API_MATH,TRUE -sentiment-medium-050,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-024,NER,Easy,223,LOCAL_NER,LOCAL_NER,TRUE -math-ast-hard-094,Math (AST),Hard,43,API_MATH,API_MATH,TRUE -codegen-medium-057,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -math-word-medium-053,Math (Word),Medium,217,API_MATH,API_MATH,TRUE -logic-easy-010,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE -summary-easy-016,Summarization,Easy,130,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-065,Summarization,Medium,420,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-052,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-028,Sentiment,Easy,164,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-079,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -logic-easy-018,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE -factual-easy-032,Factual,Easy,123,LOCAL_GENERAL,API_MATH,FALSE -sentiment-hard-091,Sentiment,Hard,165,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-hard-093,Summarization,Hard,1309,LOCAL_GENERAL,LOCAL_NER,FALSE -codegen-medium-063,Code Generation,Medium,170,API_CODE,API_CODE,TRUE -math-word-easy-019,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -math-word-easy-033,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -logic-medium-040,Logical Reasoning,Medium,157,API_LOGIC,API_LOGIC,TRUE -logic-hard-076,Logical Reasoning,Hard,219,API_LOGIC,API_LOGIC,TRUE -factual-medium-046,Factual,Medium,109,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-057,Factual,Medium,90,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-036,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -math-ast-hard-100,Math (AST),Hard,43,API_MATH,API_MATH,TRUE -codegen-hard-077,Code Generation,Hard,186,API_CODE,API_CODE,TRUE -summary-easy-013,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-002,Code Debugging,Easy,289,API_CODE,API_CODE,TRUE -math-word-hard-093,Math (Word),Hard,215,API_MATH,API_MATH,TRUE -codegen-hard-094,Code Generation,Hard,191,API_CODE,LOCAL_NER,FALSE -summary-hard-096,Summarization,Hard,1062,LOCAL_GENERAL,API_MATH,FALSE -summary-easy-011,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-043,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE -logic-medium-049,Logical Reasoning,Medium,159,API_LOGIC,API_MATH,FALSE -sentiment-medium-052,Sentiment,Medium,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-hard-077,Code Debugging,Hard,195,API_CODE,API_CODE,TRUE -ner-hard-086,NER,Hard,217,LOCAL_NER,LOCAL_NER,TRUE -codegen-hard-096,Code Generation,Hard,147,API_CODE,API_CODE,TRUE -ner-medium-044,NER,Medium,198,LOCAL_NER,LOCAL_NER,TRUE -ner-medium-068,NER,Medium,229,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-050,Factual,Medium,103,LOCAL_GENERAL,API_MATH,FALSE -math-ast-medium-048,Math (AST),Medium,42,API_MATH,API_MATH,TRUE -ner-medium-058,NER,Medium,156,LOCAL_NER,LOCAL_NER,TRUE -ner-medium-073,NER,Medium,175,LOCAL_NER,LOCAL_NER,TRUE -math-word-easy-007,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -ner-easy-031,NER,Easy,211,LOCAL_NER,LOCAL_NER,TRUE -summary-hard-078,Summarization,Hard,1354,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-049,Code Debugging,Medium,224,API_CODE,API_CODE,TRUE -logic-hard-089,Logical Reasoning,Hard,185,API_LOGIC,API_LOGIC,TRUE -summary-medium-036,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -summary-medium-056,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-easy-017,Code Generation,Easy,170,API_CODE,API_CODE,TRUE -codegen-easy-011,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -codegen-medium-065,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -ner-medium-050,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -math-ast-easy-002,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -math-ast-easy-016,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -ner-medium-057,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE -ner-hard-089,NER,Hard,233,LOCAL_NER,LOCAL_NER,TRUE -ner-easy-001,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE -math-ast-medium-072,Math (AST),Medium,45,API_MATH,API_MATH,TRUE -ner-medium-040,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-044,Factual,Medium,100,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-087,Factual,Hard,93,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-034,Code Debugging,Easy,232,API_CODE,API_CODE,TRUE -summary-easy-008,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-048,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -sentiment-medium-069,Sentiment,Medium,116,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-025,Code Debugging,Easy,224,API_CODE,API_CODE,TRUE -codegen-easy-028,Code Generation,Easy,147,API_CODE,API_CODE,TRUE -math-word-medium-065,Math (Word),Medium,217,API_MATH,API_MATH,TRUE -sentiment-hard-095,Sentiment,Hard,150,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-066,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-hard-083,Logical Reasoning,Hard,157,API_LOGIC,API_LOGIC,TRUE -math-ast-medium-056,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -debug-easy-007,Code Debugging,Easy,433,API_CODE,API_CODE,TRUE -sentiment-easy-008,Sentiment,Easy,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-019,Code Debugging,Easy,224,API_CODE,API_CODE,TRUE -codegen-hard-087,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -debug-easy-016,Code Debugging,Easy,205,API_CODE,API_CODE,TRUE -sentiment-easy-016,Sentiment,Easy,132,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-033,Logical Reasoning,Easy,152,API_LOGIC,API_LOGIC,TRUE -logic-easy-020,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE -math-ast-medium-052,Math (AST),Medium,42,API_MATH,API_MATH,TRUE -ner-hard-088,NER,Hard,156,LOCAL_NER,LOCAL_NER,TRUE -codegen-easy-007,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -sentiment-easy-010,Sentiment,Easy,130,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-017,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE -ner-medium-047,NER,Medium,234,LOCAL_NER,LOCAL_NER,TRUE -ner-easy-025,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-064,Factual,Medium,79,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-070,Code Debugging,Medium,380,API_CODE,API_CODE,TRUE -summary-easy-022,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-easy-028,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-073,Summarization,Medium,316,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-069,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -debug-medium-038,Code Debugging,Medium,232,API_CODE,API_CODE,TRUE -debug-hard-095,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE -sentiment-medium-067,Sentiment,Medium,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-026,Logical Reasoning,Easy,178,API_LOGIC,API_LOGIC,TRUE -summary-easy-009,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-097,Summarization,Hard,1219,LOCAL_GENERAL,API_LOGIC,FALSE -math-word-easy-031,Math (Word),Easy,152,API_MATH,API_MATH,TRUE -factual-hard-098,Factual,Hard,101,LOCAL_GENERAL,API_MATH,FALSE -sentiment-hard-096,Sentiment,Hard,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-hard-098,Math (AST),Hard,43,API_MATH,API_MATH,TRUE -logic-medium-074,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-062,Sentiment,Medium,163,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-081,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -ner-medium-067,NER,Medium,198,LOCAL_NER,LOCAL_NER,TRUE -math-word-hard-077,Math (Word),Hard,215,API_MATH,API_MATH,TRUE -debug-easy-014,Code Debugging,Easy,224,API_CODE,API_CODE,TRUE -logic-easy-035,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE -ner-medium-062,NER,Medium,211,LOCAL_NER,LOCAL_NER,TRUE -debug-medium-060,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE -logic-hard-086,Logical Reasoning,Hard,145,API_LOGIC,API_LOGIC,TRUE -sentiment-hard-098,Sentiment,Hard,186,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-051,Summarization,Medium,360,LOCAL_GENERAL,LOCAL_SENTIMENT,FALSE -debug-easy-004,Code Debugging,Easy,176,API_CODE,API_CODE,TRUE -ner-easy-020,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE -factual-easy-015,Factual,Easy,81,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-086,Code Generation,Hard,171,API_CODE,API_CODE,TRUE -logic-hard-094,Logical Reasoning,Hard,140,API_LOGIC,API_MATH,FALSE -ner-medium-061,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-031,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -ner-hard-076,NER,Hard,229,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-029,Summarization,Easy,139,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-043,NER,Medium,203,LOCAL_NER,LOCAL_NER,TRUE -logic-hard-081,Logical Reasoning,Hard,145,API_LOGIC,API_LOGIC,TRUE -debug-easy-026,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE -sentiment-easy-005,Sentiment,Easy,170,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-hard-099,Factual,Hard,118,LOCAL_GENERAL,API_MATH,FALSE -ner-easy-008,NER,Easy,229,LOCAL_NER,LOCAL_NER,TRUE -logic-easy-021,Logical Reasoning,Easy,185,API_LOGIC,API_LOGIC,TRUE -math-word-easy-017,Math (Word),Easy,170,API_MATH,API_MATH,TRUE -logic-easy-031,Logical Reasoning,Easy,166,API_LOGIC,API_LOGIC,TRUE -math-ast-hard-080,Math (AST),Hard,45,API_MATH,API_MATH,TRUE -debug-hard-085,Code Debugging,Hard,380,API_CODE,API_CODE,TRUE -math-word-easy-003,Math (Word),Easy,150,API_MATH,API_MATH,TRUE -math-ast-easy-014,Math (AST),Easy,41,API_MATH,API_MATH,TRUE -sentiment-medium-041,Sentiment,Medium,129,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-022,Logical Reasoning,Easy,200,API_LOGIC,API_LOGIC,TRUE -math-ast-hard-096,Math (AST),Hard,45,API_MATH,API_MATH,TRUE -debug-medium-056,Code Debugging,Medium,433,API_CODE,API_CODE,TRUE -codegen-easy-029,Code Generation,Easy,171,API_CODE,API_CODE,TRUE -math-word-medium-037,Math (Word),Medium,172,API_MATH,API_MATH,TRUE -codegen-medium-072,Code Generation,Medium,186,API_CODE,API_CODE,TRUE -ner-medium-069,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE -sentiment-easy-011,Sentiment,Easy,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-medium-058,Logical Reasoning,Medium,200,API_LOGIC,API_LOGIC,TRUE -factual-hard-092,Factual,Hard,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-013,Factual,Easy,123,LOCAL_GENERAL,API_MATH,FALSE -sentiment-easy-012,Sentiment,Easy,120,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-hard-091,Factual,Hard,93,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-084,Factual,Hard,90,LOCAL_GENERAL,API_MATH,FALSE -math-ast-easy-022,Math (AST),Easy,36,API_MATH,API_MATH,TRUE -ner-easy-016,NER,Easy,167,LOCAL_NER,LOCAL_NER,TRUE -debug-medium-057,Code Debugging,Medium,205,API_CODE,API_CODE,TRUE -sentiment-easy-009,Sentiment,Easy,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-087,Sentiment,Hard,180,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-hard-082,Math (AST),Hard,49,API_MATH,API_MATH,TRUE -factual-medium-053,Factual,Medium,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-easy-028,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-045,Code Generation,Medium,200,API_CODE,API_CODE,TRUE -summary-medium-050,Summarization,Medium,336,LOCAL_GENERAL,API_MATH,FALSE -ner-easy-012,NER,Easy,217,LOCAL_NER,LOCAL_NER,TRUE -math-ast-hard-084,Math (AST),Hard,35,API_MATH,API_MATH,TRUE -codegen-hard-100,Code Generation,Hard,162,API_CODE,API_CODE,TRUE -math-word-hard-087,Math (Word),Hard,152,API_MATH,API_MATH,TRUE -math-ast-medium-058,Math (AST),Medium,35,API_MATH,API_MATH,TRUE -debug-hard-087,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE -debug-hard-089,Code Debugging,Hard,230,API_CODE,API_CODE,TRUE -codegen-medium-058,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -factual-hard-076,Factual,Hard,101,LOCAL_GENERAL,API_MATH,FALSE -codegen-medium-044,Code Generation,Medium,200,API_CODE,API_CODE,TRUE -ner-medium-075,NER,Medium,242,LOCAL_NER,LOCAL_NER,TRUE -ner-medium-041,NER,Medium,156,LOCAL_NER,LOCAL_NER,TRUE -factual-easy-019,Factual,Easy,90,LOCAL_GENERAL,API_MATH,FALSE -sentiment-hard-080,Sentiment,Hard,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-hard-077,Factual,Hard,97,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-054,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-039,Factual,Medium,80,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-hard-099,Logical Reasoning,Hard,178,API_LOGIC,API_LOGIC,TRUE -ner-easy-029,NER,Easy,175,LOCAL_NER,LOCAL_NER,TRUE -sentiment-hard-088,Sentiment,Hard,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-029,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE -summary-medium-062,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-083,Sentiment,Hard,198,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-easy-027,Sentiment,Easy,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-easy-020,Code Generation,Easy,170,API_CODE,API_CODE,TRUE -factual-hard-083,Factual,Hard,82,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-easy-035,Math (Word),Easy,149,API_MATH,API_MATH,TRUE -summary-easy-027,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-067,Factual,Medium,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-033,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -factual-easy-002,Factual,Easy,82,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-023,Factual,Easy,123,LOCAL_GENERAL,API_MATH,FALSE -logic-medium-070,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -logic-easy-001,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE -ner-easy-007,NER,Easy,248,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-048,Factual,Medium,81,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-027,Factual,Easy,118,LOCAL_GENERAL,API_MATH,FALSE -logic-easy-009,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE -codegen-easy-008,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -sentiment-medium-059,Sentiment,Medium,162,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-058,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -summary-easy-025,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-052,Summarization,Medium,408,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-hard-084,Logical Reasoning,Hard,157,API_LOGIC,API_LOGIC,TRUE -codegen-easy-010,Code Generation,Easy,147,API_CODE,API_CODE,TRUE -codegen-medium-052,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -math-ast-easy-008,Math (AST),Easy,49,API_MATH,API_MATH,TRUE -summary-easy-010,Summarization,Easy,149,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-medium-045,Math (Word),Medium,153,API_MATH,API_MATH,TRUE -math-ast-medium-040,Math (AST),Medium,43,API_MATH,API_MATH,TRUE -factual-hard-080,Factual,Hard,90,LOCAL_GENERAL,API_MATH,FALSE -ner-easy-026,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE -summary-hard-081,Summarization,Hard,1342,LOCAL_GENERAL,API_LOGIC,FALSE -factual-medium-055,Factual,Medium,89,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-039,Logical Reasoning,Medium,119,API_LOGIC,API_LOGIC,TRUE -codegen-medium-040,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -factual-medium-069,Factual,Medium,82,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-054,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-040,Summarization,Medium,316,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-easy-009,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -sentiment-hard-084,Sentiment,Hard,163,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-019,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE -summary-medium-042,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-055,NER,Medium,202,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-049,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -summary-easy-035,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-easy-014,NER,Easy,248,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-014,Summarization,Easy,159,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-medium-055,Math (Word),Medium,136,API_MATH,API_MATH,TRUE -summary-medium-061,Summarization,Medium,416,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-079,Sentiment,Hard,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-027,NER,Easy,211,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-071,Logical Reasoning,Medium,140,API_LOGIC,API_MATH,FALSE -logic-easy-023,Logical Reasoning,Easy,164,API_LOGIC,API_LOGIC,TRUE -math-ast-medium-046,Math (AST),Medium,49,API_MATH,API_MATH,TRUE -math-ast-hard-090,Math (AST),Hard,48,API_MATH,API_MATH,TRUE -math-ast-easy-026,Math (AST),Easy,44,API_MATH,API_MATH,TRUE -ner-hard-094,NER,Hard,181,LOCAL_NER,LOCAL_NER,TRUE -math-word-hard-079,Math (Word),Hard,134,API_MATH,API_MATH,TRUE -debug-hard-093,Code Debugging,Hard,230,API_CODE,API_CODE,TRUE -logic-medium-042,Logical Reasoning,Medium,219,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-060,Sentiment,Medium,163,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-easy-010,Math (AST),Easy,49,API_MATH,API_MATH,TRUE -sentiment-hard-076,Sentiment,Hard,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-medium-042,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -debug-medium-037,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE -sentiment-hard-089,Sentiment,Hard,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-019,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-059,NER,Medium,167,LOCAL_NER,LOCAL_NER,TRUE -factual-easy-005,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-038,Factual,Medium,100,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-085,Summarization,Hard,1377,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-072,NER,Medium,229,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-080,Code Debugging,Hard,176,API_CODE,API_CODE,TRUE -math-ast-medium-054,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -summary-easy-031,Summarization,Easy,135,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-044,Summarization,Medium,404,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-090,Code Generation,Hard,186,API_CODE,API_CODE,TRUE -sentiment-easy-003,Sentiment,Easy,176,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-069,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE -math-word-easy-023,Math (Word),Easy,151,API_MATH,API_MATH,TRUE -debug-easy-010,Code Debugging,Easy,232,API_CODE,API_CODE,TRUE -summary-medium-067,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-043,Summarization,Medium,400,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-medium-056,Code Generation,Medium,171,API_CODE,API_CODE,TRUE -math-ast-easy-034,Math (AST),Easy,48,API_MATH,API_MATH,TRUE -logic-medium-063,Logical Reasoning,Medium,159,API_LOGIC,API_MATH,FALSE -sentiment-medium-068,Sentiment,Medium,199,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-090,Sentiment,Hard,181,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-099,Sentiment,Hard,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-037,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -codegen-easy-018,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -ner-hard-097,NER,Hard,234,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-018,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-021,Factual,Easy,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-094,Code Debugging,Hard,195,API_CODE,API_CODE,TRUE -codegen-easy-030,Code Generation,Easy,200,API_CODE,API_CODE,TRUE -math-word-easy-027,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -sentiment-hard-092,Sentiment,Hard,156,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-medium-056,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE -ner-hard-083,NER,Hard,166,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-062,Factual,Medium,104,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-007,Sentiment,Easy,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-word-easy-021,Math (Word),Easy,170,API_MATH,API_MATH,TRUE -debug-medium-061,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -summary-medium-046,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-035,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-easy-005,Summarization,Easy,135,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-036,Factual,Medium,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-035,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -ner-medium-036,NER,Medium,234,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-042,Factual,Medium,107,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-061,Sentiment,Medium,145,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-hard-092,NER,Hard,181,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-009,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -logic-easy-025,Logical Reasoning,Easy,219,API_LOGIC,API_LOGIC,TRUE -logic-medium-069,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -ner-medium-037,NER,Medium,202,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-047,Code Generation,Medium,186,API_CODE,API_CODE,TRUE -logic-medium-053,Logical Reasoning,Medium,145,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-045,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-100,Sentiment,Hard,150,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-098,Code Generation,Hard,186,API_CODE,API_CODE,TRUE -factual-easy-030,Factual,Easy,90,LOCAL_GENERAL,API_MATH,FALSE -math-word-medium-061,Math (Word),Medium,154,API_MATH,API_MATH,TRUE -debug-hard-096,Code Debugging,Hard,232,API_CODE,API_CODE,TRUE -debug-medium-054,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -codegen-hard-078,Code Generation,Hard,200,API_CODE,API_CODE,TRUE -sentiment-easy-023,Sentiment,Easy,184,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-004,Summarization,Easy,159,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-048,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -math-word-medium-059,Math (Word),Medium,136,API_MATH,API_MATH,TRUE -ner-hard-099,NER,Hard,175,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-071,Code Generation,Medium,147,API_CODE,API_CODE,TRUE -sentiment-hard-093,Sentiment,Hard,143,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-medium-066,NER,Medium,248,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-067,Code Generation,Medium,205,API_CODE,API_CODE,TRUE -summary-hard-100,Summarization,Hard,1042,LOCAL_GENERAL,API_MATH,FALSE -factual-easy-003,Factual,Easy,118,LOCAL_GENERAL,API_MATH,FALSE -codegen-easy-005,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -math-word-medium-069,Math (Word),Medium,137,API_MATH,API_MATH,TRUE -logic-easy-015,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE -logic-hard-095,Logical Reasoning,Hard,200,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-019,Sentiment,Easy,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-hard-093,NER,Hard,166,LOCAL_NER,LOCAL_NER,TRUE -debug-medium-072,Code Debugging,Medium,205,API_CODE,API_CODE,TRUE -debug-medium-065,Code Debugging,Medium,433,API_CODE,API_CODE,TRUE -math-word-medium-071,Math (Word),Medium,217,API_MATH,API_MATH,TRUE -sentiment-medium-043,Sentiment,Medium,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-082,Code Generation,Hard,185,API_CODE,API_CODE,TRUE -factual-easy-016,Factual,Easy,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-052,Factual,Medium,110,LOCAL_GENERAL,API_MATH,FALSE -debug-medium-055,Code Debugging,Medium,259,API_CODE,API_CODE,TRUE -ner-medium-054,NER,Medium,202,LOCAL_NER,LOCAL_NER,TRUE -factual-hard-078,Factual,Hard,108,LOCAL_GENERAL,API_MATH,FALSE -sentiment-medium-056,Sentiment,Medium,160,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-easy-006,Code Generation,Easy,200,API_CODE,API_CODE,TRUE -debug-medium-073,Code Debugging,Medium,176,API_CODE,API_CODE,TRUE -math-ast-hard-088,Math (AST),Hard,49,API_MATH,API_MATH,TRUE -logic-medium-061,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -math-ast-medium-044,Math (AST),Medium,42,API_MATH,API_MATH,TRUE -codegen-medium-068,Code Generation,Medium,162,API_CODE,API_CODE,TRUE -debug-medium-064,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE -codegen-medium-036,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -sentiment-medium-055,Sentiment,Medium,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-easy-012,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -summary-medium-048,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -ner-easy-013,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-063,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-easy-001,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -factual-medium-056,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-hard-079,NER,Hard,223,LOCAL_NER,LOCAL_NER,TRUE -ner-easy-021,NER,Easy,234,LOCAL_NER,LOCAL_NER,TRUE -logic-easy-002,Logical Reasoning,Easy,100,API_LOGIC,API_LOGIC,TRUE -summary-medium-063,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_NER,FALSE -ner-medium-042,NER,Medium,166,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-053,Code Generation,Medium,191,API_CODE,LOCAL_NER,FALSE -factual-hard-097,Factual,Hard,78,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-hard-095,Math (Word),Hard,150,API_MATH,API_MATH,TRUE -ner-easy-017,NER,Easy,217,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-086,Code Debugging,Hard,211,API_CODE,API_CODE,TRUE -sentiment-medium-049,Sentiment,Medium,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-hard-085,NER,Hard,166,LOCAL_NER,LOCAL_NER,TRUE -math-ast-medium-036,Math (AST),Medium,43,API_MATH,API_MATH,TRUE -debug-hard-100,Code Debugging,Hard,259,API_CODE,API_CODE,TRUE -sentiment-easy-020,Sentiment,Easy,162,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-012,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE -sentiment-hard-086,Sentiment,Hard,133,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-044,Sentiment,Medium,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-027,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE -debug-hard-082,Code Debugging,Hard,274,API_CODE,API_CODE,TRUE -summary-medium-039,Summarization,Medium,328,LOCAL_GENERAL,API_MATH,FALSE -logic-hard-097,Logical Reasoning,Hard,157,API_LOGIC,API_LOGIC,TRUE -factual-medium-070,Factual,Medium,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-hard-098,Logical Reasoning,Hard,152,API_LOGIC,API_LOGIC,TRUE -factual-medium-061,Factual,Medium,89,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-073,Factual,Medium,110,LOCAL_GENERAL,API_MATH,FALSE -factual-easy-008,Factual,Easy,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-medium-069,Code Generation,Medium,200,API_CODE,API_CODE,TRUE -sentiment-medium-036,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-068,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE -factual-easy-022,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-047,Summarization,Medium,372,LOCAL_GENERAL,API_LOGIC,FALSE -logic-hard-085,Logical Reasoning,Hard,119,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-031,Sentiment,Easy,164,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-054,Sentiment,Medium,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-easy-024,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -ner-medium-063,NER,Medium,217,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-030,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -ner-medium-049,NER,Medium,162,LOCAL_NER,LOCAL_NER,TRUE -logic-easy-030,Logical Reasoning,Easy,164,API_LOGIC,API_LOGIC,TRUE -logic-medium-073,Logical Reasoning,Medium,157,API_LOGIC,API_LOGIC,TRUE -summary-easy-020,Summarization,Easy,130,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-006,Sentiment,Easy,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-028,Logical Reasoning,Easy,164,API_LOGIC,API_LOGIC,TRUE -logic-hard-092,Logical Reasoning,Hard,200,API_LOGIC,API_LOGIC,TRUE -ner-easy-034,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE -debug-medium-041,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE -logic-hard-082,Logical Reasoning,Hard,185,API_LOGIC,API_LOGIC,TRUE -math-ast-medium-070,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -debug-medium-063,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -ner-hard-087,NER,Hard,162,LOCAL_NER,LOCAL_NER,TRUE -math-ast-easy-032,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -math-word-easy-029,Math (Word),Easy,150,API_MATH,API_MATH,TRUE -debug-medium-046,Code Debugging,Medium,232,API_CODE,API_CODE,TRUE -factual-medium-071,Factual,Medium,110,LOCAL_GENERAL,API_MATH,FALSE -ner-hard-098,NER,Hard,192,LOCAL_NER,LOCAL_NER,TRUE -factual-hard-088,Factual,Hard,92,LOCAL_GENERAL,API_MATH,FALSE -codegen-medium-043,Code Generation,Medium,170,API_CODE,API_CODE,TRUE -logic-hard-091,Logical Reasoning,Hard,119,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-047,Sentiment,Medium,152,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-medium-060,Factual,Medium,107,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-046,Sentiment,Medium,152,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-015,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-072,Summarization,Medium,372,LOCAL_GENERAL,API_LOGIC,FALSE -logic-medium-046,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -factual-easy-020,Factual,Easy,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-001,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -summary-easy-032,Summarization,Easy,147,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-014,Logical Reasoning,Easy,145,API_LOGIC,API_LOGIC,TRUE -factual-easy-001,Factual,Easy,102,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-044,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE -debug-medium-053,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE -math-ast-medium-066,Math (AST),Medium,43,API_MATH,API_MATH,TRUE -summary-hard-083,Summarization,Hard,1219,LOCAL_GENERAL,API_LOGIC,FALSE -codegen-easy-014,Code Generation,Easy,171,API_CODE,API_CODE,TRUE -codegen-medium-075,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -codegen-easy-012,Code Generation,Easy,170,API_CODE,API_CODE,TRUE -logic-medium-037,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE -summary-easy-006,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-easy-009,Math (Word),Easy,170,API_MATH,API_MATH,TRUE -debug-easy-020,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -math-word-hard-083,Math (Word),Hard,215,API_MATH,API_MATH,TRUE -debug-easy-013,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -logic-hard-077,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE -factual-easy-004,Factual,Easy,88,LOCAL_GENERAL,API_MATH,FALSE -codegen-easy-024,Code Generation,Easy,162,API_CODE,API_CODE,TRUE -math-ast-medium-062,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -ner-medium-046,NER,Medium,229,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-041,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -codegen-hard-099,Code Generation,Hard,185,API_CODE,API_CODE,TRUE -codegen-medium-061,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -codegen-easy-016,Code Generation,Easy,170,API_CODE,API_CODE,TRUE -codegen-medium-046,Code Generation,Medium,186,API_CODE,API_CODE,TRUE -math-ast-easy-018,Math (AST),Easy,42,API_MATH,API_MATH,TRUE -factual-easy-012,Factual,Easy,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-067,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE -sentiment-medium-058,Sentiment,Medium,198,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-042,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE -ner-hard-080,NER,Hard,156,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-060,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -summary-hard-091,Summarization,Hard,1377,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-095,Summarization,Hard,1309,LOCAL_GENERAL,LOCAL_NER,FALSE -factual-medium-066,Factual,Medium,89,LOCAL_GENERAL,API_MATH,FALSE -logic-easy-011,Logical Reasoning,Easy,145,API_LOGIC,API_LOGIC,TRUE -codegen-medium-062,Code Generation,Medium,171,API_CODE,API_CODE,TRUE -codegen-medium-064,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -logic-medium-043,Logical Reasoning,Medium,159,API_LOGIC,API_MATH,FALSE -math-word-medium-057,Math (Word),Medium,153,API_MATH,API_MATH,TRUE -math-word-easy-013,Math (Word),Easy,141,API_MATH,API_MATH,TRUE -factual-medium-049,Factual,Medium,73,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-094,Factual,Hard,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-006,Code Debugging,Easy,289,API_CODE,API_CODE,TRUE -debug-easy-022,Code Debugging,Easy,205,API_CODE,API_CODE,TRUE -math-ast-medium-042,Math (AST),Medium,45,API_MATH,API_MATH,TRUE -codegen-easy-001,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -codegen-medium-038,Code Generation,Medium,200,API_CODE,API_CODE,TRUE -ner-easy-009,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-098,Code Debugging,Hard,224,API_CODE,API_CODE,TRUE -ner-medium-039,NER,Medium,234,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-099,Code Debugging,Hard,380,API_CODE,API_CODE,TRUE -sentiment-medium-070,Sentiment,Medium,170,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-052,Code Debugging,Medium,195,API_CODE,API_CODE,TRUE -codegen-hard-076,Code Generation,Hard,191,API_CODE,LOCAL_NER,FALSE -sentiment-medium-053,Sentiment,Medium,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-093,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -sentiment-medium-072,Sentiment,Medium,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-word-medium-047,Math (Word),Medium,106,API_MATH,API_MATH,TRUE -ner-medium-070,NER,Medium,217,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-059,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE -ner-easy-006,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-012,Code Debugging,Easy,205,API_CODE,API_CODE,TRUE -logic-medium-072,Logical Reasoning,Medium,100,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-018,Sentiment,Easy,175,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-023,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-081,Code Debugging,Hard,232,API_CODE,API_CODE,TRUE -sentiment-easy-017,Sentiment,Easy,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-medium-068,Factual,Medium,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-075,Sentiment,Medium,125,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-011,NER,Easy,217,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-024,Summarization,Easy,147,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-084,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -sentiment-easy-035,Sentiment,Easy,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-medium-058,Factual,Medium,120,LOCAL_GENERAL,API_MATH,FALSE -summary-easy-033,Summarization,Easy,159,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-025,Sentiment,Easy,135,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-071,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE -logic-medium-056,Logical Reasoning,Medium,219,API_LOGIC,API_LOGIC,TRUE -sentiment-hard-077,Sentiment,Hard,129,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-hard-089,Summarization,Hard,1369,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-074,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE -math-ast-medium-068,Math (AST),Medium,48,API_MATH,API_MATH,TRUE -codegen-hard-095,Code Generation,Hard,205,API_CODE,API_CODE,TRUE -debug-medium-047,Code Debugging,Medium,259,API_CODE,API_CODE,TRUE -math-ast-hard-076,Math (AST),Hard,45,API_MATH,API_MATH,TRUE -codegen-easy-023,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -logic-easy-024,Logical Reasoning,Easy,185,API_LOGIC,API_LOGIC,TRUE -factual-medium-074,Factual,Medium,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-064,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -logic-hard-100,Logical Reasoning,Hard,119,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-074,Sentiment,Medium,145,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-hard-087,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-015,Sentiment,Easy,151,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-064,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_NER,FALSE -summary-hard-080,Summarization,Hard,1309,LOCAL_GENERAL,LOCAL_NER,FALSE -math-word-medium-067,Math (Word),Medium,172,API_MATH,API_MATH,TRUE -summary-medium-074,Summarization,Medium,416,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-051,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE -ner-medium-065,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -codegen-hard-097,Code Generation,Hard,147,API_CODE,API_CODE,TRUE -codegen-medium-054,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -math-ast-easy-004,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -codegen-hard-080,Code Generation,Hard,170,API_CODE,API_CODE,TRUE -factual-hard-082,Factual,Hard,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-097,Sentiment,Hard,130,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-easy-034,Sentiment,Easy,145,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-hard-078,Logical Reasoning,Hard,178,API_LOGIC,API_LOGIC,TRUE -factual-medium-047,Factual,Medium,92,LOCAL_GENERAL,API_MATH,FALSE -factual-medium-072,Factual,Medium,92,LOCAL_GENERAL,API_MATH,FALSE -codegen-easy-022,Code Generation,Easy,186,API_CODE,API_CODE,TRUE -math-word-hard-097,Math (Word),Hard,135,API_MATH,API_MATH,TRUE -debug-hard-092,Code Debugging,Hard,433,API_CODE,API_CODE,TRUE -summary-medium-071,Summarization,Medium,412,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-034,Logical Reasoning,Easy,178,API_LOGIC,API_LOGIC,TRUE -logic-medium-044,Logical Reasoning,Medium,219,API_LOGIC,API_LOGIC,TRUE -summary-easy-001,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-066,Code Debugging,Medium,232,API_CODE,API_CODE,TRUE -factual-hard-081,Factual,Hard,111,LOCAL_GENERAL,LOCAL_NER,FALSE -summary-hard-088,Summarization,Hard,1042,LOCAL_GENERAL,API_MATH,FALSE -math-ast-easy-028,Math (AST),Easy,48,API_MATH,API_MATH,TRUE -debug-medium-050,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE -ner-medium-071,NER,Medium,162,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-045,Factual,Medium,109,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-071,Sentiment,Medium,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-019,Logical Reasoning,Easy,140,API_LOGIC,API_MATH,FALSE -summary-medium-053,Summarization,Medium,328,LOCAL_GENERAL,API_MATH,FALSE -summary-medium-068,Summarization,Medium,416,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-100,Factual,Hard,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-090,Summarization,Hard,1207,LOCAL_GENERAL,API_LOGIC,FALSE -factual-hard-095,Factual,Hard,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-091,Code Generation,Hard,171,API_CODE,API_CODE,TRUE -summary-medium-045,Summarization,Medium,400,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-021,Sentiment,Easy,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-medium-060,Math (AST),Medium,49,API_MATH,API_MATH,TRUE -codegen-medium-066,Code Generation,Medium,162,API_CODE,API_CODE,TRUE -summary-medium-049,Summarization,Medium,372,LOCAL_GENERAL,API_LOGIC,FALSE -codegen-hard-085,Code Generation,Hard,171,API_CODE,API_CODE,TRUE -codegen-easy-015,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -math-ast-medium-064,Math (AST),Medium,45,API_MATH,API_MATH,TRUE -factual-easy-007,Factual,Easy,80,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-085,Sentiment,Hard,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-004,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE -ner-medium-060,NER,Medium,156,LOCAL_NER,LOCAL_NER,TRUE -sentiment-easy-026,Sentiment,Easy,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-030,NER,Easy,223,LOCAL_NER,LOCAL_NER,TRUE -codegen-hard-088,Code Generation,Hard,162,API_CODE,API_CODE,TRUE -codegen-easy-019,Code Generation,Easy,147,API_CODE,API_CODE,TRUE -logic-medium-066,Logical Reasoning,Medium,164,API_LOGIC,API_LOGIC,TRUE -codegen-easy-034,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -sentiment-easy-004,Sentiment,Easy,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-075,Code Debugging,Medium,380,API_CODE,API_CODE,TRUE -factual-easy-028,Factual,Easy,88,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-017,Logical Reasoning,Easy,200,API_LOGIC,API_LOGIC,TRUE -debug-easy-018,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE -sentiment-medium-039,Sentiment,Medium,184,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-easy-024,Sentiment,Easy,147,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-hard-096,Factual,Hard,118,LOCAL_GENERAL,API_MATH,FALSE -factual-easy-031,Factual,Easy,81,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-033,Factual,Easy,92,LOCAL_GENERAL,API_MATH,FALSE -ner-easy-033,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE -logic-easy-005,Logical Reasoning,Easy,112,API_LOGIC,API_LOGIC,TRUE -factual-hard-090,Factual,Hard,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-059,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE -codegen-medium-059,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -logic-hard-093,Logical Reasoning,Hard,212,API_LOGIC,API_LOGIC,TRUE -math-word-easy-005,Math (Word),Easy,149,API_MATH,API_MATH,TRUE -codegen-medium-055,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -math-word-medium-043,Math (Word),Medium,217,API_MATH,API_MATH,TRUE -sentiment-easy-029,Sentiment,Easy,152,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-051,Sentiment,Medium,127,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-082,Sentiment,Hard,180,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-medium-060,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -factual-easy-026,Factual,Easy,85,LOCAL_GENERAL,API_MATH,FALSE -math-word-easy-015,Math (Word),Easy,141,API_MATH,API_MATH,TRUE -logic-medium-055,Logical Reasoning,Medium,200,API_LOGIC,API_LOGIC,TRUE diff --git a/output/classifier_eval_results.csv b/output/classifier_eval_results.csv deleted file mode 100644 index 72a3d74..0000000 --- a/output/classifier_eval_results.csv +++ /dev/null @@ -1,801 +0,0 @@ -task_id,category,difficulty,prompt_length,expected_route,predicted_route,is_correct -codegen-easy-033,Code Generation,Easy,205,API_CODE,API_CODE,TRUE -summary-medium-075,Summarization,Medium,408,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-074,Code Debugging,Medium,380,API_CODE,API_CODE,TRUE -ner-hard-091,NER,Hard,192,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-048,Logical Reasoning,Medium,166,API_LOGIC,API_LOGIC,TRUE -debug-easy-032,Code Debugging,Easy,399,API_CODE,API_CODE,TRUE -summary-medium-059,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-050,Logical Reasoning,Medium,119,API_LOGIC,API_LOGIC,TRUE -ner-medium-051,NER,Medium,181,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-074,Code Generation,Medium,147,API_CODE,API_CODE,TRUE -ner-hard-096,NER,Hard,233,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-045,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -codegen-hard-089,Code Generation,Hard,199,API_CODE,API_CODE,TRUE -summary-medium-038,Summarization,Medium,360,LOCAL_GENERAL,LOCAL_SENTIMENT,FALSE -factual-easy-025,Factual,Easy,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-092,Code Generation,Hard,200,API_CODE,API_CODE,TRUE -math-ast-hard-078,Math (AST),Hard,36,API_MATH,API_MATH,TRUE -summary-hard-079,Summarization,Hard,1357,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-041,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -math-ast-easy-006,Math (AST),Easy,49,API_MATH,API_MATH,TRUE -summary-hard-099,Summarization,Hard,1207,LOCAL_GENERAL,API_LOGIC,FALSE -factual-hard-089,Factual,Hard,131,LOCAL_GENERAL,API_MATH,FALSE -logic-easy-016,Logical Reasoning,Easy,185,API_LOGIC,API_LOGIC,TRUE -ner-easy-022,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE -sentiment-easy-001,Sentiment,Easy,133,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-easy-002,Sentiment,Easy,156,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-hard-090,NER,Hard,248,LOCAL_NER,LOCAL_NER,TRUE -sentiment-easy-032,Sentiment,Easy,193,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-040,Sentiment,Medium,135,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-hard-077,Summarization,Hard,1342,LOCAL_GENERAL,API_LOGIC,FALSE -factual-medium-065,Factual,Medium,107,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-easy-035,Code Generation,Easy,200,API_CODE,API_CODE,TRUE -codegen-easy-031,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -sentiment-easy-030,Sentiment,Easy,114,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-045,Code Debugging,Medium,433,API_CODE,API_CODE,TRUE -logic-easy-013,Logical Reasoning,Easy,100,API_LOGIC,API_LOGIC,TRUE -debug-easy-011,Code Debugging,Easy,176,API_CODE,API_CODE,TRUE -sentiment-medium-037,Sentiment,Medium,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-032,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE -logic-hard-080,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-022,Sentiment,Easy,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-medium-045,NER,Medium,181,LOCAL_NER,LOCAL_NER,TRUE -ner-hard-100,NER,Hard,156,LOCAL_NER,LOCAL_NER,TRUE -codegen-easy-032,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -summary-easy-030,Summarization,Easy,147,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-081,Sentiment,Hard,106,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-003,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-ast-hard-092,Math (AST),Hard,42,API_MATH,API_MATH,TRUE -ner-easy-002,NER,Easy,203,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-017,Summarization,Easy,130,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-079,Factual,Hard,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-013,Sentiment,Easy,114,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-medium-075,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -summary-hard-084,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE -codegen-medium-037,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -summary-hard-092,Summarization,Hard,1317,LOCAL_GENERAL,LOCAL_NER,FALSE -logic-medium-057,Logical Reasoning,Medium,200,API_LOGIC,API_LOGIC,TRUE -codegen-medium-051,Code Generation,Medium,191,API_CODE,LOCAL_NER,FALSE -summary-easy-002,Summarization,Easy,141,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-medium-051,Math (Word),Medium,151,API_MATH,API_MATH,TRUE -debug-medium-039,Code Debugging,Medium,224,API_CODE,API_CODE,TRUE -factual-medium-037,Factual,Medium,94,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-003,Logical Reasoning,Easy,140,API_LOGIC,API_MATH,FALSE -math-ast-hard-086,Math (AST),Hard,36,API_MATH,API_MATH,TRUE -math-word-medium-073,Math (Word),Medium,152,API_MATH,API_MATH,TRUE -math-word-hard-081,Math (Word),Hard,215,API_MATH,API_MATH,TRUE -factual-hard-093,Factual,Hard,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-010,Factual,Easy,108,LOCAL_GENERAL,API_MATH,FALSE -summary-medium-058,Summarization,Medium,404,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-051,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-064,NER,Medium,248,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-075,Factual,Medium,104,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-067,Logical Reasoning,Medium,166,API_LOGIC,API_LOGIC,TRUE -factual-easy-014,Factual,Easy,76,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-medium-039,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -math-word-medium-075,Math (Word),Medium,152,API_MATH,API_MATH,TRUE -codegen-medium-070,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -debug-medium-040,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE -ner-easy-003,NER,Easy,175,LOCAL_NER,LOCAL_NER,TRUE -ner-medium-052,NER,Medium,217,LOCAL_NER,LOCAL_NER,TRUE -sentiment-hard-078,Sentiment,Hard,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-066,Sentiment,Medium,176,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-060,Summarization,Medium,412,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-076,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE -summary-easy-021,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-029,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-033,Sentiment,Easy,160,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-083,Code Generation,Hard,185,API_CODE,API_CODE,TRUE -ner-easy-035,NER,Easy,202,LOCAL_NER,LOCAL_NER,TRUE -summary-medium-057,Summarization,Medium,328,LOCAL_GENERAL,API_MATH,FALSE -codegen-easy-004,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -debug-easy-008,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -sentiment-easy-014,Sentiment,Easy,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-medium-041,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -logic-hard-090,Logical Reasoning,Hard,185,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-048,Sentiment,Medium,114,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-easy-009,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-055,Summarization,Medium,360,LOCAL_GENERAL,LOCAL_SENTIMENT,FALSE -factual-easy-029,Factual,Easy,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-medium-048,Code Generation,Medium,191,API_CODE,LOCAL_NER,FALSE -logic-medium-047,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -summary-hard-098,Summarization,Hard,1357,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-hard-095,NER,Hard,217,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-024,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE -summary-easy-026,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-011,Factual,Easy,90,LOCAL_GENERAL,API_MATH,FALSE -debug-hard-088,Code Debugging,Hard,433,API_CODE,API_CODE,TRUE -summary-hard-086,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE -ner-easy-023,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE -sentiment-medium-065,Sentiment,Medium,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-006,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE -math-ast-medium-074,Math (AST),Medium,42,API_MATH,API_MATH,TRUE -ner-easy-005,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-038,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -logic-medium-054,Logical Reasoning,Medium,100,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-063,Sentiment,Medium,176,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-027,Logical Reasoning,Easy,100,API_LOGIC,API_LOGIC,TRUE -factual-medium-059,Factual,Medium,89,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-084,Code Debugging,Hard,211,API_CODE,API_CODE,TRUE -ner-hard-078,NER,Hard,248,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-073,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -factual-easy-034,Factual,Easy,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-038,NER,Medium,192,LOCAL_NER,LOCAL_NER,TRUE -factual-easy-024,Factual,Easy,88,LOCAL_GENERAL,API_MATH,FALSE -debug-easy-023,Code Debugging,Easy,274,API_CODE,API_CODE,TRUE -math-word-easy-011,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -factual-easy-018,Factual,Easy,86,LOCAL_GENERAL,API_CODE,FALSE -math-word-hard-099,Math (Word),Hard,103,API_MATH,API_MATH,TRUE -logic-hard-088,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE -logic-easy-008,Logical Reasoning,Easy,140,API_LOGIC,API_MATH,FALSE -ner-hard-082,NER,Hard,162,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-091,Code Debugging,Hard,380,API_CODE,API_CODE,TRUE -debug-hard-078,Code Debugging,Hard,274,API_CODE,API_CODE,TRUE -math-word-medium-063,Math (Word),Medium,143,API_MATH,API_MATH,TRUE -factual-medium-043,Factual,Medium,88,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-easy-027,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -sentiment-medium-064,Sentiment,Medium,177,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-word-medium-049,Math (Word),Medium,153,API_MATH,API_MATH,TRUE -debug-hard-083,Code Debugging,Hard,230,API_CODE,API_CODE,TRUE -codegen-easy-025,Code Generation,Easy,205,API_CODE,API_CODE,TRUE -debug-medium-051,Code Debugging,Medium,176,API_CODE,API_CODE,TRUE -codegen-easy-002,Code Generation,Easy,147,API_CODE,API_CODE,TRUE -ner-easy-032,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE -ner-hard-081,NER,Hard,175,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-012,Summarization,Easy,118,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-easy-015,NER,Easy,233,LOCAL_NER,LOCAL_NER,TRUE -math-ast-medium-038,Math (AST),Medium,43,API_MATH,API_MATH,TRUE -debug-easy-005,Code Debugging,Easy,211,API_CODE,API_CODE,TRUE -summary-medium-070,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_NER,FALSE -codegen-easy-026,Code Generation,Easy,191,API_CODE,LOCAL_NER,FALSE -ner-easy-018,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-090,Code Debugging,Hard,213,API_CODE,API_CODE,TRUE -debug-easy-021,Code Debugging,Easy,289,API_CODE,API_CODE,TRUE -debug-easy-015,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE -sentiment-medium-042,Sentiment,Medium,180,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-easy-006,Factual,Easy,131,LOCAL_GENERAL,API_MATH,FALSE -math-ast-medium-050,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -debug-medium-062,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE -sentiment-medium-073,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-hard-087,Summarization,Hard,1369,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-036,Code Debugging,Medium,211,API_CODE,API_CODE,TRUE -codegen-easy-021,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -math-ast-easy-020,Math (AST),Easy,44,API_MATH,API_MATH,TRUE -ner-easy-010,NER,Easy,211,LOCAL_NER,LOCAL_NER,TRUE -factual-hard-085,Factual,Hard,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-097,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE -logic-medium-068,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE -factual-easy-017,Factual,Easy,118,LOCAL_GENERAL,API_MATH,FALSE -debug-easy-003,Code Debugging,Easy,274,API_CODE,API_CODE,TRUE -factual-medium-040,Factual,Medium,73,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-038,Sentiment,Medium,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-007,Summarization,Easy,129,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-082,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE -sentiment-hard-094,Sentiment,Hard,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-028,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE -math-word-medium-039,Math (Word),Medium,137,API_MATH,API_MATH,TRUE -logic-hard-096,Logical Reasoning,Hard,212,API_LOGIC,API_LOGIC,TRUE -factual-hard-086,Factual,Hard,108,LOCAL_GENERAL,API_MATH,FALSE -debug-hard-079,Code Debugging,Hard,211,API_CODE,API_CODE,TRUE -logic-hard-079,Logical Reasoning,Hard,200,API_LOGIC,API_LOGIC,TRUE -codegen-easy-013,Code Generation,Easy,186,API_CODE,API_CODE,TRUE -codegen-medium-050,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -logic-medium-065,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -math-word-hard-089,Math (Word),Hard,134,API_MATH,API_MATH,TRUE -logic-easy-007,Logical Reasoning,Easy,212,API_LOGIC,API_LOGIC,TRUE -ner-hard-084,NER,Hard,203,LOCAL_NER,LOCAL_NER,TRUE -math-word-easy-025,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -ner-medium-053,NER,Medium,211,LOCAL_NER,LOCAL_NER,TRUE -codegen-easy-003,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -logic-medium-062,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -ner-hard-077,NER,Hard,203,LOCAL_NER,LOCAL_NER,TRUE -summary-hard-094,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE -sentiment-medium-057,Sentiment,Medium,147,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-004,NER,Easy,233,LOCAL_NER,LOCAL_NER,TRUE -math-word-hard-091,Math (Word),Hard,150,API_MATH,API_MATH,TRUE -factual-medium-041,Factual,Medium,73,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-easy-034,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-medium-041,Math (Word),Medium,153,API_MATH,API_MATH,TRUE -summary-hard-076,Summarization,Hard,1342,LOCAL_GENERAL,API_LOGIC,FALSE -math-ast-easy-030,Math (AST),Easy,42,API_MATH,API_MATH,TRUE -math-word-hard-085,Math (Word),Hard,141,API_MATH,API_MATH,TRUE -sentiment-medium-050,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-024,NER,Easy,223,LOCAL_NER,LOCAL_NER,TRUE -math-ast-hard-094,Math (AST),Hard,43,API_MATH,API_MATH,TRUE -codegen-medium-057,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -math-word-medium-053,Math (Word),Medium,217,API_MATH,API_MATH,TRUE -logic-easy-010,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE -summary-easy-016,Summarization,Easy,130,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-065,Summarization,Medium,420,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-052,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-028,Sentiment,Easy,164,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-079,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -logic-easy-018,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE -factual-easy-032,Factual,Easy,123,LOCAL_GENERAL,API_MATH,FALSE -sentiment-hard-091,Sentiment,Hard,165,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-hard-093,Summarization,Hard,1309,LOCAL_GENERAL,LOCAL_NER,FALSE -codegen-medium-063,Code Generation,Medium,170,API_CODE,API_CODE,TRUE -math-word-easy-019,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -math-word-easy-033,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -logic-medium-040,Logical Reasoning,Medium,157,API_LOGIC,API_LOGIC,TRUE -logic-hard-076,Logical Reasoning,Hard,219,API_LOGIC,API_LOGIC,TRUE -factual-medium-046,Factual,Medium,109,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-057,Factual,Medium,90,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-036,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -math-ast-hard-100,Math (AST),Hard,43,API_MATH,API_MATH,TRUE -codegen-hard-077,Code Generation,Hard,186,API_CODE,API_CODE,TRUE -summary-easy-013,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-002,Code Debugging,Easy,289,API_CODE,API_CODE,TRUE -math-word-hard-093,Math (Word),Hard,215,API_MATH,API_MATH,TRUE -codegen-hard-094,Code Generation,Hard,191,API_CODE,LOCAL_NER,FALSE -summary-hard-096,Summarization,Hard,1062,LOCAL_GENERAL,API_MATH,FALSE -summary-easy-011,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-043,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE -logic-medium-049,Logical Reasoning,Medium,159,API_LOGIC,API_MATH,FALSE -sentiment-medium-052,Sentiment,Medium,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-hard-077,Code Debugging,Hard,195,API_CODE,API_CODE,TRUE -ner-hard-086,NER,Hard,217,LOCAL_NER,LOCAL_NER,TRUE -codegen-hard-096,Code Generation,Hard,147,API_CODE,API_CODE,TRUE -ner-medium-044,NER,Medium,198,LOCAL_NER,LOCAL_NER,TRUE -ner-medium-068,NER,Medium,229,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-050,Factual,Medium,103,LOCAL_GENERAL,API_MATH,FALSE -math-ast-medium-048,Math (AST),Medium,42,API_MATH,API_MATH,TRUE -ner-medium-058,NER,Medium,156,LOCAL_NER,LOCAL_NER,TRUE -ner-medium-073,NER,Medium,175,LOCAL_NER,LOCAL_NER,TRUE -math-word-easy-007,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -ner-easy-031,NER,Easy,211,LOCAL_NER,LOCAL_NER,TRUE -summary-hard-078,Summarization,Hard,1354,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-049,Code Debugging,Medium,224,API_CODE,API_CODE,TRUE -logic-hard-089,Logical Reasoning,Hard,185,API_LOGIC,API_LOGIC,TRUE -summary-medium-036,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -summary-medium-056,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-easy-017,Code Generation,Easy,170,API_CODE,API_CODE,TRUE -codegen-easy-011,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -codegen-medium-065,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -ner-medium-050,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -math-ast-easy-002,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -math-ast-easy-016,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -ner-medium-057,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE -ner-hard-089,NER,Hard,233,LOCAL_NER,LOCAL_NER,TRUE -ner-easy-001,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE -math-ast-medium-072,Math (AST),Medium,45,API_MATH,API_MATH,TRUE -ner-medium-040,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-044,Factual,Medium,100,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-087,Factual,Hard,93,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-034,Code Debugging,Easy,232,API_CODE,API_CODE,TRUE -summary-easy-008,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-048,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -sentiment-medium-069,Sentiment,Medium,116,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-025,Code Debugging,Easy,224,API_CODE,API_CODE,TRUE -codegen-easy-028,Code Generation,Easy,147,API_CODE,API_CODE,TRUE -math-word-medium-065,Math (Word),Medium,217,API_MATH,API_MATH,TRUE -sentiment-hard-095,Sentiment,Hard,150,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-066,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-hard-083,Logical Reasoning,Hard,157,API_LOGIC,API_LOGIC,TRUE -math-ast-medium-056,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -debug-easy-007,Code Debugging,Easy,433,API_CODE,API_CODE,TRUE -sentiment-easy-008,Sentiment,Easy,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-019,Code Debugging,Easy,224,API_CODE,API_CODE,TRUE -codegen-hard-087,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -debug-easy-016,Code Debugging,Easy,205,API_CODE,API_CODE,TRUE -sentiment-easy-016,Sentiment,Easy,132,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-033,Logical Reasoning,Easy,152,API_LOGIC,API_LOGIC,TRUE -logic-easy-020,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE -math-ast-medium-052,Math (AST),Medium,42,API_MATH,API_MATH,TRUE -ner-hard-088,NER,Hard,156,LOCAL_NER,LOCAL_NER,TRUE -codegen-easy-007,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -sentiment-easy-010,Sentiment,Easy,130,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-017,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE -ner-medium-047,NER,Medium,234,LOCAL_NER,LOCAL_NER,TRUE -ner-easy-025,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-064,Factual,Medium,79,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-070,Code Debugging,Medium,380,API_CODE,API_CODE,TRUE -summary-easy-022,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-easy-028,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-073,Summarization,Medium,316,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-069,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -debug-medium-038,Code Debugging,Medium,232,API_CODE,API_CODE,TRUE -debug-hard-095,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE -sentiment-medium-067,Sentiment,Medium,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-026,Logical Reasoning,Easy,178,API_LOGIC,API_LOGIC,TRUE -summary-easy-009,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-097,Summarization,Hard,1219,LOCAL_GENERAL,API_LOGIC,FALSE -math-word-easy-031,Math (Word),Easy,152,API_MATH,API_MATH,TRUE -factual-hard-098,Factual,Hard,101,LOCAL_GENERAL,API_MATH,FALSE -sentiment-hard-096,Sentiment,Hard,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-hard-098,Math (AST),Hard,43,API_MATH,API_MATH,TRUE -logic-medium-074,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-062,Sentiment,Medium,163,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-081,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -ner-medium-067,NER,Medium,198,LOCAL_NER,LOCAL_NER,TRUE -math-word-hard-077,Math (Word),Hard,215,API_MATH,API_MATH,TRUE -debug-easy-014,Code Debugging,Easy,224,API_CODE,API_CODE,TRUE -logic-easy-035,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE -ner-medium-062,NER,Medium,211,LOCAL_NER,LOCAL_NER,TRUE -debug-medium-060,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE -logic-hard-086,Logical Reasoning,Hard,145,API_LOGIC,API_LOGIC,TRUE -sentiment-hard-098,Sentiment,Hard,186,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-051,Summarization,Medium,360,LOCAL_GENERAL,LOCAL_SENTIMENT,FALSE -debug-easy-004,Code Debugging,Easy,176,API_CODE,API_CODE,TRUE -ner-easy-020,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE -factual-easy-015,Factual,Easy,81,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-086,Code Generation,Hard,171,API_CODE,API_CODE,TRUE -logic-hard-094,Logical Reasoning,Hard,140,API_LOGIC,API_MATH,FALSE -ner-medium-061,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-031,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -ner-hard-076,NER,Hard,229,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-029,Summarization,Easy,139,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-043,NER,Medium,203,LOCAL_NER,LOCAL_NER,TRUE -logic-hard-081,Logical Reasoning,Hard,145,API_LOGIC,API_LOGIC,TRUE -debug-easy-026,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE -sentiment-easy-005,Sentiment,Easy,170,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-hard-099,Factual,Hard,118,LOCAL_GENERAL,API_MATH,FALSE -ner-easy-008,NER,Easy,229,LOCAL_NER,LOCAL_NER,TRUE -logic-easy-021,Logical Reasoning,Easy,185,API_LOGIC,API_LOGIC,TRUE -math-word-easy-017,Math (Word),Easy,170,API_MATH,API_MATH,TRUE -logic-easy-031,Logical Reasoning,Easy,166,API_LOGIC,API_LOGIC,TRUE -math-ast-hard-080,Math (AST),Hard,45,API_MATH,API_MATH,TRUE -debug-hard-085,Code Debugging,Hard,380,API_CODE,API_CODE,TRUE -math-word-easy-003,Math (Word),Easy,150,API_MATH,API_MATH,TRUE -math-ast-easy-014,Math (AST),Easy,41,API_MATH,API_MATH,TRUE -sentiment-medium-041,Sentiment,Medium,129,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-022,Logical Reasoning,Easy,200,API_LOGIC,API_LOGIC,TRUE -math-ast-hard-096,Math (AST),Hard,45,API_MATH,API_MATH,TRUE -debug-medium-056,Code Debugging,Medium,433,API_CODE,API_CODE,TRUE -codegen-easy-029,Code Generation,Easy,171,API_CODE,API_CODE,TRUE -math-word-medium-037,Math (Word),Medium,172,API_MATH,API_MATH,TRUE -codegen-medium-072,Code Generation,Medium,186,API_CODE,API_CODE,TRUE -ner-medium-069,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE -sentiment-easy-011,Sentiment,Easy,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-medium-058,Logical Reasoning,Medium,200,API_LOGIC,API_LOGIC,TRUE -factual-hard-092,Factual,Hard,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-013,Factual,Easy,123,LOCAL_GENERAL,API_MATH,FALSE -sentiment-easy-012,Sentiment,Easy,120,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-hard-091,Factual,Hard,93,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-084,Factual,Hard,90,LOCAL_GENERAL,API_MATH,FALSE -math-ast-easy-022,Math (AST),Easy,36,API_MATH,API_MATH,TRUE -ner-easy-016,NER,Easy,167,LOCAL_NER,LOCAL_NER,TRUE -debug-medium-057,Code Debugging,Medium,205,API_CODE,API_CODE,TRUE -sentiment-easy-009,Sentiment,Easy,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-087,Sentiment,Hard,180,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-hard-082,Math (AST),Hard,49,API_MATH,API_MATH,TRUE -factual-medium-053,Factual,Medium,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-easy-028,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-045,Code Generation,Medium,200,API_CODE,API_CODE,TRUE -summary-medium-050,Summarization,Medium,336,LOCAL_GENERAL,API_MATH,FALSE -ner-easy-012,NER,Easy,217,LOCAL_NER,LOCAL_NER,TRUE -math-ast-hard-084,Math (AST),Hard,35,API_MATH,API_MATH,TRUE -codegen-hard-100,Code Generation,Hard,162,API_CODE,API_CODE,TRUE -math-word-hard-087,Math (Word),Hard,152,API_MATH,API_MATH,TRUE -math-ast-medium-058,Math (AST),Medium,35,API_MATH,API_MATH,TRUE -debug-hard-087,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE -debug-hard-089,Code Debugging,Hard,230,API_CODE,API_CODE,TRUE -codegen-medium-058,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -factual-hard-076,Factual,Hard,101,LOCAL_GENERAL,API_MATH,FALSE -codegen-medium-044,Code Generation,Medium,200,API_CODE,API_CODE,TRUE -ner-medium-075,NER,Medium,242,LOCAL_NER,LOCAL_NER,TRUE -ner-medium-041,NER,Medium,156,LOCAL_NER,LOCAL_NER,TRUE -factual-easy-019,Factual,Easy,90,LOCAL_GENERAL,API_MATH,FALSE -sentiment-hard-080,Sentiment,Hard,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-hard-077,Factual,Hard,97,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-054,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-039,Factual,Medium,80,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-hard-099,Logical Reasoning,Hard,178,API_LOGIC,API_LOGIC,TRUE -ner-easy-029,NER,Easy,175,LOCAL_NER,LOCAL_NER,TRUE -sentiment-hard-088,Sentiment,Hard,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-029,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE -summary-medium-062,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-083,Sentiment,Hard,198,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-easy-027,Sentiment,Easy,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-easy-020,Code Generation,Easy,170,API_CODE,API_CODE,TRUE -factual-hard-083,Factual,Hard,82,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-easy-035,Math (Word),Easy,149,API_MATH,API_MATH,TRUE -summary-easy-027,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-067,Factual,Medium,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-033,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -factual-easy-002,Factual,Easy,82,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-023,Factual,Easy,123,LOCAL_GENERAL,API_MATH,FALSE -logic-medium-070,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -logic-easy-001,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE -ner-easy-007,NER,Easy,248,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-048,Factual,Medium,81,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-027,Factual,Easy,118,LOCAL_GENERAL,API_MATH,FALSE -logic-easy-009,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE -codegen-easy-008,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -sentiment-medium-059,Sentiment,Medium,162,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-058,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -summary-easy-025,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-052,Summarization,Medium,408,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-hard-084,Logical Reasoning,Hard,157,API_LOGIC,API_LOGIC,TRUE -codegen-easy-010,Code Generation,Easy,147,API_CODE,API_CODE,TRUE -codegen-medium-052,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -math-ast-easy-008,Math (AST),Easy,49,API_MATH,API_MATH,TRUE -summary-easy-010,Summarization,Easy,149,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-medium-045,Math (Word),Medium,153,API_MATH,API_MATH,TRUE -math-ast-medium-040,Math (AST),Medium,43,API_MATH,API_MATH,TRUE -factual-hard-080,Factual,Hard,90,LOCAL_GENERAL,API_MATH,FALSE -ner-easy-026,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE -summary-hard-081,Summarization,Hard,1342,LOCAL_GENERAL,API_LOGIC,FALSE -factual-medium-055,Factual,Medium,89,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-039,Logical Reasoning,Medium,119,API_LOGIC,API_LOGIC,TRUE -codegen-medium-040,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -factual-medium-069,Factual,Medium,82,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-054,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-040,Summarization,Medium,316,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-easy-009,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -sentiment-hard-084,Sentiment,Hard,163,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-019,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE -summary-medium-042,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-055,NER,Medium,202,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-049,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -summary-easy-035,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-easy-014,NER,Easy,248,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-014,Summarization,Easy,159,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-medium-055,Math (Word),Medium,136,API_MATH,API_MATH,TRUE -summary-medium-061,Summarization,Medium,416,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-079,Sentiment,Hard,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-027,NER,Easy,211,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-071,Logical Reasoning,Medium,140,API_LOGIC,API_MATH,FALSE -logic-easy-023,Logical Reasoning,Easy,164,API_LOGIC,API_LOGIC,TRUE -math-ast-medium-046,Math (AST),Medium,49,API_MATH,API_MATH,TRUE -math-ast-hard-090,Math (AST),Hard,48,API_MATH,API_MATH,TRUE -math-ast-easy-026,Math (AST),Easy,44,API_MATH,API_MATH,TRUE -ner-hard-094,NER,Hard,181,LOCAL_NER,LOCAL_NER,TRUE -math-word-hard-079,Math (Word),Hard,134,API_MATH,API_MATH,TRUE -debug-hard-093,Code Debugging,Hard,230,API_CODE,API_CODE,TRUE -logic-medium-042,Logical Reasoning,Medium,219,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-060,Sentiment,Medium,163,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-easy-010,Math (AST),Easy,49,API_MATH,API_MATH,TRUE -sentiment-hard-076,Sentiment,Hard,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-medium-042,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -debug-medium-037,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE -sentiment-hard-089,Sentiment,Hard,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-019,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-059,NER,Medium,167,LOCAL_NER,LOCAL_NER,TRUE -factual-easy-005,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-038,Factual,Medium,100,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-085,Summarization,Hard,1377,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-072,NER,Medium,229,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-080,Code Debugging,Hard,176,API_CODE,API_CODE,TRUE -math-ast-medium-054,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -summary-easy-031,Summarization,Easy,135,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-044,Summarization,Medium,404,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-090,Code Generation,Hard,186,API_CODE,API_CODE,TRUE -sentiment-easy-003,Sentiment,Easy,176,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-069,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE -math-word-easy-023,Math (Word),Easy,151,API_MATH,API_MATH,TRUE -debug-easy-010,Code Debugging,Easy,232,API_CODE,API_CODE,TRUE -summary-medium-067,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-043,Summarization,Medium,400,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-medium-056,Code Generation,Medium,171,API_CODE,API_CODE,TRUE -math-ast-easy-034,Math (AST),Easy,48,API_MATH,API_MATH,TRUE -logic-medium-063,Logical Reasoning,Medium,159,API_LOGIC,API_MATH,FALSE -sentiment-medium-068,Sentiment,Medium,199,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-090,Sentiment,Hard,181,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-099,Sentiment,Hard,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-037,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -codegen-easy-018,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -ner-hard-097,NER,Hard,234,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-018,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-021,Factual,Easy,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-094,Code Debugging,Hard,195,API_CODE,API_CODE,TRUE -codegen-easy-030,Code Generation,Easy,200,API_CODE,API_CODE,TRUE -math-word-easy-027,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -sentiment-hard-092,Sentiment,Hard,156,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-medium-056,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE -ner-hard-083,NER,Hard,166,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-062,Factual,Medium,104,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-007,Sentiment,Easy,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-word-easy-021,Math (Word),Easy,170,API_MATH,API_MATH,TRUE -debug-medium-061,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -summary-medium-046,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-035,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-easy-005,Summarization,Easy,135,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-036,Factual,Medium,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-035,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -ner-medium-036,NER,Medium,234,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-042,Factual,Medium,107,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-061,Sentiment,Medium,145,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-hard-092,NER,Hard,181,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-009,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -logic-easy-025,Logical Reasoning,Easy,219,API_LOGIC,API_LOGIC,TRUE -logic-medium-069,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE -ner-medium-037,NER,Medium,202,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-047,Code Generation,Medium,186,API_CODE,API_CODE,TRUE -logic-medium-053,Logical Reasoning,Medium,145,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-045,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-100,Sentiment,Hard,150,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-098,Code Generation,Hard,186,API_CODE,API_CODE,TRUE -factual-easy-030,Factual,Easy,90,LOCAL_GENERAL,API_MATH,FALSE -math-word-medium-061,Math (Word),Medium,154,API_MATH,API_MATH,TRUE -debug-hard-096,Code Debugging,Hard,232,API_CODE,API_CODE,TRUE -debug-medium-054,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -codegen-hard-078,Code Generation,Hard,200,API_CODE,API_CODE,TRUE -sentiment-easy-023,Sentiment,Easy,184,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-004,Summarization,Easy,159,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-048,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -math-word-medium-059,Math (Word),Medium,136,API_MATH,API_MATH,TRUE -ner-hard-099,NER,Hard,175,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-071,Code Generation,Medium,147,API_CODE,API_CODE,TRUE -sentiment-hard-093,Sentiment,Hard,143,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-medium-066,NER,Medium,248,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-067,Code Generation,Medium,205,API_CODE,API_CODE,TRUE -summary-hard-100,Summarization,Hard,1042,LOCAL_GENERAL,API_MATH,FALSE -factual-easy-003,Factual,Easy,118,LOCAL_GENERAL,API_MATH,FALSE -codegen-easy-005,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -math-word-medium-069,Math (Word),Medium,137,API_MATH,API_MATH,TRUE -logic-easy-015,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE -logic-hard-095,Logical Reasoning,Hard,200,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-019,Sentiment,Easy,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-hard-093,NER,Hard,166,LOCAL_NER,LOCAL_NER,TRUE -debug-medium-072,Code Debugging,Medium,205,API_CODE,API_CODE,TRUE -debug-medium-065,Code Debugging,Medium,433,API_CODE,API_CODE,TRUE -math-word-medium-071,Math (Word),Medium,217,API_MATH,API_MATH,TRUE -sentiment-medium-043,Sentiment,Medium,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-082,Code Generation,Hard,185,API_CODE,API_CODE,TRUE -factual-easy-016,Factual,Easy,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-052,Factual,Medium,110,LOCAL_GENERAL,API_MATH,FALSE -debug-medium-055,Code Debugging,Medium,259,API_CODE,API_CODE,TRUE -ner-medium-054,NER,Medium,202,LOCAL_NER,LOCAL_NER,TRUE -factual-hard-078,Factual,Hard,108,LOCAL_GENERAL,API_MATH,FALSE -sentiment-medium-056,Sentiment,Medium,160,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-easy-006,Code Generation,Easy,200,API_CODE,API_CODE,TRUE -debug-medium-073,Code Debugging,Medium,176,API_CODE,API_CODE,TRUE -math-ast-hard-088,Math (AST),Hard,49,API_MATH,API_MATH,TRUE -logic-medium-061,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -math-ast-medium-044,Math (AST),Medium,42,API_MATH,API_MATH,TRUE -codegen-medium-068,Code Generation,Medium,162,API_CODE,API_CODE,TRUE -debug-medium-064,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE -codegen-medium-036,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -sentiment-medium-055,Sentiment,Medium,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-easy-012,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -summary-medium-048,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE -ner-easy-013,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-063,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-easy-001,Math (Word),Easy,215,API_MATH,API_MATH,TRUE -factual-medium-056,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-hard-079,NER,Hard,223,LOCAL_NER,LOCAL_NER,TRUE -ner-easy-021,NER,Easy,234,LOCAL_NER,LOCAL_NER,TRUE -logic-easy-002,Logical Reasoning,Easy,100,API_LOGIC,API_LOGIC,TRUE -summary-medium-063,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_NER,FALSE -ner-medium-042,NER,Medium,166,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-053,Code Generation,Medium,191,API_CODE,LOCAL_NER,FALSE -factual-hard-097,Factual,Hard,78,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-hard-095,Math (Word),Hard,150,API_MATH,API_MATH,TRUE -ner-easy-017,NER,Easy,217,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-086,Code Debugging,Hard,211,API_CODE,API_CODE,TRUE -sentiment-medium-049,Sentiment,Medium,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-hard-085,NER,Hard,166,LOCAL_NER,LOCAL_NER,TRUE -math-ast-medium-036,Math (AST),Medium,43,API_MATH,API_MATH,TRUE -debug-hard-100,Code Debugging,Hard,259,API_CODE,API_CODE,TRUE -sentiment-easy-020,Sentiment,Easy,162,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-012,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE -sentiment-hard-086,Sentiment,Hard,133,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-044,Sentiment,Medium,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-easy-027,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE -debug-hard-082,Code Debugging,Hard,274,API_CODE,API_CODE,TRUE -summary-medium-039,Summarization,Medium,328,LOCAL_GENERAL,API_MATH,FALSE -logic-hard-097,Logical Reasoning,Hard,157,API_LOGIC,API_LOGIC,TRUE -factual-medium-070,Factual,Medium,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-hard-098,Logical Reasoning,Hard,152,API_LOGIC,API_LOGIC,TRUE -factual-medium-061,Factual,Medium,89,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-medium-073,Factual,Medium,110,LOCAL_GENERAL,API_MATH,FALSE -factual-easy-008,Factual,Easy,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-medium-069,Code Generation,Medium,200,API_CODE,API_CODE,TRUE -sentiment-medium-036,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-068,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE -factual-easy-022,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-047,Summarization,Medium,372,LOCAL_GENERAL,API_LOGIC,FALSE -logic-hard-085,Logical Reasoning,Hard,119,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-031,Sentiment,Easy,164,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-054,Sentiment,Medium,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-easy-024,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -ner-medium-063,NER,Medium,217,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-030,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -ner-medium-049,NER,Medium,162,LOCAL_NER,LOCAL_NER,TRUE -logic-easy-030,Logical Reasoning,Easy,164,API_LOGIC,API_LOGIC,TRUE -logic-medium-073,Logical Reasoning,Medium,157,API_LOGIC,API_LOGIC,TRUE -summary-easy-020,Summarization,Easy,130,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-006,Sentiment,Easy,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-028,Logical Reasoning,Easy,164,API_LOGIC,API_LOGIC,TRUE -logic-hard-092,Logical Reasoning,Hard,200,API_LOGIC,API_LOGIC,TRUE -ner-easy-034,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE -debug-medium-041,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE -logic-hard-082,Logical Reasoning,Hard,185,API_LOGIC,API_LOGIC,TRUE -math-ast-medium-070,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -debug-medium-063,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE -ner-hard-087,NER,Hard,162,LOCAL_NER,LOCAL_NER,TRUE -math-ast-easy-032,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -math-word-easy-029,Math (Word),Easy,150,API_MATH,API_MATH,TRUE -debug-medium-046,Code Debugging,Medium,232,API_CODE,API_CODE,TRUE -factual-medium-071,Factual,Medium,110,LOCAL_GENERAL,API_MATH,FALSE -ner-hard-098,NER,Hard,192,LOCAL_NER,LOCAL_NER,TRUE -factual-hard-088,Factual,Hard,92,LOCAL_GENERAL,API_MATH,FALSE -codegen-medium-043,Code Generation,Medium,170,API_CODE,API_CODE,TRUE -logic-hard-091,Logical Reasoning,Hard,119,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-047,Sentiment,Medium,152,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-medium-060,Factual,Medium,107,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-046,Sentiment,Medium,152,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-015,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-medium-072,Summarization,Medium,372,LOCAL_GENERAL,API_LOGIC,FALSE -logic-medium-046,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -factual-easy-020,Factual,Easy,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-001,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -summary-easy-032,Summarization,Easy,147,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-014,Logical Reasoning,Easy,145,API_LOGIC,API_LOGIC,TRUE -factual-easy-001,Factual,Easy,102,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-044,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE -debug-medium-053,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE -math-ast-medium-066,Math (AST),Medium,43,API_MATH,API_MATH,TRUE -summary-hard-083,Summarization,Hard,1219,LOCAL_GENERAL,API_LOGIC,FALSE -codegen-easy-014,Code Generation,Easy,171,API_CODE,API_CODE,TRUE -codegen-medium-075,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -codegen-easy-012,Code Generation,Easy,170,API_CODE,API_CODE,TRUE -logic-medium-037,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE -summary-easy-006,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -math-word-easy-009,Math (Word),Easy,170,API_MATH,API_MATH,TRUE -debug-easy-020,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -math-word-hard-083,Math (Word),Hard,215,API_MATH,API_MATH,TRUE -debug-easy-013,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE -logic-hard-077,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE -factual-easy-004,Factual,Easy,88,LOCAL_GENERAL,API_MATH,FALSE -codegen-easy-024,Code Generation,Easy,162,API_CODE,API_CODE,TRUE -math-ast-medium-062,Math (AST),Medium,36,API_MATH,API_MATH,TRUE -ner-medium-046,NER,Medium,229,LOCAL_NER,LOCAL_NER,TRUE -codegen-medium-041,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -codegen-hard-099,Code Generation,Hard,185,API_CODE,API_CODE,TRUE -codegen-medium-061,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -codegen-easy-016,Code Generation,Easy,170,API_CODE,API_CODE,TRUE -codegen-medium-046,Code Generation,Medium,186,API_CODE,API_CODE,TRUE -math-ast-easy-018,Math (AST),Easy,42,API_MATH,API_MATH,TRUE -factual-easy-012,Factual,Easy,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-067,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE -sentiment-medium-058,Sentiment,Medium,198,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-042,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE -ner-hard-080,NER,Hard,156,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-060,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE -summary-hard-091,Summarization,Hard,1377,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-095,Summarization,Hard,1309,LOCAL_GENERAL,LOCAL_NER,FALSE -factual-medium-066,Factual,Medium,89,LOCAL_GENERAL,API_MATH,FALSE -logic-easy-011,Logical Reasoning,Easy,145,API_LOGIC,API_LOGIC,TRUE -codegen-medium-062,Code Generation,Medium,171,API_CODE,API_CODE,TRUE -codegen-medium-064,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -logic-medium-043,Logical Reasoning,Medium,159,API_LOGIC,API_MATH,FALSE -math-word-medium-057,Math (Word),Medium,153,API_MATH,API_MATH,TRUE -math-word-easy-013,Math (Word),Easy,141,API_MATH,API_MATH,TRUE -factual-medium-049,Factual,Medium,73,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-094,Factual,Hard,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-easy-006,Code Debugging,Easy,289,API_CODE,API_CODE,TRUE -debug-easy-022,Code Debugging,Easy,205,API_CODE,API_CODE,TRUE -math-ast-medium-042,Math (AST),Medium,45,API_MATH,API_MATH,TRUE -codegen-easy-001,Code Generation,Easy,176,API_CODE,API_CODE,TRUE -codegen-medium-038,Code Generation,Medium,200,API_CODE,API_CODE,TRUE -ner-easy-009,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-098,Code Debugging,Hard,224,API_CODE,API_CODE,TRUE -ner-medium-039,NER,Medium,234,LOCAL_NER,LOCAL_NER,TRUE -debug-hard-099,Code Debugging,Hard,380,API_CODE,API_CODE,TRUE -sentiment-medium-070,Sentiment,Medium,170,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-052,Code Debugging,Medium,195,API_CODE,API_CODE,TRUE -codegen-hard-076,Code Generation,Hard,191,API_CODE,LOCAL_NER,FALSE -sentiment-medium-053,Sentiment,Medium,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-hard-093,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -sentiment-medium-072,Sentiment,Medium,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-word-medium-047,Math (Word),Medium,106,API_MATH,API_MATH,TRUE -ner-medium-070,NER,Medium,217,LOCAL_NER,LOCAL_NER,TRUE -logic-medium-059,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE -ner-easy-006,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE -debug-easy-012,Code Debugging,Easy,205,API_CODE,API_CODE,TRUE -logic-medium-072,Logical Reasoning,Medium,100,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-018,Sentiment,Easy,175,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-easy-023,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-hard-081,Code Debugging,Hard,232,API_CODE,API_CODE,TRUE -sentiment-easy-017,Sentiment,Easy,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-medium-068,Factual,Medium,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-075,Sentiment,Medium,125,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-011,NER,Easy,217,LOCAL_NER,LOCAL_NER,TRUE -summary-easy-024,Summarization,Easy,147,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-084,Code Generation,Hard,176,API_CODE,API_CODE,TRUE -sentiment-easy-035,Sentiment,Easy,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-medium-058,Factual,Medium,120,LOCAL_GENERAL,API_MATH,FALSE -summary-easy-033,Summarization,Easy,159,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-025,Sentiment,Easy,135,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-071,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE -logic-medium-056,Logical Reasoning,Medium,219,API_LOGIC,API_LOGIC,TRUE -sentiment-hard-077,Sentiment,Hard,129,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-hard-089,Summarization,Hard,1369,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -ner-medium-074,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE -math-ast-medium-068,Math (AST),Medium,48,API_MATH,API_MATH,TRUE -codegen-hard-095,Code Generation,Hard,205,API_CODE,API_CODE,TRUE -debug-medium-047,Code Debugging,Medium,259,API_CODE,API_CODE,TRUE -math-ast-hard-076,Math (AST),Hard,45,API_MATH,API_MATH,TRUE -codegen-easy-023,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -logic-easy-024,Logical Reasoning,Easy,185,API_LOGIC,API_LOGIC,TRUE -factual-medium-074,Factual,Medium,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-064,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE -logic-hard-100,Logical Reasoning,Hard,119,API_LOGIC,API_LOGIC,TRUE -sentiment-medium-074,Sentiment,Medium,145,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-hard-087,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE -sentiment-easy-015,Sentiment,Easy,151,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -summary-medium-064,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_NER,FALSE -summary-hard-080,Summarization,Hard,1309,LOCAL_GENERAL,LOCAL_NER,FALSE -math-word-medium-067,Math (Word),Medium,172,API_MATH,API_MATH,TRUE -summary-medium-074,Summarization,Medium,416,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-medium-051,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE -ner-medium-065,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE -codegen-hard-097,Code Generation,Hard,147,API_CODE,API_CODE,TRUE -codegen-medium-054,Code Generation,Medium,176,API_CODE,API_CODE,TRUE -math-ast-easy-004,Math (AST),Easy,43,API_MATH,API_MATH,TRUE -codegen-hard-080,Code Generation,Hard,170,API_CODE,API_CODE,TRUE -factual-hard-082,Factual,Hard,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-097,Sentiment,Hard,130,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-easy-034,Sentiment,Easy,145,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-hard-078,Logical Reasoning,Hard,178,API_LOGIC,API_LOGIC,TRUE -factual-medium-047,Factual,Medium,92,LOCAL_GENERAL,API_MATH,FALSE -factual-medium-072,Factual,Medium,92,LOCAL_GENERAL,API_MATH,FALSE -codegen-easy-022,Code Generation,Easy,186,API_CODE,API_CODE,TRUE -math-word-hard-097,Math (Word),Hard,135,API_MATH,API_MATH,TRUE -debug-hard-092,Code Debugging,Hard,433,API_CODE,API_CODE,TRUE -summary-medium-071,Summarization,Medium,412,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-034,Logical Reasoning,Easy,178,API_LOGIC,API_LOGIC,TRUE -logic-medium-044,Logical Reasoning,Medium,219,API_LOGIC,API_LOGIC,TRUE -summary-easy-001,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-066,Code Debugging,Medium,232,API_CODE,API_CODE,TRUE -factual-hard-081,Factual,Hard,111,LOCAL_GENERAL,LOCAL_NER,FALSE -summary-hard-088,Summarization,Hard,1042,LOCAL_GENERAL,API_MATH,FALSE -math-ast-easy-028,Math (AST),Easy,48,API_MATH,API_MATH,TRUE -debug-medium-050,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE -ner-medium-071,NER,Medium,162,LOCAL_NER,LOCAL_NER,TRUE -factual-medium-045,Factual,Medium,109,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-medium-071,Sentiment,Medium,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-019,Logical Reasoning,Easy,140,API_LOGIC,API_MATH,FALSE -summary-medium-053,Summarization,Medium,328,LOCAL_GENERAL,API_MATH,FALSE -summary-medium-068,Summarization,Medium,416,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-hard-100,Factual,Hard,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -summary-hard-090,Summarization,Hard,1207,LOCAL_GENERAL,API_LOGIC,FALSE -factual-hard-095,Factual,Hard,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -codegen-hard-091,Code Generation,Hard,171,API_CODE,API_CODE,TRUE -summary-medium-045,Summarization,Medium,400,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-easy-021,Sentiment,Easy,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -math-ast-medium-060,Math (AST),Medium,49,API_MATH,API_MATH,TRUE -codegen-medium-066,Code Generation,Medium,162,API_CODE,API_CODE,TRUE -summary-medium-049,Summarization,Medium,372,LOCAL_GENERAL,API_LOGIC,FALSE -codegen-hard-085,Code Generation,Hard,171,API_CODE,API_CODE,TRUE -codegen-easy-015,Code Generation,Easy,185,API_CODE,API_CODE,TRUE -math-ast-medium-064,Math (AST),Medium,45,API_MATH,API_MATH,TRUE -factual-easy-007,Factual,Easy,80,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -sentiment-hard-085,Sentiment,Hard,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -logic-easy-004,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE -ner-medium-060,NER,Medium,156,LOCAL_NER,LOCAL_NER,TRUE -sentiment-easy-026,Sentiment,Easy,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -ner-easy-030,NER,Easy,223,LOCAL_NER,LOCAL_NER,TRUE -codegen-hard-088,Code Generation,Hard,162,API_CODE,API_CODE,TRUE -codegen-easy-019,Code Generation,Easy,147,API_CODE,API_CODE,TRUE -logic-medium-066,Logical Reasoning,Medium,164,API_LOGIC,API_LOGIC,TRUE -codegen-easy-034,Code Generation,Easy,199,API_CODE,API_CODE,TRUE -sentiment-easy-004,Sentiment,Easy,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -debug-medium-075,Code Debugging,Medium,380,API_CODE,API_CODE,TRUE -factual-easy-028,Factual,Easy,88,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -logic-easy-017,Logical Reasoning,Easy,200,API_LOGIC,API_LOGIC,TRUE -debug-easy-018,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE -sentiment-medium-039,Sentiment,Medium,184,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-easy-024,Sentiment,Easy,147,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -factual-hard-096,Factual,Hard,118,LOCAL_GENERAL,API_MATH,FALSE -factual-easy-031,Factual,Easy,81,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -factual-easy-033,Factual,Easy,92,LOCAL_GENERAL,API_MATH,FALSE -ner-easy-033,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE -logic-easy-005,Logical Reasoning,Easy,112,API_LOGIC,API_LOGIC,TRUE -factual-hard-090,Factual,Hard,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE -debug-medium-059,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE -codegen-medium-059,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -logic-hard-093,Logical Reasoning,Hard,212,API_LOGIC,API_LOGIC,TRUE -math-word-easy-005,Math (Word),Easy,149,API_MATH,API_MATH,TRUE -codegen-medium-055,Code Generation,Medium,199,API_CODE,API_CODE,TRUE -math-word-medium-043,Math (Word),Medium,217,API_MATH,API_MATH,TRUE -sentiment-easy-029,Sentiment,Easy,152,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-medium-051,Sentiment,Medium,127,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -sentiment-hard-082,Sentiment,Hard,180,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE -codegen-medium-060,Code Generation,Medium,185,API_CODE,API_CODE,TRUE -factual-easy-026,Factual,Easy,85,LOCAL_GENERAL,API_MATH,FALSE -math-word-easy-015,Math (Word),Easy,141,API_MATH,API_MATH,TRUE -logic-medium-055,Logical Reasoning,Medium,200,API_LOGIC,API_LOGIC,TRUE diff --git a/output/math_analysis/math_benchmark.csv b/output/math_analysis/math_benchmark.csv deleted file mode 100644 index 4fa4304..0000000 --- a/output/math_analysis/math_benchmark.csv +++ /dev/null @@ -1,21 +0,0 @@ -task_id,prompt,difficulty,expected_answer,predicted_answer,is_correct,latency_seconds,input_tokens,output_tokens,total_tokens -task-gen-02-006,Calculate the exact value: 312 + 226.,easy,538,538,1,10.607,311,34,345 -task-gen-02-031,Calculate the exact value: 152 * 242.,easy,36784,36784,1,1.435,311,80,391 -task-gen-02-015,Calculate the exact value: 367 - 226.,easy,141,141,1,1.537,311,30,341 -task-gen-02-019,Calculate the exact value: 193 - 59.,easy,134,134,1,1.624,311,30,341 -task-gen-02-055,A store has 543 items. It sells 20% on Monday and 27 more on Tuesday. How many items remain? (Difficulty: medium),medium,407.4,407.4,1,3.989,331,354,685 -task-gen-02-037,A store has 344 items. It sells 20% on Monday and 79 more on Tuesday. How many items remain? (Difficulty: medium),medium,196.2,196.2,1,3.149,331,139,470 -task-gen-02-064,A store has 599 items. It sells 30% on Monday and 80 more on Tuesday. How many items remain? (Difficulty: medium),medium,339.3,339.3,1,1.575,331,112,443 -task-gen-02-052,A store has 347 items. It sells 25% on Monday and 58 more on Tuesday. How many items remain? (Difficulty: medium),medium,202.25,202.25,1,3.159,331,439,770 -task-gen-02-034,A store has 493 items. It sells 20% on Monday and 64 more on Tuesday. How many items remain? (Difficulty: medium),medium,330.4,330.4,1,3.193,331,297,628 -task-gen-02-059,An investor deposits $11244 in a savings account with 5% annual interest compounded annually. What will the balance be after 6 years? (Difficulty: medium),medium,15068.04,15068.04,1,5.176,334,744,1078 -task-gen-02-053,An investor deposits $8668 in a savings account with 7% annual interest compounded annually. What will the balance be after 7 years? (Difficulty: medium),medium,13918.91,13918.91,1,3.319,334,436,770 -task-gen-02-038,An investor deposits $11485 in a savings account with 7% annual interest compounded annually. What will the balance be after 6 years? (Difficulty: medium),medium,17235.89,17235.89,1,3.834,334,479,813 -task-gen-02-081,"Solve the following problem: A factory produces 46 units per hour. After 12 hours, 10% of the units are inspected. Of those, 5% are defective. How many non-defective inspected units are there? (Difficulty: hard)",hard,52.44,52.44,1,4.464,353,387,740 -task-gen-02-084,"Solve the following problem: A factory produces 32 units per hour. After 11 hours, 20% of the units are inspected. Of those, 5% are defective. How many non-defective inspected units are there? (Difficulty: hard)",hard,66.88,66.88,1,3.686,353,561,914 -task-gen-02-100,"A train leaves Station A traveling at 49 mph. 2 hours later, another train leaves Station A traveling in the same direction at 71 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,49/11,49/11,1,2.824,350,206,556 -task-gen-02-079,"A train leaves Station A traveling at 48 mph. 4 hours later, another train leaves Station A traveling in the same direction at 76 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,48/7,48/7,1,2.971,350,234,584 -task-gen-02-082,"A train leaves Station A traveling at 53 mph. 2 hours later, another train leaves Station A traveling in the same direction at 73 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,5.3,5.3,1,2.396,350,232,582 -task-gen-02-067,"A train leaves Station A traveling at 55 mph. 2 hours later, another train leaves Station A traveling in the same direction at 78 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,110/23,110/23,1,3.77,350,251,601 -task-gen-02-074,"If the price of an item increases from $120 to $317, what is the percentage increase? (Difficulty: hard)",hard,164.17,164.17,1,4.569,325,204,529 -task-gen-02-095,"If the price of an item increases from $77 to $405, what is the percentage increase? (Difficulty: hard)",hard,425.97,425.97,1,4.564,325,313,638 diff --git a/output/math_benchmark.csv b/output/math_benchmark.csv deleted file mode 100644 index 91b3ae0..0000000 --- a/output/math_benchmark.csv +++ /dev/null @@ -1,21 +0,0 @@ -task_id,prompt,difficulty,expected_answer,predicted_answer,is_correct,latency_seconds,input_tokens,output_tokens,total_tokens -task-gen-02-006,Calculate the exact value: 312 + 226.,easy,538,538,1,10.607,311,34,345 -task-gen-02-031,Calculate the exact value: 152 * 242.,easy,36784,36784,1,1.435,311,80,391 -task-gen-02-015,Calculate the exact value: 367 - 226.,easy,141,141,1,1.537,311,30,341 -task-gen-02-019,Calculate the exact value: 193 - 59.,easy,134,134,1,1.624,311,30,341 -task-gen-02-055,A store has 543 items. It sells 20% on Monday and 27 more on Tuesday. How many items remain? (Difficulty: medium),medium,407.4,407.4,1,3.989,331,354,685 -task-gen-02-037,A store has 344 items. It sells 20% on Monday and 79 more on Tuesday. How many items remain? (Difficulty: medium),medium,196.2,196.2,1,3.149,331,139,470 -task-gen-02-064,A store has 599 items. It sells 30% on Monday and 80 more on Tuesday. How many items remain? (Difficulty: medium),medium,339.3,339.3,1,1.575,331,112,443 -task-gen-02-052,A store has 347 items. It sells 25% on Monday and 58 more on Tuesday. How many items remain? (Difficulty: medium),medium,202.25,202.25,1,3.159,331,439,770 -task-gen-02-034,A store has 493 items. It sells 20% on Monday and 64 more on Tuesday. How many items remain? (Difficulty: medium),medium,330.4,330.4,1,3.193,331,297,628 -task-gen-02-059,An investor deposits $11244 in a savings account with 5% annual interest compounded annually. What will the balance be after 6 years? (Difficulty: medium),medium,15068.04,15068.04,1,5.176,334,744,1078 -task-gen-02-053,An investor deposits $8668 in a savings account with 7% annual interest compounded annually. What will the balance be after 7 years? (Difficulty: medium),medium,13918.91,13918.91,1,3.319,334,436,770 -task-gen-02-038,An investor deposits $11485 in a savings account with 7% annual interest compounded annually. What will the balance be after 6 years? (Difficulty: medium),medium,17235.89,17235.89,1,3.834,334,479,813 -task-gen-02-081,"Solve the following problem: A factory produces 46 units per hour. After 12 hours, 10% of the units are inspected. Of those, 5% are defective. How many non-defective inspected units are there? (Difficulty: hard)",hard,52.44,52.44,1,4.464,353,387,740 -task-gen-02-084,"Solve the following problem: A factory produces 32 units per hour. After 11 hours, 20% of the units are inspected. Of those, 5% are defective. How many non-defective inspected units are there? (Difficulty: hard)",hard,66.88,66.88,1,3.686,353,561,914 -task-gen-02-100,"A train leaves Station A traveling at 49 mph. 2 hours later, another train leaves Station A traveling in the same direction at 71 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,49/11,49/11,1,2.824,350,206,556 -task-gen-02-079,"A train leaves Station A traveling at 48 mph. 4 hours later, another train leaves Station A traveling in the same direction at 76 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,48/7,48/7,1,2.971,350,234,584 -task-gen-02-082,"A train leaves Station A traveling at 53 mph. 2 hours later, another train leaves Station A traveling in the same direction at 73 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,5.3,5.3,1,2.396,350,232,582 -task-gen-02-067,"A train leaves Station A traveling at 55 mph. 2 hours later, another train leaves Station A traveling in the same direction at 78 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,110/23,110/23,1,3.77,350,251,601 -task-gen-02-074,"If the price of an item increases from $120 to $317, what is the percentage increase? (Difficulty: hard)",hard,164.17,164.17,1,4.569,325,204,529 -task-gen-02-095,"If the price of an item increases from $77 to $405, what is the percentage increase? (Difficulty: hard)",hard,425.97,425.97,1,4.564,325,313,638 diff --git a/output/practice_results.json b/output/practice_results.json deleted file mode 100644 index 81b52e2..0000000 --- a/output/practice_results.json +++ /dev/null @@ -1,34 +0,0 @@ -[ - { - "task_id": "practice-01", - "answer": "The capital of Australia is Canberra, and it is near the Australian Capital Territory." - }, - { - "task_id": "practice-02", - "answer": "__ESCALATE__" - }, - { - "task_id": "practice-03", - "answer": "Neutral" - }, - { - "task_id": "practice-04", - "answer": "Artificial intelligence has transformed various industries, enhancing tasks and decision-making while raising significant ethical and regulatory concerns." - }, - { - "task_id": "practice-05", - "answer": "[\n {\"entity\": \"Maria Sanchez\", \"type\": \"Person\"},\n {\"entity\": \"Fireworks AI\", \"type\": \"Organization\"},\n {\"entity\": \"Berlin\", \"type\": \"Location\"},\n {\"entity\": \"March\", \"type\": \"Date\"}\n]" - }, - { - "task_id": "practice-06", - "answer": "```python\ndef get_max(nums):\n if not nums: # Check if the list is empty\n return None # Return None if the list is empty\n max_num = nums[0]\n for num in nums:\n if num > max_num:\n max_num = num\n return max_num\n```" - }, - { - "task_id": "practice-07", - "answer": "Let's analyze the given information step by step:\n\n1. **Premises:**\n - Sam does not own the bird.\n - Jo owns the dog.\n - Sam, Jo, and Lee each own a different pet: cat, dog, bird.\n\n2. **Intermediate facts:**\n - Since Jo owns the dog, Jo cannot own the cat or the bird.\n - Sam does not own the bird, so Sam must own either the cat or the dog. But since Jo owns the dog, Sam must own the cat.\n - Lee must own the bird, as it is the only pet left.\n\n3. **Conclusion:**\n - Sam owns the cat.\n\nThus, the final conclusion is that Sam owns the cat." - }, - { - "task_id": "practice-08", - "answer": "```python\ndef second_largest(nums):\n if len(nums) < 2:\n return None\n first_max = second_max = float('-inf')\n for num in nums:\n if num > first_max:\n second_max = first_max\n first_max = num\n elif first_max > num > second_max:\n second_max = num\n return second_max if second_max != float('-inf') else None\n```" - } -] diff --git a/output/results.json b/output/results.json deleted file mode 100644 index e69de29..0000000 diff --git a/output/sentiment_summarization_analysis/prompt_benchmark.csv b/output/sentiment_summarization_analysis/prompt_benchmark.csv deleted file mode 100644 index aa69439..0000000 --- a/output/sentiment_summarization_analysis/prompt_benchmark.csv +++ /dev/null @@ -1,41 +0,0 @@ -strategy,task,prompt_id,expected,predicted,exact_match,jaccard_sim,remote_tokens,latency_ms,max_tokens_used,escalated -baseline,sentiment,sent_p1,Negative,Negative,1,,0,276.6,20,0 -baseline,sentiment,sent_p2,Positive,Positive,1,,0,112.7,20,0 -baseline,sentiment,sent_p3,Negative,Negative,1,,0,89.1,20,0 -baseline,sentiment,sent_p4,Neutral,Neutral,1,,0,91.0,20,0 -baseline,summarization,sum_p1,"AI has transformed industries by automating tasks and enabling better decisions, but also raises concerns about privacy, bias, and regulation.","Artificial intelligence has advanced rapidly across various industries, enhancing automation, decision-making, and business opportunities. However, it also raises concerns about privacy, bias, job displacement, and the need for regulation.",,0.2703,0,1257.9,250,0 -baseline,summarization,sum_p2,"Climate change drives extreme weather, melting ice, and biodiversity loss, and scientists warn of catastrophic consequences without cutting greenhouse gas emissions.","Climate change is a major global issue causing extreme weather, melting ice, and endangering biodiversity. Scientists urge urgent action to reduce emissions to prevent catastrophic consequences.",,0.3636,0,941.2,250,0 -baseline,summarization,sum_p3,"Private companies like SpaceX are lowering launch costs with reusable rockets, making space more accessible and accelerating lunar and Martian missions.","Space exploration is experiencing a new era with the help of private companies like SpaceX and Blue Origin. These companies are developing reusable rockets, which significantly reduces launch costs. This commercial revolution is making space more affordable and could accelerate missions to the Moon, Mars, and beyond.",,0.3261,0,1536.6,250,0 -baseline,summarization,sum_p4,"Remote work is now permanent for many firms, boosting productivity and cutting costs, though challenges around collaboration and work-life boundaries persist.","Remote work has become a permanent fixture due to the pandemic, offering higher productivity and better work-life balance for employees, while saving companies on office overhead. However, challenges remain around collaboration, company culture, and maintaining boundaries between professional and personal life.",,0.2292,0,1470.8,250,0 -zero_shot_strict,sentiment,sent_p1,Negative,Negative,1,,0,155.6,2,0 -zero_shot_strict,sentiment,sent_p2,Positive,Positive,1,,0,82.4,2,0 -zero_shot_strict,sentiment,sent_p3,Negative,Negative,1,,0,86.0,2,0 -zero_shot_strict,sentiment,sent_p4,Neutral,Neutral,1,,0,93.9,2,0 -zero_shot_strict,summarization,sum_p1,"AI has transformed industries by automating tasks and enabling better decisions, but also raises concerns about privacy, bias, and regulation.","Artificial intelligence has transformed many industries, but raises concerns about privacy, bias, job displacement, and regulation.",,0.4583,0,759.8,90,0 -zero_shot_strict,summarization,sum_p2,"Climate change drives extreme weather, melting ice, and biodiversity loss, and scientists warn of catastrophic consequences without cutting greenhouse gas emissions.","Climate change poses significant threats to the environment and human societies. Rising temperatures and extreme weather events are becoming more frequent, while biodiversity is threatened. Scientists warn that urgent reductions in greenhouse gas emissions are needed to avoid catastrophic consequences.",,0.2955,0,1276.1,90,0 -zero_shot_strict,summarization,sum_p3,"Private companies like SpaceX are lowering launch costs with reusable rockets, making space more accessible and accelerating lunar and Martian missions.","Space exploration is experiencing a new era with private companies developing reusable rockets, making space more affordable and accelerating missions to the Moon and Mars.",,0.3438,0,848.4,90,0 -zero_shot_strict,summarization,sum_p4,"Remote work is now permanent for many firms, boosting productivity and cutting costs, though challenges around collaboration and work-life boundaries persist.","Remote work has become a permanent fixture in many organizations following the COVID-19 pandemic. Employees report higher productivity and better work-life balance, while companies save on office overhead. Challenges remain around collaboration, company culture, and maintaining boundaries between professional and personal life.",,0.2245,0,1501.4,90,0 -few_shot_3,sentiment,sent_p1,Negative,Negative,1,,0,236.7,5,0 -few_shot_3,sentiment,sent_p2,Positive,Positive,1,,0,84.3,5,0 -few_shot_3,sentiment,sent_p3,Negative,Negative,1,,0,85.7,5,0 -few_shot_3,sentiment,sent_p4,Neutral,Neutral,1,,0,89.9,5,0 -few_shot_3,summarization,sum_p1,"AI has transformed industries by automating tasks and enabling better decisions, but also raises concerns about privacy, bias, and regulation.","AI has rapidly transformed industries such as healthcare, finance, transportation, and entertainment, automating tasks, improving decision-making, and creating new business opportunities. However, it also raises concerns about privacy, bias, job displacement, and the need for robust regulation.",,0.35,0,1657.4,150,0 -few_shot_3,summarization,sum_p2,"Climate change drives extreme weather, melting ice, and biodiversity loss, and scientists warn of catastrophic consequences without cutting greenhouse gas emissions.","Climate change poses significant threats to ecosystems and human societies, with rising temperatures leading to more frequent extreme weather events, melting polar ice caps, and biodiversity loss. Scientists warn that urgent reductions in greenhouse gas emissions are necessary to prevent catastrophic consequences.",,0.3721,0,1430.7,150,0 -few_shot_3,summarization,sum_p3,"Private companies like SpaceX are lowering launch costs with reusable rockets, making space more accessible and accelerating lunar and Martian missions.","Space exploration is advancing rapidly with private companies developing reusable rockets, making space more affordable and potentially accelerating missions to the Moon, Mars, and beyond.",,0.3438,0,944.5,150,0 -few_shot_3,summarization,sum_p4,"Remote work is now permanent for many firms, boosting productivity and cutting costs, though challenges around collaboration and work-life boundaries persist.","Remote work has become a permanent fixture in many organizations following the COVID-19 pandemic, offering higher productivity and better work-life balance for employees, while companies save on office overhead. However, challenges remain around collaboration, maintaining company culture, and maintaining boundaries between professional and personal life.",,0.24,0,1663.5,150,0 -chain_of_thought,sentiment,sent_p1,Negative,Negative,1,,0,594.3,150,0 -chain_of_thought,sentiment,sent_p2,Positive,Positive,1,,0,84.4,150,0 -chain_of_thought,sentiment,sent_p3,Negative,Negative,1,,0,91.7,150,0 -chain_of_thought,sentiment,sent_p4,Neutral,Neutral,1,,0,87.4,150,0 -chain_of_thought,summarization,sum_p1,"AI has transformed industries by automating tasks and enabling better decisions, but also raises concerns about privacy, bias, and regulation.","Artificial intelligence has significantly impacted various industries, enhancing efficiency and innovation. However, it also raises concerns about privacy, bias, and the need for regulatory oversight.",,0.2647,0,1881.2,200,0 -chain_of_thought,summarization,sum_p2,"Climate change drives extreme weather, melting ice, and biodiversity loss, and scientists warn of catastrophic consequences without cutting greenhouse gas emissions.","Climate change poses significant threats to ecosystems and human societies, with urgent need for reductions in greenhouse gas emissions.",,0.1818,0,1702.2,200,0 -chain_of_thought,summarization,sum_p3,"Private companies like SpaceX are lowering launch costs with reusable rockets, making space more accessible and accelerating lunar and Martian missions.","Space exploration is experiencing a significant shift with the involvement of private companies, leading to more affordable access to space and potentially accelerating missions to the Moon and Mars.",,0.2222,0,2328.3,200,0 -chain_of_thought,summarization,sum_p4,"Remote work is now permanent for many firms, boosting productivity and cutting costs, though challenges around collaboration and work-life boundaries persist.","Remote work has become a permanent fixture in many organizations due to the pandemic, with employees reporting higher productivity and better work-life balance, while companies save on office overhead. However, challenges remain around collaboration, company culture, and maintaining boundaries between professional and personal life.",,0.2157,0,2996.5,200,0 -self_consistency_3x,sentiment,sent_p1,Negative,Negative,1,,0,80.8,5,0 -self_consistency_3x,sentiment,sent_p2,Positive,Positive,1,,0,64.4,5,0 -self_consistency_3x,sentiment,sent_p3,Negative,Negative,1,,0,64.2,5,0 -self_consistency_3x,sentiment,sent_p4,Neutral,Neutral,1,,0,65.3,5,0 -self_consistency_3x,summarization,sum_p1,"AI has transformed industries by automating tasks and enabling better decisions, but also raises concerns about privacy, bias, and regulation.","AI has transformed many industries, but raises concerns about privacy, bias, job displacement, and regulation.",,0.5455,0,646.5,90,0 -self_consistency_3x,summarization,sum_p2,"Climate change drives extreme weather, melting ice, and biodiversity loss, and scientists warn of catastrophic consequences without cutting greenhouse gas emissions.","Climate change poses significant risks to Earth's ecosystems and human societies. Rising temperatures and extreme weather events are occurring more frequently, leading to biodiversity loss. Scientists warn that significant reductions in greenhouse gas emissions are needed to avoid catastrophic consequences.",,0.3415,0,1019.2,90,0 -self_consistency_3x,summarization,sum_p3,"Private companies like SpaceX are lowering launch costs with reusable rockets, making space more accessible and accelerating lunar and Martian missions.","Space exploration is advancing due to private companies developing reusable rockets, reducing launch costs. This could lead to more affordable access to space, accelerating missions to the Moon, Mars, and beyond.",,0.3056,0,866.2,90,0 -self_consistency_3x,summarization,sum_p4,"Remote work is now permanent for many firms, boosting productivity and cutting costs, though challenges around collaboration and work-life boundaries persist.","Remote work has become common due to the pandemic, leading to higher productivity and better work-life balance for employees while saving companies on office overhead. However, challenges such as collaboration, maintaining company culture, and boundaries between professional and personal life persist.",,0.2083,0,1400.6,90,0 diff --git a/output/sentiment_summarization_analysis/prompt_benchmark_summary.md b/output/sentiment_summarization_analysis/prompt_benchmark_summary.md deleted file mode 100644 index 5a10847..0000000 --- a/output/sentiment_summarization_analysis/prompt_benchmark_summary.md +++ /dev/null @@ -1,23 +0,0 @@ -# Prompting Strategy Benchmark — Summary -> Remote tokens consumed: **0** for all strategies (local-only inference) - -## Sentiment -| Strategy | Accuracy (4 probes) | Avg Latency ms | Remote Tokens | -|---|---|---|---| -| self_consistency_3x | 100.00% | 69 | 0 | -| zero_shot_strict | 100.00% | 104 | 0 | -| few_shot_3 | 100.00% | 124 | 0 | -| baseline | 100.00% | 142 | 0 | -| chain_of_thought | 100.00% | 214 | 0 | - -## Summarization -| Strategy | Avg Jaccard Sim | Avg Latency ms | Remote Tokens | -|---|---|---|---| -| self_consistency_3x | 0.3502 | 983 | 0 | -| zero_shot_strict | 0.3305 | 1096 | 0 | -| few_shot_3 | 0.3265 | 1424 | 0 | -| baseline | 0.2973 | 1302 | 0 | -| chain_of_thought | 0.2211 | 2227 | 0 | - ---- -**Scoring note**: Local tokens cost zero toward the hackathon score. Latency is recorded for the 10-minute wall-clock budget constraint only. diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b12b991 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,48 @@ +import os +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from agent.classifier import ( + ROUTE_API_CODE, + ROUTE_API_LOGIC, + ROUTE_API_MATH, + ROUTE_LOCAL_GENERAL, + ROUTE_LOCAL_NER, + ROUTE_LOCAL_SENTIMENT, +) + + +def fake_classify_generate(prompt: str, **kwargs: Any) -> str: + p = prompt.lower() + if "code" in p or "python" in p or "c++" in p or "bug" in p or "script" in p or "error" in p: + return ROUTE_API_CODE + if "sentiment" in p or "emotion" in p or "tone" in p: + return ROUTE_LOCAL_SENTIMENT + if "entity" in p or "entities" in p or "name" in p: + return ROUTE_LOCAL_NER + if "calculate" in p or "solve for x" in p or "derivative" in p or "percentage" in p: + return ROUTE_API_MATH + if "taller" in p or "riddle" in p or "constraint" in p or ">" in p or "logic" in p: + return ROUTE_API_LOGIC + return ROUTE_LOCAL_GENERAL + + +@pytest.fixture(autouse=True) +def mock_engines(monkeypatch: pytest.MonkeyPatch) -> None: + """ + Automatically mock the ClassifierEngine and LocalSLMEngine if the model file is not present. + This allows the test suite to pass in CI environments (like GitHub Actions) + where the 2GB .gguf file is excluded by .dockerignore/.gitignore. + """ + model_path = os.environ.get("LOCAL_MODEL_PATH", "models/qwen2.5-3b-instruct-q4_k_m.gguf") + if not os.path.exists(model_path): + fake_classifier = MagicMock() + fake_classifier.generate.side_effect = fake_classify_generate + + fake_local = MagicMock() + fake_local.generate.return_value = "Mocked Local Generation" + + monkeypatch.setattr("engines.local_slm.ClassifierEngine.get_instance", lambda: fake_classifier) + monkeypatch.setattr("engines.local_slm.LocalSLMEngine.get_instance", lambda: fake_local)