diff --git a/.dockerignore b/.dockerignore index dbef81f..d4d45b4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,6 +26,7 @@ __pycache__/ htmlcov/ # Local output / temp +output/ output_test/ *.tmp *.bak diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1afa6da..dbb5733 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: hooks: - id: mypy name: mypy - entry: mypy + entry: python -m mypy language: system types: [python] diff --git a/Dockerfile b/Dockerfile index 568476f..3312d5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,12 +30,7 @@ RUN uv pip install llama-cpp-python \ # Step 4: Pre-cache sentence-transformer model weights at build time # Prevents runtime download within the 10-minute container limit -RUN python -c " -from sentence_transformers import SentenceTransformer -print('Pre-caching all-MiniLM-L6-v2...') -SentenceTransformer('all-MiniLM-L6-v2') -print('Model cached successfully.') -" +RUN python -c "from sentence_transformers import SentenceTransformer; print('Pre-caching...'); SentenceTransformer('all-MiniLM-L6-v2'); print('Model cached successfully.')" # Bundle GGUF model weights (~986 MB) # Make sure to run scripts/download_model.sh before building @@ -61,6 +56,8 @@ ENV PYTHONUNBUFFERED=1 \ LOCAL_MODEL_PATH=/app/models/qwen2.5-3b-instruct-q4_k_m.gguf \ LOCAL_N_GPU_LAYERS=0 \ LOCAL_N_THREADS=2 \ - LOCAL_N_CTX=2048 + LOCAL_N_CTX=2048 \ + HF_HUB_OFFLINE=1 \ + TRANSFORMERS_OFFLINE=1 -CMD ["python", "/app/main.py"] +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/agent/classifier.py b/agent/classifier.py index c27ffd5..706f609 100644 --- a/agent/classifier.py +++ b/agent/classifier.py @@ -1,8 +1,8 @@ """ agent/classifier.py — Supervised Neural Network Classifier ========================================================= -Uses sentence-transformers (all-MiniLM-L6-v2) to encode prompts into dense -384-dimensional embeddings, then passes them through a PyTorch classification +Uses sentence-transformers (all-MiniLM-L6-v2) to encode prompts into dense +384-dimensional embeddings, then passes them through a PyTorch classification head to predict the optimal routing category. Routes (constants unchanged): @@ -12,8 +12,8 @@ import logging import os -import re -from typing import Optional +from typing import Any, Optional + import torch import torch.nn as nn @@ -43,25 +43,23 @@ _LONG_CONTEXT_THRESHOLD = 6000 # chars; above this → API_LONG to avoid CPU OOM WEIGHTS_PATH = os.path.join(os.path.dirname(__file__), "supervised_model.pt") + class LinearClassifier(nn.Module): def __init__(self, input_dim: int, num_classes: int): super().__init__() # Simple MLP head: 384 -> 64 -> 6 classes - self.net = nn.Sequential( - nn.Linear(input_dim, 64), - nn.ReLU(), - nn.Dropout(0.1), - nn.Linear(64, num_classes) - ) - - def forward(self, x): + self.net = nn.Sequential(nn.Linear(input_dim, 64), nn.ReLU(), nn.Dropout(0.1), nn.Linear(64, num_classes)) + + def forward(self, x: Any) -> Any: return self.net(x) + class SemanticClassifier: """ Loads all-MiniLM-L6-v2 and the trained PyTorch classification head. Performs fast, offline inference by passing the prompt embedding through the MLP. """ + _instance: Optional["SemanticClassifier"] = None @classmethod @@ -75,12 +73,11 @@ def __init__(self) -> None: try: from sentence_transformers import SentenceTransformer except ImportError as exc: - raise ImportError("sentence-transformers is required. " - "Run: pip install sentence-transformers") from exc + raise ImportError("sentence-transformers is required. " "Run: pip install sentence-transformers") from exc self.encoder = SentenceTransformer(_MODEL_NAME) self.model = LinearClassifier(input_dim=384, num_classes=len(ROUTES_MAP)) - + if os.path.exists(WEIGHTS_PATH): logger.info(f"Loading supervised weights from {WEIGHTS_PATH}") self.model.load_state_dict(torch.load(WEIGHTS_PATH, map_location="cpu")) @@ -92,33 +89,34 @@ def __init__(self) -> None: self._train_from_data(task_path) else: logger.error(f"Supervised weights not found at {WEIGHTS_PATH} and no training data found. Using untrained weights.") - + self.model.eval() def _train_from_data(self, data_path: str) -> None: - import torch.optim as optim import json - + + import torch.optim as optim + with open(data_path, encoding="utf-8") as f: tasks = json.load(f) - + prompts = [t["prompt"] for t in tasks] labels = [ROUTES_MAP.index(t["expected_route"]) for t in tasks] - + embeddings = self.encoder.encode(prompts, convert_to_tensor=True).cpu() labels_tensor = torch.tensor(labels, dtype=torch.long) - + criterion = nn.CrossEntropyLoss() optimizer = optim.AdamW(self.model.parameters(), lr=0.01, weight_decay=0.01) - + self.model.train() - for epoch in range(150): + for _ in range(150): optimizer.zero_grad() outputs = self.model(embeddings) loss = criterion(outputs, labels_tensor) loss.backward() optimizer.step() - + torch.save(self.model.state_dict(), WEIGHTS_PATH) logger.info(f"Successfully trained and saved new classifier weights to {WEIGHTS_PATH}!") @@ -129,6 +127,7 @@ def classify(self, prompt: str) -> str: pred_idx = int(logits.argmax(dim=1).item()) return ROUTES_MAP[pred_idx] + # --------------------------------------------------------------------------- # Public API — drop-in replacement for the old classify() # --------------------------------------------------------------------------- @@ -142,7 +141,7 @@ def classify(prompt: str) -> str: return ROUTE_API_LONG classifier = SemanticClassifier.get_instance() - + # --- Execute Supervised PyTorch Classification --- route = classifier.classify(prompt) logger.debug("Classifier: supervised PyTorch → %s", route) diff --git a/engines/remote_llm.py b/engines/remote_llm.py index d06840b..a57f82f 100644 --- a/engines/remote_llm.py +++ b/engines/remote_llm.py @@ -65,7 +65,7 @@ def __contains__(self, item: object) -> bool: _OUTPUT_SUFFIX: dict[str, str] = { "API_CODE": " Return ONLY raw code. No markdown, no explanation.", "API_MATH": "", # Handled natively in CoT system prompt - "API_LOGIC": "", # Handled natively in Direct system prompt + "API_LOGIC": "", # Handled natively in Direct system prompt "API_LONG_CONTEXT": " Summarize in 3 sentences max.", } diff --git a/handlers/code_utils.py b/handlers/code_utils.py index c3531be..8d1ca97 100644 --- a/handlers/code_utils.py +++ b/handlers/code_utils.py @@ -10,6 +10,7 @@ import ast import logging import re +from typing import Any logger = logging.getLogger(__name__) @@ -29,11 +30,26 @@ def classify_code_difficulty(prompt: str) -> str: length = len(prompt) hard_kw = ( - "dynamic programming", "dijkstra", "graph", "tree", "trie", - "heap", "linked list", "lru", "backtracking", "topological", - "n-queens", "knapsack", "memoization", "bfs", "dfs", - "binary search tree", "cycle detection", "floyd", - "strongly connected", "minimax", + "dynamic programming", + "dijkstra", + "graph", + "tree", + "trie", + "heap", + "linked list", + "lru", + "backtracking", + "topological", + "n-queens", + "knapsack", + "memoization", + "bfs", + "dfs", + "binary search tree", + "cycle detection", + "floyd", + "strongly connected", + "minimax", ) if any(kw in low for kw in hard_kw): return HARD @@ -58,22 +74,19 @@ def extract_code(text: str) -> str: text = text.strip() matches = _FENCE_RE.findall(text) if matches: - return max(matches, key=len).strip() + return str(max(matches, key=len)).strip() # No fences — strip leading prose until we hit a code-like line. lines = text.splitlines() for i, line in enumerate(lines): s = line.strip() - if s and ( - s.startswith(("def ", "class ", "import ", "from ", "if __", "#")) - or re.match(r"^[a-zA-Z_]\w*\s*=", s) - ): + if s and (s.startswith(("def ", "class ", "import ", "from ", "if __", "#")) or re.match(r"^[a-zA-Z_]\w*\s*=", s)): return "\n".join(lines[i:]).strip() return text -def validate_code(code: str) -> dict: +def validate_code(code: str) -> dict[str, Any]: """Run syntax + completeness checks. Returns {is_valid, reason}.""" if not code or not code.strip(): return {"is_valid": False, "reason": "empty"} diff --git a/handlers/logic.py b/handlers/logic.py index 0cc23e5..e60ad04 100644 --- a/handlers/logic.py +++ b/handlers/logic.py @@ -11,30 +11,30 @@ def extract_logic(text: str) -> str: """Cleans up the final logic answer based on strict format.""" text = text.strip() - m = re.search(r'ANSWER:\s*(.+?)(?:\n|$)', text, re.IGNORECASE) + m = re.search(r"ANSWER:\s*(.+?)(?:\n|$)", text, re.IGNORECASE) if m: - return m.group(1).strip().rstrip('.') + return m.group(1).strip().rstrip(".") if len(text.split()) <= 3: return text - - bold = re.findall(r'\*\*([^*]+?)\*\*', text) + + bold = re.findall(r"\*\*([^*]+?)\*\*", text) if bold: - return bold[-1].strip() - - lines = text.strip().split('\n') + return str(bold[-1]).strip() + + lines = text.strip().split("\n") last = lines[-1].strip() - if re.match(r'^(yes|no)\.?$', last, re.IGNORECASE): - return last.rstrip('.') - - if re.search(r'\byes\b', text, re.I) and not re.search(r'\bno\b', text, re.I): + if re.match(r"^(yes|no)\.?$", last, re.IGNORECASE): + return last.rstrip(".") + + if re.search(r"\byes\b", text, re.I) and not re.search(r"\bno\b", text, re.I): return "Yes" - if re.search(r'\bno\b', text, re.I) and not re.search(r'\byes\b', text, re.I): + if re.search(r"\bno\b", text, re.I) and not re.search(r"\byes\b", text, re.I): return "No" - + ans = re.findall(r'(?:answer|conclusion|therefore)\s*(?:is|:)\s*["\']?(.+?)(?:["\']?\s*(?:\.|$|!))', text, re.I) if ans and len(ans[-1].split()) <= 6: - return ans[-1].strip() - + return str(ans[-1]).strip() + return last if len(last.split()) <= 6 else text[:60] diff --git a/handlers/math_handler.py b/handlers/math_handler.py index 2f483c7..e17e5ed 100644 --- a/handlers/math_handler.py +++ b/handlers/math_handler.py @@ -11,26 +11,26 @@ def extract_number(text: str) -> str: """Extracts numeric answer from a verbose Chain-of-Thought response.""" text = text.strip() - if re.match(r'^-?\d+(?:/\d+)?(?:\.\d+)?$', text): + if re.match(r"^-?\d+(?:/\d+)?(?:\.\d+)?$", text): return text - + # Try different pattern fallbacks patterns = [ - r'ANSWER:\s*\$?\s*(-?\d+(?:/\d+)?(?:\.\d+)?)', - r'(?:answer|result|total|sum|area|speed|price|cost|value|average|remain|left|volume|profit|capacity|units|pieces|girls|seconds|original|radius)\s*(?:is|=|:)\s*\$?\s*(-?\d+(?:/\d+)?(?:\.\d+)?)', - r'\*\*(-?\d+(?:/\d+)?(?:\.\d+)?)\*\*', - r'\\boxed\{(-?\d+(?:/\d+)?(?:\.\d+)?)\}', - r'`(-?\d+(?:/\d+)?(?:\.\d+)?)`' + r"ANSWER:\s*\$?\s*(-?\d+(?:/\d+)?(?:\.\d+)?)", + r"(?:answer|result|total|sum|area|speed|price|cost|value|average|remain|left|volume|profit|capacity|units|pieces|girls|seconds|original|radius)\s*(?:is|=|:)\s*\$?\s*(-?\d+(?:/\d+)?(?:\.\d+)?)", + r"\*\*(-?\d+(?:/\d+)?(?:\.\d+)?)\*\*", + r"\\boxed\{(-?\d+(?:/\d+)?(?:\.\d+)?)\}", + r"`(-?\d+(?:/\d+)?(?:\.\d+)?)`", ] for p in patterns: m = re.findall(p, text, re.IGNORECASE) if m: val = m[-1] if isinstance(m[-1], str) else [x for x in m[-1] if x][-1] - return val - + return str(val) + # Final fallback: last number in text - nums = re.findall(r'-?\d+(?:/\d+)?(?:\.\d+)?', text) - return nums[-1] if nums else text + nums = re.findall(r"-?\d+(?:/\d+)?(?:\.\d+)?", text) + return str(nums[-1]) if nums else text class MathHandler: diff --git a/output/classifier_analysis/classifier_eval_results.csv b/output/classifier_analysis/classifier_eval_results.csv new file mode 100644 index 0000000..4c007de --- /dev/null +++ b/output/classifier_analysis/classifier_eval_results.csv @@ -0,0 +1,801 @@ +task_id,category,difficulty,prompt_length,expected_route,predicted_route,is_correct +codegen-easy-033,Code Generation,Easy,205,API_CODE,API_CODE,TRUE +summary-medium-075,Summarization,Medium,408,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-074,Code Debugging,Medium,380,API_CODE,API_CODE,TRUE +ner-hard-091,NER,Hard,192,LOCAL_NER,LOCAL_NER,TRUE +logic-medium-048,Logical Reasoning,Medium,166,API_LOGIC,API_LOGIC,TRUE +debug-easy-032,Code Debugging,Easy,399,API_CODE,API_CODE,TRUE +summary-medium-059,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-medium-050,Logical Reasoning,Medium,119,API_LOGIC,API_LOGIC,TRUE +ner-medium-051,NER,Medium,181,LOCAL_NER,LOCAL_NER,TRUE +codegen-medium-074,Code Generation,Medium,147,API_CODE,API_CODE,TRUE +ner-hard-096,NER,Hard,233,LOCAL_NER,LOCAL_NER,TRUE +logic-medium-045,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE +codegen-hard-089,Code Generation,Hard,199,API_CODE,API_CODE,TRUE +summary-medium-038,Summarization,Medium,360,LOCAL_GENERAL,LOCAL_SENTIMENT,FALSE +factual-easy-025,Factual,Easy,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-hard-092,Code Generation,Hard,200,API_CODE,API_CODE,TRUE +math-ast-hard-078,Math (AST),Hard,36,API_MATH,API_MATH,TRUE +summary-hard-079,Summarization,Hard,1357,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-041,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE +math-ast-easy-006,Math (AST),Easy,49,API_MATH,API_MATH,TRUE +summary-hard-099,Summarization,Hard,1207,LOCAL_GENERAL,API_LOGIC,FALSE +factual-hard-089,Factual,Hard,131,LOCAL_GENERAL,API_MATH,FALSE +logic-easy-016,Logical Reasoning,Easy,185,API_LOGIC,API_LOGIC,TRUE +ner-easy-022,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE +sentiment-easy-001,Sentiment,Easy,133,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-easy-002,Sentiment,Easy,156,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-hard-090,NER,Hard,248,LOCAL_NER,LOCAL_NER,TRUE +sentiment-easy-032,Sentiment,Easy,193,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-medium-040,Sentiment,Medium,135,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-hard-077,Summarization,Hard,1342,LOCAL_GENERAL,API_LOGIC,FALSE +factual-medium-065,Factual,Medium,107,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-easy-035,Code Generation,Easy,200,API_CODE,API_CODE,TRUE +codegen-easy-031,Code Generation,Easy,176,API_CODE,API_CODE,TRUE +sentiment-easy-030,Sentiment,Easy,114,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-medium-045,Code Debugging,Medium,433,API_CODE,API_CODE,TRUE +logic-easy-013,Logical Reasoning,Easy,100,API_LOGIC,API_LOGIC,TRUE +debug-easy-011,Code Debugging,Easy,176,API_CODE,API_CODE,TRUE +sentiment-medium-037,Sentiment,Medium,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-032,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE +logic-hard-080,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE +sentiment-easy-022,Sentiment,Easy,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-medium-045,NER,Medium,181,LOCAL_NER,LOCAL_NER,TRUE +ner-hard-100,NER,Hard,156,LOCAL_NER,LOCAL_NER,TRUE +codegen-easy-032,Code Generation,Easy,199,API_CODE,API_CODE,TRUE +summary-easy-030,Summarization,Easy,147,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-hard-081,Sentiment,Hard,106,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-easy-003,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +math-ast-hard-092,Math (AST),Hard,42,API_MATH,API_MATH,TRUE +ner-easy-002,NER,Easy,203,LOCAL_NER,LOCAL_NER,TRUE +summary-easy-017,Summarization,Easy,130,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-hard-079,Factual,Hard,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-easy-013,Sentiment,Easy,114,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-medium-075,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE +summary-hard-084,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE +codegen-medium-037,Code Generation,Medium,199,API_CODE,API_CODE,TRUE +summary-hard-092,Summarization,Hard,1317,LOCAL_GENERAL,LOCAL_NER,FALSE +logic-medium-057,Logical Reasoning,Medium,200,API_LOGIC,API_LOGIC,TRUE +codegen-medium-051,Code Generation,Medium,191,API_CODE,LOCAL_NER,FALSE +summary-easy-002,Summarization,Easy,141,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +math-word-medium-051,Math (Word),Medium,151,API_MATH,API_MATH,TRUE +debug-medium-039,Code Debugging,Medium,224,API_CODE,API_CODE,TRUE +factual-medium-037,Factual,Medium,94,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-easy-003,Logical Reasoning,Easy,140,API_LOGIC,API_MATH,FALSE +math-ast-hard-086,Math (AST),Hard,36,API_MATH,API_MATH,TRUE +math-word-medium-073,Math (Word),Medium,152,API_MATH,API_MATH,TRUE +math-word-hard-081,Math (Word),Hard,215,API_MATH,API_MATH,TRUE +factual-hard-093,Factual,Hard,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-easy-010,Factual,Easy,108,LOCAL_GENERAL,API_MATH,FALSE +summary-medium-058,Summarization,Medium,404,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-medium-051,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-medium-064,NER,Medium,248,LOCAL_NER,LOCAL_NER,TRUE +factual-medium-075,Factual,Medium,104,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-medium-067,Logical Reasoning,Medium,166,API_LOGIC,API_LOGIC,TRUE +factual-easy-014,Factual,Easy,76,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-medium-039,Code Generation,Medium,199,API_CODE,API_CODE,TRUE +math-word-medium-075,Math (Word),Medium,152,API_MATH,API_MATH,TRUE +codegen-medium-070,Code Generation,Medium,199,API_CODE,API_CODE,TRUE +debug-medium-040,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE +ner-easy-003,NER,Easy,175,LOCAL_NER,LOCAL_NER,TRUE +ner-medium-052,NER,Medium,217,LOCAL_NER,LOCAL_NER,TRUE +sentiment-hard-078,Sentiment,Hard,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-medium-066,Sentiment,Medium,176,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-medium-060,Summarization,Medium,412,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-hard-076,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE +summary-easy-021,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-easy-029,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE +sentiment-easy-033,Sentiment,Easy,160,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-hard-083,Code Generation,Hard,185,API_CODE,API_CODE,TRUE +ner-easy-035,NER,Easy,202,LOCAL_NER,LOCAL_NER,TRUE +summary-medium-057,Summarization,Medium,328,LOCAL_GENERAL,API_MATH,FALSE +codegen-easy-004,Code Generation,Easy,199,API_CODE,API_CODE,TRUE +debug-easy-008,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE +sentiment-easy-014,Sentiment,Easy,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-medium-041,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE +logic-hard-090,Logical Reasoning,Hard,185,API_LOGIC,API_LOGIC,TRUE +sentiment-medium-048,Sentiment,Medium,114,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +factual-easy-009,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-055,Summarization,Medium,360,LOCAL_GENERAL,LOCAL_SENTIMENT,FALSE +factual-easy-029,Factual,Easy,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-medium-048,Code Generation,Medium,191,API_CODE,LOCAL_NER,FALSE +logic-medium-047,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE +summary-hard-098,Summarization,Hard,1357,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-hard-095,NER,Hard,217,LOCAL_NER,LOCAL_NER,TRUE +debug-easy-024,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE +summary-easy-026,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-easy-011,Factual,Easy,90,LOCAL_GENERAL,API_MATH,FALSE +debug-hard-088,Code Debugging,Hard,433,API_CODE,API_CODE,TRUE +summary-hard-086,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE +ner-easy-023,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE +sentiment-medium-065,Sentiment,Medium,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-006,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE +math-ast-medium-074,Math (AST),Medium,42,API_MATH,API_MATH,TRUE +ner-easy-005,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE +logic-medium-038,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE +logic-medium-054,Logical Reasoning,Medium,100,API_LOGIC,API_LOGIC,TRUE +sentiment-medium-063,Sentiment,Medium,176,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-027,Logical Reasoning,Easy,100,API_LOGIC,API_LOGIC,TRUE +factual-medium-059,Factual,Medium,89,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-hard-084,Code Debugging,Hard,211,API_CODE,API_CODE,TRUE +ner-hard-078,NER,Hard,248,LOCAL_NER,LOCAL_NER,TRUE +codegen-medium-073,Code Generation,Medium,199,API_CODE,API_CODE,TRUE +factual-easy-034,Factual,Easy,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-medium-038,NER,Medium,192,LOCAL_NER,LOCAL_NER,TRUE +factual-easy-024,Factual,Easy,88,LOCAL_GENERAL,API_MATH,FALSE +debug-easy-023,Code Debugging,Easy,274,API_CODE,API_CODE,TRUE +math-word-easy-011,Math (Word),Easy,215,API_MATH,API_MATH,TRUE +factual-easy-018,Factual,Easy,86,LOCAL_GENERAL,API_CODE,FALSE +math-word-hard-099,Math (Word),Hard,103,API_MATH,API_MATH,TRUE +logic-hard-088,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE +logic-easy-008,Logical Reasoning,Easy,140,API_LOGIC,API_MATH,FALSE +ner-hard-082,NER,Hard,162,LOCAL_NER,LOCAL_NER,TRUE +debug-hard-091,Code Debugging,Hard,380,API_CODE,API_CODE,TRUE +debug-hard-078,Code Debugging,Hard,274,API_CODE,API_CODE,TRUE +math-word-medium-063,Math (Word),Medium,143,API_MATH,API_MATH,TRUE +factual-medium-043,Factual,Medium,88,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-easy-027,Code Generation,Easy,185,API_CODE,API_CODE,TRUE +sentiment-medium-064,Sentiment,Medium,177,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +math-word-medium-049,Math (Word),Medium,153,API_MATH,API_MATH,TRUE +debug-hard-083,Code Debugging,Hard,230,API_CODE,API_CODE,TRUE +codegen-easy-025,Code Generation,Easy,205,API_CODE,API_CODE,TRUE +debug-medium-051,Code Debugging,Medium,176,API_CODE,API_CODE,TRUE +codegen-easy-002,Code Generation,Easy,147,API_CODE,API_CODE,TRUE +ner-easy-032,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE +ner-hard-081,NER,Hard,175,LOCAL_NER,LOCAL_NER,TRUE +summary-easy-012,Summarization,Easy,118,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-easy-015,NER,Easy,233,LOCAL_NER,LOCAL_NER,TRUE +math-ast-medium-038,Math (AST),Medium,43,API_MATH,API_MATH,TRUE +debug-easy-005,Code Debugging,Easy,211,API_CODE,API_CODE,TRUE +summary-medium-070,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_NER,FALSE +codegen-easy-026,Code Generation,Easy,191,API_CODE,LOCAL_NER,FALSE +ner-easy-018,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE +debug-hard-090,Code Debugging,Hard,213,API_CODE,API_CODE,TRUE +debug-easy-021,Code Debugging,Easy,289,API_CODE,API_CODE,TRUE +debug-easy-015,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE +sentiment-medium-042,Sentiment,Medium,180,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +factual-easy-006,Factual,Easy,131,LOCAL_GENERAL,API_MATH,FALSE +math-ast-medium-050,Math (AST),Medium,36,API_MATH,API_MATH,TRUE +debug-medium-062,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE +sentiment-medium-073,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-hard-087,Summarization,Hard,1369,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-036,Code Debugging,Medium,211,API_CODE,API_CODE,TRUE +codegen-easy-021,Code Generation,Easy,176,API_CODE,API_CODE,TRUE +math-ast-easy-020,Math (AST),Easy,44,API_MATH,API_MATH,TRUE +ner-easy-010,NER,Easy,211,LOCAL_NER,LOCAL_NER,TRUE +factual-hard-085,Factual,Hard,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-hard-097,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE +logic-medium-068,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE +factual-easy-017,Factual,Easy,118,LOCAL_GENERAL,API_MATH,FALSE +debug-easy-003,Code Debugging,Easy,274,API_CODE,API_CODE,TRUE +factual-medium-040,Factual,Medium,73,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-medium-038,Sentiment,Medium,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-easy-007,Summarization,Easy,129,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-hard-082,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE +sentiment-hard-094,Sentiment,Hard,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-easy-028,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE +math-word-medium-039,Math (Word),Medium,137,API_MATH,API_MATH,TRUE +logic-hard-096,Logical Reasoning,Hard,212,API_LOGIC,API_LOGIC,TRUE +factual-hard-086,Factual,Hard,108,LOCAL_GENERAL,API_MATH,FALSE +debug-hard-079,Code Debugging,Hard,211,API_CODE,API_CODE,TRUE +logic-hard-079,Logical Reasoning,Hard,200,API_LOGIC,API_LOGIC,TRUE +codegen-easy-013,Code Generation,Easy,186,API_CODE,API_CODE,TRUE +codegen-medium-050,Code Generation,Medium,185,API_CODE,API_CODE,TRUE +logic-medium-065,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE +math-word-hard-089,Math (Word),Hard,134,API_MATH,API_MATH,TRUE +logic-easy-007,Logical Reasoning,Easy,212,API_LOGIC,API_LOGIC,TRUE +ner-hard-084,NER,Hard,203,LOCAL_NER,LOCAL_NER,TRUE +math-word-easy-025,Math (Word),Easy,215,API_MATH,API_MATH,TRUE +ner-medium-053,NER,Medium,211,LOCAL_NER,LOCAL_NER,TRUE +codegen-easy-003,Code Generation,Easy,185,API_CODE,API_CODE,TRUE +logic-medium-062,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE +ner-hard-077,NER,Hard,203,LOCAL_NER,LOCAL_NER,TRUE +summary-hard-094,Summarization,Hard,1227,LOCAL_GENERAL,API_LOGIC,FALSE +sentiment-medium-057,Sentiment,Medium,147,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-easy-004,NER,Easy,233,LOCAL_NER,LOCAL_NER,TRUE +math-word-hard-091,Math (Word),Hard,150,API_MATH,API_MATH,TRUE +factual-medium-041,Factual,Medium,73,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-easy-034,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +math-word-medium-041,Math (Word),Medium,153,API_MATH,API_MATH,TRUE +summary-hard-076,Summarization,Hard,1342,LOCAL_GENERAL,API_LOGIC,FALSE +math-ast-easy-030,Math (AST),Easy,42,API_MATH,API_MATH,TRUE +math-word-hard-085,Math (Word),Hard,141,API_MATH,API_MATH,TRUE +sentiment-medium-050,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-easy-024,NER,Easy,223,LOCAL_NER,LOCAL_NER,TRUE +math-ast-hard-094,Math (AST),Hard,43,API_MATH,API_MATH,TRUE +codegen-medium-057,Code Generation,Medium,185,API_CODE,API_CODE,TRUE +math-word-medium-053,Math (Word),Medium,217,API_MATH,API_MATH,TRUE +logic-easy-010,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE +summary-easy-016,Summarization,Easy,130,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-065,Summarization,Medium,420,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-medium-052,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE +sentiment-easy-028,Sentiment,Easy,164,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-hard-079,Code Generation,Hard,176,API_CODE,API_CODE,TRUE +logic-easy-018,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE +factual-easy-032,Factual,Easy,123,LOCAL_GENERAL,API_MATH,FALSE +sentiment-hard-091,Sentiment,Hard,165,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-hard-093,Summarization,Hard,1309,LOCAL_GENERAL,LOCAL_NER,FALSE +codegen-medium-063,Code Generation,Medium,170,API_CODE,API_CODE,TRUE +math-word-easy-019,Math (Word),Easy,215,API_MATH,API_MATH,TRUE +math-word-easy-033,Math (Word),Easy,215,API_MATH,API_MATH,TRUE +logic-medium-040,Logical Reasoning,Medium,157,API_LOGIC,API_LOGIC,TRUE +logic-hard-076,Logical Reasoning,Hard,219,API_LOGIC,API_LOGIC,TRUE +factual-medium-046,Factual,Medium,109,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-medium-057,Factual,Medium,90,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-medium-036,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE +math-ast-hard-100,Math (AST),Hard,43,API_MATH,API_MATH,TRUE +codegen-hard-077,Code Generation,Hard,186,API_CODE,API_CODE,TRUE +summary-easy-013,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-easy-002,Code Debugging,Easy,289,API_CODE,API_CODE,TRUE +math-word-hard-093,Math (Word),Hard,215,API_MATH,API_MATH,TRUE +codegen-hard-094,Code Generation,Hard,191,API_CODE,LOCAL_NER,FALSE +summary-hard-096,Summarization,Hard,1062,LOCAL_GENERAL,API_MATH,FALSE +summary-easy-011,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-043,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE +logic-medium-049,Logical Reasoning,Medium,159,API_LOGIC,API_MATH,FALSE +sentiment-medium-052,Sentiment,Medium,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-hard-077,Code Debugging,Hard,195,API_CODE,API_CODE,TRUE +ner-hard-086,NER,Hard,217,LOCAL_NER,LOCAL_NER,TRUE +codegen-hard-096,Code Generation,Hard,147,API_CODE,API_CODE,TRUE +ner-medium-044,NER,Medium,198,LOCAL_NER,LOCAL_NER,TRUE +ner-medium-068,NER,Medium,229,LOCAL_NER,LOCAL_NER,TRUE +factual-medium-050,Factual,Medium,103,LOCAL_GENERAL,API_MATH,FALSE +math-ast-medium-048,Math (AST),Medium,42,API_MATH,API_MATH,TRUE +ner-medium-058,NER,Medium,156,LOCAL_NER,LOCAL_NER,TRUE +ner-medium-073,NER,Medium,175,LOCAL_NER,LOCAL_NER,TRUE +math-word-easy-007,Math (Word),Easy,215,API_MATH,API_MATH,TRUE +ner-easy-031,NER,Easy,211,LOCAL_NER,LOCAL_NER,TRUE +summary-hard-078,Summarization,Hard,1354,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-049,Code Debugging,Medium,224,API_CODE,API_CODE,TRUE +logic-hard-089,Logical Reasoning,Hard,185,API_LOGIC,API_LOGIC,TRUE +summary-medium-036,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE +summary-medium-056,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-easy-017,Code Generation,Easy,170,API_CODE,API_CODE,TRUE +codegen-easy-011,Code Generation,Easy,176,API_CODE,API_CODE,TRUE +codegen-medium-065,Code Generation,Medium,199,API_CODE,API_CODE,TRUE +ner-medium-050,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE +math-ast-easy-002,Math (AST),Easy,43,API_MATH,API_MATH,TRUE +math-ast-easy-016,Math (AST),Easy,43,API_MATH,API_MATH,TRUE +ner-medium-057,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE +ner-hard-089,NER,Hard,233,LOCAL_NER,LOCAL_NER,TRUE +ner-easy-001,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE +math-ast-medium-072,Math (AST),Medium,45,API_MATH,API_MATH,TRUE +ner-medium-040,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE +factual-medium-044,Factual,Medium,100,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-hard-087,Factual,Hard,93,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-easy-034,Code Debugging,Easy,232,API_CODE,API_CODE,TRUE +summary-easy-008,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-medium-048,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE +sentiment-medium-069,Sentiment,Medium,116,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-easy-025,Code Debugging,Easy,224,API_CODE,API_CODE,TRUE +codegen-easy-028,Code Generation,Easy,147,API_CODE,API_CODE,TRUE +math-word-medium-065,Math (Word),Medium,217,API_MATH,API_MATH,TRUE +sentiment-hard-095,Sentiment,Hard,150,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-medium-066,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-hard-083,Logical Reasoning,Hard,157,API_LOGIC,API_LOGIC,TRUE +math-ast-medium-056,Math (AST),Medium,36,API_MATH,API_MATH,TRUE +debug-easy-007,Code Debugging,Easy,433,API_CODE,API_CODE,TRUE +sentiment-easy-008,Sentiment,Easy,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-easy-019,Code Debugging,Easy,224,API_CODE,API_CODE,TRUE +codegen-hard-087,Code Generation,Hard,176,API_CODE,API_CODE,TRUE +debug-easy-016,Code Debugging,Easy,205,API_CODE,API_CODE,TRUE +sentiment-easy-016,Sentiment,Easy,132,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-033,Logical Reasoning,Easy,152,API_LOGIC,API_LOGIC,TRUE +logic-easy-020,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE +math-ast-medium-052,Math (AST),Medium,42,API_MATH,API_MATH,TRUE +ner-hard-088,NER,Hard,156,LOCAL_NER,LOCAL_NER,TRUE +codegen-easy-007,Code Generation,Easy,185,API_CODE,API_CODE,TRUE +sentiment-easy-010,Sentiment,Easy,130,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-easy-017,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE +ner-medium-047,NER,Medium,234,LOCAL_NER,LOCAL_NER,TRUE +ner-easy-025,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE +factual-medium-064,Factual,Medium,79,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-070,Code Debugging,Medium,380,API_CODE,API_CODE,TRUE +summary-easy-022,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-easy-028,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-073,Summarization,Medium,316,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-069,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE +debug-medium-038,Code Debugging,Medium,232,API_CODE,API_CODE,TRUE +debug-hard-095,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE +sentiment-medium-067,Sentiment,Medium,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-026,Logical Reasoning,Easy,178,API_LOGIC,API_LOGIC,TRUE +summary-easy-009,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-hard-097,Summarization,Hard,1219,LOCAL_GENERAL,API_LOGIC,FALSE +math-word-easy-031,Math (Word),Easy,152,API_MATH,API_MATH,TRUE +factual-hard-098,Factual,Hard,101,LOCAL_GENERAL,API_MATH,FALSE +sentiment-hard-096,Sentiment,Hard,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +math-ast-hard-098,Math (AST),Hard,43,API_MATH,API_MATH,TRUE +logic-medium-074,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE +sentiment-medium-062,Sentiment,Medium,163,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-hard-081,Code Generation,Hard,176,API_CODE,API_CODE,TRUE +ner-medium-067,NER,Medium,198,LOCAL_NER,LOCAL_NER,TRUE +math-word-hard-077,Math (Word),Hard,215,API_MATH,API_MATH,TRUE +debug-easy-014,Code Debugging,Easy,224,API_CODE,API_CODE,TRUE +logic-easy-035,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE +ner-medium-062,NER,Medium,211,LOCAL_NER,LOCAL_NER,TRUE +debug-medium-060,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE +logic-hard-086,Logical Reasoning,Hard,145,API_LOGIC,API_LOGIC,TRUE +sentiment-hard-098,Sentiment,Hard,186,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-medium-051,Summarization,Medium,360,LOCAL_GENERAL,LOCAL_SENTIMENT,FALSE +debug-easy-004,Code Debugging,Easy,176,API_CODE,API_CODE,TRUE +ner-easy-020,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE +factual-easy-015,Factual,Easy,81,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-hard-086,Code Generation,Hard,171,API_CODE,API_CODE,TRUE +logic-hard-094,Logical Reasoning,Hard,140,API_LOGIC,API_MATH,FALSE +ner-medium-061,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE +debug-easy-031,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE +ner-hard-076,NER,Hard,229,LOCAL_NER,LOCAL_NER,TRUE +summary-easy-029,Summarization,Easy,139,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-medium-043,NER,Medium,203,LOCAL_NER,LOCAL_NER,TRUE +logic-hard-081,Logical Reasoning,Hard,145,API_LOGIC,API_LOGIC,TRUE +debug-easy-026,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE +sentiment-easy-005,Sentiment,Easy,170,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +factual-hard-099,Factual,Hard,118,LOCAL_GENERAL,API_MATH,FALSE +ner-easy-008,NER,Easy,229,LOCAL_NER,LOCAL_NER,TRUE +logic-easy-021,Logical Reasoning,Easy,185,API_LOGIC,API_LOGIC,TRUE +math-word-easy-017,Math (Word),Easy,170,API_MATH,API_MATH,TRUE +logic-easy-031,Logical Reasoning,Easy,166,API_LOGIC,API_LOGIC,TRUE +math-ast-hard-080,Math (AST),Hard,45,API_MATH,API_MATH,TRUE +debug-hard-085,Code Debugging,Hard,380,API_CODE,API_CODE,TRUE +math-word-easy-003,Math (Word),Easy,150,API_MATH,API_MATH,TRUE +math-ast-easy-014,Math (AST),Easy,41,API_MATH,API_MATH,TRUE +sentiment-medium-041,Sentiment,Medium,129,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-022,Logical Reasoning,Easy,200,API_LOGIC,API_LOGIC,TRUE +math-ast-hard-096,Math (AST),Hard,45,API_MATH,API_MATH,TRUE +debug-medium-056,Code Debugging,Medium,433,API_CODE,API_CODE,TRUE +codegen-easy-029,Code Generation,Easy,171,API_CODE,API_CODE,TRUE +math-word-medium-037,Math (Word),Medium,172,API_MATH,API_MATH,TRUE +codegen-medium-072,Code Generation,Medium,186,API_CODE,API_CODE,TRUE +ner-medium-069,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE +sentiment-easy-011,Sentiment,Easy,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-medium-058,Logical Reasoning,Medium,200,API_LOGIC,API_LOGIC,TRUE +factual-hard-092,Factual,Hard,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-easy-013,Factual,Easy,123,LOCAL_GENERAL,API_MATH,FALSE +sentiment-easy-012,Sentiment,Easy,120,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +factual-hard-091,Factual,Hard,93,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-hard-084,Factual,Hard,90,LOCAL_GENERAL,API_MATH,FALSE +math-ast-easy-022,Math (AST),Easy,36,API_MATH,API_MATH,TRUE +ner-easy-016,NER,Easy,167,LOCAL_NER,LOCAL_NER,TRUE +debug-medium-057,Code Debugging,Medium,205,API_CODE,API_CODE,TRUE +sentiment-easy-009,Sentiment,Easy,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-hard-087,Sentiment,Hard,180,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +math-ast-hard-082,Math (AST),Hard,49,API_MATH,API_MATH,TRUE +factual-medium-053,Factual,Medium,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-easy-028,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE +codegen-medium-045,Code Generation,Medium,200,API_CODE,API_CODE,TRUE +summary-medium-050,Summarization,Medium,336,LOCAL_GENERAL,API_MATH,FALSE +ner-easy-012,NER,Easy,217,LOCAL_NER,LOCAL_NER,TRUE +math-ast-hard-084,Math (AST),Hard,35,API_MATH,API_MATH,TRUE +codegen-hard-100,Code Generation,Hard,162,API_CODE,API_CODE,TRUE +math-word-hard-087,Math (Word),Hard,152,API_MATH,API_MATH,TRUE +math-ast-medium-058,Math (AST),Medium,35,API_MATH,API_MATH,TRUE +debug-hard-087,Code Debugging,Hard,399,API_CODE,API_CODE,TRUE +debug-hard-089,Code Debugging,Hard,230,API_CODE,API_CODE,TRUE +codegen-medium-058,Code Generation,Medium,185,API_CODE,API_CODE,TRUE +factual-hard-076,Factual,Hard,101,LOCAL_GENERAL,API_MATH,FALSE +codegen-medium-044,Code Generation,Medium,200,API_CODE,API_CODE,TRUE +ner-medium-075,NER,Medium,242,LOCAL_NER,LOCAL_NER,TRUE +ner-medium-041,NER,Medium,156,LOCAL_NER,LOCAL_NER,TRUE +factual-easy-019,Factual,Easy,90,LOCAL_GENERAL,API_MATH,FALSE +sentiment-hard-080,Sentiment,Hard,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +factual-hard-077,Factual,Hard,97,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-054,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-medium-039,Factual,Medium,80,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-hard-099,Logical Reasoning,Hard,178,API_LOGIC,API_LOGIC,TRUE +ner-easy-029,NER,Easy,175,LOCAL_NER,LOCAL_NER,TRUE +sentiment-hard-088,Sentiment,Hard,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-easy-029,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE +summary-medium-062,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-hard-083,Sentiment,Hard,198,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-easy-027,Sentiment,Easy,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-easy-020,Code Generation,Easy,170,API_CODE,API_CODE,TRUE +factual-hard-083,Factual,Hard,82,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +math-word-easy-035,Math (Word),Easy,149,API_MATH,API_MATH,TRUE +summary-easy-027,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-medium-067,Factual,Medium,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-easy-033,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE +factual-easy-002,Factual,Easy,82,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-easy-023,Factual,Easy,123,LOCAL_GENERAL,API_MATH,FALSE +logic-medium-070,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE +logic-easy-001,Logical Reasoning,Easy,119,API_LOGIC,API_LOGIC,TRUE +ner-easy-007,NER,Easy,248,LOCAL_NER,LOCAL_NER,TRUE +factual-medium-048,Factual,Medium,81,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-easy-027,Factual,Easy,118,LOCAL_GENERAL,API_MATH,FALSE +logic-easy-009,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE +codegen-easy-008,Code Generation,Easy,176,API_CODE,API_CODE,TRUE +sentiment-medium-059,Sentiment,Medium,162,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-medium-058,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE +summary-easy-025,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-052,Summarization,Medium,408,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-hard-084,Logical Reasoning,Hard,157,API_LOGIC,API_LOGIC,TRUE +codegen-easy-010,Code Generation,Easy,147,API_CODE,API_CODE,TRUE +codegen-medium-052,Code Generation,Medium,176,API_CODE,API_CODE,TRUE +math-ast-easy-008,Math (AST),Easy,49,API_MATH,API_MATH,TRUE +summary-easy-010,Summarization,Easy,149,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +math-word-medium-045,Math (Word),Medium,153,API_MATH,API_MATH,TRUE +math-ast-medium-040,Math (AST),Medium,43,API_MATH,API_MATH,TRUE +factual-hard-080,Factual,Hard,90,LOCAL_GENERAL,API_MATH,FALSE +ner-easy-026,NER,Easy,162,LOCAL_NER,LOCAL_NER,TRUE +summary-hard-081,Summarization,Hard,1342,LOCAL_GENERAL,API_LOGIC,FALSE +factual-medium-055,Factual,Medium,89,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-medium-039,Logical Reasoning,Medium,119,API_LOGIC,API_LOGIC,TRUE +codegen-medium-040,Code Generation,Medium,176,API_CODE,API_CODE,TRUE +factual-medium-069,Factual,Medium,82,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-medium-054,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-040,Summarization,Medium,316,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-easy-009,Code Generation,Easy,185,API_CODE,API_CODE,TRUE +sentiment-hard-084,Sentiment,Hard,163,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-easy-019,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE +summary-medium-042,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-medium-055,NER,Medium,202,LOCAL_NER,LOCAL_NER,TRUE +codegen-medium-049,Code Generation,Medium,185,API_CODE,API_CODE,TRUE +summary-easy-035,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-easy-014,NER,Easy,248,LOCAL_NER,LOCAL_NER,TRUE +summary-easy-014,Summarization,Easy,159,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +math-word-medium-055,Math (Word),Medium,136,API_MATH,API_MATH,TRUE +summary-medium-061,Summarization,Medium,416,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-hard-079,Sentiment,Hard,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-easy-027,NER,Easy,211,LOCAL_NER,LOCAL_NER,TRUE +logic-medium-071,Logical Reasoning,Medium,140,API_LOGIC,API_MATH,FALSE +logic-easy-023,Logical Reasoning,Easy,164,API_LOGIC,API_LOGIC,TRUE +math-ast-medium-046,Math (AST),Medium,49,API_MATH,API_MATH,TRUE +math-ast-hard-090,Math (AST),Hard,48,API_MATH,API_MATH,TRUE +math-ast-easy-026,Math (AST),Easy,44,API_MATH,API_MATH,TRUE +ner-hard-094,NER,Hard,181,LOCAL_NER,LOCAL_NER,TRUE +math-word-hard-079,Math (Word),Hard,134,API_MATH,API_MATH,TRUE +debug-hard-093,Code Debugging,Hard,230,API_CODE,API_CODE,TRUE +logic-medium-042,Logical Reasoning,Medium,219,API_LOGIC,API_LOGIC,TRUE +sentiment-medium-060,Sentiment,Medium,163,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +math-ast-easy-010,Math (AST),Easy,49,API_MATH,API_MATH,TRUE +sentiment-hard-076,Sentiment,Hard,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-medium-042,Code Generation,Medium,176,API_CODE,API_CODE,TRUE +debug-medium-037,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE +sentiment-hard-089,Sentiment,Hard,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-easy-019,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-medium-059,NER,Medium,167,LOCAL_NER,LOCAL_NER,TRUE +factual-easy-005,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-medium-038,Factual,Medium,100,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-hard-085,Summarization,Hard,1377,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-medium-072,NER,Medium,229,LOCAL_NER,LOCAL_NER,TRUE +debug-hard-080,Code Debugging,Hard,176,API_CODE,API_CODE,TRUE +math-ast-medium-054,Math (AST),Medium,36,API_MATH,API_MATH,TRUE +summary-easy-031,Summarization,Easy,135,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-044,Summarization,Medium,404,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-hard-090,Code Generation,Hard,186,API_CODE,API_CODE,TRUE +sentiment-easy-003,Sentiment,Easy,176,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-medium-069,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE +math-word-easy-023,Math (Word),Easy,151,API_MATH,API_MATH,TRUE +debug-easy-010,Code Debugging,Easy,232,API_CODE,API_CODE,TRUE +summary-medium-067,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-043,Summarization,Medium,400,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-medium-056,Code Generation,Medium,171,API_CODE,API_CODE,TRUE +math-ast-easy-034,Math (AST),Easy,48,API_MATH,API_MATH,TRUE +logic-medium-063,Logical Reasoning,Medium,159,API_LOGIC,API_MATH,FALSE +sentiment-medium-068,Sentiment,Medium,199,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-hard-090,Sentiment,Hard,181,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-hard-099,Sentiment,Hard,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-medium-037,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE +codegen-easy-018,Code Generation,Easy,199,API_CODE,API_CODE,TRUE +ner-hard-097,NER,Hard,234,LOCAL_NER,LOCAL_NER,TRUE +summary-easy-018,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-easy-021,Factual,Easy,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-hard-094,Code Debugging,Hard,195,API_CODE,API_CODE,TRUE +codegen-easy-030,Code Generation,Easy,200,API_CODE,API_CODE,TRUE +math-word-easy-027,Math (Word),Easy,215,API_MATH,API_MATH,TRUE +sentiment-hard-092,Sentiment,Hard,156,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-medium-056,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE +ner-hard-083,NER,Hard,166,LOCAL_NER,LOCAL_NER,TRUE +factual-medium-062,Factual,Medium,104,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-easy-007,Sentiment,Easy,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +math-word-easy-021,Math (Word),Easy,170,API_MATH,API_MATH,TRUE +debug-medium-061,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE +summary-medium-046,Summarization,Medium,384,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-easy-035,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-easy-005,Summarization,Easy,135,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-medium-036,Factual,Medium,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-easy-035,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE +ner-medium-036,NER,Medium,234,LOCAL_NER,LOCAL_NER,TRUE +factual-medium-042,Factual,Medium,107,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-medium-061,Sentiment,Medium,145,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-hard-092,NER,Hard,181,LOCAL_NER,LOCAL_NER,TRUE +debug-easy-009,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE +logic-easy-025,Logical Reasoning,Easy,219,API_LOGIC,API_LOGIC,TRUE +logic-medium-069,Logical Reasoning,Medium,152,API_LOGIC,API_LOGIC,TRUE +ner-medium-037,NER,Medium,202,LOCAL_NER,LOCAL_NER,TRUE +codegen-medium-047,Code Generation,Medium,186,API_CODE,API_CODE,TRUE +logic-medium-053,Logical Reasoning,Medium,145,API_LOGIC,API_LOGIC,TRUE +sentiment-medium-045,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-hard-100,Sentiment,Hard,150,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-hard-098,Code Generation,Hard,186,API_CODE,API_CODE,TRUE +factual-easy-030,Factual,Easy,90,LOCAL_GENERAL,API_MATH,FALSE +math-word-medium-061,Math (Word),Medium,154,API_MATH,API_MATH,TRUE +debug-hard-096,Code Debugging,Hard,232,API_CODE,API_CODE,TRUE +debug-medium-054,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE +codegen-hard-078,Code Generation,Hard,200,API_CODE,API_CODE,TRUE +sentiment-easy-023,Sentiment,Easy,184,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-easy-004,Summarization,Easy,159,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-048,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE +math-word-medium-059,Math (Word),Medium,136,API_MATH,API_MATH,TRUE +ner-hard-099,NER,Hard,175,LOCAL_NER,LOCAL_NER,TRUE +codegen-medium-071,Code Generation,Medium,147,API_CODE,API_CODE,TRUE +sentiment-hard-093,Sentiment,Hard,143,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-medium-066,NER,Medium,248,LOCAL_NER,LOCAL_NER,TRUE +codegen-medium-067,Code Generation,Medium,205,API_CODE,API_CODE,TRUE +summary-hard-100,Summarization,Hard,1042,LOCAL_GENERAL,API_MATH,FALSE +factual-easy-003,Factual,Easy,118,LOCAL_GENERAL,API_MATH,FALSE +codegen-easy-005,Code Generation,Easy,176,API_CODE,API_CODE,TRUE +math-word-medium-069,Math (Word),Medium,137,API_MATH,API_MATH,TRUE +logic-easy-015,Logical Reasoning,Easy,159,API_LOGIC,API_MATH,FALSE +logic-hard-095,Logical Reasoning,Hard,200,API_LOGIC,API_LOGIC,TRUE +sentiment-easy-019,Sentiment,Easy,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-hard-093,NER,Hard,166,LOCAL_NER,LOCAL_NER,TRUE +debug-medium-072,Code Debugging,Medium,205,API_CODE,API_CODE,TRUE +debug-medium-065,Code Debugging,Medium,433,API_CODE,API_CODE,TRUE +math-word-medium-071,Math (Word),Medium,217,API_MATH,API_MATH,TRUE +sentiment-medium-043,Sentiment,Medium,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-hard-082,Code Generation,Hard,185,API_CODE,API_CODE,TRUE +factual-easy-016,Factual,Easy,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-medium-052,Factual,Medium,110,LOCAL_GENERAL,API_MATH,FALSE +debug-medium-055,Code Debugging,Medium,259,API_CODE,API_CODE,TRUE +ner-medium-054,NER,Medium,202,LOCAL_NER,LOCAL_NER,TRUE +factual-hard-078,Factual,Hard,108,LOCAL_GENERAL,API_MATH,FALSE +sentiment-medium-056,Sentiment,Medium,160,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-easy-006,Code Generation,Easy,200,API_CODE,API_CODE,TRUE +debug-medium-073,Code Debugging,Medium,176,API_CODE,API_CODE,TRUE +math-ast-hard-088,Math (AST),Hard,49,API_MATH,API_MATH,TRUE +logic-medium-061,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE +math-ast-medium-044,Math (AST),Medium,42,API_MATH,API_MATH,TRUE +codegen-medium-068,Code Generation,Medium,162,API_CODE,API_CODE,TRUE +debug-medium-064,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE +codegen-medium-036,Code Generation,Medium,176,API_CODE,API_CODE,TRUE +sentiment-medium-055,Sentiment,Medium,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +math-ast-easy-012,Math (AST),Easy,43,API_MATH,API_MATH,TRUE +summary-medium-048,Summarization,Medium,380,LOCAL_GENERAL,API_LOGIC,FALSE +ner-easy-013,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE +factual-medium-063,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +math-word-easy-001,Math (Word),Easy,215,API_MATH,API_MATH,TRUE +factual-medium-056,Factual,Medium,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-hard-079,NER,Hard,223,LOCAL_NER,LOCAL_NER,TRUE +ner-easy-021,NER,Easy,234,LOCAL_NER,LOCAL_NER,TRUE +logic-easy-002,Logical Reasoning,Easy,100,API_LOGIC,API_LOGIC,TRUE +summary-medium-063,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_NER,FALSE +ner-medium-042,NER,Medium,166,LOCAL_NER,LOCAL_NER,TRUE +codegen-medium-053,Code Generation,Medium,191,API_CODE,LOCAL_NER,FALSE +factual-hard-097,Factual,Hard,78,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +math-word-hard-095,Math (Word),Hard,150,API_MATH,API_MATH,TRUE +ner-easy-017,NER,Easy,217,LOCAL_NER,LOCAL_NER,TRUE +debug-hard-086,Code Debugging,Hard,211,API_CODE,API_CODE,TRUE +sentiment-medium-049,Sentiment,Medium,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-hard-085,NER,Hard,166,LOCAL_NER,LOCAL_NER,TRUE +math-ast-medium-036,Math (AST),Medium,43,API_MATH,API_MATH,TRUE +debug-hard-100,Code Debugging,Hard,259,API_CODE,API_CODE,TRUE +sentiment-easy-020,Sentiment,Easy,162,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-012,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE +sentiment-hard-086,Sentiment,Hard,133,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-medium-044,Sentiment,Medium,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-easy-027,Code Debugging,Easy,230,API_CODE,API_CODE,TRUE +debug-hard-082,Code Debugging,Hard,274,API_CODE,API_CODE,TRUE +summary-medium-039,Summarization,Medium,328,LOCAL_GENERAL,API_MATH,FALSE +logic-hard-097,Logical Reasoning,Hard,157,API_LOGIC,API_LOGIC,TRUE +factual-medium-070,Factual,Medium,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-hard-098,Logical Reasoning,Hard,152,API_LOGIC,API_LOGIC,TRUE +factual-medium-061,Factual,Medium,89,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-medium-073,Factual,Medium,110,LOCAL_GENERAL,API_MATH,FALSE +factual-easy-008,Factual,Easy,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-medium-069,Code Generation,Medium,200,API_CODE,API_CODE,TRUE +sentiment-medium-036,Sentiment,Medium,139,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-medium-068,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE +factual-easy-022,Factual,Easy,96,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-047,Summarization,Medium,372,LOCAL_GENERAL,API_LOGIC,FALSE +logic-hard-085,Logical Reasoning,Hard,119,API_LOGIC,API_LOGIC,TRUE +sentiment-easy-031,Sentiment,Easy,164,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-medium-054,Sentiment,Medium,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +math-ast-easy-024,Math (AST),Easy,43,API_MATH,API_MATH,TRUE +ner-medium-063,NER,Medium,217,LOCAL_NER,LOCAL_NER,TRUE +debug-easy-030,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE +ner-medium-049,NER,Medium,162,LOCAL_NER,LOCAL_NER,TRUE +logic-easy-030,Logical Reasoning,Easy,164,API_LOGIC,API_LOGIC,TRUE +logic-medium-073,Logical Reasoning,Medium,157,API_LOGIC,API_LOGIC,TRUE +summary-easy-020,Summarization,Easy,130,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-easy-006,Sentiment,Easy,144,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-028,Logical Reasoning,Easy,164,API_LOGIC,API_LOGIC,TRUE +logic-hard-092,Logical Reasoning,Hard,200,API_LOGIC,API_LOGIC,TRUE +ner-easy-034,NER,Easy,192,LOCAL_NER,LOCAL_NER,TRUE +debug-medium-041,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE +logic-hard-082,Logical Reasoning,Hard,185,API_LOGIC,API_LOGIC,TRUE +math-ast-medium-070,Math (AST),Medium,36,API_MATH,API_MATH,TRUE +debug-medium-063,Code Debugging,Medium,213,API_CODE,API_CODE,TRUE +ner-hard-087,NER,Hard,162,LOCAL_NER,LOCAL_NER,TRUE +math-ast-easy-032,Math (AST),Easy,43,API_MATH,API_MATH,TRUE +math-word-easy-029,Math (Word),Easy,150,API_MATH,API_MATH,TRUE +debug-medium-046,Code Debugging,Medium,232,API_CODE,API_CODE,TRUE +factual-medium-071,Factual,Medium,110,LOCAL_GENERAL,API_MATH,FALSE +ner-hard-098,NER,Hard,192,LOCAL_NER,LOCAL_NER,TRUE +factual-hard-088,Factual,Hard,92,LOCAL_GENERAL,API_MATH,FALSE +codegen-medium-043,Code Generation,Medium,170,API_CODE,API_CODE,TRUE +logic-hard-091,Logical Reasoning,Hard,119,API_LOGIC,API_LOGIC,TRUE +sentiment-medium-047,Sentiment,Medium,152,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +factual-medium-060,Factual,Medium,107,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-medium-046,Sentiment,Medium,152,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-easy-015,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-medium-072,Summarization,Medium,372,LOCAL_GENERAL,API_LOGIC,FALSE +logic-medium-046,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE +factual-easy-020,Factual,Easy,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-easy-001,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE +summary-easy-032,Summarization,Easy,147,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-easy-014,Logical Reasoning,Easy,145,API_LOGIC,API_LOGIC,TRUE +factual-easy-001,Factual,Easy,102,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-044,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE +debug-medium-053,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE +math-ast-medium-066,Math (AST),Medium,43,API_MATH,API_MATH,TRUE +summary-hard-083,Summarization,Hard,1219,LOCAL_GENERAL,API_LOGIC,FALSE +codegen-easy-014,Code Generation,Easy,171,API_CODE,API_CODE,TRUE +codegen-medium-075,Code Generation,Medium,185,API_CODE,API_CODE,TRUE +codegen-easy-012,Code Generation,Easy,170,API_CODE,API_CODE,TRUE +logic-medium-037,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE +summary-easy-006,Summarization,Easy,138,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +math-word-easy-009,Math (Word),Easy,170,API_MATH,API_MATH,TRUE +debug-easy-020,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE +math-word-hard-083,Math (Word),Hard,215,API_MATH,API_MATH,TRUE +debug-easy-013,Code Debugging,Easy,213,API_CODE,API_CODE,TRUE +logic-hard-077,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE +factual-easy-004,Factual,Easy,88,LOCAL_GENERAL,API_MATH,FALSE +codegen-easy-024,Code Generation,Easy,162,API_CODE,API_CODE,TRUE +math-ast-medium-062,Math (AST),Medium,36,API_MATH,API_MATH,TRUE +ner-medium-046,NER,Medium,229,LOCAL_NER,LOCAL_NER,TRUE +codegen-medium-041,Code Generation,Medium,176,API_CODE,API_CODE,TRUE +codegen-hard-099,Code Generation,Hard,185,API_CODE,API_CODE,TRUE +codegen-medium-061,Code Generation,Medium,185,API_CODE,API_CODE,TRUE +codegen-easy-016,Code Generation,Easy,170,API_CODE,API_CODE,TRUE +codegen-medium-046,Code Generation,Medium,186,API_CODE,API_CODE,TRUE +math-ast-easy-018,Math (AST),Easy,42,API_MATH,API_MATH,TRUE +factual-easy-012,Factual,Easy,87,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-067,Code Debugging,Medium,274,API_CODE,API_CODE,TRUE +sentiment-medium-058,Sentiment,Medium,198,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-medium-042,Code Debugging,Medium,230,API_CODE,API_CODE,TRUE +ner-hard-080,NER,Hard,156,LOCAL_NER,LOCAL_NER,TRUE +logic-medium-060,Logical Reasoning,Medium,212,API_LOGIC,API_LOGIC,TRUE +summary-hard-091,Summarization,Hard,1377,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-hard-095,Summarization,Hard,1309,LOCAL_GENERAL,LOCAL_NER,FALSE +factual-medium-066,Factual,Medium,89,LOCAL_GENERAL,API_MATH,FALSE +logic-easy-011,Logical Reasoning,Easy,145,API_LOGIC,API_LOGIC,TRUE +codegen-medium-062,Code Generation,Medium,171,API_CODE,API_CODE,TRUE +codegen-medium-064,Code Generation,Medium,199,API_CODE,API_CODE,TRUE +logic-medium-043,Logical Reasoning,Medium,159,API_LOGIC,API_MATH,FALSE +math-word-medium-057,Math (Word),Medium,153,API_MATH,API_MATH,TRUE +math-word-easy-013,Math (Word),Easy,141,API_MATH,API_MATH,TRUE +factual-medium-049,Factual,Medium,73,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-hard-094,Factual,Hard,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-easy-006,Code Debugging,Easy,289,API_CODE,API_CODE,TRUE +debug-easy-022,Code Debugging,Easy,205,API_CODE,API_CODE,TRUE +math-ast-medium-042,Math (AST),Medium,45,API_MATH,API_MATH,TRUE +codegen-easy-001,Code Generation,Easy,176,API_CODE,API_CODE,TRUE +codegen-medium-038,Code Generation,Medium,200,API_CODE,API_CODE,TRUE +ner-easy-009,NER,Easy,242,LOCAL_NER,LOCAL_NER,TRUE +debug-hard-098,Code Debugging,Hard,224,API_CODE,API_CODE,TRUE +ner-medium-039,NER,Medium,234,LOCAL_NER,LOCAL_NER,TRUE +debug-hard-099,Code Debugging,Hard,380,API_CODE,API_CODE,TRUE +sentiment-medium-070,Sentiment,Medium,170,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-medium-052,Code Debugging,Medium,195,API_CODE,API_CODE,TRUE +codegen-hard-076,Code Generation,Hard,191,API_CODE,LOCAL_NER,FALSE +sentiment-medium-053,Sentiment,Medium,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-hard-093,Code Generation,Hard,176,API_CODE,API_CODE,TRUE +sentiment-medium-072,Sentiment,Medium,178,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +math-word-medium-047,Math (Word),Medium,106,API_MATH,API_MATH,TRUE +ner-medium-070,NER,Medium,217,LOCAL_NER,LOCAL_NER,TRUE +logic-medium-059,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE +ner-easy-006,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE +debug-easy-012,Code Debugging,Easy,205,API_CODE,API_CODE,TRUE +logic-medium-072,Logical Reasoning,Medium,100,API_LOGIC,API_LOGIC,TRUE +sentiment-easy-018,Sentiment,Easy,175,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-easy-023,Summarization,Easy,150,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-hard-081,Code Debugging,Hard,232,API_CODE,API_CODE,TRUE +sentiment-easy-017,Sentiment,Easy,158,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +factual-medium-068,Factual,Medium,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-medium-075,Sentiment,Medium,125,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-easy-011,NER,Easy,217,LOCAL_NER,LOCAL_NER,TRUE +summary-easy-024,Summarization,Easy,147,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-hard-084,Code Generation,Hard,176,API_CODE,API_CODE,TRUE +sentiment-easy-035,Sentiment,Easy,142,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +factual-medium-058,Factual,Medium,120,LOCAL_GENERAL,API_MATH,FALSE +summary-easy-033,Summarization,Easy,159,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-easy-025,Sentiment,Easy,135,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-medium-071,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE +logic-medium-056,Logical Reasoning,Medium,219,API_LOGIC,API_LOGIC,TRUE +sentiment-hard-077,Sentiment,Hard,129,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-hard-089,Summarization,Hard,1369,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +ner-medium-074,NER,Medium,233,LOCAL_NER,LOCAL_NER,TRUE +math-ast-medium-068,Math (AST),Medium,48,API_MATH,API_MATH,TRUE +codegen-hard-095,Code Generation,Hard,205,API_CODE,API_CODE,TRUE +debug-medium-047,Code Debugging,Medium,259,API_CODE,API_CODE,TRUE +math-ast-hard-076,Math (AST),Hard,45,API_MATH,API_MATH,TRUE +codegen-easy-023,Code Generation,Easy,199,API_CODE,API_CODE,TRUE +logic-easy-024,Logical Reasoning,Easy,185,API_LOGIC,API_LOGIC,TRUE +factual-medium-074,Factual,Medium,85,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-medium-064,Logical Reasoning,Medium,185,API_LOGIC,API_LOGIC,TRUE +logic-hard-100,Logical Reasoning,Hard,119,API_LOGIC,API_LOGIC,TRUE +sentiment-medium-074,Sentiment,Medium,145,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-hard-087,Logical Reasoning,Hard,166,API_LOGIC,API_LOGIC,TRUE +sentiment-easy-015,Sentiment,Easy,151,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +summary-medium-064,Summarization,Medium,396,LOCAL_GENERAL,LOCAL_NER,FALSE +summary-hard-080,Summarization,Hard,1309,LOCAL_GENERAL,LOCAL_NER,FALSE +math-word-medium-067,Math (Word),Medium,172,API_MATH,API_MATH,TRUE +summary-medium-074,Summarization,Medium,416,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-medium-051,Logical Reasoning,Medium,112,API_LOGIC,API_LOGIC,TRUE +ner-medium-065,NER,Medium,223,LOCAL_NER,LOCAL_NER,TRUE +codegen-hard-097,Code Generation,Hard,147,API_CODE,API_CODE,TRUE +codegen-medium-054,Code Generation,Medium,176,API_CODE,API_CODE,TRUE +math-ast-easy-004,Math (AST),Easy,43,API_MATH,API_MATH,TRUE +codegen-hard-080,Code Generation,Hard,170,API_CODE,API_CODE,TRUE +factual-hard-082,Factual,Hard,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-hard-097,Sentiment,Hard,130,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-easy-034,Sentiment,Easy,145,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-hard-078,Logical Reasoning,Hard,178,API_LOGIC,API_LOGIC,TRUE +factual-medium-047,Factual,Medium,92,LOCAL_GENERAL,API_MATH,FALSE +factual-medium-072,Factual,Medium,92,LOCAL_GENERAL,API_MATH,FALSE +codegen-easy-022,Code Generation,Easy,186,API_CODE,API_CODE,TRUE +math-word-hard-097,Math (Word),Hard,135,API_MATH,API_MATH,TRUE +debug-hard-092,Code Debugging,Hard,433,API_CODE,API_CODE,TRUE +summary-medium-071,Summarization,Medium,412,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-easy-034,Logical Reasoning,Easy,178,API_LOGIC,API_LOGIC,TRUE +logic-medium-044,Logical Reasoning,Medium,219,API_LOGIC,API_LOGIC,TRUE +summary-easy-001,Summarization,Easy,151,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-066,Code Debugging,Medium,232,API_CODE,API_CODE,TRUE +factual-hard-081,Factual,Hard,111,LOCAL_GENERAL,LOCAL_NER,FALSE +summary-hard-088,Summarization,Hard,1042,LOCAL_GENERAL,API_MATH,FALSE +math-ast-easy-028,Math (AST),Easy,48,API_MATH,API_MATH,TRUE +debug-medium-050,Code Debugging,Medium,289,API_CODE,API_CODE,TRUE +ner-medium-071,NER,Medium,162,LOCAL_NER,LOCAL_NER,TRUE +factual-medium-045,Factual,Medium,109,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-medium-071,Sentiment,Medium,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-019,Logical Reasoning,Easy,140,API_LOGIC,API_MATH,FALSE +summary-medium-053,Summarization,Medium,328,LOCAL_GENERAL,API_MATH,FALSE +summary-medium-068,Summarization,Medium,416,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-hard-100,Factual,Hard,98,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +summary-hard-090,Summarization,Hard,1207,LOCAL_GENERAL,API_LOGIC,FALSE +factual-hard-095,Factual,Hard,84,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +codegen-hard-091,Code Generation,Hard,171,API_CODE,API_CODE,TRUE +summary-medium-045,Summarization,Medium,400,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-easy-021,Sentiment,Easy,183,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +math-ast-medium-060,Math (AST),Medium,49,API_MATH,API_MATH,TRUE +codegen-medium-066,Code Generation,Medium,162,API_CODE,API_CODE,TRUE +summary-medium-049,Summarization,Medium,372,LOCAL_GENERAL,API_LOGIC,FALSE +codegen-hard-085,Code Generation,Hard,171,API_CODE,API_CODE,TRUE +codegen-easy-015,Code Generation,Easy,185,API_CODE,API_CODE,TRUE +math-ast-medium-064,Math (AST),Medium,45,API_MATH,API_MATH,TRUE +factual-easy-007,Factual,Easy,80,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +sentiment-hard-085,Sentiment,Hard,141,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +logic-easy-004,Logical Reasoning,Easy,157,API_LOGIC,API_LOGIC,TRUE +ner-medium-060,NER,Medium,156,LOCAL_NER,LOCAL_NER,TRUE +sentiment-easy-026,Sentiment,Easy,166,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +ner-easy-030,NER,Easy,223,LOCAL_NER,LOCAL_NER,TRUE +codegen-hard-088,Code Generation,Hard,162,API_CODE,API_CODE,TRUE +codegen-easy-019,Code Generation,Easy,147,API_CODE,API_CODE,TRUE +logic-medium-066,Logical Reasoning,Medium,164,API_LOGIC,API_LOGIC,TRUE +codegen-easy-034,Code Generation,Easy,199,API_CODE,API_CODE,TRUE +sentiment-easy-004,Sentiment,Easy,168,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +debug-medium-075,Code Debugging,Medium,380,API_CODE,API_CODE,TRUE +factual-easy-028,Factual,Easy,88,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +logic-easy-017,Logical Reasoning,Easy,200,API_LOGIC,API_LOGIC,TRUE +debug-easy-018,Code Debugging,Easy,380,API_CODE,API_CODE,TRUE +sentiment-medium-039,Sentiment,Medium,184,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-easy-024,Sentiment,Easy,147,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +factual-hard-096,Factual,Hard,118,LOCAL_GENERAL,API_MATH,FALSE +factual-easy-031,Factual,Easy,81,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +factual-easy-033,Factual,Easy,92,LOCAL_GENERAL,API_MATH,FALSE +ner-easy-033,NER,Easy,198,LOCAL_NER,LOCAL_NER,TRUE +logic-easy-005,Logical Reasoning,Easy,112,API_LOGIC,API_LOGIC,TRUE +factual-hard-090,Factual,Hard,83,LOCAL_GENERAL,LOCAL_GENERAL,TRUE +debug-medium-059,Code Debugging,Medium,399,API_CODE,API_CODE,TRUE +codegen-medium-059,Code Generation,Medium,199,API_CODE,API_CODE,TRUE +logic-hard-093,Logical Reasoning,Hard,212,API_LOGIC,API_LOGIC,TRUE +math-word-easy-005,Math (Word),Easy,149,API_MATH,API_MATH,TRUE +codegen-medium-055,Code Generation,Medium,199,API_CODE,API_CODE,TRUE +math-word-medium-043,Math (Word),Medium,217,API_MATH,API_MATH,TRUE +sentiment-easy-029,Sentiment,Easy,152,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-medium-051,Sentiment,Medium,127,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +sentiment-hard-082,Sentiment,Hard,180,LOCAL_SENTIMENT,LOCAL_SENTIMENT,TRUE +codegen-medium-060,Code Generation,Medium,185,API_CODE,API_CODE,TRUE +factual-easy-026,Factual,Easy,85,LOCAL_GENERAL,API_MATH,FALSE +math-word-easy-015,Math (Word),Easy,141,API_MATH,API_MATH,TRUE +logic-medium-055,Logical Reasoning,Medium,200,API_LOGIC,API_LOGIC,TRUE diff --git a/output/math_analysis/math_benchmark.csv b/output/math_analysis/math_benchmark.csv new file mode 100644 index 0000000..4fa4304 --- /dev/null +++ b/output/math_analysis/math_benchmark.csv @@ -0,0 +1,21 @@ +task_id,prompt,difficulty,expected_answer,predicted_answer,is_correct,latency_seconds,input_tokens,output_tokens,total_tokens +task-gen-02-006,Calculate the exact value: 312 + 226.,easy,538,538,1,10.607,311,34,345 +task-gen-02-031,Calculate the exact value: 152 * 242.,easy,36784,36784,1,1.435,311,80,391 +task-gen-02-015,Calculate the exact value: 367 - 226.,easy,141,141,1,1.537,311,30,341 +task-gen-02-019,Calculate the exact value: 193 - 59.,easy,134,134,1,1.624,311,30,341 +task-gen-02-055,A store has 543 items. It sells 20% on Monday and 27 more on Tuesday. How many items remain? (Difficulty: medium),medium,407.4,407.4,1,3.989,331,354,685 +task-gen-02-037,A store has 344 items. It sells 20% on Monday and 79 more on Tuesday. How many items remain? (Difficulty: medium),medium,196.2,196.2,1,3.149,331,139,470 +task-gen-02-064,A store has 599 items. It sells 30% on Monday and 80 more on Tuesday. How many items remain? (Difficulty: medium),medium,339.3,339.3,1,1.575,331,112,443 +task-gen-02-052,A store has 347 items. It sells 25% on Monday and 58 more on Tuesday. How many items remain? (Difficulty: medium),medium,202.25,202.25,1,3.159,331,439,770 +task-gen-02-034,A store has 493 items. It sells 20% on Monday and 64 more on Tuesday. How many items remain? (Difficulty: medium),medium,330.4,330.4,1,3.193,331,297,628 +task-gen-02-059,An investor deposits $11244 in a savings account with 5% annual interest compounded annually. What will the balance be after 6 years? (Difficulty: medium),medium,15068.04,15068.04,1,5.176,334,744,1078 +task-gen-02-053,An investor deposits $8668 in a savings account with 7% annual interest compounded annually. What will the balance be after 7 years? (Difficulty: medium),medium,13918.91,13918.91,1,3.319,334,436,770 +task-gen-02-038,An investor deposits $11485 in a savings account with 7% annual interest compounded annually. What will the balance be after 6 years? (Difficulty: medium),medium,17235.89,17235.89,1,3.834,334,479,813 +task-gen-02-081,"Solve the following problem: A factory produces 46 units per hour. After 12 hours, 10% of the units are inspected. Of those, 5% are defective. How many non-defective inspected units are there? (Difficulty: hard)",hard,52.44,52.44,1,4.464,353,387,740 +task-gen-02-084,"Solve the following problem: A factory produces 32 units per hour. After 11 hours, 20% of the units are inspected. Of those, 5% are defective. How many non-defective inspected units are there? (Difficulty: hard)",hard,66.88,66.88,1,3.686,353,561,914 +task-gen-02-100,"A train leaves Station A traveling at 49 mph. 2 hours later, another train leaves Station A traveling in the same direction at 71 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,49/11,49/11,1,2.824,350,206,556 +task-gen-02-079,"A train leaves Station A traveling at 48 mph. 4 hours later, another train leaves Station A traveling in the same direction at 76 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,48/7,48/7,1,2.971,350,234,584 +task-gen-02-082,"A train leaves Station A traveling at 53 mph. 2 hours later, another train leaves Station A traveling in the same direction at 73 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,5.3,5.3,1,2.396,350,232,582 +task-gen-02-067,"A train leaves Station A traveling at 55 mph. 2 hours later, another train leaves Station A traveling in the same direction at 78 mph. How long will it take the second train to overtake the first? (Difficulty: hard)",hard,110/23,110/23,1,3.77,350,251,601 +task-gen-02-074,"If the price of an item increases from $120 to $317, what is the percentage increase? (Difficulty: hard)",hard,164.17,164.17,1,4.569,325,204,529 +task-gen-02-095,"If the price of an item increases from $77 to $405, what is the percentage increase? (Difficulty: hard)",hard,425.97,425.97,1,4.564,325,313,638 diff --git a/output/practice_results.json b/output/practice_results.json index 12219a6..81b52e2 100644 --- a/output/practice_results.json +++ b/output/practice_results.json @@ -31,4 +31,4 @@ "task_id": "practice-08", "answer": "```python\ndef second_largest(nums):\n if len(nums) < 2:\n return None\n first_max = second_max = float('-inf')\n for num in nums:\n if num > first_max:\n second_max = first_max\n first_max = num\n elif first_max > num > second_max:\n second_max = num\n return second_max if second_max != float('-inf') else None\n```" } -] \ No newline at end of file +] diff --git a/output/results.json b/output/results.json index 3f2a6a1..b5781a7 100644 --- a/output/results.json +++ b/output/results.json @@ -31,4 +31,4 @@ "task_id": "practice-02", "answer": "Error: processing failed (ClientResponseError)." } -] \ No newline at end of file +] diff --git a/prompts/local_code_debug.txt b/prompts/local_code_debug.txt index e99de7f..dd3a116 100644 --- a/prompts/local_code_debug.txt +++ b/prompts/local_code_debug.txt @@ -1 +1 @@ -Output only the complete corrected Python code. No markdown fences. No explanation. \ No newline at end of file +Output only the complete corrected Python code. No markdown fences. No explanation. diff --git a/prompts/local_code_gen.txt b/prompts/local_code_gen.txt index 4dae4ab..84d2e90 100644 --- a/prompts/local_code_gen.txt +++ b/prompts/local_code_gen.txt @@ -1 +1 @@ -Output only valid Python code. No markdown fences. No explanation. \ No newline at end of file +Output only valid Python code. No markdown fences. No explanation. diff --git a/prompts/remote_code.txt b/prompts/remote_code.txt index da92763..6e23e29 100644 --- a/prompts/remote_code.txt +++ b/prompts/remote_code.txt @@ -1,2 +1,2 @@ You are a code assistant. Output ONLY the raw Python code. No markdown fences. No explanation. -If fixing a bug, output the complete corrected code. \ No newline at end of file +If fixing a bug, output the complete corrected code. diff --git a/prompts/remote_logic.txt b/prompts/remote_logic.txt index f64fd15..4a8aecc 100644 --- a/prompts/remote_logic.txt +++ b/prompts/remote_logic.txt @@ -1,2 +1,2 @@ -You are a logic solver. Think through the puzzle, then output ONLY the final answer. +You are a logic solver. Think through the puzzle, then output ONLY the final answer. If Yes/No, output only Yes or No. If a name/order, output only that. No explanation. diff --git a/tests/eval_200_tasks.json b/tests/eval_200_tasks.json index 8b22d18..29310ba 100644 --- a/tests/eval_200_tasks.json +++ b/tests/eval_200_tasks.json @@ -1399,4 +1399,4 @@ "difficulty": "hard", "expected_answer": "The development of mRNA vaccines offers a rapid and effective alternative to traditional vaccines, enabling the production of antibodies without causing disease. In contrast, space exploration has evolved from Cold War competition to international collaboration, with private companies now playing a significant role in space missions. Renewable energy technologies are expanding globally to combat climate change, but challenges such as grid integration and battery storage remain. The global economy is undergoing significant shifts due to nearshoring and central bank interest rate adjustments, impacting international trade and currency valuations." } -] \ No newline at end of file +] diff --git a/tests/eval_results.json b/tests/eval_results.json index 9ca4877..3414e85 100644 --- a/tests/eval_results.json +++ b/tests/eval_results.json @@ -2399,4 +2399,4 @@ "jaccard_score": 0.3093, "is_correct": 1 } -] \ No newline at end of file +] diff --git a/tests/evaluation_report.md b/tests/evaluation_report.md index b62c99b..d1f494c 100644 --- a/tests/evaluation_report.md +++ b/tests/evaluation_report.md @@ -34,14 +34,14 @@ graph TD D -- Là biểu thức Toán đơn giản --> E[Tính toán trực tiếp - 0ms] D -- Không phải Toán đơn giản --> F{Lớp 2: Weighted Classifier} F -- Dự đoán Route --> G{Lớp 3 & 4: Dispatcher} - + G -- LOCAL_GENERAL --> H{Chứa từ khóa Summarize?} H -- Có --> I[SummarizationHandler - Chạy Qwen 2.5 3B cục bộ] H -- Không --> J[FactualHandler - Chạy Qwen 2.5 3B cục bộ] - + I --> K{Mô hình trả về __ESCALATE__?} J --> K - + K -- Có (Tràn cửa sổ ngữ cảnh/lỗi) --> L[RemoteGeneralHandler - Gọi API Fireworks] K -- Không --> M[Lưu kết quả vào Cache & Trả về cho User] L --> M diff --git a/tests/fixtures/task.json b/tests/fixtures/task.json index 1b497e8..21d58af 100644 --- a/tests/fixtures/task.json +++ b/tests/fixtures/task.json @@ -18359,4 +18359,4 @@ "prompt": "Classify the sentiment of this text: 'Outstanding performance! It handles heavy workloads with ease and never slows down.'. Give the sentiment as Positive, Negative, or Neutral.", "expected_route": "LOCAL_SENTIMENT" } -] \ No newline at end of file +] diff --git a/tests/test_remote_llm.py b/tests/test_remote_llm.py index b19f381..cd0c501 100644 --- a/tests/test_remote_llm.py +++ b/tests/test_remote_llm.py @@ -54,7 +54,6 @@ def test_model_selection_logic_prefers_kimi_then_minimax() -> None: assert select_remote_model("API_LOGIC") == "kimi-k2p7-code" - def test_model_selection_fallback_empty() -> None: with patch("engines.remote_llm.ALLOWED_MODELS", []): assert select_remote_model("API_MATH") == "gemma-4-31b-it" @@ -96,4 +95,3 @@ async def test_remote_llm_generate_completions_mocked() -> None: result = await engine.generate("Solve logic riddle", "API_LOGIC") assert result == "99" - diff --git a/tests/test_router.py b/tests/test_router.py index 6aa486c..21b6bcd 100644 --- a/tests/test_router.py +++ b/tests/test_router.py @@ -113,6 +113,7 @@ async def test_route_api_code() -> None: patch("handlers.sentiment.LocalSLMEngine.get_instance"), patch("handlers.ner.LocalSLMEngine.get_instance"), patch("handlers.summarization.LocalSLMEngine.get_instance"), + patch("handlers.code_gen.classify_code_difficulty", return_value="hard"), patch("engines.remote_llm.RemoteLLMEngine.generate", new_callable=AsyncMock) as mock_remote, ): mock_remote.return_value = "def add(a, b): return a + b" @@ -139,7 +140,6 @@ async def test_route_api_logic() -> None: assert result == "Yes" - # --------------------------------------------------------------------------- # Local sentiment: unrecognised output → 'Neutral' locally, zero remote calls # (PI-DEV-009: __ESCALATE__ path removed from SentimentHandler)