update

Dockerfile

@@ -17,19 +17,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 COPY requirements.txt /app/
 
-# Step 1: Install CPU-only torch first to avoid pulling 2.5 GB CUDA wheels
-RUN uv pip install --no-cache torch \
-        --index-url https://download.pytorch.org/whl/cpu
-
-# Step 2: Install remaining deps (sentence-transformers will reuse the torch above)
+# Install remaining deps (llama-cpp-python installed separately via CPU wheel).
+# torch + sentence-transformers were removed: classification now uses the local
+# Qwen GGUF model directly, so no embedding model / PyTorch head is needed.
 RUN uv pip install --no-cache -r requirements.txt
 
-# Step 3: Install llama-cpp-python via precompiled CPU wheel (avoids C++ compilation)
+# Install llama-cpp-python via precompiled CPU wheel (avoids C++ compilation)
 RUN uv pip install --no-cache llama-cpp-python \
         --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
 
-# Bundle GGUF model weights (~986 MB)
-# Make sure to run scripts/download_model.sh before building
+# Bundle GGUF model weights (~2 GB)
 COPY models/ /app/models/
 
 # System prompt templates
@@ -49,11 +46,9 @@ ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     INPUT_PATH=/input/tasks.json \
     OUTPUT_PATH=/output/results.json \
-    LOCAL_MODEL_PATH=/app/models/qwen2.5-1.5b-instruct-q4_k_m.gguf \
+    LOCAL_MODEL_PATH=/app/models/qwen2.5-3b-instruct-q4_k_m.gguf \
     LOCAL_N_GPU_LAYERS=0 \
     LOCAL_N_THREADS=2 \
-    LOCAL_N_CTX=2048 \
-    HF_HUB_OFFLINE=1 \
-    TRANSFORMERS_OFFLINE=1
+    LOCAL_N_CTX=2048
 
 ENTRYPOINT ["/app/entrypoint.sh"]

README.md

@@ -20,7 +20,7 @@ Input Task
 [L1b] AST Math Evaluator              → pure expression → 0 tokens (deterministic)
     │ NOT PURE MATH
     ▼
-[L2]  Supervised PyTorch Classifier   → 0 tokens, ~9ms (all-MiniLM-L6-v2 + PyTorch MLP)
+[L2]  LLM Classifier (local Qwen)        → 0 tokens, grammar-constrained label
     │
     ├─► LOCAL_SENTIMENT / LOCAL_NER / LOCAL_GENERAL
     │       ▼
@@ -51,15 +51,13 @@ Input Task
 
 > **Local-first code strategy**: Code debugging and code generation tasks use a difficulty classifier (`code_utils.py`). Easy/medium tasks attempt Qwen2.5-3B locally first, validated with `ast.parse` + completeness checks — only falling back to the remote API if local output is invalid. Hard tasks go directly to the remote API to avoid wasting wall-clock time.
 
-### Semantic Classifier (L2)
+### LLM Classifier (L2)
 
-Layer 2 uses **`all-MiniLM-L6-v2`** (sentence-transformers) combined with a **Supervised Neural Network Head (PyTorch MLP)** for highly robust semantic classification:
-- **Embedding Generation**: Encodes the prompt into a dense 384-dimensional vector embedding (~8-10ms, CPU-only).
-- **Neural Network Head**: Passes the embedding through a trained Multi-Layer Perceptron (MLP) head (`384 -> 64 -> ReLU -> Dropout -> 6 Classes`).
-- **Consolidated Training**: Trained on a diverse combined dataset of **3,235 tasks** (covering standard, adversarial, and conversational phrasings).
-- **Accuracy**: Achieves **100.00% classification accuracy** across all task categories, including tricky inputs with overlapping keywords (e.g., historical numbers or code snippets).
-- **Efficiency**: Runs entirely local with **0 Fireworks API tokens** and extremely low memory footprint (weights are only ~100 KB).
-- **Auto-training**: If pre-trained weights are missing, the classifier automatically trains from `tests/fixtures/task.json` at startup.
+Layer 2 uses the **same local Qwen2.5-3B GGUF model** as the local handlers to pick the routing category — no separate embedding model or neural head:
+- **Grammar-constrained output**: a GBNF grammar forces the model to emit exactly one of the valid route labels (e.g. `LOCAL_SENTIMENT`, `API_CODE`). There is no free-form prose to parse, so mislabels from a stray token are impossible.
+- **Zero tokens**: runs entirely on the bundled model, 0 Fireworks API tokens.
+- **Concurrency-safe**: the classifier and the local handlers share one llama.cpp context, serialized by a lock inside `LocalSLMEngine`; `router.py` offloads every local call (classify + handlers) to worker threads so the asyncio event loop stays responsive while remote API tasks run concurrently.
+- **Long-context override**: prompts longer than 6,000 chars skip the model and route directly to `API_LONG_CONTEXT` to avoid CPU OOM.
 
 ### Remote Model Selection
 
@@ -93,8 +91,7 @@ This reduces input + output tokens on every remote call.
 │   ├── schemas.py            # Pydantic Task & Result models
 │   ├── cache.py              # SHA-256 semantic dedup cache (thread-safe)
 │   ├── ast_eval.py           # Safe deterministic math evaluator (AST whitelist)
-│   ├── classifier.py         # Supervised PyTorch classifier (all-MiniLM-L6-v2 + MLP)
-│   ├── supervised_model.pt   # Pre-trained classifier weights (~100 KB)
+│   ├── classifier.py         # LLM classifier (local Qwen, GBNF grammar-constrained)
 │   ├── router.py             # AgentRouter — orchestrates all 4 layers
 │   └── watchdog.py           # Daemon thread: fires at 570s, flushes partial output
 │
@@ -180,10 +177,8 @@ bash scripts/setup.sh
 
 Script này tự động:
 - Tạo virtual environment (`.venv`)
-- Cài `torch` CPU-only (tránh CUDA wheels 2.5 GB)
 - Cài tất cả dependencies từ `requirements.txt`
 - Cài `llama-cpp-python` (CPU wheel, không cần C++ compiler)
-- Pre-cache `all-MiniLM-L6-v2` (~90 MB)
 - Download `Qwen2.5-3B Q4_K_M` GGUF (~2 GB)
 - Tạo `.env` từ `.env.example`
 
@@ -246,7 +241,7 @@ Benchmarks 5 strategies (baseline, zero-shot strict, few-shot, chain-of-thought,
 PYTHONPATH=. pytest tests/test_ast_eval.py tests/test_cache.py \
     tests/test_remote_llm.py tests/test_router.py -v
 
-# Classifier test (requires all-MiniLM-L6-v2 + task.json)
+# Classifier test (loads local GGUF model)
 PYTHONPATH=. pytest tests/test_classifier.py -v
 
 # Local SLM test (requires GGUF model)
@@ -282,9 +277,8 @@ docker push <your-dockerhub-username>/develarper-agent:latest
 
 **Docker image features:**
 - Uses `entrypoint.sh` (loads `.env` if present, then runs `main.py`)
-- Sets `HF_HUB_OFFLINE=1` and `TRANSFORMERS_OFFLINE=1` — all model weights are pre-cached at build time, no runtime downloads
-- Pre-caches `all-MiniLM-L6-v2` sentence-transformer during build
-- Bundles `Qwen2.5-3B Q4_K_M` GGUF (~986 MB) in `/app/models/`
+- Bundles `Qwen2.5-3B Q4_K_M` GGUF (~2 GB) in `/app/models/` — the only model needed (used for both classification and local handlers)
+- No torch / sentence-transformers / embedding model — smaller image, faster cold start
 
 ---
 
@@ -355,7 +349,7 @@ See [`tests/evaluation_report.md`](tests/evaluation_report.md) for detailed anal
 
 - **Python version**: 3.12 (Docker) / 3.11+ (host dev)
 - **Package manager**: `uv` (in Docker), `pip` (host dev)
-- **Classifier**: `all-MiniLM-L6-v2` (SentenceTransformer) + PyTorch MLP head — trained locally on 3,235 consolidated tasks (including test suite prompts)
+- **Classifier**: local Qwen2.5-3B with a GBNF grammar constraining output to one of the route labels (0 tokens, no separate embedding model / PyTorch head)
 - **Local SLM**: `Qwen2.5-3B-Instruct Q4_K_M` via `llama-cpp-python`
 - **Remote API**: `aiohttp` + `tenacity` retry (3 attempts, exponential backoff)
 - **Math prompting**: CoT with few-shot examples, handles fractions/decimals, answer extraction via regex

agent/classifier.py

@@ -1,22 +1,14 @@
 """
-agent/classifier.py — Supervised Neural Network Classifier
-=========================================================
-Uses sentence-transformers (all-MiniLM-L6-v2) to encode prompts into dense
-384-dimensional embeddings, then passes them through a PyTorch classification
-head to predict the optimal routing category.
-
-Routes (constants unchanged):
-  LOCAL_SENTIMENT, LOCAL_NER, LOCAL_GENERAL
-  API_MATH, API_CODE, API_LOGIC, API_LONG_CONTEXT
+agent/classifier.py — LLM-driven zero-token classifier
+======================================================
+The local Qwen model chooses the routing category. Output is constrained by a
+GBNF grammar so the model can ONLY emit a valid label.
 """
 
 import logging
 import os
-import threading
-from typing import Any, Optional
 
-import torch
-import torch.nn as nn
+from engines.local_slm import LocalSLMEngine
 
 logger = logging.getLogger(__name__)
 
@@ -31,7 +23,7 @@
 ROUTE_API_LOGIC = "API_LOGIC"
 ROUTE_API_LONG = "API_LONG_CONTEXT"
 
-ROUTES_MAP = [
+_LABELS = [
     ROUTE_LOCAL_SENTIMENT,
     ROUTE_LOCAL_NER,
     ROUTE_LOCAL_GENERAL,
@@ -40,113 +32,64 @@
     ROUTE_API_LOGIC,
 ]
 
-_MODEL_NAME = os.path.join(os.path.dirname(os.path.dirname(__file__)), "models", "all-MiniLM-L6-v2")
 _LONG_CONTEXT_THRESHOLD = 6000  # chars; above this → API_LONG to avoid CPU OOM
-WEIGHTS_PATH = os.path.join(os.path.dirname(__file__), "supervised_model.pt")
 
+_SYSTEM_PROMPT = (
+    "You are a task router. Read the user task and output EXACTLY ONE label "
+    "describing its type. Definitions:\n"
+    "LOCAL_SENTIMENT = classify sentiment or emotion of text.\n"
+    "LOCAL_NER = extract named entities (people, places, organizations, dates).\n"
+    "LOCAL_GENERAL = factual knowledge question, definition, or summarization request.\n"
+    "API_MATH = arithmetic calculation or math word problem that needs a numeric answer.\n"
+    "API_CODE = write, generate, fix, or debug programming code.\n"
+    "API_LOGIC = logical reasoning puzzle, deduction, or constraint problem.\n"
+    "Examples:\n"
+    "Q: What is the capital of France? → LOCAL_GENERAL\n"
+    "Q: What is the boiling point of water in degrees Celsius? → LOCAL_GENERAL\n"
+    "Q: Who wrote 'One Hundred Years of Solitude'? → LOCAL_GENERAL\n"
+    "Q: Classify the sentiment of this review → LOCAL_SENTIMENT\n"
+    "Q: Extract named entities from this text → LOCAL_NER\n"
+    "Q: Calculate 342 * 12 → API_MATH\n"
+    "Q: Write a Python function to sort a list → API_CODE\n"
+    "Q: If A is taller than B and B is taller than C, is A taller than C? → API_LOGIC\n"
+    "Output only the label, nothing else."
+)
+
+# GBNF grammar forces output to be one of the valid route labels.
+_GRAMMAR_STR = "root ::= " + " | ".join(f'"{label}"' for label in _LABELS)
+
+_grammar = None
+
+
+def _get_grammar():
+    global _grammar
+    if _grammar is None:
+        from llama_cpp import LlamaGrammar
+
+        _grammar = LlamaGrammar.from_string(_GRAMMAR_STR)
+    return _grammar
 
-class LinearClassifier(nn.Module):
-    def __init__(self, input_dim: int, num_classes: int):
-        super().__init__()
-        # Simple MLP head: 384 -> 64 -> 6 classes
-        self.net = nn.Sequential(nn.Linear(input_dim, 64), nn.ReLU(), nn.Dropout(0.1), nn.Linear(64, num_classes))
 
-    def forward(self, x: Any) -> Any:
-        return self.net(x)
-
-
-class SemanticClassifier:
-    """
-    Loads all-MiniLM-L6-v2 and the trained PyTorch classification head.
-    Performs fast, offline inference by passing the prompt embedding through the MLP.
-    """
-
-    _instance: Optional["SemanticClassifier"] = None
-    _init_lock: threading.Lock = threading.Lock()  # guards concurrent thread-pool callers
-
-    @classmethod
-    def get_instance(cls) -> "SemanticClassifier":
-        if cls._instance is None:
-            with cls._init_lock:  # only one thread enters at a time
-                if cls._instance is None:  # re-check inside lock (double-checked locking)
-                    logger.info("Initializing SemanticClassifier (loading model & PyTorch head)...")
-                    cls._instance = cls()
-        return cls._instance
-
-    def __init__(self) -> None:
-        try:
-            from sentence_transformers import SentenceTransformer
-        except ImportError as exc:
-            raise ImportError("sentence-transformers is required. " "Run: pip install sentence-transformers") from exc
-
-        self.encoder = SentenceTransformer(_MODEL_NAME)
-        self.model = LinearClassifier(input_dim=384, num_classes=len(ROUTES_MAP))
-
-        if os.path.exists(WEIGHTS_PATH):
-            logger.info(f"Loading supervised weights from {WEIGHTS_PATH}")
-            self.model.load_state_dict(torch.load(WEIGHTS_PATH, map_location="cpu"))
-        else:
-            # Fallback if weights not found: check if task.json exists to auto-train
-            task_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tests", "fixtures", "task.json")
-            if os.path.exists(task_path):
-                logger.warning(f"Supervised weights not found at {WEIGHTS_PATH}. Auto-training from {task_path}...")
-                self._train_from_data(task_path)
-            else:
-                logger.error(f"Supervised weights not found at {WEIGHTS_PATH} and no training data found. Using untrained weights.")
-
-        self.model.eval()
-
-    def _train_from_data(self, data_path: str) -> None:
-        import json
-
-        import torch.optim as optim
-
-        with open(data_path, encoding="utf-8") as f:
-            tasks = json.load(f)
-
-        prompts = [t["prompt"] for t in tasks]
-        labels = [ROUTES_MAP.index(t["expected_route"]) for t in tasks]
-
-        embeddings = self.encoder.encode(prompts, convert_to_tensor=True).cpu()
-        labels_tensor = torch.tensor(labels, dtype=torch.long)
-
-        criterion = nn.CrossEntropyLoss()
-        optimizer = optim.AdamW(self.model.parameters(), lr=0.01, weight_decay=0.01)
-
-        self.model.train()
-        for _ in range(150):
-            optimizer.zero_grad()
-            outputs = self.model(embeddings)
-            loss = criterion(outputs, labels_tensor)
-            loss.backward()
-            optimizer.step()
-
-        torch.save(self.model.state_dict(), WEIGHTS_PATH)
-        logger.info(f"Successfully trained and saved new classifier weights to {WEIGHTS_PATH}!")
-
-    def classify(self, prompt: str) -> str:
-        prompt_emb = self.encoder.encode(prompt, convert_to_tensor=True).cpu()
-        with torch.no_grad():
-            logits = self.model(prompt_emb.unsqueeze(0))
-            pred_idx = int(logits.argmax(dim=1).item())
-        return ROUTES_MAP[pred_idx]
-
-
-# ---------------------------------------------------------------------------
-# Public API — drop-in replacement for the old classify()
-# ---------------------------------------------------------------------------
 def classify(prompt: str) -> str:
     """
     Classify a prompt into one of the routing destinations.
     """
-    # --- Override: Long context ---
     if len(prompt) > _LONG_CONTEXT_THRESHOLD:
         logger.debug("Classifier: long context (%d chars) → %s", len(prompt), ROUTE_API_LONG)
         return ROUTE_API_LONG
 
-    classifier = SemanticClassifier.get_instance()
-
-    # --- Execute Supervised PyTorch Classification ---
-    route = classifier.classify(prompt)
-    logger.debug("Classifier: supervised PyTorch → %s", route)
-    return route
+    engine = LocalSLMEngine.get_instance()
+    raw = engine.generate(
+        prompt=f"Task:\n{prompt}\n\nLabel:",
+        system_prompt=_SYSTEM_PROMPT,
+        max_tokens=8,
+        temperature=0.0,
+        grammar=_get_grammar(),
+    ).strip()
+
+    if raw in _LABELS:
+        logger.debug("Classifier: LLM → %s", raw)
+        return raw
+
+    logger.warning("Classifier: invalid LLM label '%s' → fallback LOCAL_GENERAL", raw)
+    return ROUTE_LOCAL_GENERAL

agent/router.py

@@ -3,8 +3,8 @@
 4-Layer AgentRouter
     Layer 1a: Semantic Cache (0 tokens)
     Layer 1b: AST Math Evaluator (0 tokens, deterministic)
-    Layer 2:  Weighted Scoring Classifier (~0ms, no model)
-    Layer 3:  Local SLM — Qwen2.5 via llama.cpp
+    Layer 2:  LLM Classifier (local Qwen, grammar-constrained, 0 tokens)
+    Layer 3:  Local SLM — Qwen2.5-3B via llama.cpp (serialized by engine lock)
     Layer 4:  Remote Fireworks API (category-aware model + prompt compression)
 """
 
@@ -79,7 +79,9 @@ async def route(self, task_id: str, prompt: str) -> str:
             return math_result
 
         # -------------------------------------------------------------------
-        # Layer 2: Weighted Classifier (offloaded to thread pool)
+        # Layer 2: LLM Classifier (offloaded to thread pool — local llama.cpp
+        # call is synchronous; offloading keeps the event loop responsive so
+        # concurrent API tasks can progress while classification runs).
         # -------------------------------------------------------------------
         loop = asyncio.get_running_loop()
         route = await loop.run_in_executor(None, classify, prompt)
@@ -94,16 +96,22 @@ async def route(self, task_id: str, prompt: str) -> str:
 
     async def _dispatch(self, task_id: str, prompt: str, route: str) -> str:
         try:
+            loop = asyncio.get_running_loop()
             # ---- Local routes ----
+            # Local handlers are synchronous llama.cpp calls. They are serialized
+            # by the engine's internal lock, so we offload each to a worker thread
+            # via run_in_executor — this lets remote API tasks (and other local
+            # tasks waiting for the lock) keep the event loop alive instead of
+            # freezing it for the duration of one local generation.
             if route == ROUTE_LOCAL_SENTIMENT:
-                res = self._sentiment.handle(prompt)
+                res = await loop.run_in_executor(None, self._sentiment.handle, prompt)
                 if res == "__ESCALATE__":
                     logger.info("[%s] Sentiment escalated → remote", task_id)
                     res = await self._remote_general.handle(prompt, category=ROUTE_LOCAL_SENTIMENT)
                 return res
 
             if route == ROUTE_LOCAL_NER:
-                res = self._ner.handle(prompt)
+                res = await loop.run_in_executor(None, self._ner.handle, prompt)
                 if res == "__ESCALATE__":
                     logger.info("[%s] NER escalated → remote", task_id)
                     res = await self._remote_general.handle(prompt, category=ROUTE_LOCAL_NER)
@@ -112,9 +120,9 @@ async def _dispatch(self, task_id: str, prompt: str, route: str) -> str:
             if route == ROUTE_LOCAL_GENERAL:
                 p = prompt.lower()
                 if any(w in p for w in ["summarize", "summary", "tldr"]):
-                    res = self._summarization.handle(prompt)
+                    res = await loop.run_in_executor(None, self._summarization.handle, prompt)
                 else:
-                    res = self._factual.handle(prompt)
+                    res = await loop.run_in_executor(None, self._factual.handle, prompt)
                 if res == "__ESCALATE__":
                     logger.info("[%s] Local general escalated → remote", task_id)
                     res = await self._remote_general.handle(prompt, category=ROUTE_LOCAL_GENERAL)