diff --git a/environments/qqr/config.py b/environments/qqr/config.py index 532b4fd..b8493ef 100644 --- a/environments/qqr/config.py +++ b/environments/qqr/config.py @@ -295,7 +295,7 @@ LLM_MODELS = [ "Qwen/Qwen3-235B-A22B-Instruct-2507-TEE", "Qwen/Qwen2.5-72B-Instruct", - "Qwen/Qwen3-32B", + "Qwen/Qwen3-32B-TEE", ] # Backward compat aliases LLM_QUALITY_MODELS = LLM_MODELS diff --git a/environments/qqr/llm_validator.py b/environments/qqr/llm_validator.py index e6a15f1..529d202 100644 --- a/environments/qqr/llm_validator.py +++ b/environments/qqr/llm_validator.py @@ -225,6 +225,7 @@ def to_dict(self) -> dict: 加分项:所有事实均可在工具数据中找到出处 → 维持满分''', } +NUM_FIXED_JUDGES = 3 # ============================================================================ # Abstract Base Class @@ -276,40 +277,64 @@ async def evaluate(self, context: Dict[str, Any]) -> Dict[str, Any]: prompt = self.build_prompt(context) total_attempts = 0 - for round_idx in range(self.max_rounds): - for model in self.models: - total_attempts += 1 - try: - response = await asyncio.wait_for( - self.client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": prompt}], - temperature=0, - max_tokens=2000, - ), - timeout=self.timeout, - ) - content = response.choices[0].message.content - parsed = self.parse_response(content) - if parsed is not None: - parsed["_model"] = model - parsed["_success"] = True - return parsed - # Parse failed — treat as transient, try next model - print(f"[{self.group_name}] {model} parse failed (round {round_idx+1})") - except asyncio.TimeoutError: - print(f"[{self.group_name}] {model} timeout (round {round_idx+1})") - except Exception as e: - print(f"[{self.group_name}] {model} error: {e} (round {round_idx+1})") - - # Brief backoff between attempts; longer between rounds - await asyncio.sleep(1 if round_idx == 0 else 2) - - print(f"[{self.group_name}] round {round_idx+1}/{self.max_rounds} exhausted") - - # All rounds exhausted - print(f"[{self.group_name}] all {total_attempts} attempts failed across {self.max_rounds} rounds") - return {dim: {"score": 0.0, "reason": "all models failed"} + parsed_results = [] + + for _ in range(NUM_FIXED_JUDGES): + for round_idx in range(self.max_rounds): + judged = False + + for model in self.models: + total_attempts += 1 + try: + response = await asyncio.wait_for( + self.client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": prompt}], + temperature=0, + max_tokens=2000, + ), + timeout=self.timeout, + ) + content = response.choices[0].message.content + parsed = self.parse_response(content) + if parsed is not None: + parsed["_model"] = model + parsed["_success"] = True + parsed_results.append(parsed) + judged = True + break + # Parse failed — treat as transient, try next model + print(f"[{self.group_name}] {model} parse failed (round {round_idx+1})") + except asyncio.TimeoutError: + print(f"[{self.group_name}] {model} timeout (round {round_idx+1})") + except Exception as e: + print(f"[{self.group_name}] {model} error: {e} (round {round_idx+1})") + + # Brief backoff between attempts; longer between rounds + await asyncio.sleep(1 if round_idx == 0 else 2) + + if judged: + break # Move to next fixed judge after a successful evaluation + + print(f"[{self.group_name}] round {round_idx+1}/{self.max_rounds} exhausted") + + print(f"[{self.group_name}] - {len(parsed_results)}/{NUM_FIXED_JUDGES} judges succeeded") + + final_result = {} + final_result["_success"] = len(parsed_results) > 0 + final_result["_model"] = parsed_results[0].get("_model") if parsed_results else None + + for dim in self.dimension_names: + avg_score = 0.0 + reason = parsed_results[0].get(dim, {}).get("reason", "") if parsed_results else "" + + for parsed_result in parsed_results: + if dim in parsed_result: + avg_score += parsed_result[dim]["score"] + + final_result[dim] = {"score": avg_score / len(parsed_results), "reason": reason} + + return final_result if len(parsed_results) > 0 else {dim: {"score": 0.0, "reason": "all models failed"} for dim in self.dimension_names} | {"_success": False, "_model": None} @@ -1183,7 +1208,7 @@ def get_llm_evaluator( # Backward compat: keep get_llm_validator working for any external callers def get_llm_validator( - model: str = "Qwen/Qwen3-32B", + model: str = "Qwen/Qwen3-32B-TEE", base_url: str = "https://llm.chutes.ai/v1", api_key: Optional[str] = None, ) -> Optional['LLMEvaluator']: