fix: G-Eval timeout — cap max_geval_responses=1, disable inter-judge variance by default, increase DeepEval timeout to 180s

Author: Pegah Zargarian

agentscope/dashboard/app.py

@@ -95,7 +95,7 @@ def run_evaluation(
         label="Evaluation inputs (one query per line)",
         lines=4,
         placeholder="What is the refund policy?\nHow do I reset my password?",
-        value="What was the revenue for Q3?\nWhat are the key features of the product?\nWho is the CEO of XYZ Corporation?",
+        value="What was the revenue for Q3?",
     )
 
     with gr.Row():

agentscope/tools/geval_tool.py

@@ -1,3 +1,4 @@
+import os
 import warnings
 from deepeval.metrics import GEval, ConversationalGEval
 from deepeval.test_case import LLMTestCase, LLMTestCaseParams, ConversationalTestCase, Turn
@@ -130,14 +131,15 @@ def run(state: AgentState) -> AgentState:
                 primary_model=model_name,
             ))
 
-    # Inter-judge variance — single-turn metrics only
+    # Inter-judge variance disabled by default — enable by setting AGENTSCOPE_VARIANCE=1
     variance_results = []
-    for sm in scored_metrics:
-        if sm.test_cases:
-            try:
-                variance_results.append(measure_inter_judge_variance(sm))
-            except Exception as e:
-                warnings.warn(f"variance measurement failed for {sm.name}: {e}")
+    if os.environ.get("AGENTSCOPE_VARIANCE") == "1":
+        for sm in scored_metrics:
+            if sm.test_cases:
+                try:
+                    variance_results.append(measure_inter_judge_variance(sm))
+                except Exception as e:
+                    warnings.warn(f"variance measurement failed for {sm.name}: {e}")
 
     # Calibration drift — compare against prior run baseline if provided
     baseline     = state.get("baseline_geval_scores") or {}

config.yaml

@@ -21,5 +21,5 @@ eval:
   latency_orange: 5.0
   # Evaluation pipeline cost limits
   max_synth_pairs: 50
-  max_geval_responses: 100
+  max_geval_responses: 1    # score only 1 response to avoid rate limits during testing
   eval_budget_usd: 2.00

run_dashboard.sh

@@ -5,9 +5,17 @@ cd /Users/pegahzargarian/projects/MCP-1/workspace/AgenticScope
 # Load AgentScope keys
 set -a
 source .env
-# Load capstone-rag DB + model config (overwrites model with Haiku)
+# Load capstone-rag DB + model config
 source /Users/pegahzargarian/projects/MCP-1/workspace/capstone-rag/.env
 set +a
 
+# Give DeepEval more time per API call (default 88s is too short for Haiku under load)
+export DEEPEVAL_PER_ATTEMPT_TIMEOUT_SECONDS_OVERRIDE=180
+
+# Disable inter-judge variance by default (prevents OpenAI timeout)
+export AGENTSCOPE_VARIANCE=0
+
+export PYTHONPATH=/Users/pegahzargarian/projects/MCP-1/workspace/AgenticScope
+
 source .venv/bin/activate
 python -m agentscope.dashboard.app