diff --git a/gaia/cli.py b/gaia/cli.py index 54d64157..65db096d 100644 --- a/gaia/cli.py +++ b/gaia/cli.py @@ -71,7 +71,24 @@ CONTROL_CHOICES = ("local", "telegram") TELEGRAM_MODE_CHOICES = ("polling", "webhook") TELEGRAM_SETUP_CHOICES = ("reuse", "fresh") -TERMINAL_PURPOSE_CHOICES = ("실제 사용 모드 실행", "벤치마크 용도 실행") +TERMINAL_ACTUAL_PURPOSE_LABEL = "실제 사용 모드 실행" +TERMINAL_BENCHMARK_PURPOSE_LABEL = "벤치마크 용도 실행" +TERMINAL_DEEP_QA_BENCHMARK_PURPOSE_LABEL = "Deep QA 전용 벤치마크 실행" +TERMINAL_PURPOSE_CHOICES = ( + TERMINAL_ACTUAL_PURPOSE_LABEL, + TERMINAL_BENCHMARK_PURPOSE_LABEL, + TERMINAL_DEEP_QA_BENCHMARK_PURPOSE_LABEL, +) +TERMINAL_PURPOSE_BY_LABEL = { + TERMINAL_ACTUAL_PURPOSE_LABEL: "actual", + TERMINAL_BENCHMARK_PURPOSE_LABEL: "benchmark", + TERMINAL_DEEP_QA_BENCHMARK_PURPOSE_LABEL: "deep_qa_benchmark", +} +TERMINAL_PURPOSE_PROFILE_LABEL = { + "actual": TERMINAL_ACTUAL_PURPOSE_LABEL, + "benchmark": TERMINAL_BENCHMARK_PURPOSE_LABEL, + "deep_qa_benchmark": TERMINAL_DEEP_QA_BENCHMARK_PURPOSE_LABEL, +} PROVIDER_CHOICES = ("openai", "gemini", "ollama") DEFAULT_TELEGRAM_TOKEN_FILE = str(Path.home() / ".gaia" / "telegram_bot_token") TELEGRAM_BRIDGE_PID_FILE = Path.home() / ".gaia" / "telegram_bridge.pid" @@ -750,17 +767,17 @@ def _resolve_terminal_launch_purpose( return "actual" if not sys.stdin.isatty(): return "actual" - default = TERMINAL_PURPOSE_CHOICES[0] - if str(profile.get("last_terminal_purpose") or "").strip().lower() == "benchmark": - default = TERMINAL_PURPOSE_CHOICES[1] + last_purpose = str(profile.get("last_terminal_purpose") or "").strip().lower() + default = TERMINAL_PURPOSE_PROFILE_LABEL.get(last_purpose, TERMINAL_ACTUAL_PURPOSE_LABEL) selected = _prompt_select( "테스트 용도 인가요?", TERMINAL_PURPOSE_CHOICES, default=default, ) - profile["last_terminal_purpose"] = "benchmark" if selected == TERMINAL_PURPOSE_CHOICES[1] else "actual" + purpose = TERMINAL_PURPOSE_BY_LABEL.get(selected, "actual") + profile["last_terminal_purpose"] = purpose _save_profile(profile) - return "benchmark" if selected == TERMINAL_PURPOSE_CHOICES[1] else "actual" + return purpose def _resolve_telegram_setup_strategy(parsed: argparse.Namespace, profile: dict[str, str]) -> str: @@ -1234,7 +1251,12 @@ def _dispatch_plan( return run_gui(forwarded) -def _run_terminal_benchmark_mode(*, workspace_root: Path, push_metrics: bool = False) -> int: +def _run_terminal_benchmark_mode( + *, + workspace_root: Path, + push_metrics: bool = False, + qa_mode: str | None = None, +) -> int: from gaia.src.terminal_benchmark_mode import run_terminal_benchmark_mode return run_terminal_benchmark_mode( @@ -1244,6 +1266,7 @@ def _run_terminal_benchmark_mode(*, workspace_root: Path, push_metrics: bool = F prompt_non_empty=_prompt_non_empty, emit=print, push_metrics=push_metrics, + qa_mode=qa_mode, ) @@ -1593,11 +1616,14 @@ def run_launcher(argv: Sequence[str] | None = None) -> int: pending_user_input = dict(saved_state.pending_user_input) if saved_state else {} profile = _load_profile() terminal_purpose = _resolve_terminal_launch_purpose(args, profile, runtime=runtime) - if terminal_purpose == "benchmark": - return _run_terminal_benchmark_mode( - workspace_root=Path(__file__).resolve().parent.parent, - push_metrics=bool(getattr(args, "push_metrics", False)), - ) + if terminal_purpose in {"benchmark", "deep_qa_benchmark"}: + benchmark_kwargs = { + "workspace_root": Path(__file__).resolve().parent.parent, + "push_metrics": bool(getattr(args, "push_metrics", False)), + } + if terminal_purpose == "deep_qa_benchmark": + benchmark_kwargs["qa_mode"] = DEEP_ADAPTIVE_QA_MODE + return _run_terminal_benchmark_mode(**benchmark_kwargs) url = _resolve_url(args, profile, required=True) if not url: diff --git a/gaia/harness/cli_runtime.py b/gaia/harness/cli_runtime.py index a4b3a94a..942c5428 100644 --- a/gaia/harness/cli_runtime.py +++ b/gaia/harness/cli_runtime.py @@ -10,11 +10,13 @@ from gaia.harness.registry import HarnessTask, TaskRegistry, load_builtin_registry, load_registry from gaia.harness.runner import ( ARTIFACT_ROOT, + benchmark_mode_label, _grade_task_result, _latest_report_path, _summarize_grades, _summarize_results, _write_markdown, + normalize_harness_qa_mode, run_task, ) @@ -71,6 +73,12 @@ def build_harness_parser(prog: str = "gaia harness") -> argparse.ArgumentParser: run_parser.add_argument("--repeats", type=int, default=1, help="Run the selected tasks multiple times.") run_parser.add_argument("--timeout-sec", type=int, default=180) run_parser.add_argument("--session-prefix", default="harness") + run_parser.add_argument( + "--qa-mode", + choices=("off", "adaptive", "deep", "adaptive_qa", "deep_qa", "deep_adaptive_qa"), + default="off", + help="Run selected harness tasks with adaptive QA expansion.", + ) run_parser.add_argument("--json", action="store_true", help="Emit JSON instead of a human-readable summary.") report_parser = subparsers.add_parser("report", help="Show a harness report.") @@ -409,6 +417,7 @@ def _run_registry( timeout_sec: int = 1800, env: Mapping[str, str] | None = None, session_prefix: str = "harness", + qa_mode: str | None = None, ) -> dict[str, Any]: tasks = _select_tasks( registry, @@ -421,6 +430,7 @@ def _run_registry( repeat_count = max(int(repeats), 1) results: list[dict[str, Any]] = [] task_groups: dict[str, list[dict[str, Any]]] = {} + normalized_qa_mode = normalize_harness_qa_mode(qa_mode) ARTIFACT_ROOT.mkdir(parents=True, exist_ok=True) for repeat_index in range(1, repeat_count + 1): @@ -432,6 +442,7 @@ def _run_registry( timeout_sec=timeout_sec, env=env, session_id=session_id, + qa_mode=normalized_qa_mode, ) grades = _grade_task_result(task, row) row["task_index"] = index @@ -453,6 +464,8 @@ def _run_registry( "suite_id": task.suite_id, "goal": task.goal, "url": task.url, + "qa_mode": normalized_qa_mode or "off", + "benchmark_mode": benchmark_mode_label(normalized_qa_mode), "repeats": repeat_count, "rows": task_rows, "attempts": task_rows, @@ -495,7 +508,10 @@ def _run_registry( "suite_ids": list(_normalize_values(suite_ids)), "tags": list(_normalize_values(tags)), "contains": list(_normalize_values(contains)), + "qa_mode": normalized_qa_mode or "off", }, + "qa_mode": normalized_qa_mode or "off", + "benchmark_mode": benchmark_mode_label(normalized_qa_mode), "results": results, "tasks": task_reports, "summary": summary, @@ -557,6 +573,7 @@ def run_harness_cli(argv: Sequence[str] | None = None) -> int: contains=getattr(parsed, "contains", None), timeout_sec=max(10, int(parsed.timeout_sec)), session_prefix=str(parsed.session_prefix or "harness"), + qa_mode=str(getattr(parsed, "qa_mode", "off") or "off"), ) if parsed.json: print(json.dumps(payload, ensure_ascii=False, indent=2)) diff --git a/gaia/harness/runner.py b/gaia/harness/runner.py index 8a1f4665..24856d8e 100644 --- a/gaia/harness/runner.py +++ b/gaia/harness/runner.py @@ -26,6 +26,28 @@ sys.path.insert(0, str(WORKSPACE_ROOT)) ARTIFACT_ROOT = WORKSPACE_ROOT / "artifacts" / "harness" +ADAPTIVE_QA_MODE = "adaptive_qa" +DEEP_ADAPTIVE_QA_MODE = "deep_adaptive_qa" + + +def normalize_harness_qa_mode(value: str | None) -> str | None: + raw = str(value or "").strip().lower() + if raw in {"", "off", "none", "default", "false", "0"}: + return None + if raw in {"adaptive", ADAPTIVE_QA_MODE, "progressive_qa"}: + return ADAPTIVE_QA_MODE + if raw in {"deep", "deep_qa", "aggressive_qa", DEEP_ADAPTIVE_QA_MODE}: + return DEEP_ADAPTIVE_QA_MODE + return None + + +def benchmark_mode_label(qa_mode: str | None) -> str: + normalized = normalize_harness_qa_mode(qa_mode) + if normalized == DEEP_ADAPTIVE_QA_MODE: + return "deep_qa" + if normalized == ADAPTIVE_QA_MODE: + return "adaptive_qa" + return "standard" def _normalize_status(summary: Mapping[str, Any], exit_code: int) -> str: @@ -39,18 +61,34 @@ def _task_payload(task: HarnessTask) -> dict[str, Any]: return task.as_dict() -def _build_child_code(task: Mapping[str, Any], session_id: str) -> str: - payload = json.dumps({"task": task, "session_id": session_id}, ensure_ascii=False) +def _build_child_code(task: Mapping[str, Any], session_id: str, qa_mode: str | None = None) -> str: + payload = json.dumps( + { + "task": task, + "session_id": session_id, + "qa_mode": normalize_harness_qa_mode(qa_mode) or "", + }, + ensure_ascii=False, + ) return f""" import contextlib, io, json, os from gaia.terminal import _build_test_goal, run_chat_terminal_once payload = json.loads({payload!r}) task = payload["task"] session_id = payload["session_id"] +benchmark_qa_mode = str(payload.get("qa_mode") or "").strip() prepared_goal = _build_test_goal(url=task["url"], query=task["goal"]) constraints = task.get("constraints") if isinstance(task.get("constraints"), dict) else {{}} expected_signals = task.get("expected_signals") if isinstance(task.get("expected_signals"), list) else [] goal_test_data = dict(getattr(prepared_goal, "test_data", {{}}) or {{}}) +if benchmark_qa_mode: + goal_test_data["qa_mode"] = benchmark_qa_mode + if benchmark_qa_mode == "deep_adaptive_qa": + goal_test_data.pop("adaptive_qa", None) + goal_test_data["deep_adaptive_qa"] = {{"enabled": True}} + elif benchmark_qa_mode == "adaptive_qa": + goal_test_data.pop("deep_adaptive_qa", None) + goal_test_data["adaptive_qa"] = {{"enabled": True}} prepared_goal.expected_signals = [str(item) for item in expected_signals if str(item).strip()] if prepared_goal.expected_signals: goal_test_data["harness_expected_signals"] = list(prepared_goal.expected_signals) @@ -90,13 +128,21 @@ def run_task( timeout_sec: int = 1800, env: Mapping[str, str] | None = None, session_id: str | None = None, + qa_mode: str | None = None, ) -> Dict[str, Any]: task_payload = _task_payload(task) - code = _build_child_code(task_payload, session_id or task.id) + normalized_qa_mode = normalize_harness_qa_mode(qa_mode) + code = _build_child_code(task_payload, session_id or task.id, qa_mode=normalized_qa_mode) started = time.monotonic() run_env = dict(os.environ) if env: run_env.update(env) + run_env.pop("GAIA_ADAPTIVE_QA", None) + run_env.pop("GAIA_DEEP_ADAPTIVE_QA", None) + if normalized_qa_mode == DEEP_ADAPTIVE_QA_MODE: + run_env["GAIA_DEEP_ADAPTIVE_QA"] = "1" + elif normalized_qa_mode == ADAPTIVE_QA_MODE: + run_env["GAIA_ADAPTIVE_QA"] = "1" run_env.setdefault("PYTHONUNBUFFERED", "1") try: proc = subprocess.run( @@ -116,6 +162,8 @@ def run_task( "goal": task.goal, "url": task.url, "constraints": dict(task.constraints), + "qa_mode": normalized_qa_mode or "off", + "benchmark_mode": benchmark_mode_label(normalized_qa_mode), "status": "FAIL", "final_status": "FAIL", "reason": f"harness_timeout({timeout_sec}s)", @@ -152,6 +200,8 @@ def run_task( "goal": task.goal, "url": task.url, "constraints": dict(task.constraints), + "qa_mode": normalized_qa_mode or "off", + "benchmark_mode": benchmark_mode_label(normalized_qa_mode), "status": status, "final_status": status, "reason": reason, @@ -355,6 +405,8 @@ def _write_markdown(path: Path, payload: Mapping[str, Any]) -> None: f"- generated_at: {payload.get('generated_at')}", f"- task_count: {payload.get('task_count')}", f"- repeats: {payload.get('repeats')}", + f"- qa_mode: {payload.get('qa_mode', 'off')}", + f"- benchmark_mode: {payload.get('benchmark_mode', 'standard')}", "", "## Summary", "", @@ -391,6 +443,7 @@ def run_registry( env: Mapping[str, str] | None = None, session_prefix: str = "harness", repeats: int = 1, + qa_mode: str | None = None, ) -> dict[str, Any]: tasks = _select_tasks(registry, task_id=task_id, limit=limit) if suite_id is not None: @@ -400,6 +453,7 @@ def run_registry( ARTIFACT_ROOT.mkdir(parents=True, exist_ok=True) task_reports = [] repeats = max(1, int(repeats)) + normalized_qa_mode = normalize_harness_qa_mode(qa_mode) for index, task in enumerate(tasks, start=1): task_rows = [] for attempt in range(1, repeats + 1): @@ -410,6 +464,7 @@ def run_registry( timeout_sec=timeout_sec, env=env, session_id=session_id, + qa_mode=normalized_qa_mode, ) row["attempt"] = attempt row["attempt_index"] = attempt @@ -432,6 +487,8 @@ def run_registry( "suite_id": task.suite_id, "goal": task.goal, "url": task.url, + "qa_mode": normalized_qa_mode or "off", + "benchmark_mode": benchmark_mode_label(normalized_qa_mode), "repeats": repeats, "rows": task_rows, "attempts": task_rows, @@ -474,6 +531,8 @@ def run_registry( "registry": str(registry.source) if registry.source else None, "task_count": len(tasks), "repeats": repeats, + "qa_mode": normalized_qa_mode or "off", + "benchmark_mode": benchmark_mode_label(normalized_qa_mode), "results": results, "tasks": task_reports, "summary": summary, @@ -492,6 +551,7 @@ def run_registry( "_latest_report_path", "load_builtin_registry", "load_registry", + "normalize_harness_qa_mode", "run_registry", "run_task", ] diff --git a/gaia/src/terminal_benchmark_mode.py b/gaia/src/terminal_benchmark_mode.py index 4f115918..3cd71008 100644 --- a/gaia/src/terminal_benchmark_mode.py +++ b/gaia/src/terminal_benchmark_mode.py @@ -51,6 +51,10 @@ merge_shared_suite_payload, upload_shared_suite, ) +from gaia.src.phase4.goal_driven.adaptive_qa_runtime import ( + ADAPTIVE_QA_MODE, + DEEP_ADAPTIVE_QA_MODE, +) from scripts.runner_identity import resolve_runner_id PromptSelectFn = Callable[[str, Sequence[str], str | None], str] @@ -80,6 +84,15 @@ SHARE_TESTS_OPTION = "팀 테스트 공유" UPLOAD_SHARED_TESTS_OPTION = "내 테스트 올리기" PULL_SHARED_TESTS_OPTION = "팀 테스트 가져오기" +BENCHMARK_QA_MODE_CHOICES = { + "adaptive": ADAPTIVE_QA_MODE, + ADAPTIVE_QA_MODE: ADAPTIVE_QA_MODE, + "progressive_qa": ADAPTIVE_QA_MODE, + "deep": DEEP_ADAPTIVE_QA_MODE, + "deep_qa": DEEP_ADAPTIVE_QA_MODE, + "aggressive_qa": DEEP_ADAPTIVE_QA_MODE, + DEEP_ADAPTIVE_QA_MODE: DEEP_ADAPTIVE_QA_MODE, +} def build_terminal_benchmark_catalog( @@ -88,6 +101,31 @@ def build_terminal_benchmark_catalog( return build_benchmark_site_catalog(payload) +def normalize_benchmark_qa_mode(value: str | None) -> str | None: + raw = str(value or "").strip().lower() + if raw in {"", "off", "none", "default", "false", "0"}: + return None + return BENCHMARK_QA_MODE_CHOICES.get(raw) + + +def benchmark_qa_mode_label(value: str | None) -> str: + normalized = normalize_benchmark_qa_mode(value) + if normalized == DEEP_ADAPTIVE_QA_MODE: + return "Deep QA" + if normalized == ADAPTIVE_QA_MODE: + return "QA 확장" + return "기본" + + +def benchmark_qa_mode_run_tag(value: str | None, run_tag: str) -> str: + normalized = normalize_benchmark_qa_mode(value) + if normalized == DEEP_ADAPTIVE_QA_MODE: + return f"deep_qa_{run_tag}" + if normalized == ADAPTIVE_QA_MODE: + return f"adaptive_qa_{run_tag}" + return run_tag + + def prompt_scenario_fields( *, prompt_select: PromptSelectFn, @@ -652,17 +690,19 @@ def run_benchmark_suite( process_factory: ProcessFactory = subprocess.Popen, push_metrics: bool = False, runner_id: str = "", + qa_mode: str | None = None, ) -> dict[str, Any]: scenarios = [dict(row) for row in list(suite_payload.get("scenarios") or []) if isinstance(row, Mapping)] if not scenarios: emit("등록된 테스트가 없습니다. 먼저 테스트를 추가해주세요.") return {"status": "empty", "summary": {}, "results": [], "output_dir": ""} + normalized_qa_mode = normalize_benchmark_qa_mode(qa_mode) overridden = override_suite_urls(suite_payload, target_url) started = int(time.time()) tmp_root = workspace_root / "artifacts" / "tmp" / "terminal_benchmark_mode" tmp_root.mkdir(parents=True, exist_ok=True) - suite_slug = _slugify(run_tag) + suite_slug = _slugify(benchmark_qa_mode_run_tag(normalized_qa_mode, run_tag)) tmp_suite_path = tmp_root / f"{preset.key}_{suite_slug}_{started}.json" tmp_suite_path.write_text(json.dumps(overridden, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") @@ -681,6 +721,8 @@ def run_benchmark_suite( "--output-dir", str(output_dir), ] + if normalized_qa_mode: + cmd.extend(["--qa-mode", normalized_qa_mode]) if push_metrics: cmd.append("--push-metrics") env = os.environ.copy() @@ -697,6 +739,7 @@ def run_benchmark_suite( emit(f" - target: {target_url}") emit(f" - suite: {tmp_suite_path}") emit(f" - runner_id: {resolved_runner_id}") + emit(f" - qa_mode: {benchmark_qa_mode_label(normalized_qa_mode)}") if push_metrics: emit(" - metrics: upload enabled (--push-metrics)") @@ -740,6 +783,7 @@ def run_benchmark_suite( "output_dir": str(output_dir), "cmd": cmd, "captured": captured, + "qa_mode": normalized_qa_mode or "off", } @@ -754,12 +798,14 @@ def run_external_public_benchmark_pack( push_metrics: bool = True, runner_id: str = "", process_factory: ProcessFactory = subprocess.Popen, + qa_mode: str | None = None, ) -> dict[str, Any]: resolved_manifest = (workspace_root / manifest_path).resolve() if not resolved_manifest.exists(): emit(f"전체 benchmark manifest를 찾지 못했습니다: {resolved_manifest}") return {"status": "missing_manifest", "summary": {}, "results": [], "output_dir": ""} + normalized_qa_mode = normalize_benchmark_qa_mode(qa_mode) env = os.environ.copy() resolved_runner_id = resolve_runner_id(runner_id, env) env["GAIA_RUNNER_ID"] = resolved_runner_id @@ -784,6 +830,8 @@ def run_external_public_benchmark_pack( "--runner-id", resolved_runner_id, ] + if normalized_qa_mode: + cmd.extend(["--qa-mode", normalized_qa_mode]) if push_metrics: cmd.append("--push-metrics") @@ -791,6 +839,7 @@ def run_external_public_benchmark_pack( emit(f" - manifest: {resolved_manifest}") emit(f" - runner_id: {resolved_runner_id}") emit(" - headless: enabled") + emit(f" - qa_mode: {benchmark_qa_mode_label(normalized_qa_mode)}") if push_metrics: emit(" - metrics: upload enabled (--push-metrics)") @@ -841,6 +890,7 @@ def run_external_public_benchmark_pack( "output_dir": output_dir, "cmd": cmd, "captured": captured, + "qa_mode": normalized_qa_mode or "off", } @@ -962,10 +1012,14 @@ def run_terminal_benchmark_mode( push_metrics: bool = False, monitoring_config_path: Path | None = None, auto_pull_shared_tests: bool = True, + qa_mode: str | None = None, ) -> int: registry = load_benchmark_registry(registry_path) auto_pull_attempted: set[str] = set() runner_id = resolve_runner_id(env=os.environ) + normalized_qa_mode = normalize_benchmark_qa_mode(qa_mode) + if normalized_qa_mode: + emit(f"Deep QA 벤치마크 프로필: {benchmark_qa_mode_label(normalized_qa_mode)} 모드로 실행합니다.") while True: catalog, preset_map = build_terminal_benchmark_catalog(registry) @@ -989,6 +1043,7 @@ def run_terminal_benchmark_mode( run_pack_handler=run_pack_handler, monitoring_config_path=monitoring_config_path, runner_id=runner_id, + qa_mode=normalized_qa_mode, ) continue if selected_site == SITE_EXIT_OPTION: @@ -1143,6 +1198,7 @@ def run_terminal_benchmark_mode( run_tag="full_suite", push_metrics=push_metrics_for_run, runner_id=runner_id, + qa_mode=normalized_qa_mode, ) continue @@ -1171,6 +1227,7 @@ def run_terminal_benchmark_mode( run_tag=scenario_id, push_metrics=push_metrics_for_run, runner_id=runner_id, + qa_mode=normalized_qa_mode, ) continue @@ -1227,6 +1284,7 @@ def _handle_all_sites_all_cases_run( run_pack_handler: Callable[..., dict[str, Any]], monitoring_config_path: Path | None = None, runner_id: str = "", + qa_mode: str | None = None, ) -> None: if not _ensure_monitoring_connection_required( prompt_select=prompt_select, @@ -1247,6 +1305,7 @@ def _handle_all_sites_all_cases_run( session_prefix="terminal-external-public", push_metrics=True, runner_id=runner_id, + qa_mode=qa_mode, ) diff --git a/gaia/telegram_bridge.py b/gaia/telegram_bridge.py index 70fd4eb4..eda72040 100644 --- a/gaia/telegram_bridge.py +++ b/gaia/telegram_bridge.py @@ -11,7 +11,7 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, Mapping, Optional from gaia.chat_hub import HubContext, build_command_payload, dispatch_command from gaia.src.phase4.memory.store import MemoryStore @@ -52,7 +52,7 @@ class _PendingIntervention: fields: list[str] event: threading.Event response_text: str = "" - ack_text: str = "응답을 받았습니다. 실행을 계속합니다." + ack_text: str = "좋아요, 이어서 진행해볼게요." @dataclass(slots=True) @@ -853,6 +853,104 @@ def _has_field(name: str) -> bool: helper_lines.append("중단하려면 /cancel") return "\n".join([question, *helper_lines]).strip() + @staticmethod + def _compact_goal_context(payload: Mapping[str, Any] | None, question: str = "") -> str: + source = payload if isinstance(payload, Mapping) else {} + candidates = ( + source.get("goal_description"), + source.get("goal_name"), + source.get("reason"), + question, + ) + for candidate in candidates: + text = re.sub(r"\s+", " ", str(candidate or "")).strip() + if not text: + continue + return text[:90] + ("…" if len(text) > 90 else "") + return "" + + @classmethod + def _fallback_login_credentials_message( + cls, + *, + payload: Mapping[str, Any] | None, + question: str, + username_label: str, + stage: str, + ) -> str: + context = cls._compact_goal_context(payload, question) + if stage == "password": + lines = [ + f"{username_label} 받았어요.", + "이제 비밀번호만 보내주시면 로그인 후 바로 이어서 진행할게요.", + "중단하려면 /cancel", + ] + else: + lines = [ + "좋아요, 제가 이어서 진행해볼게요.", + ( + f"{context} 작업을 계속하려면 로그인이 필요해요." + if context + else "현재 화면에서 로그인이 필요해요." + ), + f"{username_label}만 먼저 보내주세요. 비밀번호는 다음 메시지에서 따로 받을게요.", + "중단하려면 /cancel", + ] + return "\n".join(line for line in lines if line).strip() + + @classmethod + def _compose_login_credentials_message( + cls, + *, + payload: Mapping[str, Any] | None, + question: str, + username_label: str, + stage: str, + ) -> str: + fallback = cls._fallback_login_credentials_message( + payload=payload, + question=question, + username_label=username_label, + stage=stage, + ) + if os.getenv("GAIA_TELEGRAM_LLM_INTERVENTION_MESSAGE", "1").strip().lower() in {"0", "false", "off", "no"}: + return fallback + try: + from gaia import chat_hub + + client = chat_hub._get_chat_router_client() + if client is None or not hasattr(client, "analyze_text"): + return fallback + safe_payload = { + "stage": stage, + "username_label": username_label, + "question": question, + "goal_context": cls._compact_goal_context(payload, question), + "kind": str((payload or {}).get("kind") or "").strip(), + "reason_code": str((payload or {}).get("reason_code") or "").strip(), + } + prompt = ( + "당신은 GAIA Telegram 봇의 로그인 도움말 메시지를 작성합니다.\n" + "사용자는 자연스럽게 답장하고 싶어합니다. 한국어로 짧고 친근하게 쓰세요.\n" + "반드시 JSON만 출력하세요: {\"message\":\"...\"}\n\n" + "작성 규칙:\n" + "- 지금 필요한 값 하나만 요청합니다.\n" + "- 내부 필드명(username, password, proceed, manual_done)은 쓰지 않습니다.\n" + "- 아이디 단계에서는 비밀번호를 한꺼번에 요구하지 말고 다음 메시지에서 받는다고 안내합니다.\n" + "- 비밀번호 단계에서는 받은 아이디 값을 반복하거나 노출하지 않습니다.\n" + "- /cancel 안내는 마지막 줄에 포함합니다.\n" + "- 500자 이하로 작성합니다.\n\n" + f"요청 payload:\n{json.dumps(safe_payload, ensure_ascii=False, indent=2)}" + ) + response = client.analyze_text(prompt, max_completion_tokens=500, temperature=0.25) + data = json.loads(chat_hub._extract_json_object(str(response))) + message = str(data.get("message") or "").strip() if isinstance(data, dict) else "" + if message: + return message[:1200] + except Exception: + return fallback + return fallback + @staticmethod def _sequential_login_field(fields: list[str]) -> str: normalized = {str(field or "").strip().lower() for field in fields} @@ -997,6 +1095,7 @@ def _collect_login_credentials( question: str, fields: list[str], attachments: list[dict], + payload: Mapping[str, Any] | None = None, ) -> Dict[str, Any]: username_field = self._sequential_login_field(fields) username_label = "이메일" if username_field == "email" else "아이디" @@ -1008,7 +1107,12 @@ def _collect_login_credentials( kind=kind, question=question, fields=[username_field], - prompt=f"로그인이 필요해요.\n먼저 {username_label}를 보내주세요.\n중단하려면 /cancel", + prompt=self._compose_login_credentials_message( + payload=payload, + question=question, + username_label=username_label, + stage="username", + ), attachments=attachments, ) if status != "ok": @@ -1032,7 +1136,12 @@ def _collect_login_credentials( kind=kind, question=question, fields=["password"], - prompt=f"{username_label} 확인했습니다.\n이제 비밀번호를 보내주세요.\n중단하려면 /cancel", + prompt=self._compose_login_credentials_message( + payload=payload, + question=question, + username_label=username_label, + stage="password", + ), ) if status != "ok": return {"action": "cancel", "proceed": False} @@ -1093,6 +1202,7 @@ def _callback(payload: Dict[str, Any]) -> Optional[Dict[str, Any]]: question=question, fields=normalized_fields, attachments=attachment_items, + payload=payload, ) text = self._compose_intervention_message(payload, normalized_fields) @@ -1153,23 +1263,289 @@ def _callback(payload: Dict[str, Any]) -> Optional[Dict[str, Any]]: return _callback + @classmethod + def _normalize_freeform_intent(cls, value: Any) -> str: + intent = str(value or "").strip().lower() + if intent in {"goal", "run", "execute", "test", "task"}: + return "goal" + if intent in {"help", "usage", "guide", "howto"}: + return "help" + if intent in {"status", "screen", "progress", "current"}: + return "status" + if intent in {"pending", "pending_response", "handoff", "handoff_reply", "answer"}: + return "pending_response" + if intent in {"cancel", "stop"}: + return "cancel" + if intent in {"casual", "chat", "smalltalk", "greeting", "thanks"}: + return "casual" + return "" + + @classmethod + def _fallback_chatbot_route(cls, text: str, *, pending: Mapping[str, Any] | None = None) -> dict[str, Any]: + raw = re.sub(r"\s+", " ", str(text or "")).strip() + if pending: + return { + "intent": "pending_response", + "confidence": 0.0, + "reply": "", + "goal_text": "", + "pending_text": raw, + "skill": "answer_pending_input", + } + return { + "intent": "goal", + "confidence": 0.0, + "reply": "", + "goal_text": raw, + "pending_text": "", + "skill": "run_gaia_goal", + } + + @classmethod + def _llm_route_telegram_message( + cls, + text: str, + *, + pending: Mapping[str, Any] | None = None, + active: bool = False, + queued: int = 0, + ) -> dict[str, Any]: + if os.getenv("GAIA_TELEGRAM_LLM_MESSAGE_INTENT", "1").strip().lower() in {"0", "false", "off", "no"}: + return {} + try: + from gaia import chat_hub + + client = chat_hub._get_chat_router_client() + if client is None or not hasattr(client, "analyze_text"): + return {} + pending_fields = [] + pending_kind = "" + pending_question = "" + if isinstance(pending, Mapping): + pending_kind = str(pending.get("kind") or "").strip() + pending_question = str(pending.get("question") or "").strip() + fields = pending.get("fields") + if isinstance(fields, list): + pending_fields = [str(item) for item in fields if str(item).strip()] + context_payload = { + "active_run": bool(active), + "queued_count": max(0, int(queued or 0)), + "pending": { + "kind": pending_kind, + "question": pending_question, + "fields": pending_fields, + }, + } + prompt = ( + "당신은 GAIA Telegram 챗봇의 저지연 라우터입니다. reasoning은 low로, 빠르게 판단하세요.\n" + "사용자 메시지를 아래 skill 중 하나에 연결합니다. 반드시 JSON만 출력하세요.\n\n" + "사용 가능한 skill:\n" + "- run_gaia_goal: 사용자가 외부 사이트/브라우저에서 수행하거나 검증할 웹 QA 목표를 준 경우. " + "짧은 명령이라도 현재 화면에서 할 행동이면 이 skill입니다. 예: 로그인해줘, 재생해줘, 네이버에서 사용법 검색해줘.\n" + "- explain_gaia: 사용법, GAIA가 무엇인지, 어떤 문장을 보내야 하는지 설명해야 하는 경우.\n" + "- show_status: 현재 상태, 현재 화면, 진행 상황, 스크린샷을 묻는 경우.\n" + "- casual_chat: 인사, 감사, 사과, 농담, 짧은 반응처럼 실행할 목표가 없는 대화.\n" + "- answer_pending_input: pending이 있고 사용자가 필요한 값이나 답변을 보낸 경우. 숫자/아이디/비밀번호/OTP처럼 짧아도 이 skill입니다.\n" + "- cancel_run: 사용자가 취소/중단을 명확히 요청한 경우.\n\n" + "출력 스키마:\n" + "{\n" + ' "intent": "goal|help|status|casual|pending_response|cancel",\n' + ' "skill": "run_gaia_goal|explain_gaia|show_status|casual_chat|answer_pending_input|cancel_run",\n' + ' "confidence": 0.0,\n' + ' "reply": "",\n' + ' "goal_text": "",\n' + ' "pending_text": ""\n' + "}\n\n" + "규칙:\n" + "- goal이면 goal_text에 실행할 목표 문장을 넣습니다. 원문을 보존하되 필요하면 살짝 정리합니다.\n" + "- help/casual이면 reply를 한국어 Telegram 답장으로 짧게 작성합니다.\n" + "- status이면 reply는 비워도 됩니다. 시스템이 실제 상태를 붙입니다.\n" + "- pending_response이면 pending_text에 사용자가 보낸 값을 그대로 보존합니다. 내부 필드명은 만들지 않습니다.\n" + "- '사용법'이라는 단어가 있어도 '네이버에서 사용법 검색해줘'처럼 사이트 행동이면 goal입니다.\n" + "- pending이 있으면 잡담/도움말/상태 조회가 아닌 한 answer_pending_input을 우선합니다.\n\n" + f"현재 컨텍스트:\n{json.dumps(context_payload, ensure_ascii=False, indent=2)}\n\n" + f"사용자 메시지:\n{text}" + ) + reasoning_key = "GAIA_CODEX_REASONING_EFFORT" + previous_reasoning = os.environ.get(reasoning_key) + os.environ[reasoning_key] = str( + os.getenv("GAIA_TELEGRAM_CHATBOT_REASONING_EFFORT", "low") or "low" + ) + try: + response = client.analyze_text(prompt, max_completion_tokens=360, temperature=0.0) + finally: + if previous_reasoning is None: + os.environ.pop(reasoning_key, None) + else: + os.environ[reasoning_key] = previous_reasoning + data = json.loads(chat_hub._extract_json_object(str(response))) + if isinstance(data, dict): + intent = cls._normalize_freeform_intent(data.get("intent")) + if not intent: + return {} + try: + confidence = float(data.get("confidence") or 0.0) + except Exception: + confidence = 0.0 + return { + "intent": intent, + "confidence": max(0.0, min(1.0, confidence)), + "reply": str(data.get("reply") or "").strip(), + "goal_text": str(data.get("goal_text") or "").strip(), + "pending_text": str(data.get("pending_text") or "").strip(), + "skill": str(data.get("skill") or "").strip(), + } + except Exception: + return {} + return {} + + def _route_telegram_message( + self, + text: str, + *, + chat_id: int, + pending: Mapping[str, Any] | None = None, + ) -> dict[str, Any]: + with self._state_lock: + active = self._active_runs.get(chat_id) is not None + queued = int(self._queued_count_by_chat.get(chat_id, 0) or 0) + route = self._llm_route_telegram_message(text, pending=pending, active=active, queued=queued) + if route: + return route + return self._fallback_chatbot_route(text, pending=pending) + + @classmethod + def _classify_freeform_message_intent(cls, text: str) -> str: + route = cls._llm_route_telegram_message(text) + if route: + return str(route.get("intent") or "goal") + return "goal" + @staticmethod - def _looks_like_live_status_query(text: str) -> bool: - normalized = re.sub(r"\s+", "", (text or "").strip().lower()) - if not normalized: - return False - tokens = ( - "지금뭐하고있어", - "뭐하고있어", - "현재뭐해", - "현재상태", - "진행상황", - "어디까지했어", - "상태어때", - "whatyoudoing", - "currentstatus", + def _field_label(field: str) -> str: + normalized = str(field or "").strip().lower() + labels = { + "username": "아이디", + "email": "이메일", + "password": "비밀번호", + "otp": "인증번호", + "captcha": "보안문자", + "answer": "정답", + } + return labels.get(normalized, normalized or "필요한 값") + + @classmethod + def _format_pending_help(cls, kind: str, question: str, fields: list[str]) -> str: + visible_fields = [ + str(field or "").strip() + for field in fields + if str(field or "").strip().lower() + not in {"action", "proceed", "manual_done", "instruction", "auth_mode", "return_credentials"} + ] + if visible_fields: + labels = [cls._field_label(field) for field in visible_fields] + if "password" in {field.lower() for field in visible_fields} and ( + "username" in {field.lower() for field in visible_fields} + or "email" in {field.lower() for field in visible_fields} + ): + labels = [cls._field_label(cls._sequential_login_field(visible_fields))] + ask = ", ".join(labels) + return ( + "지금은 실행을 잠깐 멈추고 사용자 입력을 기다리는 중이에요.\n" + f"필요한 것: {ask}\n" + "그 값만 답장으로 보내주시면 제가 이어서 처리할게요.\n" + "중단하려면 /cancel" + ) + context = re.sub(r"\s+", " ", str(question or "")).strip() + return ( + "지금은 추가 안내를 기다리는 중이에요.\n" + + (f"요청 내용: {context}\n" if context else "") + + "원하는 처리 방향을 한 문장으로 답장해 주세요.\n" + "중단하려면 /cancel" + ) + + def _format_casual_reply(self, chat_id: int, text: str) -> str: + with self._state_lock: + active = self._active_runs.get(chat_id) + queued = int(self._queued_count_by_chat.get(chat_id, 0) or 0) + if active is not None: + cmd = (active.raw_command or "").strip() + if len(cmd) > 70: + cmd = cmd[:67] + "..." + return ( + "괜찮아요. 지금 요청을 계속 처리하고 있어요.\n" + f"진행 중: {cmd or '현재 작업'}\n" + "상태가 궁금하면 `현재 상태`라고 보내주세요." + ) + if queued > 0: + return ( + "좋아요, 앞선 요청이 있어서 순서대로 처리할게요.\n" + "새 테스트 목표를 보내면 그 다음 순서로 넣어둘게요." + ) + return ( + "안녕하세요! GAIA예요.\n" + "새 테스트 목표를 문장으로 보내면 바로 실행 준비할게요.\n" + "상태가 궁금하면 `현재 상태`라고 보내주세요." + ) + + def _format_help_message(self, chat_id: int) -> str: + with self._pending_lock: + pending = self._pending_interventions.get(chat_id) + if pending is not None: + return self._format_pending_help( + pending.kind, + pending.question, + list(pending.fields or []), + ) + hub_pending = dict(getattr(self.hub_context, "pending_user_input", {}) or {}) + if hub_pending: + fields = hub_pending.get("fields") + if not isinstance(fields, list): + fields = [] + return self._format_pending_help( + str(hub_pending.get("kind") or "input"), + str(hub_pending.get("question") or "추가 입력이 필요합니다."), + [str(item) for item in fields], + ) + + with self._state_lock: + active = self._active_runs.get(chat_id) + queued = int(self._queued_count_by_chat.get(chat_id, 0) or 0) + if active is not None: + return ( + "지금은 요청을 실행 중이에요.\n" + "`현재 상태`라고 보내면 진행 상황을 볼 수 있고, 새 테스트 목표를 보내면 다음 작업으로 처리할게요.\n" + "로그인/인증이 필요하면 제가 그때 필요한 값만 물어볼게요." + ) + if queued > 0: + return ( + "앞선 요청이 있어서 순서대로 처리 중이에요.\n" + "`현재 상태`라고 보내면 대기/진행 상황을 볼 수 있어요.\n" + "새 테스트 목표는 한 문장으로 보내면 이어서 실행할게요." + ) + return ( + "GAIA에게는 테스트 목표를 한 문장으로 보내면 돼요.\n" + "예: 인천대 사이버캠퍼스에서 12주차 첫 번째 강의를 열고 재생되는지 확인해줘\n" + "예: 네이버 뉴스에서 스포츠 > 축구로 이동한 뒤 순위표 상위 3개 팀이 보이는지 확인해줘\n" + "실행 중에는 `현재 상태` 또는 `현재 화면`이라고 물어볼 수 있어요." + ) + + @staticmethod + def _format_command_received_message(raw: str, *, queued_ahead: int = 0) -> str: + preview = re.sub(r"\s+", " ", str(raw or "")).strip() + if len(preview) > 90: + preview = preview[:89] + "…" + if queued_ahead > 0: + return ( + "앞선 요청이 끝나면 이어서 실행할게요.\n" + + (f"다음 요청: {preview}\n" if preview else "") + + "진행 상황은 `현재 상태`라고 보내면 확인할 수 있어요." + ) + return ( + "좋아요, 실행해볼게요.\n" + + (f"요청: {preview}\n" if preview else "") + + "진행 중 궁금하면 `현재 상태`라고 보내주세요." ) - return any(token in normalized for token in tokens) def _format_live_status(self, chat_id: int) -> str: with self._pending_lock: @@ -1502,17 +1878,53 @@ async def handle_message(self, update, context) -> None: with self._pending_lock: pending = self._pending_interventions.get(chat_id) if pending is not None: - if self._looks_like_live_status_query(raw): - await message.reply_text(self._format_live_status(chat_id)) - return lowered = raw.strip().lower() + if lowered in {"/cancel", "cancel", "취소"}: + pending.response_text = "cancel" + pending.event.set() + if pending.ack_text: + await message.reply_text(pending.ack_text) + return if lowered.startswith("/pair"): await message.reply_text("현재 실행이 추가 입력을 기다리는 중입니다. 응답 텍스트 또는 /cancel을 보내주세요.") return if lowered.startswith("/") and lowered not in {"/cancel"}: await message.reply_text("현재 실행이 추가 입력을 기다리는 중입니다. 응답 텍스트를 보내거나 /cancel을 입력하세요.") return - pending.response_text = "cancel" if lowered in {"/cancel", "cancel", "취소"} else raw + + pending_context = { + "kind": pending.kind, + "question": pending.question, + "fields": list(pending.fields or []), + } + route = self._route_telegram_message(raw, chat_id=chat_id, pending=pending_context) + intent = str(route.get("intent") or "pending_response") + if intent == "status": + await message.reply_text(self._format_live_status(chat_id)) + return + if intent == "help": + await message.reply_text(str(route.get("reply") or "").strip() or self._format_help_message(chat_id)) + return + if intent == "casual": + await message.reply_text( + str(route.get("reply") or "").strip() + or self._format_pending_help( + pending.kind, + pending.question, + list(pending.fields or []), + ) + ) + return + if intent == "cancel": + pending.response_text = "cancel" + pending.event.set() + if pending.ack_text: + await message.reply_text(pending.ack_text) + return + pending.response_text = ( + str(route.get("pending_text") or route.get("goal_text") or "").strip() + or raw + ) pending.event.set() if pending.ack_text: await message.reply_text(pending.ack_text) @@ -1528,17 +1940,47 @@ async def handle_message(self, update, context) -> None: ) return - if self._looks_like_live_status_query(raw): - await message.reply_text(self._format_live_status(chat_id)) - return - hub_pending = dict(getattr(self.hub_context, "pending_user_input", {}) or {}) + route_pending = None if hub_pending: kind = str(hub_pending.get("kind") or "input").strip().lower() fields = hub_pending.get("fields") if not isinstance(fields, list): fields = [] - response = self._parse_intervention_response(kind, raw, fields=[str(item) for item in fields]) + normalized_fields = [str(item) for item in fields] + route_pending = { + "kind": kind, + "question": str(hub_pending.get("question") or "추가 입력이 필요합니다."), + "fields": normalized_fields, + } + + route = self._route_telegram_message(raw, chat_id=chat_id, pending=route_pending) + intent = str(route.get("intent") or "goal") + if intent == "status": + await message.reply_text(self._format_live_status(chat_id)) + return + + if intent == "help": + await message.reply_text(str(route.get("reply") or "").strip() or self._format_help_message(chat_id)) + return + + if hub_pending: + kind = str(route_pending.get("kind") or "input") if isinstance(route_pending, dict) else "input" + normalized_fields = list(route_pending.get("fields") or []) if isinstance(route_pending, dict) else [] + if intent == "casual": + await message.reply_text( + str(route.get("reply") or "").strip() + or self._format_pending_help( + kind, + str(route_pending.get("question") or "추가 입력이 필요합니다.") if isinstance(route_pending, dict) else "", + normalized_fields, + ) + ) + return + response_text = str(route.get("pending_text") or route.get("goal_text") or "").strip() or raw + if intent == "cancel": + response_text = "/cancel" + response = self._parse_intervention_response(kind, response_text, fields=normalized_fields) self.hub_context.pending_user_response = response self.hub_context.pending_user_input = {} if self.hub_context.on_session_update: @@ -1546,21 +1988,26 @@ async def handle_message(self, update, context) -> None: self.hub_context.on_session_update(self.hub_context) except Exception: pass - await message.reply_text("응답을 받았습니다. 실행을 계속합니다.") + await message.reply_text("좋아요, 답장 받았어요. 이어서 진행해볼게요.") + return + + if intent == "casual": + await message.reply_text(str(route.get("reply") or "").strip() or self._format_casual_reply(chat_id, raw)) return - position = self.queue.qsize() + 1 + queued_ahead = self.queue.qsize() + command_text = str(route.get("goal_text") or "").strip() or raw with self._state_lock: queued_now = int(self._queued_count_by_chat.get(chat_id, 0) or 0) self._queued_count_by_chat[chat_id] = queued_now + 1 await self.queue.put( _CommandEnvelope( chat_id=chat_id, - raw_command=raw, + raw_command=command_text, reply_to_message_id=message.message_id, ) ) - await self._safe_reply_text(message, f"queued #{position}: {raw[:120]}") + await self._safe_reply_text(message, self._format_command_received_message(command_text, queued_ahead=queued_ahead)) def _split_text(text: str, limit: int = 3900) -> list[str]: diff --git a/gaia/tests/unit/test_cli_adaptive_qa_mode.py b/gaia/tests/unit/test_cli_adaptive_qa_mode.py index 31c113ff..f54b884e 100644 --- a/gaia/tests/unit/test_cli_adaptive_qa_mode.py +++ b/gaia/tests/unit/test_cli_adaptive_qa_mode.py @@ -2,9 +2,12 @@ from gaia.cli import ( DEEP_ADAPTIVE_QA_MODE, + TERMINAL_DEEP_QA_BENCHMARK_PURPOSE_LABEL, + TERMINAL_PURPOSE_CHOICES, QUICK_DEEP_QA_LABEL, QUICK_RUN_MODE_CHOICES, _dispatch_chat, + _resolve_terminal_launch_purpose, run_launcher, ) @@ -70,6 +73,51 @@ def fake_dispatch_chat(runtime, url, feature_query, repl, *, session_id, qa_mode assert profile["last_quick_mode"] == DEEP_ADAPTIVE_QA_MODE +def test_terminal_purpose_menu_can_select_deep_qa_benchmark(monkeypatch) -> None: + profile: dict[str, str] = {} + captured: dict[str, object] = {} + + monkeypatch.setattr("gaia.cli.sys.stdin", _TTY()) + monkeypatch.setattr("gaia.cli._save_profile", lambda payload: captured.setdefault("profile", dict(payload))) + + def fake_select(prompt: str, options, default=None): + captured["prompt"] = prompt + captured["options"] = tuple(options) + captured["default"] = default + return TERMINAL_DEEP_QA_BENCHMARK_PURPOSE_LABEL + + monkeypatch.setattr("gaia.cli._prompt_select", fake_select) + + selected = _resolve_terminal_launch_purpose(object(), profile, runtime="terminal") + + assert selected == "deep_qa_benchmark" + assert captured["prompt"] == "테스트 용도 인가요?" + assert captured["options"] == TERMINAL_PURPOSE_CHOICES + assert profile["last_terminal_purpose"] == "deep_qa_benchmark" + + +def test_launcher_routes_deep_qa_benchmark_to_benchmark_mode(monkeypatch) -> None: + captured: dict[str, object] = {} + + monkeypatch.setattr("gaia.cli._configure_session", lambda parsed, require_url: _stub_configured_terminal()) + monkeypatch.setattr("gaia.cli.load_session_state", lambda session_key: None) + monkeypatch.setattr("gaia.cli._load_profile", lambda: {}) + monkeypatch.setattr("gaia.cli._resolve_terminal_launch_purpose", lambda *args, **kwargs: "deep_qa_benchmark") + + def fake_run_terminal_benchmark_mode(*, workspace_root, push_metrics=False, qa_mode=None): + captured["workspace_root"] = workspace_root + captured["push_metrics"] = push_metrics + captured["qa_mode"] = qa_mode + return 0 + + monkeypatch.setattr("gaia.cli._run_terminal_benchmark_mode", fake_run_terminal_benchmark_mode) + + assert run_launcher(["--terminal"]) == 0 + + assert captured["qa_mode"] == DEEP_ADAPTIVE_QA_MODE + assert captured["push_metrics"] is False + + def test_dispatch_chat_terminal_applies_deep_qa_env_for_run(monkeypatch) -> None: captured: dict[str, object] = {} monkeypatch.setenv("GAIA_ADAPTIVE_QA", "old-adaptive") diff --git a/gaia/tests/unit/test_gui_benchmark_sync.py b/gaia/tests/unit/test_gui_benchmark_sync.py index 1534e1b9..8a1a1608 100644 --- a/gaia/tests/unit/test_gui_benchmark_sync.py +++ b/gaia/tests/unit/test_gui_benchmark_sync.py @@ -322,8 +322,8 @@ def test_benchmark_worker_supports_in_memory_suite_payload(tmp_path: Path, monke progress: list[str] = [] class _FakeProcess: - def __init__(self, cmd, cwd=None, stdout=None, stderr=None, text=None, bufsize=None, env=None): - del cwd, stdout, stderr, text, bufsize, env + def __init__(self, cmd, cwd=None, stdout=None, stderr=None, text=None, bufsize=None, env=None, **kwargs): + del cwd, stdout, stderr, text, bufsize, env, kwargs output_dir = Path(cmd[cmd.index("--output-dir") + 1]) output_dir.mkdir(parents=True, exist_ok=True) (output_dir / "summary.json").write_text( @@ -383,8 +383,8 @@ def test_benchmark_worker_appends_push_metrics_flag(tmp_path: Path, monkeypatch) captured_cmd: list[str] = [] class _FakeProcess: - def __init__(self, cmd, cwd=None, stdout=None, stderr=None, text=None, bufsize=None, env=None): - del cwd, stdout, stderr, text, bufsize, env + def __init__(self, cmd, cwd=None, stdout=None, stderr=None, text=None, bufsize=None, env=None, **kwargs): + del cwd, stdout, stderr, text, bufsize, env, kwargs captured_cmd[:] = list(cmd) output_dir = Path(cmd[cmd.index("--output-dir") + 1]) output_dir.mkdir(parents=True, exist_ok=True) @@ -436,7 +436,7 @@ def terminate(self): assert "--push-metrics" in captured_cmd -def test_main_window_benchmark_mode_uses_dedicated_stage_and_emits_manager_open(monkeypatch) -> None: +def test_main_window_benchmark_mode_emits_manager_open_without_auto_stage_jump(monkeypatch) -> None: _app() monkeypatch.setattr(MainWindow, "_setup_screencast", lambda self: None) @@ -447,7 +447,7 @@ def test_main_window_benchmark_mode_uses_dedicated_stage_and_emits_manager_open( window._benchmark_mode_button.click() assert window.get_selected_run_mode() == "benchmark" - assert window._workflow_stack.currentWidget() is window._benchmark_page + assert window._standard_action_container.isVisible() is False assert emitted == [("", "")] diff --git a/gaia/tests/unit/test_run_goal_benchmark_script.py b/gaia/tests/unit/test_run_goal_benchmark_script.py index 870450ef..eb083861 100644 --- a/gaia/tests/unit/test_run_goal_benchmark_script.py +++ b/gaia/tests/unit/test_run_goal_benchmark_script.py @@ -5,10 +5,14 @@ import pytest from scripts.run_goal_benchmark import ( + DEEP_ADAPTIVE_QA_MODE, + _apply_qa_mode_env, _build_child_code, + _benchmark_mode_label, _compute_kpi_metrics, _compute_metrics, _infer_provider_from_model, + _normalize_qa_mode, _prepare_scenario_env, _provider_credential_error, _run_scenario_once, @@ -58,6 +62,38 @@ def test_build_child_code_propagates_expected_signals_without_mcp_host_guard() - assert "sys.__stdout__" in code +def test_build_child_code_forces_deep_qa_mode_for_benchmark_runs() -> None: + scenario = { + "id": "DEEP_001", + "url": "https://example.com", + "goal": "상품 필터 동작을 검증한다", + "test_data": {"qa_mode": "adaptive"}, + } + + code = _build_child_code(scenario, "session-1", qa_mode="deep") + + assert '"qa_mode": "deep_adaptive_qa"' in code + assert "goal_test_data['qa_mode'] = benchmark_qa_mode" in code + assert "goal_test_data['deep_adaptive_qa'] = {'enabled': True}" in code + + +def test_qa_mode_helpers_normalize_and_apply_env() -> None: + env = {"GAIA_ADAPTIVE_QA": "1", "GAIA_DEEP_ADAPTIVE_QA": "1"} + + assert _normalize_qa_mode("deep") == DEEP_ADAPTIVE_QA_MODE + assert _benchmark_mode_label(DEEP_ADAPTIVE_QA_MODE) == "deep_qa" + + _apply_qa_mode_env(env, "deep") + + assert "GAIA_ADAPTIVE_QA" not in env + assert env["GAIA_DEEP_ADAPTIVE_QA"] == "1" + + _apply_qa_mode_env(env, "off") + + assert "GAIA_ADAPTIVE_QA" not in env + assert "GAIA_DEEP_ADAPTIVE_QA" not in env + + def test_timeout_floor_applies_by_default() -> None: budget = _resolve_scenario_timeout_budget( scenario_budget=180, diff --git a/gaia/tests/unit/test_run_kpi_benchmark_pack_script.py b/gaia/tests/unit/test_run_kpi_benchmark_pack_script.py index 80922ba0..cdaf4b75 100644 --- a/gaia/tests/unit/test_run_kpi_benchmark_pack_script.py +++ b/gaia/tests/unit/test_run_kpi_benchmark_pack_script.py @@ -6,11 +6,15 @@ from scripts import run_kpi_benchmark_pack as kpi_pack from scripts.run_kpi_benchmark_pack import ( + DEEP_ADAPTIVE_QA_MODE, MIN_BENCHMARK_TIMEOUT_SEC, _build_run_suite_command, + _benchmark_mode_label, _compute_pack_kpis, _effective_timeout_cap, + _run_harness, _is_blocked_user_action, + _normalize_qa_mode, _load_suite_manifest, _resolve_suite_paths, _try_push_pack_metrics, @@ -78,6 +82,50 @@ def test_build_run_suite_command_forwards_push_metrics(tmp_path: Path) -> None: assert with_push[-1] == "--push-metrics" +def test_build_run_suite_command_forwards_deep_qa_mode(tmp_path: Path) -> None: + suite_path = tmp_path / "suite.json" + + cmd = _build_run_suite_command( + suite_path, + repeats=1, + timeout_cap=600, + session_prefix="external-public", + push_metrics=False, + runner_id="macmini", + qa_mode="deep", + ) + + assert _normalize_qa_mode("deep") == DEEP_ADAPTIVE_QA_MODE + assert _benchmark_mode_label(DEEP_ADAPTIVE_QA_MODE) == "deep_qa" + assert cmd[cmd.index("--qa-mode") + 1] == DEEP_ADAPTIVE_QA_MODE + + +def test_run_harness_forwards_deep_qa_mode(monkeypatch) -> None: + captured: dict[str, object] = {} + + def fake_run(cmd, **kwargs): + captured["cmd"] = list(cmd) + captured["kwargs"] = kwargs + return SimpleNamespace(returncode=0, stdout=json.dumps({"summary": {}}), stderr="") + + monkeypatch.setattr(kpi_pack.subprocess, "run", fake_run) + + payload = _run_harness( + task_ids=["TASK_001"], + suite_ids=[], + tags=[], + contains=[], + repeats=1, + timeout_sec=600, + env={}, + qa_mode="deep", + ) + + cmd = captured["cmd"] + assert payload == {"summary": {}} + assert cmd[cmd.index("--qa-mode") + 1] == DEEP_ADAPTIVE_QA_MODE + + def test_try_push_pack_metrics_uploads_final_pack_artifact(tmp_path: Path, monkeypatch) -> None: monitoring_config = tmp_path / "monitoring.json" monitoring_config.write_text("{}", encoding="utf-8") diff --git a/gaia/tests/unit/test_telegram_bridge.py b/gaia/tests/unit/test_telegram_bridge.py index 2ae75368..0c64e567 100644 --- a/gaia/tests/unit/test_telegram_bridge.py +++ b/gaia/tests/unit/test_telegram_bridge.py @@ -117,6 +117,68 @@ def test_compose_intervention_message_falls_back_when_llm_disabled(monkeypatch) assert "manual_done" not in message +def test_login_prompt_fallback_is_contextual_and_hides_internal_fields(monkeypatch) -> None: + monkeypatch.setenv("GAIA_TELEGRAM_LLM_INTERVENTION_MESSAGE", "0") + + first = _TelegramBridge._compose_login_credentials_message( + payload={ + "kind": "human_answer", + "goal_description": "대중매체속바이오테크놀로지 강의 12주차의 첫번째 강의를 누르고 재생 확인", + }, + question="로그인 정보가 필요합니다.", + username_label="아이디", + stage="username", + ) + second = _TelegramBridge._compose_login_credentials_message( + payload={"kind": "human_answer", "goal_description": "강의 재생 확인"}, + question="로그인 정보가 필요합니다.", + username_label="아이디", + stage="password", + ) + + assert "대중매체속바이오테크놀로지" in first + assert "아이디만 먼저" in first + assert "비밀번호는 다음 메시지" in first + assert "username" not in first + assert "proceed" not in first + assert "/cancel" in first + assert "비밀번호만" in second + assert "아이디 받았어요" in second + + +def test_login_prompt_can_use_llm_for_more_chatbot_like_message(monkeypatch) -> None: + seen: dict[str, str] = {} + + class _FakeClient: + def analyze_text(self, prompt: str, max_completion_tokens: int, temperature: float): + seen["prompt"] = prompt + seen["max_completion_tokens"] = str(max_completion_tokens) + seen["temperature"] = str(temperature) + return json.dumps( + { + "message": ( + "좋아요, 로그인만 도와주시면 제가 바로 이어서 확인할게요.\n" + "아이디만 먼저 보내주세요.\n" + "중단하려면 /cancel" + ) + }, + ensure_ascii=False, + ) + + monkeypatch.setattr(chat_hub, "_get_chat_router_client", lambda: _FakeClient()) + + message = _TelegramBridge._compose_login_credentials_message( + payload={"kind": "human_answer", "goal_description": "강의 12주차 첫 번째 영상 재생 확인"}, + question="로그인 정보가 필요합니다.", + username_label="아이디", + stage="username", + ) + + assert "아이디만 먼저" in message + assert "stage" in seen["prompt"] + assert "내부 필드명" in seen["prompt"] + + def test_fallback_human_answer_login_message_hides_internal_proceed() -> None: message = _TelegramBridge._fallback_intervention_message( "human_answer", @@ -141,3 +203,122 @@ def test_login_credentials_are_collected_sequentially() -> None: assert _TelegramBridge._sequential_login_field(["email", "password"]) == "email" assert _TelegramBridge._sequential_login_field(["username", "password"]) == "username" assert not _TelegramBridge._should_collect_login_credentials("clarification", ["username", "password"]) + + +def test_freeform_intent_fallback_defaults_to_goal_without_heuristics(monkeypatch) -> None: + monkeypatch.setenv("GAIA_TELEGRAM_LLM_MESSAGE_INTENT", "0") + + assert _TelegramBridge._classify_freeform_message_intent("보이루") == "goal" + assert _TelegramBridge._classify_freeform_message_intent("로그인해줘") == "goal" + assert _TelegramBridge._classify_freeform_message_intent("재생해줘") == "goal" + + +def test_freeform_intent_can_use_llm_to_avoid_false_goal_queue(monkeypatch) -> None: + seen: dict[str, str] = {} + + class _FakeClient: + def analyze_text(self, prompt: str, max_completion_tokens: int, temperature: float): + seen["prompt"] = prompt + seen["max_completion_tokens"] = str(max_completion_tokens) + seen["temperature"] = str(temperature) + return json.dumps({"intent": "casual", "reason": "농담성 인사"}, ensure_ascii=False) + + monkeypatch.setattr(chat_hub, "_get_chat_router_client", lambda: _FakeClient()) + + assert _TelegramBridge._classify_freeform_message_intent("부스에서 가볍게 인사 한번 해줘") == "casual" + assert "run_gaia_goal" in seen["prompt"] + + +def test_freeform_intent_llm_keeps_search_goal_even_with_usage_word(monkeypatch) -> None: + class _FakeClient: + def analyze_text(self, prompt: str, max_completion_tokens: int, temperature: float): + assert "네이버에서 사용법 검색해줘" in prompt + return json.dumps( + { + "intent": "goal", + "skill": "run_gaia_goal", + "confidence": 0.91, + "goal_text": "네이버에서 챗GPT 사용법 검색해줘", + }, + ensure_ascii=False, + ) + + monkeypatch.setattr(chat_hub, "_get_chat_router_client", lambda: _FakeClient()) + + assert _TelegramBridge._classify_freeform_message_intent("네이버에서 챗GPT 사용법 검색해줘") == "goal" + + +def test_help_request_is_answered_without_queue_language() -> None: + bridge = _TelegramBridge( + hub_context=HubContext( + provider="openai", + model="gpt-5.5", + auth_strategy="reuse", + url="https://example.com", + runtime="terminal", + control_channel="telegram", + ), + config=TelegramConfig(), + memory_store=MemoryStore(enabled=False), + ) + + message = bridge._format_help_message(123) + + assert "테스트 목표를 한 문장" in message + assert "현재 상태" in message + assert "대기열" not in message + assert "queued" not in message.lower() + + +def test_current_screen_request_is_routed_by_llm_to_status(monkeypatch) -> None: + class _FakeClient: + def analyze_text(self, prompt: str, max_completion_tokens: int, temperature: float): + assert "현재 화면" in prompt + return json.dumps( + {"intent": "status", "skill": "show_status", "confidence": 0.95}, + ensure_ascii=False, + ) + + monkeypatch.setattr(chat_hub, "_get_chat_router_client", lambda: _FakeClient()) + + assert _TelegramBridge._classify_freeform_message_intent("현재 화면") == "status" + + +def test_help_request_during_pending_input_explains_needed_field() -> None: + context = HubContext( + provider="openai", + model="gpt-5.5", + auth_strategy="reuse", + url="https://example.com", + runtime="terminal", + control_channel="telegram", + pending_user_input={ + "kind": "human_answer", + "question": "사이버캠퍼스 로그인이 필요합니다.", + "fields": ["proceed", "manual_done", "username", "password", "instruction"], + }, + ) + bridge = _TelegramBridge( + hub_context=context, + config=TelegramConfig(), + memory_store=MemoryStore(enabled=False), + ) + + message = bridge._format_help_message(123) + + assert "사용자 입력을 기다리는 중" in message + assert "아이디" in message + assert "password" not in message + assert "proceed" not in message + + +def test_command_received_message_hides_internal_queue_number() -> None: + message = _TelegramBridge._format_command_received_message( + "12주차 첫 번째 강의를 재생 확인해줘", + queued_ahead=0, + ) + + assert "실행해볼게요" in message + assert "queue" not in message.lower() + assert "대기열" not in message + assert "#1" not in message diff --git a/gaia/tests/unit/test_terminal_benchmark_mode.py b/gaia/tests/unit/test_terminal_benchmark_mode.py index 63666ced..77133ca2 100644 --- a/gaia/tests/unit/test_terminal_benchmark_mode.py +++ b/gaia/tests/unit/test_terminal_benchmark_mode.py @@ -519,6 +519,77 @@ def wait(self): assert "--push-metrics" in captured_cmd +def test_run_benchmark_suite_forwards_deep_qa_mode_and_tags_artifact(tmp_path: Path) -> None: + preset = find_preset("inu_timetable") + assert preset is not None + captured_cmd: list[str] = [] + emitted: list[str] = [] + + class _FakeProcess: + def __init__(self, cmd, **kwargs): + del kwargs + captured_cmd[:] = list(cmd) + output_dir = Path(cmd[cmd.index("--output-dir") + 1]) + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "summary.json").write_text( + json.dumps( + { + "scenario_count": 1, + "status_counts": {"SUCCESS": 1}, + "qa_mode": "deep_adaptive_qa", + "benchmark_mode": "deep_qa", + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + (output_dir / "results.json").write_text( + json.dumps( + [ + { + "scenario_id": "INUU_001_HOME_LOGIN_VISIBLE", + "status": "SUCCESS", + "qa_mode": "deep_adaptive_qa", + "benchmark_mode": "deep_qa", + } + ], + ensure_ascii=False, + ), + encoding="utf-8", + ) + self.stdout = iter([]) + + def wait(self): + return 0 + + result = run_benchmark_suite( + workspace_root=tmp_path, + preset=preset, + target_url="https://inuu-timetable.vercel.app/", + suite_payload={ + "suite_id": "inu_timetable_public_v1", + "site": {"name": "INU TIMETABLE", "base_url": "https://inuu-timetable.vercel.app/"}, + "scenarios": [ + { + "id": "INUU_001_HOME_LOGIN_VISIBLE", + "url": "https://inuu-timetable.vercel.app/", + "goal": "홈 화면 확인", + } + ], + }, + emit=emitted.append, + run_tag="full_suite", + process_factory=_FakeProcess, + qa_mode="deep", + ) + + assert captured_cmd[captured_cmd.index("--qa-mode") + 1] == "deep_adaptive_qa" + output_dir = Path(captured_cmd[captured_cmd.index("--output-dir") + 1]) + assert "deep_qa_full_suite" in output_dir.name + assert result["qa_mode"] == "deep_adaptive_qa" + assert any("qa_mode: Deep QA" in message for message in emitted) + + def test_run_terminal_benchmark_mode_dispatches_single_scenario(tmp_path: Path) -> None: script = _PromptScript( selections=[ @@ -553,6 +624,37 @@ def fake_run_suite_handler(**kwargs): assert [row["id"] for row in calls[0]["suite_payload"]["scenarios"]] == ["INUU_001_HOME_LOGIN_VISIBLE"] +def test_run_terminal_benchmark_mode_forwards_deep_qa_to_suite_runs(tmp_path: Path) -> None: + script = _PromptScript( + selections=[ + "INU TIMETABLE", + "https://inuu-timetable.vercel.app/", + "기존 테스트 실행", + "기존 테스트 전체 실행", + "로컬만 저장", + "이전으로", + "종료", + ] + ) + calls: list[dict[str, object]] = [] + emitted: list[str] = [] + + run_terminal_benchmark_mode( + workspace_root=_repo_root(), + prompt_select=script.select, + prompt=script.text, + prompt_non_empty=script.non_empty_prompt, + emit=emitted.append, + registry_path=tmp_path / "benchmark_registry.json", + run_suite_handler=lambda **kwargs: calls.append(kwargs) or {}, + qa_mode="deep", + ) + + assert len(calls) == 1 + assert calls[0]["qa_mode"] == "deep_adaptive_qa" + assert any("Deep QA 벤치마크 프로필" in message for message in emitted) + + def test_run_terminal_benchmark_mode_recovers_missing_custom_suite_before_run(tmp_path: Path) -> None: registry_path = tmp_path / "benchmark_registry.json" suite_path = tmp_path / "gaia/tests/scenarios/custom_story_docs_suite.json" diff --git a/scripts/run_goal_benchmark.py b/scripts/run_goal_benchmark.py index bde9597c..06ef6864 100755 --- a/scripts/run_goal_benchmark.py +++ b/scripts/run_goal_benchmark.py @@ -40,6 +40,16 @@ _MIN_CODEX_EXEC_TIMEOUT_SEC = 180 _MAX_CODEX_EXEC_TIMEOUT_SEC = 300 _BENCHMARK_CODEX_REASONING_EFFORT = "low" +ADAPTIVE_QA_MODE = "adaptive_qa" +DEEP_ADAPTIVE_QA_MODE = "deep_adaptive_qa" +QA_MODE_CHOICES = ( + "off", + "adaptive", + "deep", + ADAPTIVE_QA_MODE, + "deep_qa", + DEEP_ADAPTIVE_QA_MODE, +) _LIVE_TRACE_MARKERS = ( "🎯 목표 시작", "--- Step ", @@ -119,8 +129,45 @@ def _tail_text(text: str, *, max_lines: int = 20, max_chars: int = 4000) -> str: return tail -def _build_child_code(scenario: Dict[str, Any], session_id: str) -> str: - payload = json.dumps({"scenario": scenario, "session_id": session_id}, ensure_ascii=False) +def _normalize_qa_mode(value: str | None) -> str | None: + raw = str(value or "").strip().lower() + if raw in {"", "off", "none", "default", "false", "0"}: + return None + if raw in {"adaptive", ADAPTIVE_QA_MODE, "progressive_qa"}: + return ADAPTIVE_QA_MODE + if raw in {"deep", "deep_qa", "aggressive_qa", DEEP_ADAPTIVE_QA_MODE}: + return DEEP_ADAPTIVE_QA_MODE + return None + + +def _benchmark_mode_label(qa_mode: str | None) -> str: + normalized = _normalize_qa_mode(qa_mode) + if normalized == DEEP_ADAPTIVE_QA_MODE: + return "deep_qa" + if normalized == ADAPTIVE_QA_MODE: + return "adaptive_qa" + return "standard" + + +def _apply_qa_mode_env(env: Dict[str, str], qa_mode: str | None) -> None: + normalized = _normalize_qa_mode(qa_mode) + env.pop("GAIA_ADAPTIVE_QA", None) + env.pop("GAIA_DEEP_ADAPTIVE_QA", None) + if normalized == DEEP_ADAPTIVE_QA_MODE: + env["GAIA_DEEP_ADAPTIVE_QA"] = "1" + elif normalized == ADAPTIVE_QA_MODE: + env["GAIA_ADAPTIVE_QA"] = "1" + + +def _build_child_code(scenario: Dict[str, Any], session_id: str, qa_mode: str | None = None) -> str: + payload = json.dumps( + { + "scenario": scenario, + "session_id": session_id, + "qa_mode": _normalize_qa_mode(qa_mode) or "", + }, + ensure_ascii=False, + ) return f""" import contextlib, io, json, sys import os @@ -134,6 +181,7 @@ def _build_child_code(scenario: Dict[str, Any], session_id: str) -> str: payload = json.loads({payload!r}) scenario = payload['scenario'] session_id = payload['session_id'] +benchmark_qa_mode = str(payload.get('qa_mode') or '').strip() prepared_goal = _build_test_goal(url=scenario['url'], query=scenario['goal']) constraints = scenario.get('constraints') if isinstance(scenario.get('constraints'), dict) else {{}} expected_signals = scenario.get('expected_signals') if isinstance(scenario.get('expected_signals'), list) else [] @@ -141,6 +189,14 @@ def _build_child_code(scenario: Dict[str, Any], session_id: str) -> str: scenario_test_data = scenario.get('test_data') if isinstance(scenario.get('test_data'), dict) else {{}} if scenario_test_data: goal_test_data.update(scenario_test_data) +if benchmark_qa_mode: + goal_test_data['qa_mode'] = benchmark_qa_mode + if benchmark_qa_mode == 'deep_adaptive_qa': + goal_test_data.pop('adaptive_qa', None) + goal_test_data['deep_adaptive_qa'] = {{'enabled': True}} + elif benchmark_qa_mode == 'adaptive_qa': + goal_test_data.pop('deep_adaptive_qa', None) + goal_test_data['adaptive_qa'] = {{'enabled': True}} prepared_goal.expected_signals = [str(item) for item in expected_signals if str(item).strip()] if prepared_goal.expected_signals: goal_test_data['harness_expected_signals'] = list(prepared_goal.expected_signals) @@ -196,9 +252,10 @@ def _run_scenario_once( session_id: str, timeout_sec: int, env: Dict[str, str], + qa_mode: str | None = None, ) -> Dict[str, Any]: scenario_env = _prepare_scenario_env(env, timeout_sec) - code = _build_child_code(scenario, session_id) + code = _build_child_code(scenario, session_id, qa_mode=qa_mode) started = time.monotonic() try: proc = subprocess.Popen( @@ -551,6 +608,12 @@ def main() -> int: parser.add_argument("--timeout-cap", type=int, default=600) parser.add_argument("--session-prefix", default="benchmark") parser.add_argument("--output-dir", default="") + parser.add_argument( + "--qa-mode", + choices=QA_MODE_CHOICES, + default="off", + help="Run every scenario with adaptive QA expansion enabled; use deep/deep_adaptive_qa for human-comparison Deep QA benches.", + ) parser.add_argument( "--push-metrics", action="store_true", @@ -565,6 +628,11 @@ def main() -> int: scenarios = scenarios[: int(args.limit)] repeats = max(1, int(args.repeats)) timeout_cap = max(_MIN_BENCHMARK_TIMEOUT_SEC, int(args.timeout_cap)) + requested_qa_mode = str(args.qa_mode or "").strip() + if not requested_qa_mode or requested_qa_mode.lower() in {"off", "none", "default", "false", "0"}: + requested_qa_mode = str(suite.get("qa_mode") or requested_qa_mode).strip() + normalized_qa_mode = _normalize_qa_mode(requested_qa_mode) + benchmark_mode = _benchmark_mode_label(normalized_qa_mode) started_at = datetime.now().astimezone() run_id = f"{Path(args.suite).stem}_{started_at.strftime('%Y%m%d_%H%M%S')}" @@ -574,6 +642,7 @@ def main() -> int: env = os.environ.copy() runner_id = resolve_runner_id(args.runner_id, env) env["GAIA_RUNNER_ID"] = runner_id + _apply_qa_mode_env(env, normalized_qa_mode) provider = str(args.provider or "").strip().lower() if not provider: @@ -597,6 +666,8 @@ def main() -> int: "provider": provider, "model": args.model, "runner_id": runner_id, + "qa_mode": normalized_qa_mode or "off", + "benchmark_mode": benchmark_mode, "metrics": empty_metrics, "kpi_metrics": empty_kpis, "status_counts": {}, @@ -613,6 +684,8 @@ def main() -> int: f"- provider: {provider or '-'}\n" f"- model: {args.model}\n" f"- runner_id: {runner_id}\n" + f"- qa_mode: {normalized_qa_mode or 'off'}\n" + f"- benchmark_mode: {benchmark_mode}\n" f"- fatal_error: {credential_error}\n", encoding="utf-8", ) @@ -637,11 +710,14 @@ def main() -> int: session_id=sid, timeout_sec=budget, env=env, + qa_mode=normalized_qa_mode, ) row["repeat"] = repeat_idx row["provider"] = provider row["model"] = str(args.model) row["runner_id"] = runner_id + row["qa_mode"] = normalized_qa_mode or "off" + row["benchmark_mode"] = benchmark_mode row["constraints"] = scenario.get("constraints") if isinstance(scenario.get("constraints"), dict) else {} row["expected_signals"] = scenario.get("expected_signals") if isinstance(scenario.get("expected_signals"), list) else [] rows.append(row) @@ -660,6 +736,8 @@ def main() -> int: "provider": provider, "model": args.model, "runner_id": runner_id, + "qa_mode": normalized_qa_mode or "off", + "benchmark_mode": benchmark_mode, "metrics": metrics, "kpi_metrics": kpi_metrics, "status_counts": dict(status_counts), @@ -695,6 +773,8 @@ def main() -> int: md.write(f"- provider: {provider or '-'}\n") md.write(f"- model: {args.model}\n") md.write(f"- runner_id: {runner_id}\n") + md.write(f"- qa_mode: {normalized_qa_mode or 'off'}\n") + md.write(f"- benchmark_mode: {benchmark_mode}\n") md.write(f"- success_rate: {metrics['success_rate']}\n") md.write(f"- primary_success_rate: {metrics['primary_success_rate']}\n") md.write(f"- avg_time_seconds: {metrics['avg_time_seconds']}\n") diff --git a/scripts/run_kpi_benchmark_pack.py b/scripts/run_kpi_benchmark_pack.py index 2acdb93d..0c50f2fc 100644 --- a/scripts/run_kpi_benchmark_pack.py +++ b/scripts/run_kpi_benchmark_pack.py @@ -20,6 +20,16 @@ PUSH_METRICS = ROOT / "scripts" / "push_metrics.py" MONITORING_CONFIG = Path.home() / ".gaia" / "monitoring.json" MIN_BENCHMARK_TIMEOUT_SEC = 600 +ADAPTIVE_QA_MODE = "adaptive_qa" +DEEP_ADAPTIVE_QA_MODE = "deep_adaptive_qa" +QA_MODE_CHOICES = ( + "off", + "adaptive", + "deep", + ADAPTIVE_QA_MODE, + "deep_qa", + DEEP_ADAPTIVE_QA_MODE, +) if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) @@ -31,6 +41,26 @@ from scripts.runner_identity import resolve_runner_id +def _normalize_qa_mode(value: str | None) -> str | None: + raw = str(value or "").strip().lower() + if raw in {"", "off", "none", "default", "false", "0"}: + return None + if raw in {"adaptive", ADAPTIVE_QA_MODE, "progressive_qa"}: + return ADAPTIVE_QA_MODE + if raw in {"deep", "deep_qa", "aggressive_qa", DEEP_ADAPTIVE_QA_MODE}: + return DEEP_ADAPTIVE_QA_MODE + return None + + +def _benchmark_mode_label(qa_mode: str | None) -> str: + normalized = _normalize_qa_mode(qa_mode) + if normalized == DEEP_ADAPTIVE_QA_MODE: + return "deep_qa" + if normalized == ADAPTIVE_QA_MODE: + return "adaptive_qa" + return "standard" + + def _load_json(path: Path) -> Dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) @@ -208,6 +238,7 @@ def _run_suite( push_metrics: bool, runner_id: str, env: Dict[str, str], + qa_mode: str | None = None, ) -> Dict[str, Any]: started = time.time() before = time.time() @@ -218,6 +249,7 @@ def _run_suite( session_prefix=session_prefix, push_metrics=push_metrics, runner_id=runner_id, + qa_mode=qa_mode, ) proc = subprocess.Popen( cmd, @@ -267,7 +299,9 @@ def _build_run_suite_command( session_prefix: str, push_metrics: bool, runner_id: str = "", + qa_mode: str | None = None, ) -> List[str]: + normalized_qa_mode = _normalize_qa_mode(qa_mode) cmd = [ sys.executable, str(RUN_SINGLE), @@ -282,6 +316,8 @@ def _build_run_suite_command( ] if str(runner_id or "").strip(): cmd.extend(["--runner-id", str(runner_id)]) + if normalized_qa_mode: + cmd.extend(["--qa-mode", normalized_qa_mode]) if push_metrics: cmd.append("--push-metrics") return cmd @@ -296,7 +332,9 @@ def _run_harness( repeats: int, timeout_sec: int, env: Dict[str, str], + qa_mode: str | None = None, ) -> Dict[str, Any]: + normalized_qa_mode = _normalize_qa_mode(qa_mode) cmd = [ sys.executable, "-m", @@ -309,6 +347,8 @@ def _run_harness( "--timeout-sec", str(timeout_sec), ] + if normalized_qa_mode: + cmd.extend(["--qa-mode", normalized_qa_mode]) for task_id in task_ids: cmd.extend(["--task-id", task_id]) for suite_id in suite_ids: @@ -341,6 +381,8 @@ def _write_markdown(path: Path, report: Dict[str, Any]) -> None: lines.append(f"- repeats: {report['repeats']}") lines.append(f"- timeout_cap: {report['timeout_cap']}") lines.append(f"- runner_id: {report.get('runner_id', 'unknown')}") + lines.append(f"- qa_mode: {report.get('qa_mode', 'off')}") + lines.append(f"- benchmark_mode: {report.get('benchmark_mode', 'standard')}") lines.append("") lines.append("## Overall KPI") lines.append("") @@ -443,6 +485,12 @@ def main() -> None: default="", help="Human/team runner identifier recorded in artifacts and metrics. Defaults to GAIA_RUNNER_ID or user@host.", ) + parser.add_argument( + "--qa-mode", + choices=QA_MODE_CHOICES, + default="off", + help="Forward adaptive QA mode to every suite run; deep/deep_adaptive_qa is the human-comparison Deep QA benchmark profile.", + ) parser.add_argument("--push-metrics", action="store_true", help="Forward metrics upload to each suite run.") parser.add_argument("--harness-task-id", action="append", default=[], dest="harness_task_ids") parser.add_argument("--harness-suite-id", action="append", default=[], dest="harness_suite_ids") @@ -455,6 +503,8 @@ def main() -> None: env = os.environ.copy() runner_id = resolve_runner_id(args.runner_id, env) env["GAIA_RUNNER_ID"] = runner_id + normalized_qa_mode = _normalize_qa_mode(str(args.qa_mode or "")) + benchmark_mode = _benchmark_mode_label(normalized_qa_mode) try: suite_paths = _resolve_suite_paths(suite_args=args.suite, suite_manifest=args.suite_manifest) except ValueError as exc: @@ -475,6 +525,7 @@ def main() -> None: push_metrics=bool(args.push_metrics), runner_id=runner_id, env=env, + qa_mode=normalized_qa_mode, ) suite_reports.append(suite_report) all_rows.extend(suite_report["rows"]) @@ -498,6 +549,7 @@ def main() -> None: repeats=max(1, int(args.harness_repeats or args.repeats)), timeout_sec=_effective_timeout_cap(int(args.harness_timeout_sec or args.timeout_cap)), env=env, + qa_mode=normalized_qa_mode, ) harness_report = { "run_id": harness_payload.get("run_id"), @@ -515,6 +567,8 @@ def main() -> None: "timeout_cap": _effective_timeout_cap(int(args.timeout_cap)), "push_metrics": bool(args.push_metrics), "runner_id": runner_id, + "qa_mode": normalized_qa_mode or "off", + "benchmark_mode": benchmark_mode, "suites": [ { "suite_id": suite["suite_id"],