From a3ecda571b0c0e9afd13dafa59f1c74e7b71e896 Mon Sep 17 00:00:00 2001 From: BQBBLZ Date: Thu, 13 Aug 2026 15:29:08 +0800 Subject: [PATCH] feat: add minimal PI baseline agent experiment --- benchmarks/agent/agent_ablation_v1.json | 10 +- python/agent_experiment.py | 40 ++- python/test_agent_experiment.py | 13 +- src/cli/agent_experiment.rs | 326 +++++++++++++++++++++++- 4 files changed, 368 insertions(+), 21 deletions(-) diff --git a/benchmarks/agent/agent_ablation_v1.json b/benchmarks/agent/agent_ablation_v1.json index 78702b7..a4272c8 100644 --- a/benchmarks/agent/agent_ablation_v1.json +++ b/benchmarks/agent/agent_ablation_v1.json @@ -20,13 +20,21 @@ "timeout_ms": 120000 }, "arms": [ + { + "id": "baseline_loop", + "task_tool_enabled": false, + "baseline_loop": true, + "description": "Minimal PI-style perceive-act loop: model call, tool execution, observation append, no candle-cli context/memory/subagent enhancements." + }, { "id": "single", - "task_tool_enabled": false + "task_tool_enabled": false, + "baseline_loop": false }, { "id": "delegated", "task_tool_enabled": true, + "baseline_loop": false, "subagent_permissions": "read_only", "subagent_max_steps": 3 } diff --git a/python/agent_experiment.py b/python/agent_experiment.py index b1d1a0a..67d3139 100644 --- a/python/agent_experiment.py +++ b/python/agent_experiment.py @@ -59,8 +59,10 @@ def load_config(path: str | Path) -> ExperimentConfig: arms = tuple(_required_text(item, "id") for item in payload.get("arms", [])) if len(scenarios) < 1 or len(set(scenarios)) != len(scenarios): raise ValueError("agent experiment scenarios must be non-empty and unique") - if set(arms) != {"single", "delegated"}: - raise ValueError("agent experiment requires single and delegated arms") + if set(arms) != {"baseline_loop", "single", "delegated"}: + raise ValueError( + "agent experiment requires baseline_loop, single, and delegated arms" + ) provider_name = _required_text(provider, "name") model = _required_text(provider, "model") price_date = _required_text(pricing, "price_date") @@ -148,7 +150,7 @@ def evaluate_runs(config_path: str | Path, runs_path: str | Path) -> dict[str, A return { "schema_version": "1.0", "benchmark_version": config.benchmark_version, - "dataset_kind": "paired_equal_budget_agent_ablation", + "dataset_kind": "paired_equal_budget_agent_ablation_with_pi_baseline", "provider": config.provider, "model": config.model, "temperature": config.temperature, @@ -177,7 +179,8 @@ def evaluate_runs(config_path: str | Path, runs_path: str | Path) -> dict[str, A "passed": passed, "limitations": [ "Cache metrics use provider-returned token fields only; unsupported providers remain null.", - "A delegated-agent resume claim is allowed only when comparison.claim_supported is true.", + "The baseline_loop arm is a minimal PI-style loop and excludes candle-cli context, memory, migration workflow, and sub-agent enhancements.", + "A delegated-agent benefit claim is allowed only when comparison.claim_supported is true.", "Cost uses the frozen price snapshot and excludes network, host, and accelerator costs.", ], } @@ -327,31 +330,52 @@ def _compare_arms( usage_state: dict[str, Any], repetitions: int, ) -> dict[str, Any]: + baseline = arms["baseline_loop"] single = arms["single"] delegated = arms["delegated"] pass_rate_delta = delegated["pass_rate"] - single["pass_rate"] elapsed_delta = delegated["mean_elapsed_ms"] - single["mean_elapsed_ms"] + single_vs_baseline_pass_rate_delta = single["pass_rate"] - baseline["pass_rate"] + delegated_vs_baseline_pass_rate_delta = delegated["pass_rate"] - baseline["pass_rate"] token_delta = None + single_vs_baseline_token_delta = None + delegated_vs_baseline_token_delta = None if usage_state["usage_complete"]: token_delta = delegated["total_tokens"] - single["total_tokens"] + single_vs_baseline_token_delta = single["total_tokens"] - baseline["total_tokens"] + delegated_vs_baseline_token_delta = ( + delegated["total_tokens"] - baseline["total_tokens"] + ) paired = {} for run in records: paired[(run.scenario_id, run.trial, run.arm_id)] = run delegated_wins = single_wins = ties = 0 + single_over_baseline_wins = baseline_over_single_wins = baseline_single_ties = 0 for scenario, trial, arm in list(paired): if arm != "single": continue left = paired[(scenario, trial, "single")] right = paired[(scenario, trial, "delegated")] + base = paired[(scenario, trial, "baseline_loop")] if right.passed and not left.passed: delegated_wins += 1 elif left.passed and not right.passed: single_wins += 1 else: ties += 1 + if left.passed and not base.passed: + single_over_baseline_wins += 1 + elif base.passed and not left.passed: + baseline_over_single_wins += 1 + else: + baseline_single_ties += 1 success_benefit = pass_rate_delta > 0 and delegated_wins > single_wins + full_loop_beats_baseline = ( + single_vs_baseline_pass_rate_delta > 0 + and single_over_baseline_wins > baseline_over_single_wins + ) efficiency_benefit = False if pass_rate_delta == 0 and single["pass_rate"] == 1.0: token_benefit = ( @@ -363,6 +387,14 @@ def _compare_arms( efficiency_benefit = token_benefit or latency_benefit claim_supported = repetitions >= 3 and (success_benefit or efficiency_benefit) return { + "single_vs_baseline_pass_rate_delta": single_vs_baseline_pass_rate_delta, + "delegated_vs_baseline_pass_rate_delta": delegated_vs_baseline_pass_rate_delta, + "single_vs_baseline_total_tokens_delta": single_vs_baseline_token_delta, + "delegated_vs_baseline_total_tokens_delta": delegated_vs_baseline_token_delta, + "paired_single_over_baseline_wins": single_over_baseline_wins, + "paired_baseline_over_single_wins": baseline_over_single_wins, + "paired_baseline_single_ties": baseline_single_ties, + "full_loop_beats_baseline": full_loop_beats_baseline, "delegated_pass_rate_delta": pass_rate_delta, "delegated_mean_elapsed_ms_delta": elapsed_delta, "delegated_total_tokens_delta": token_delta, diff --git a/python/test_agent_experiment.py b/python/test_agent_experiment.py index 33ef366..ac00bce 100644 --- a/python/test_agent_experiment.py +++ b/python/test_agent_experiment.py @@ -19,8 +19,9 @@ def _config(): "repetitions": 3, "budgets": {"max_model_requests": 8, "max_tool_steps": 8, "timeout_ms": 1000}, "arms": [ - {"id": "single", "task_tool_enabled": False}, - {"id": "delegated", "task_tool_enabled": True}, + {"id": "baseline_loop", "task_tool_enabled": False, "baseline_loop": True}, + {"id": "single", "task_tool_enabled": False, "baseline_loop": False}, + {"id": "delegated", "task_tool_enabled": True, "baseline_loop": False}, ], "scenarios": [{"id": "case-a"}, {"id": "case-b"}], } @@ -43,7 +44,7 @@ def _runs(*, cached=None, delegated_passes=True): records = [] for scenario in ("case-a", "case-b"): for trial in range(1, 4): - for arm in ("single", "delegated"): + for arm in ("baseline_loop", "single", "delegated"): passed = arm == "delegated" and delegated_passes records.append( { @@ -51,7 +52,7 @@ def _runs(*, cached=None, delegated_passes=True): "arm_id": arm, "trial": trial, "passed": passed, - "elapsed_ms": 100 if arm == "single" else 80, + "elapsed_ms": {"baseline_loop": 120, "single": 100, "delegated": 80}[arm], "tool_steps": 4, "model_requests": 5, "human_interventions": 0, @@ -91,8 +92,10 @@ def test_evaluator_reports_unsupported_cache_as_null(tmp_path): assert report["arms"]["single"]["retry_count"] == 6 assert report["arms"]["single"]["retry_rate"] == 0.2 assert report["arms"]["single"]["provider_latency_ms"] == 300 - assert report["provider_retry_count"] == 12 + assert report["arms"]["baseline_loop"]["retry_count"] == 6 + assert report["provider_retry_count"] == 18 assert report["provider_retry_rate"] == 0.2 + assert report["comparison"]["delegated_vs_baseline_pass_rate_delta"] == 1.0 assert report["passed"] is True diff --git a/src/cli/agent_experiment.rs b/src/cli/agent_experiment.rs index 497e27c..a2a2364 100644 --- a/src/cli/agent_experiment.rs +++ b/src/cli/agent_experiment.rs @@ -1,6 +1,9 @@ use crate::agent::r#loop::run_single_turn_with_budget; use crate::agent::state::AgentRunBudget; +use crate::agent::tool_call::{parse_tool_call, ToolCallParseError}; use crate::model::configured::ConfiguredRuntime; +use crate::model::runtime::CandleTargetRuntime; +use crate::model::types::{TokenUsage, TurnRequest, TurnResult}; use crate::permissions::mode::PermissionMode; use crate::permissions::policy::PermissionPolicy; use crate::session::model::{ContentBlock, Message, MessageRole, Session}; @@ -49,6 +52,8 @@ struct BudgetConfig { struct ArmConfig { id: String, task_tool_enabled: bool, + #[serde(default)] + baseline_loop: bool, } #[derive(Debug, Deserialize)] @@ -208,13 +213,16 @@ fn load_manifest(path: &Path) -> Result { } let arm_ids: std::collections::HashSet<_> = manifest.arms.iter().map(|arm| arm.id.as_str()).collect(); - if arm_ids != std::collections::HashSet::from(["single", "delegated"]) - || manifest - .arms - .iter() - .any(|arm| arm.task_tool_enabled != (arm.id == "delegated")) + if arm_ids != std::collections::HashSet::from(["baseline_loop", "single", "delegated"]) + || manifest.arms.iter().any(|arm| { + arm.task_tool_enabled != (arm.id == "delegated") + || arm.baseline_loop != (arm.id == "baseline_loop") + }) { - return Err("agent experiment arms must be single(no task) and delegated(task)".into()); + return Err( + "agent experiment arms must be baseline_loop(minimal PI), single(no task), and delegated(task)" + .into(), + ); } Ok(manifest) } @@ -322,6 +330,12 @@ fn execute_run( tools: &ToolRegistry, runtime: &mut ConfiguredRuntime, ) -> RawRunRecord { + if arm.baseline_loop { + return execute_baseline_loop_run( + manifest, scenario, arm, trial, workspace, tools, runtime, + ); + } + let policy = PermissionPolicy::new(if arm.task_tool_enabled { PermissionMode::ReadOnlyWithTask } else { @@ -420,11 +434,238 @@ fn execute_run( } } -fn balanced_arm_order(scenario_index: usize, trial: usize) -> [&'static str; 2] { - if (scenario_index + trial) & 1 == 0 { - ["single", "delegated"] - } else { - ["delegated", "single"] +#[allow(clippy::too_many_arguments)] +fn execute_baseline_loop_run( + manifest: &ExperimentManifest, + scenario: &ScenarioConfig, + arm: &ArmConfig, + trial: usize, + workspace: &Path, + tools: &ToolRegistry, + runtime: &mut ConfiguredRuntime, +) -> RawRunRecord { + let mut session = Session::new(workspace.display().to_string()); + session.messages.push(Message { + role: MessageRole::User, + blocks: vec![ContentBlock::Text { + text: format!( + "Task: {}\n\nUse only read/grep/glob/pwd if needed. Return a concise answer containing exact code identifiers as evidence.", + scenario.goal + ), + }], + }); + let mut budget = AgentRunBudget::with_timeout( + manifest.budgets.max_model_requests, + manifest.budgets.max_tool_steps, + Duration::from_millis(manifest.budgets.timeout_ms), + ); + let started = Instant::now(); + let result = run_minimal_pi_loop(runtime, tools, &mut session, &mut budget); + let elapsed_ms = started.elapsed().as_millis() as u64; + + match result { + Ok(result) => { + let missing_evidence = + missing_evidence(&result.final_text, &scenario.required_evidence); + let budget_exhausted = result.final_text.contains("minimal PI loop stopped after"); + let timed_out = budget.timed_out() || elapsed_ms > manifest.budgets.timeout_ms; + let passed = missing_evidence.is_empty() && !budget_exhausted && !timed_out; + let failure_type = if timed_out { + Some("timeout".to_string()) + } else if budget_exhausted { + Some("budget_exhausted".to_string()) + } else if !missing_evidence.is_empty() { + Some("missing_evidence".to_string()) + } else { + None + }; + RawRunRecord { + scenario_id: scenario.id.clone(), + arm_id: arm.id.clone(), + trial, + passed, + elapsed_ms, + tool_steps: budget.tool_steps_used(), + model_requests: budget.model_requests_used(), + subagent_invocations: 0, + human_interventions: 0, + failure_type, + missing_evidence, + final_answer_digest: digest(&result.final_text), + usage: result.usage.to_json(), + } + } + Err(error) => { + let timed_out = budget.timed_out() + || error.to_ascii_lowercase().contains("timed out") + || elapsed_ms > manifest.budgets.timeout_ms; + RawRunRecord { + scenario_id: scenario.id.clone(), + arm_id: arm.id.clone(), + trial, + passed: false, + elapsed_ms, + tool_steps: budget.tool_steps_used(), + model_requests: budget.model_requests_used(), + subagent_invocations: 0, + human_interventions: 0, + failure_type: Some( + if timed_out { + "timeout" + } else { + "runtime_error" + } + .to_string(), + ), + missing_evidence: scenario.required_evidence.clone(), + final_answer_digest: digest(&error), + usage: serde_json::Value::Null, + } + } + } +} + +fn run_minimal_pi_loop( + runtime: &mut R, + tools: &ToolRegistry, + session: &mut Session, + budget: &mut AgentRunBudget, +) -> Result { + let mut usage = TokenUsage::default(); + while !budget.timed_out() { + if !budget.consume_model_request() { + let final_text = format!( + "minimal PI loop stopped after reaching model request budget ({})", + budget.max_model_requests() + ); + append_baseline_text(session, final_text.clone()); + return Ok(TurnResult { + final_text, + tool_calls: Vec::new(), + usage, + }); + } + let request = TurnRequest { + system_prompt: baseline_system_prompt(), + messages_json: serde_json::to_string(&session.messages).map_err(|e| e.to_string())?, + tools_json: baseline_tools_json().to_string(), + timeout_ms: budget.remaining_timeout_ms(), + deadline_unix_ms: budget.deadline_unix_ms(), + }; + let result = runtime.generate_turn(request)?; + usage.merge(&result.usage); + match parse_tool_call(&result.final_text) { + Ok(Some(tool_call)) => { + if !budget.consume_tool_step() { + let final_text = format!( + "minimal PI loop stopped after reaching tool step budget ({})", + budget.max_tool_steps() + ); + append_baseline_text(session, final_text.clone()); + return Ok(TurnResult { + final_text, + tool_calls: Vec::new(), + usage, + }); + } + append_baseline_tool_call(session, &tool_call); + let (output, is_error) = if tool_call.name == "task" { + ( + "status: error\nmessage: task is disabled in baseline_loop".to_string(), + true, + ) + } else { + match tools.execute(&tool_call.name, &tool_call.input_json) { + Ok(output) => (format!("status: ok\noutput:\n{output}"), false), + Err(error) => (format!("status: error\nmessage: {error}"), true), + } + }; + append_baseline_tool_result(session, &tool_call.id, output, is_error); + } + Ok(None) => { + append_baseline_text(session, result.final_text.clone()); + return Ok(TurnResult { + final_text: result.final_text, + tool_calls: Vec::new(), + usage, + }); + } + Err(error) => { + append_baseline_text(session, baseline_parse_error_message(&error)); + } + } + } + let final_text = "minimal PI loop stopped after reaching wall-clock timeout".to_string(); + append_baseline_text(session, final_text.clone()); + Ok(TurnResult { + final_text, + tool_calls: Vec::new(), + usage, + }) +} + +fn baseline_system_prompt() -> String { + format!( + "You are a minimal PI baseline code agent. PI means a simple perceive-act loop: read the task, optionally call one tool, observe the result, and continue until a final answer.\n\ +This baseline intentionally excludes candle-cli enhancements such as grep-RAG, project memory, sub-agents, migration-specific workflow orchestration, trace diagnostics, and transactional patch rollback.\n\ +Allowed tools are pwd, read, glob, and grep. To call a tool, output exactly one raw {{\"id\":\"call-1\",\"name\":\"read\",\"input\":{{\"file_path\":\"README.md\"}}}} block and no other text.\n\ +When you have enough evidence, return the final answer directly.\n\nAvailable tools JSON: {}", + baseline_tools_json() + ) +} + +fn baseline_tools_json() -> &'static str { + r#"[{"name":"pwd"},{"name":"read"},{"name":"glob"},{"name":"grep"}]"# +} + +fn baseline_parse_error_message(error: &ToolCallParseError) -> String { + format!( + "Your previous tool call was malformed: {error}. Return exactly one valid block or a final answer." + ) +} + +fn append_baseline_tool_call( + session: &mut Session, + tool_call: &crate::model::types::ToolCallIntent, +) { + session.messages.push(Message { + role: MessageRole::Assistant, + blocks: vec![ContentBlock::ToolCall { + id: tool_call.id.clone(), + name: tool_call.name.clone(), + input: tool_call.input_json.clone(), + }], + }); +} + +fn append_baseline_tool_result( + session: &mut Session, + tool_call_id: &str, + output: String, + is_error: bool, +) { + session.messages.push(Message { + role: MessageRole::Tool, + blocks: vec![ContentBlock::ToolResult { + tool_call_id: tool_call_id.to_string(), + output, + is_error, + }], + }); +} + +fn append_baseline_text(session: &mut Session, text: String) { + session.messages.push(Message { + role: MessageRole::Assistant, + blocks: vec![ContentBlock::Text { text }], + }); +} + +fn balanced_arm_order(scenario_index: usize, trial: usize) -> [&'static str; 3] { + match (scenario_index + trial) % 3 { + 0 => ["baseline_loop", "single", "delegated"], + 1 => ["single", "delegated", "baseline_loop"], + _ => ["delegated", "baseline_loop", "single"], } } @@ -470,6 +711,9 @@ mod tests { fn paired_order_alternates_across_scenarios_and_trials() { assert_ne!(balanced_arm_order(0, 1), balanced_arm_order(0, 2)); assert_ne!(balanced_arm_order(0, 1), balanced_arm_order(1, 1)); + assert!(balanced_arm_order(0, 1).contains(&"baseline_loop")); + assert!(balanced_arm_order(0, 1).contains(&"single")); + assert!(balanced_arm_order(0, 1).contains(&"delegated")); } #[test] @@ -514,4 +758,64 @@ mod tests { assert_eq!(execution_limits(&manifest, true), (1, 1)); assert_eq!(execution_limits(&manifest, false), (2, 3)); } + + #[test] + fn baseline_arm_must_be_marked_as_pi_loop() { + let manifest = ExperimentManifest { + schema_version: "1.0".into(), + benchmark_version: "fixture".into(), + experiment_status: "ready".into(), + provider: ProviderConfig { + name: "fixture".into(), + model: "fixture".into(), + temperature: 0.0, + }, + pricing: PricingConfig { + price_date: "2026-08-06".into(), + input_per_million_tokens: Some(0.0), + output_per_million_tokens: Some(0.0), + }, + repetitions: 3, + budgets: BudgetConfig { + max_model_requests: 8, + max_tool_steps: 8, + timeout_ms: 120_000, + }, + arms: vec![ + ArmConfig { + id: "baseline_loop".into(), + task_tool_enabled: false, + baseline_loop: true, + }, + ArmConfig { + id: "single".into(), + task_tool_enabled: false, + baseline_loop: false, + }, + ArmConfig { + id: "delegated".into(), + task_tool_enabled: true, + baseline_loop: false, + }, + ], + scenarios: (0..10) + .map(|index| ScenarioConfig { + id: format!("scenario-{index}"), + goal: "goal".into(), + evidence_paths: vec!["README.md".into()], + required_evidence: vec!["candle-cli".into()], + }) + .collect(), + }; + + let arm_ids: std::collections::HashSet<_> = + manifest.arms.iter().map(|arm| arm.id.as_str()).collect(); + assert_eq!( + arm_ids, + std::collections::HashSet::from(["baseline_loop", "single", "delegated"]) + ); + assert!(manifest.arms[0].baseline_loop); + assert!(!manifest.arms[1].baseline_loop); + assert!(manifest.arms[2].task_tool_enabled); + } }