Skip to content

feat: add minimal PI baseline agent experiment #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DuangZ-GR merged 1 commit into from
Aug 13, 2026
Merged

feat: add minimal PI baseline agent experiment #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion benchmarks/agent/agent_ablation_v1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,21 @@
"timeout_ms": 120000
},
"arms": [
{
"id": "baseline_loop",
"task_tool_enabled": false,
"baseline_loop": true,
"description": "Minimal PI-style perceive-act loop: model call, tool execution, observation append, no candle-cli context/memory/subagent enhancements."
},
{
"id": "single",
"task_tool_enabled": false
"task_tool_enabled": false,
"baseline_loop": false
},
{
"id": "delegated",
"task_tool_enabled": true,
"baseline_loop": false,
"subagent_permissions": "read_only",
"subagent_max_steps": 3
}
Expand Down
40 changes: 36 additions & 4 deletions python/agent_experiment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,10 @@ def load_config(path: str | Path) -> ExperimentConfig:
arms = tuple(_required_text(item, "id") for item in payload.get("arms", []))
if len(scenarios) < 1 or len(set(scenarios)) != len(scenarios):
raise ValueError("agent experiment scenarios must be non-empty and unique")
if set(arms) != {"single", "delegated"}:
raise ValueError("agent experiment requires single and delegated arms")
if set(arms) != {"baseline_loop", "single", "delegated"}:
raise ValueError(
"agent experiment requires baseline_loop, single, and delegated arms"
)
provider_name = _required_text(provider, "name")
model = _required_text(provider, "model")
price_date = _required_text(pricing, "price_date")
Expand Down Expand Up @@ -148,7 +150,7 @@ def evaluate_runs(config_path: str | Path, runs_path: str | Path) -> dict[str, A
return {
"schema_version": "1.0",
"benchmark_version": config.benchmark_version,
"dataset_kind": "paired_equal_budget_agent_ablation",
"dataset_kind": "paired_equal_budget_agent_ablation_with_pi_baseline",
"provider": config.provider,
"model": config.model,
"temperature": config.temperature,
Expand Down Expand Up @@ -177,7 +179,8 @@ def evaluate_runs(config_path: str | Path, runs_path: str | Path) -> dict[str, A
"passed": passed,
"limitations": [
"Cache metrics use provider-returned token fields only; unsupported providers remain null.",
"A delegated-agent resume claim is allowed only when comparison.claim_supported is true.",
"The baseline_loop arm is a minimal PI-style loop and excludes candle-cli context, memory, migration workflow, and sub-agent enhancements.",
"A delegated-agent benefit claim is allowed only when comparison.claim_supported is true.",
"Cost uses the frozen price snapshot and excludes network, host, and accelerator costs.",
],
}
Expand Down Expand Up @@ -327,31 +330,52 @@ def _compare_arms(
usage_state: dict[str, Any],
repetitions: int,
) -> dict[str, Any]:
baseline = arms["baseline_loop"]
single = arms["single"]
delegated = arms["delegated"]
pass_rate_delta = delegated["pass_rate"] - single["pass_rate"]
elapsed_delta = delegated["mean_elapsed_ms"] - single["mean_elapsed_ms"]
single_vs_baseline_pass_rate_delta = single["pass_rate"] - baseline["pass_rate"]
delegated_vs_baseline_pass_rate_delta = delegated["pass_rate"] - baseline["pass_rate"]
token_delta = None
single_vs_baseline_token_delta = None
delegated_vs_baseline_token_delta = None
if usage_state["usage_complete"]:
token_delta = delegated["total_tokens"] - single["total_tokens"]
single_vs_baseline_token_delta = single["total_tokens"] - baseline["total_tokens"]
delegated_vs_baseline_token_delta = (
delegated["total_tokens"] - baseline["total_tokens"]
)

paired = {}
for run in records:
paired[(run.scenario_id, run.trial, run.arm_id)] = run
delegated_wins = single_wins = ties = 0
single_over_baseline_wins = baseline_over_single_wins = baseline_single_ties = 0
for scenario, trial, arm in list(paired):
if arm != "single":
continue
left = paired[(scenario, trial, "single")]
right = paired[(scenario, trial, "delegated")]
base = paired[(scenario, trial, "baseline_loop")]
if right.passed and not left.passed:
delegated_wins += 1
elif left.passed and not right.passed:
single_wins += 1
else:
ties += 1
if left.passed and not base.passed:
single_over_baseline_wins += 1
elif base.passed and not left.passed:
baseline_over_single_wins += 1
else:
baseline_single_ties += 1

success_benefit = pass_rate_delta > 0 and delegated_wins > single_wins
full_loop_beats_baseline = (
single_vs_baseline_pass_rate_delta > 0
and single_over_baseline_wins > baseline_over_single_wins
)
efficiency_benefit = False
if pass_rate_delta == 0 and single["pass_rate"] == 1.0:
token_benefit = (
Expand All @@ -363,6 +387,14 @@ def _compare_arms(
efficiency_benefit = token_benefit or latency_benefit
claim_supported = repetitions >= 3 and (success_benefit or efficiency_benefit)
return {
"single_vs_baseline_pass_rate_delta": single_vs_baseline_pass_rate_delta,
"delegated_vs_baseline_pass_rate_delta": delegated_vs_baseline_pass_rate_delta,
"single_vs_baseline_total_tokens_delta": single_vs_baseline_token_delta,
"delegated_vs_baseline_total_tokens_delta": delegated_vs_baseline_token_delta,
"paired_single_over_baseline_wins": single_over_baseline_wins,
"paired_baseline_over_single_wins": baseline_over_single_wins,
"paired_baseline_single_ties": baseline_single_ties,
"full_loop_beats_baseline": full_loop_beats_baseline,
"delegated_pass_rate_delta": pass_rate_delta,
"delegated_mean_elapsed_ms_delta": elapsed_delta,
"delegated_total_tokens_delta": token_delta,
Expand Down
13 changes: 8 additions & 5 deletions python/test_agent_experiment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,9 @@ def _config():
"repetitions": 3,
"budgets": {"max_model_requests": 8, "max_tool_steps": 8, "timeout_ms": 1000},
"arms": [
{"id": "single", "task_tool_enabled": False},
{"id": "delegated", "task_tool_enabled": True},
{"id": "baseline_loop", "task_tool_enabled": False, "baseline_loop": True},
{"id": "single", "task_tool_enabled": False, "baseline_loop": False},
{"id": "delegated", "task_tool_enabled": True, "baseline_loop": False},
],
"scenarios": [{"id": "case-a"}, {"id": "case-b"}],
}
Expand All @@ -43,15 +44,15 @@ def _runs(*, cached=None, delegated_passes=True):
records = []
for scenario in ("case-a", "case-b"):
for trial in range(1, 4):
for arm in ("single", "delegated"):
for arm in ("baseline_loop", "single", "delegated"):
passed = arm == "delegated" and delegated_passes
records.append(
{
"scenario_id": scenario,
"arm_id": arm,
"trial": trial,
"passed": passed,
"elapsed_ms": 100 if arm == "single" else 80,
"elapsed_ms": {"baseline_loop": 120, "single": 100, "delegated": 80}[arm],
"tool_steps": 4,
"model_requests": 5,
"human_interventions": 0,
Expand Down Expand Up @@ -91,8 +92,10 @@ def test_evaluator_reports_unsupported_cache_as_null(tmp_path):
assert report["arms"]["single"]["retry_count"] == 6
assert report["arms"]["single"]["retry_rate"] == 0.2
assert report["arms"]["single"]["provider_latency_ms"] == 300
assert report["provider_retry_count"] == 12
assert report["arms"]["baseline_loop"]["retry_count"] == 6
assert report["provider_retry_count"] == 18
assert report["provider_retry_rate"] == 0.2
assert report["comparison"]["delegated_vs_baseline_pass_rate_delta"] == 1.0
assert report["passed"] is True


Expand Down
Loading
Loading