From 8c8bf760829c60e81666561193de9ea1b352d03f Mon Sep 17 00:00:00 2001 From: guoweihua Date: Fri, 14 Aug 2026 12:24:50 +0800 Subject: [PATCH] detect delivery acceptance failures --- CHANGELOG.md | 11 +++++++++++ README.md | 6 +++--- docs/GOLDEN_FAILURE_INDEX.md | 2 +- docs/VALUE.md | 2 +- pyproject.toml | 2 +- tests/test_acceptance_failure.py | 28 ++++++++++++++++++++++++++++ trace_debugger/analyzer.py | 16 +++++++++++++++- trace_debugger/record.py | 1 + 8 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 tests/test_acceptance_failure.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9630eb0..6b199e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +## 0.5.0 (2026-08-14) + +### Added + +- `acceptance_failed` classification for failed delivery test and release-validation steps +- Regression coverage from the real GitHub delivery sandbox fault-injection trajectory + +### Changed + +- Acceptance failures now fail the path instead of being reported as a successful tool step + ### Documentation - Added Episode v1 and portable failure-log capabilities to the project overview diff --git a/README.md b/README.md index 996e6f2..f4dc193 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ | 业务环节 | 项目交付 | 决策用途 | |----------|----------|----------| | 运行采集 | Format B 轨迹、StepWatcher、Artifact 引用 | 保留可复盘的执行证据 | -| 失败识别 | 7 类可解释启发式、JSONL findings、统计聚合 | 定位工具、检索、策略和轨迹问题 | +| 失败识别 | 8 类可解释启发式、JSONL findings、统计聚合 | 定位工具、检索、验收、策略和轨迹问题 | | 版本比较 | baseline、`scan --compare`、golden CI | 检查发版后失败分布是否退化 | | 发布协作 | 可读报告、修复边界、intervention ledger | 支持 review/hold 与后续复盘 | @@ -26,7 +26,7 @@ Agent 团队把运行轨迹接入 trace-debugger 之后： -1. **自动识别** 7 类常见失败（工具报错、搜索空结果、重复调用等） +1. **自动识别** 8 类常见失败（工具报错、验收失败、搜索空结果、重复调用等） 2. **形成记录** — JSONL + 可读 log，便于复盘 3. **发版前对比** — `tdebug scan` + `--compare` 发现失败分布是否变差 4. **结构化 findings** — `--findings-out` 输出门禁判定 + 修复边界（Harness Health，v0.2.7+） @@ -85,7 +85,7 @@ python -m pytest tests/test_failure_golden.py # CI 同款 | 已交付 | 说明 | |--------|------| -| 7 类启发式 + CLI | `tdebug` / `stats` / `validate` | +| 8 类启发式 + CLI | `tdebug` / `stats` / `validate` | | 黄金集 + CI | 27/27 — 规则回归 | | 发版 compare | `--compare` + 试点 baseline / 案例 | | **Harness Health** (v0.2.7) | 五维 Agent Work Loop · 证据状态 · `findings.json` · intervention ledger | diff --git a/docs/GOLDEN_FAILURE_INDEX.md b/docs/GOLDEN_FAILURE_INDEX.md index ca5f163..dea7a5a 100644 --- a/docs/GOLDEN_FAILURE_INDEX.md +++ b/docs/GOLDEN_FAILURE_INDEX.md @@ -13,7 +13,7 @@ ## Taxonomy 覆盖 -7 类失败均有独立负例：`tool_error` · `search_empty` · `search_timeout` · `duplicate` · `no_answer` · `llm_offtrack` · `context_overflow` +原有 7 类失败均有独立 golden 负例：`tool_error` · `search_empty` · `search_timeout` · `duplicate` · `no_answer` · `llm_offtrack` · `context_overflow`。`acceptance_failed` 由真实交付故障轨迹对应的独立回归测试覆盖，尚未并入历史 golden manifest。 另含：正例（无失败）、多路径、中文搜索工具、工具错误后恢复、Harness 阻止重复等。 diff --git a/docs/VALUE.md b/docs/VALUE.md index df9c430..1dfffae 100644 --- a/docs/VALUE.md +++ b/docs/VALUE.md @@ -43,7 +43,7 @@ Agent 跑挂之后，轨迹散在 JSON 里，难分类、难汇总、难在发 **Agent 回归测试与失败治理门禁** ``` -轨迹 JSON → 7 类失败标签 → JSONL / log 记录 +轨迹 JSON → 8 类失败标签 → JSONL / log 记录 → 发版前 scan + --compare baseline → CI golden 27 条 ``` diff --git a/pyproject.toml b/pyproject.toml index 4f74617..1fdcbc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "trace-debugger" -version = "0.4.0" +version = "0.5.0" description = "Agent 回归测试与失败治理门禁 — Format B 轨迹、scan/compare baseline、7 类启发式检测、golden CI" readme = "README.md" requires-python = ">=3.10" diff --git a/tests/test_acceptance_failure.py b/tests/test_acceptance_failure.py new file mode 100644 index 0000000..7251a41 --- /dev/null +++ b/tests/test_acceptance_failure.py @@ -0,0 +1,28 @@ +from trace_debugger.analyzer import Analyzer, FailureType +from trace_debugger.reader import parse + + +def test_delivery_acceptance_failure_is_not_reported_as_pass(): + trajectory = parse({ + "session_id": "delivery-failed-1", + "query": "change a policy and run tests", + "model": "operator-plan-executor-v1", + "steps": [ + { + "step": 1, + "thought": "Run the business acceptance suite.", + "action": { + "name": "run_acceptance_tests", + "arguments": '{"command":["python","-m","pytest"]}', + }, + "observation": "failed", + } + ], + "final_answer": "delivery status: test_failed", + }) + + analysis = Analyzer().analyze(trajectory) + + assert analysis.needs_fix is True + assert FailureType.ACCEPTANCE_FAILED in analysis.paths[0].failure_types + assert analysis.paths[0].success is False diff --git a/trace_debugger/analyzer.py b/trace_debugger/analyzer.py index 962f02b..cd16b2f 100644 --- a/trace_debugger/analyzer.py +++ b/trace_debugger/analyzer.py @@ -22,6 +22,7 @@ class FailureType: """失败原因分类""" TOOL_ERROR = "tool_error" # 工具调用报错 + ACCEPTANCE_FAILED = "acceptance_failed" # 验收测试失败 SEARCH_EMPTY = "search_empty" # 搜索无结果 SEARCH_TIMEOUT = "search_timeout" # 搜索超时 LLM_OFFTRACK = "llm_offtrack" # LLM 跑偏（答非所问） @@ -32,6 +33,7 @@ class FailureType: LABELS = { "tool_error": "工具调用报错", + "acceptance_failed": "验收测试失败", "search_empty": "搜索无有效结果", "search_timeout": "搜索超时", "llm_offtrack": "LLM 偏离用户意图", @@ -318,6 +320,8 @@ def _analyze_path(self, path: Path, index: int, traj: Trajectory) -> PathAnalysi # offtrack / overflow 视为「完成但有问题」仍可能 path.success=True from parse if FailureType.LLM_OFFTRACK in failure_types: path_ok = False + if FailureType.ACCEPTANCE_FAILED in failure_types: + path_ok = False summary_parts = [] if path_ok and not failure_types: @@ -455,7 +459,16 @@ def _analyze_step( suggestion = "压缩上下文或启用摘要/窗口滑动" if not failure_type and step.is_action: - if step.has_error: + observation = (step.observation or "").strip().lower() + if step.action_name in { + "run_acceptance_tests", + "run_tests", + "pytest", + } and observation in {"failed", "test_failed", "failure"}: + failure_type = FailureType.ACCEPTANCE_FAILED + failure_detail = f"{step.action_name} 返回验收失败" + suggestion = "保留失败测试输出，修复候选变更后重新执行验收" + elif step.has_error: failure_type = FailureType.TOOL_ERROR failure_detail = f"{step.action_name} 调用失败: {step.error_message[:100]}" suggestion = f"检查 {step.action_name} 的参数或重试" @@ -527,6 +540,7 @@ def _generate_suggestions(self, traj: Trajectory, analyses: list[PathAnalysis]) def _suggestion_for(self, failure_type: str) -> str: mapping = { FailureType.TOOL_ERROR: "检查工具参数是否正确，或增加参数校验", + FailureType.ACCEPTANCE_FAILED: "检查失败断言和候选差异，修复后重新验收", FailureType.SEARCH_EMPTY: "调整搜索词策略，先确认需求再搜索", FailureType.SEARCH_TIMEOUT: "限制搜索范围或添加缓存层", FailureType.LLM_OFFTRACK: "在 system prompt 中强化约束，或增加意图校验", diff --git a/trace_debugger/record.py b/trace_debugger/record.py index 4798e04..d850d38 100644 --- a/trace_debugger/record.py +++ b/trace_debugger/record.py @@ -526,6 +526,7 @@ def failure_events_from_analysis( def _suggestion_for_type(failure_type: str) -> str: mapping = { FailureType.TOOL_ERROR: "检查工具参数是否正确，或增加参数校验", + FailureType.ACCEPTANCE_FAILED: "检查失败断言和候选差异，修复后重新验收", FailureType.SEARCH_EMPTY: "调整搜索词策略，先确认需求再搜索", FailureType.SEARCH_TIMEOUT: "限制搜索范围或添加缓存层", FailureType.LLM_OFFTRACK: "在 system prompt 中强化约束，或增加意图校验",