diff --git a/README.md b/README.md index f4dc193..bbf6a47 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,9 @@ | 版本比较 | baseline、`scan --compare`、golden CI | 检查发版后失败分布是否退化 | | 发布协作 | 可读报告、修复边界、intervention ledger | 支持 review/hold 与后续复盘 | -**当前阶段:** 适合本地或 CI 的低成本回归门禁,不是完整 APM、云 tracing 或自动修复系统;真实团队接入仍需脱敏、权限、时序存储和外部复现验证。 +**当前阶段:** 适合本地或 CI 的低成本回归门禁。已在独立 GitHub 沙箱中复现验收失败,输出 +`acceptance_failed` 并交给评测引擎形成 `hold`;这属于 `external_real_sandbox`,不是生产团队接入。 +项目仍不是完整 APM、云 tracing 或自动修复系统;真实团队接入仍需脱敏、权限和时序存储。 --- @@ -100,7 +102,10 @@ python -m pytest tests/test_failure_golden.py # CI 同款 | 干预 ledger(Learning Capture) | [docs/intervention_ledger.json](docs/intervention_ledger.json) | | 业务证明自评 ~65% | [docs/VALUE.md](docs/VALUE.md) | -仍缺:真人秒表、非模拟 PR hold、外部团队复现。 +外部沙箱证据:[`agent-delivery-sandbox`](https://github.com/weihuaguo270-ops/agent-delivery-sandbox) +已完成非模拟 PR 的接受、拒绝、回滚,以及 `acceptance_failed -> hold` 故障回流。 + +仍缺:真人执行耗时基线、生产团队接入复现、长期时序数据和告警闭环。 Golden CI:[docs/golden_evidence_baseline.md](docs/golden_evidence_baseline.md) diff --git a/pyproject.toml b/pyproject.toml index 1fdcbc6..e374b04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "trace-debugger" version = "0.5.0" -description = "Agent 回归测试与失败治理门禁 — Format B 轨迹、scan/compare baseline、7 类启发式检测、golden CI" +description = "Agent 回归测试与失败治理门禁 — Format B 轨迹、scan/compare baseline、8 类启发式检测、golden CI" readme = "README.md" requires-python = ">=3.10" diff --git a/trace_debugger/analyzer.py b/trace_debugger/analyzer.py index cd16b2f..0499a3f 100644 --- a/trace_debugger/analyzer.py +++ b/trace_debugger/analyzer.py @@ -86,6 +86,7 @@ def content_tokens(text: str) -> set[str]: def looks_like_overflow_text(text: str) -> bool: + """按已知中英文服务错误文案识别上下文溢出信号。""" if not text: return False low = text.lower() @@ -93,6 +94,7 @@ def looks_like_overflow_text(text: str) -> bool: def is_search_tool(name: str, *, substrings: tuple[str, ...] = ("search",), extra_names: tuple[str, ...] = ()) -> bool: + """按可配置名称规则判断工具是否属于搜索类。""" n = (name or "").lower() if any(sub in n for sub in substrings): return True @@ -102,6 +104,7 @@ def is_search_tool(name: str, *, substrings: tuple[str, ...] = ("search",), extr def is_final_thought(thought: str, markers: tuple[str, ...]) -> bool: + """按调用方提供的大小写不敏感标记识别终答思考。""" upper = (thought or "").upper() return any(m.upper() in upper for m in markers if m) @@ -195,9 +198,11 @@ def __init__( self.search_tool_names = search_tool_names def step_is_final(self, step: Step) -> bool: + """按当前分析器终答标记判断步骤。""" return is_final_thought(step.thought, self.final_answer_markers) def tool_is_search(self, name: str) -> bool: + """按当前分析器搜索工具配置判断名称。""" return is_search_tool( name, substrings=self.search_tool_substrings, diff --git a/trace_debugger/episode.py b/trace_debugger/episode.py index 3187f30..b0d508d 100644 --- a/trace_debugger/episode.py +++ b/trace_debugger/episode.py @@ -14,6 +14,8 @@ @dataclass(frozen=True) class ImportedEpisode: + """从跨框架 EvaluationEpisode 提取的只读调试证据。""" + episode_id: str framework: str agent_version: str diff --git a/trace_debugger/golden.py b/trace_debugger/golden.py index 61486c2..cc641f2 100644 --- a/trace_debugger/golden.py +++ b/trace_debugger/golden.py @@ -14,6 +14,8 @@ @dataclass class GoldenCase: + """一个轨迹夹具及其必须命中、不得命中的失败断言。""" + id: str file: str split: str @@ -25,6 +27,7 @@ class GoldenCase: @classmethod def from_dict(cls, data: dict[str, Any]) -> "GoldenCase": + """从 manifest 条目构建用例,缺省值保持旧清单兼容。""" return cls( id=data["id"], file=data["file"], @@ -39,12 +42,15 @@ def from_dict(cls, data: dict[str, Any]) -> "GoldenCase": @dataclass class GoldenManifest: + """带 schema 版本的失败回归集清单。""" + schema_version: str description: str cases: list[GoldenCase] @classmethod def load(cls, path: Optional[Path] = None) -> "GoldenManifest": + """从指定路径或内置夹具目录加载清单。""" root = path or (DEFAULT_GOLDEN_DIR / "manifest.json") with open(root, encoding="utf-8") as f: data = json.load(f) @@ -56,11 +62,13 @@ def load(cls, path: Optional[Path] = None) -> "GoldenManifest": def load_manifest(manifest_path: Optional[str] = None) -> GoldenManifest: + """加载默认或显式指定的 Golden 清单。""" path = Path(manifest_path) if manifest_path else DEFAULT_GOLDEN_DIR / "manifest.json" return GoldenManifest.load(path) def analyze_case(case: GoldenCase, *, golden_dir: Optional[Path] = None) -> TrajectoryAnalysis: + """加载一个 Golden 轨迹并运行默认分析器。""" base = golden_dir or DEFAULT_GOLDEN_DIR return Analyzer().analyze(load(str(base / case.file))) diff --git a/trace_debugger/harness.py b/trace_debugger/harness.py index 1ada249..e2f166a 100644 --- a/trace_debugger/harness.py +++ b/trace_debugger/harness.py @@ -51,6 +51,7 @@ class StepEvent: error_message: str = "" def tool_args_str(self) -> str: + """将工具参数序列化为 Format B 使用的字符串。""" return normalize_tool_input(self.tool_input) @@ -210,6 +211,7 @@ def finish( total_duration: float = 0.0, metadata: Optional[dict[str, Any]] = None, ) -> TrajectoryAnalysis: + """结束运行,记录路径级失败并返回完整轨迹分析。""" return self._watcher.on_finish( final_answer=final_answer, total_duration=total_duration, @@ -217,10 +219,12 @@ def finish( ) def trajectory_dict(self) -> dict[str, Any]: + """返回当前运行的 Format B 轨迹快照。""" data = self._watcher.to_trajectory_dict() data.setdefault("timestamp", self.context.timestamp) return data @property def record_path(self) -> str: + """返回 FailureHarness 实际使用的失败事件文件。""" return self._watcher.record_path diff --git a/trace_debugger/reader.py b/trace_debugger/reader.py index 17fd5fa..90a471a 100644 --- a/trace_debugger/reader.py +++ b/trace_debugger/reader.py @@ -24,18 +24,22 @@ class Step: @property def is_action(self) -> bool: + """返回该步是否包含工具动作。""" return bool(self.action_name) @property def is_final(self) -> bool: + """按兼容格式中的 FINAL ANSWER 标记识别终答步骤。""" return "FINAL ANSWER" in self.thought.upper() @property def is_thought(self) -> bool: + """返回该步是否为非终答的有效思考。""" return bool(self.thought.strip()) and not self.is_final @property def summary(self) -> str: + """生成用于终端和报告的截断单行摘要。""" if self.is_final: return f"输出答案: {self.thought[:80]}" if self.is_action: @@ -59,18 +63,22 @@ class Path: @property def num_steps(self) -> int: + """返回路径中的步骤数。""" return len(self.steps) @property def tools_used(self) -> list[str]: + """按调用顺序返回工具名,保留重复调用。""" return [s.action_name for s in self.steps if s.action_name] @property def has_errors(self) -> bool: + """返回路径是否含解析器识别出的工具错误。""" return any(s.has_error for s in self.steps) @property def error_summary(self) -> list[str]: + """返回路径内各错误步骤的截断消息。""" return [s.error_message for s in self.steps if s.has_error] @@ -89,14 +97,17 @@ class Trajectory: @property def num_steps(self) -> int: + """返回顶层轨迹步骤数。""" return len(self.steps) @property def num_paths(self) -> int: + """返回解析后的执行路径数。""" return len(self.paths) @property def main_path(self) -> Optional[Path]: + """返回显式主路径;旧数据未标记时回退到最后一条。""" for p in self.paths: if p.is_main_path: return p @@ -104,6 +115,7 @@ def main_path(self) -> Optional[Path]: @property def failed_paths(self) -> list[Path]: + """返回失败的非主路径,避免把最终输出路径重复计为失败分支。""" return [p for p in self.paths if not p.success and not p.is_main_path] diff --git a/trace_debugger/record.py b/trace_debugger/record.py index d850d38..0b5597b 100644 --- a/trace_debugger/record.py +++ b/trace_debugger/record.py @@ -50,10 +50,12 @@ def _preview(text: str, limit: int = 160) -> str: def failure_label(failure_type: str) -> str: + """返回稳定失败类型对应的展示标签。""" return FailureType.LABELS.get(failure_type, failure_type) def failure_severity(failure_type: str, *, event_type: str = "step_failure") -> str: + """将失败类型映射为报告严重级别,不改变分类结果。""" if failure_type in (FailureType.NO_FINAL_ANSWER, FailureType.LLM_OFFTRACK): return "fail" if failure_type in (FailureType.CONTEXT_OVERFLOW, FailureType.SEARCH_TIMEOUT): @@ -70,6 +72,7 @@ def build_failure_summary( action: str = "", event_type: str = "step_failure", ) -> str: + """生成包含失败类型、步骤和动作的单行摘要。""" label = failure_label(failure_type) if event_type == "path_failure": return f"路径级 · {label}" @@ -88,6 +91,7 @@ def build_failure_context( observation: str = "", duration: float = 0.0, ) -> dict[str, Any]: + """提取用于定位失败的最小轨迹上下文。""" ctx: dict[str, Any] = {} if action: ctx["action"] = action @@ -163,11 +167,13 @@ def format_event_readable(ev: dict[str, Any]) -> str: def readable_log_path(record_path: str) -> Path: + """返回机器可读 JSONL 对应的人类可读日志路径。""" p = Path(record_path) return p.with_name(p.stem + ".log") def session_summary_path(record_path: str, session_id: str) -> Path: + """返回指定会话的独立摘要文件路径。""" base = Path(record_path).parent / "sessions" safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in session_id) return base / f"{safe}.md" @@ -249,6 +255,7 @@ def write_session_summary( def load_failure_events(record_path: str, *, session_id: Optional[str] = None) -> list[dict[str, Any]]: + """按写入顺序读取有效事件,并可限定到单个会话。""" path = Path(record_path) if not path.is_file(): return [] @@ -606,6 +613,7 @@ def build_scan_snapshot( def load_snapshot(path: str) -> dict[str, Any]: + """加载一次扫描快照,供回归对比使用。""" with open(path, encoding="utf-8") as f: return json.load(f) diff --git a/trace_debugger/validate.py b/trace_debugger/validate.py index 75845fe..e0c9b05 100644 --- a/trace_debugger/validate.py +++ b/trace_debugger/validate.py @@ -48,6 +48,7 @@ def validate_trajectory_file( use_schema: bool = False, schema_path: Optional[Path] = None, ) -> list[str]: + """加载并校验一个轨迹文件,读取或格式错误均作为消息返回。""" p = Path(path) if not p.is_file(): return [f"file not found: {path}"] @@ -62,6 +63,7 @@ def validate_trajectory_file( def format_validation_report(errors: list[str], *, path: str = "") -> str: + """将校验错误格式化为稳定的终端报告。""" if not errors: prefix = f"{path}: " if path else "" return f"{prefix}OK — Format B validation passed"