From 170de085b161f7cec2891583daf606df9eeadeaa Mon Sep 17 00:00:00 2001 From: Snowson <15168246+Snowson@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:37:11 +0800 Subject: [PATCH] =?UTF-8?q?harness=EF=BC=9A=E5=BC=95=E6=93=8E=E5=8D=87?= =?UTF-8?q?=E7=BA=A7=E5=88=B0=20delivery-harness=20c3524ff=EF=BC=88B46=20P?= =?UTF-8?q?3=20=E5=85=A8=E9=87=8F=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .harness/engine.lock | 4 +- .harness/engine/agents/dispatch.py | 227 ++--- .harness/engine/agents/dispatch_host.py | 2 + .../engine/agents/dispatch_observation.py | 5 +- .harness/engine/agents/github.py | 149 ++++ .harness/engine/agents/plan_review.py | 160 ++++ .harness/engine/agents/review.py | 5 +- .harness/engine/agents/review_pack.py | 22 +- .harness/engine/agents/review_pack_io.py | 66 ++ .harness/engine/agents/run_timeline.py | 230 +++-- .harness/engine/cli.py | 5 + .harness/engine/core/alerts.py | 181 ++++ .harness/engine/core/events.py | 55 +- .harness/engine/core/events_db.py | 355 ++++++++ .harness/engine/core/events_io.py | 797 ++++++++++++++++++ .harness/engine/prompts/review_prompt.md | 2 +- .harness/engine/reports/ci_events.py | 191 +++++ .harness/engine/reports/github_events.py | 386 +++++++++ .harness/engine/reports/ledger.py | 560 ++++++++++++ .harness/engine/reports/trace.py | 455 ++++++++++ .harness/engine/routing/run_check.py | 15 +- 21 files changed, 3633 insertions(+), 239 deletions(-) create mode 100644 .harness/engine/agents/github.py create mode 100644 .harness/engine/agents/plan_review.py create mode 100644 .harness/engine/agents/review_pack_io.py create mode 100644 .harness/engine/core/alerts.py create mode 100644 .harness/engine/core/events_io.py create mode 100644 .harness/engine/reports/ci_events.py create mode 100644 .harness/engine/reports/github_events.py create mode 100644 .harness/engine/reports/ledger.py create mode 100644 .harness/engine/reports/trace.py diff --git a/.harness/engine.lock b/.harness/engine.lock index 250c944..104e996 100644 --- a/.harness/engine.lock +++ b/.harness/engine.lock @@ -1,6 +1,6 @@ { "engine": "delivery-harness", "version": "0.1.0", - "commit": "d253875b40fd2907f25732890727a092e63febcc", - "tree": "sha256:1ed5c4fd917aaf55c23baa290bd5fa3d24ee9ac57929f1f12d8d067e2c5d7f49" + "commit": "a1f73548fd6b61dcac7fe670233b152c3cd1c88a", + "tree": "sha256:6f39a109a94ae624c5c9cfbcc20f215bc8829069079d2f1237c768cb39eec8a6" } diff --git a/.harness/engine/agents/dispatch.py b/.harness/engine/agents/dispatch.py index 3db56dc..1eabab6 100644 --- a/.harness/engine/agents/dispatch.py +++ b/.harness/engine/agents/dispatch.py @@ -43,13 +43,13 @@ from engine.agents import dispatch_host, run_timeline from engine.agents import dispatch_observation as observation +from engine.agents.github import GitHub from engine.checks import acceptance, taskbook -from engine.core.common import ENGINE_DIR, ROOT, ci_workflows, git, load_rules, setting +from engine.core import alerts +from engine.core.common import ENGINE_DIR, ROOT, git, load_rules, setting PROMPT_TEMPLATE = ENGINE_DIR / "prompts" / "dispatch_prompt.md" ESCALATION_FILE = "build/dispatch/escalation.md" -CI_QUERY_ATTEMPTS = 3 # 等 CI 时对 gh 查询的最多尝试次数 -CI_QUERY_RETRY_SECONDS = 5 def agent_identity() -> tuple[str, str]: @@ -206,6 +206,85 @@ def release_slot(root: Path, index: int) -> None: (state_dir(root) / "slots" / f"{index}.json").unlink(missing_ok=True) +def _registered_worktrees(root: Path) -> set[str]: + """登记在案的工作树绝对路径（porcelain 输出是规范化路径，按 resolve 后比对，免符号路径误差）。""" + out = git("worktree", "list", "--porcelain", cwd=root) + return {str(Path(line[len("worktree "):]).resolve()) + for line in out.splitlines() if line.startswith("worktree ")} + + +def _salvage_and_remove(root: Path, slot: Path, push) -> None: + """归还一个槽位工作树（B77）：分支有未推提交先推送保全，再 worktree remove。 + + 不是登记在案的工作树（普通残留目录）不动；推送/取远端状态失败原样抛出，由调用方决定 + 停止派发（回收路径）或提示后继续（结束路径）。 + """ + if not slot.exists() or str(slot.resolve()) not in _registered_worktrees(root): + return + branch = git("rev-parse", "--abbrev-ref", "HEAD", cwd=slot, check=False) + if branch and branch != "HEAD": + # fetch 失败（远端无该分支——认领推送前崩溃/推送静默失败）不拦回收：直接尝试推送保全 + try: + git("fetch", "--quiet", "origin", branch, cwd=slot) + except RuntimeError: + pass + pushed = git("rev-parse", "--verify", "--quiet", f"refs/remotes/origin/{branch}", + cwd=slot, check=False) + if not pushed or git("rev-list", f"origin/{branch}..{branch}", cwd=slot).split(): + push(slot, branch) + git("worktree", "remove", "--force", str(slot), cwd=root) + + +def reclaim_stale_slots(root: Path, config: Config, push) -> list[str]: + """崩溃路径无法归还时的槽位回收（B77）：锁文件 pid 不存活的槽在下次派发前强制回收。 + + 槽内分支有未推提交先推送保全再删工作树；推送失败停止派发（保数据优先，不静默丢提交）， + 现场保留待人工处理。返回被回收槽位曾认领的分支名（链路自愈提示用）。 + """ + locks = state_dir(root) / "slots" + if not locks.is_dir(): + return [] + reclaimed: list[str] = [] + for lock in sorted(locks.glob("*.json"), key=lambda path: int(path.stem) if path.stem.isdigit() else 0): + if not lock.stem.isdigit(): + continue + try: + data = json.loads(lock.read_text() or "{}") + except ValueError: + data = {} + if _alive(data.get("pid", 0)): + continue + try: + _salvage_and_remove(root, slot_path(root, config, int(lock.stem)), push) + except (RuntimeError, subprocess.CalledProcessError) as error: + raise Stop(f"槽位 {lock.stem} 回收失败（分支提交保全未完成，工作树保留待人工处理）：{error}") from error + reclaimed.append(str(data.get("branch") or "")) + lock.unlink(missing_ok=True) + return reclaimed + + +def return_slot(root: Path, config: Config, index: int, push) -> None: + """派发结束路径统一归还槽位（B77）：worktree remove + 锁清理。 + + run 结束时锁已先释放；归还前先重新独占槽位锁——拿不到说明已有下一次派发认领该槽， + 不删它正准备使用的工作树。删除失败只提示，残留交给下次派发的回收路径。 + """ + lock = state_dir(root) / "slots" / f"{index}.json" + lock.parent.mkdir(parents=True, exist_ok=True) + try: + fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + return + with os.fdopen(fd, "w") as handle: + json.dump({"pid": os.getpid(), "task": "return", "started_at": _now()}, handle) + try: + _salvage_and_remove(root, slot_path(root, config, index), push) + except (RuntimeError, subprocess.CalledProcessError) as error: + print(f"槽位 {index} 归还未完成（工作树保留，下次派发会回收）：{error}", file=sys.stderr) + finally: + release_slot(root, index) + + def prepare_slot(root: Path, slot: Path, branch: str, resume: bool) -> None: if not slot.exists(): git("worktree", "add", "--detach", str(slot), "origin/main", cwd=root) @@ -312,6 +391,7 @@ def __init__(self, root: Path, config: Config, github, host, identity: dict[str, } self.identity = identity self.used_seconds = 0.0 + self.acquired: int | None = None # 已认领的槽位号；归还（return_slot）由派发进程结束路径统一做（B77） def run_executor(self, task: Task, slot: Path, guard: Path, prompt: str, events: Path) -> dispatch_host.RunResult: budget_seconds = task.budget["wall_clock_min"] * 60 @@ -330,12 +410,14 @@ def run_executor(self, task: Task, slot: Path, guard: Path, prompt: str, events: result = dispatch_host.RunResult(exit_kind, code, seconds, model, usage, denials, missing_context=context["items"], missing_context_status=context["status"], + guard_denied=counts["guard_denied"], guard_allowed=counts["guard_allowed"]) observation.executor_round(task.branch, slot, prompt, events, result, {"name": self.host.name, "version": self.host.version()}, counts) return result - def local_rounds(self, task: Task, slot: Path, guard: Path, feedback: str, trace_dir: Path) -> tuple[Attempt, list[str]]: + def local_rounds(self, task: Task, slot: Path, guard: Path, feedback: str, trace_dir: Path, + pr: int | None = None) -> tuple[Attempt, list[str]]: """5 与 6：执行方 → verify，失败带摘要重试；返回本次尝试与每轮提示词。""" attempt, prompts = Attempt(ok=False, exit="retries"), [] base = git("rev-parse", "HEAD", cwd=slot) @@ -345,6 +427,7 @@ def local_rounds(self, task: Task, slot: Path, guard: Path, feedback: str, trace prompt = render_prompt(task, task.budget["wall_clock_min"], feedback) prompts.append(prompt) result = self.run_executor(task, slot, guard, prompt, trace_dir / f"round-{round_no + 1}.jsonl") + alerts.guard_round_alert(task.branch, task.id, result.guard_denied, pr=pr, gh=self.github) attempt.executor_seconds += result.seconds attempt.model = result.model or attempt.model for key, value in result.usage.items(): @@ -391,7 +474,8 @@ def write_record(self, task: Task, slot: Path, number: int, attempt: Attempt, pr guard_ref: str, started: str, ci_rounds: int) -> str: folder = slot / "docs" / "runs" / task.path.split("/")[-1].removesuffix(".md") folder.mkdir(parents=True, exist_ok=True) - prompt_text = "\n\n---\n\n".join(prompts) + # B77 落盘前自检：prompt 快照的 CI/runner 工作区路径先占位，sha256 按占位后字节计算。 + prompt_text = run_timeline.placeholder_workspace("\n\n---\n\n".join(prompts)) (folder / f"{number}.prompt.md").write_text(prompt_text, encoding="utf-8") record = { "task": task.id, @@ -422,6 +506,7 @@ def write_record(self, task: Task, slot: Path, number: int, attempt: Attempt, pr # T201 时间线：落盘前取本机链头，stages/anchors 安全摘要写进记录；记录提交后固定 run_record 锚点。 timeline, head = run_timeline.record_fields(task.branch) record.update(timeline) + run_timeline.fit_record(record) # B77 自检：记录超 512KB 先截断 stages 并标注，不让卫生守卫在提交步杀派发 (folder / f"{number}.json").write_text(json.dumps(record, ensure_ascii=False, indent=2) + "\n") paths = [str(folder / f"{number}.json"), str(folder / f"{number}.prompt.md")] git("add", "--", *paths, cwd=slot) @@ -439,11 +524,15 @@ def write_record(self, task: Task, slot: Path, number: int, attempt: Attempt, pr def run(self, path: str, resume: bool = False) -> int: task = admit(path, self.root) + reclaimed = reclaim_stale_slots(self.root, self.config, self.github.push) + if resume and task.branch in reclaimed: + self._chain_hint(task.branch) if self.github.remote_branch_exists(task.branch) and not resume: observation.claim(task.branch, False) raise Stop(f"远端已有 {task.branch}：该任务已被认领（续跑加 --resume）") guard, guard_ref = prepare_guard(self.root, task.branch) index, slot = acquire_slot(self.root, self.config, task) + self.acquired = index try: prepare_slot(self.root, slot, task.branch, resume) observation.slot(task.branch, index, git("rev-parse", "HEAD", cwd=slot), @@ -455,6 +544,13 @@ def run(self, path: str, resume: bool = False) -> int: finally: release_slot(self.root, index) + def _chain_hint(self, branch: str) -> None: + """等待链自愈提示（B77）：上一轮派发进程异常退出（槽位锁 pid 不存活）、分支 head 已保全到远端时， + 提示设计方核对 CI 与评审结论；不自动重跑评审，缺链由设计方手动接链。""" + print(f"链路自检：上一轮派发异常退出，分支 {branch} 的 head 已在远端；若该 head 没有 CI 结论或" + "独立评审结论，请设计方手动接链（重跑 CI 或 bin/dispatch review ），评审不会自动重跑。", + file=sys.stderr) + def _loop(self, task: Task, index: int, slot: Path, guard: Path, guard_ref: str) -> int: runs = state_dir(self.root) / "runs" / task.id folder = task.path.split("/")[-1].removesuffix(".md") @@ -464,12 +560,18 @@ def _loop(self, task: Task, index: int, slot: Path, guard: Path, guard_ref: str) while True: started = _now() update_slot(self.root, index, attempt=number) - attempt, prompts = self.local_rounds(task, slot, guard, feedback, runs / str(number)) + attempt, prompts = self.local_rounds(task, slot, guard, feedback, runs / str(number), pr=pr) prompt_sha = self.write_record(task, slot, number, attempt, prompts, guard_ref, started, ci_rounds) self.github.push(slot, task.branch) if not attempt.ok: self.escalate(task, pr, attempt, f"本地未完成（{EXIT_TEXT.get(attempt.exit, attempt.exit)}）") return 1 + if pr is None: + # B70：开新 PR 前先查该分支是否已有开放 PR（resume 时上一轮已开出），有则复用编号， + # 后续 CI 轮次反馈照旧注入既有 PR；查询为空或适配器没有这个查询（既有桩）才走原新建路径。 + lookup = getattr(self.github, "existing_pr", None) + if lookup is not None: + pr = lookup(task.branch) if pr is None: body = pr_body(task, attempt, number, prompt_sha, root=self.root) pr = self.github.open_pr(slot, task.branch, _pr_title(self.root, task), body) @@ -477,6 +579,7 @@ def _loop(self, task: Task, index: int, slot: Path, guard: Path, guard_ref: str) ci_rounds += 1 head = git("rev-parse", "HEAD", cwd=slot) detail: dict = {} + alerts.last_round_alert(task.branch, task.id, ci_rounds, task.budget["ci_rounds"], pr=pr, gh=self.github) ok, summary = self.github.wait_ci(task.branch, head, self.config.ci_timeout_seconds, detail) observation.ci_wait(task.branch, pr, head, ci_rounds, ok, summary, detail.get("run_ids")) if ok: @@ -490,7 +593,7 @@ def _loop(self, task: Task, index: int, slot: Path, guard: Path, guard_ref: str) feedback, number = summary, number + 1 def escalate(self, task: Task, pr: int | None, attempt: Attempt, reason: str) -> None: - body = escalation_body(task, attempt, reason) + body = escalation_body(task, attempt, reason, pr=pr) target = "pr" if pr is not None else "issue" sent = False try: @@ -558,7 +661,7 @@ def pr_body(task: Task, attempt: Attempt, number: int, prompt_sha: str, root: Pa ]) + "\n" -def escalation_body(task: Task, attempt: Attempt, reason: str) -> str: +def escalation_body(task: Task, attempt: Attempt, reason: str, pr: int | None = None) -> str: lines = [ f"### 升级：{task.id}（{reason}）", "", f"- **当前状态与目标差距**：任务书 `{task.path}`，分支 `{task.branch}`；退出方式：{EXIT_TEXT.get(attempt.exit, attempt.exit)}。", @@ -571,6 +674,7 @@ def escalation_body(task: Task, attempt: Attempt, reason: str) -> str: lines += ["#### 执行方的说明（`build/dispatch/escalation.md`）", "", attempt.executor_note, ""] else: lines += ["- **可选方案与推荐**、**需要决定的问题**：执行方未提供，由设计方判断。", ""] + lines += alerts.timeline_lines(task.branch, pr) # T205：trace、安全阶段摘要与时间线入口 lines += ["完整事件流在派发机器本地（git 公共目录下 `dispatch/runs/`），不入库。"] return "\n".join(lines) + "\n" @@ -579,102 +683,6 @@ def _now() -> str: return dt.datetime.now(dt.UTC).isoformat(timespec="seconds") -# ---- GitHub（推送与写操作以 Agent 身份经 bin/as-agent） ---- - -class GitHub: - def __init__(self, root: Path = ROOT): - self.root = root - self.as_agent = [str(root / "bin" / "as-agent")] - - def _run(self, argv: list[str], cwd: Path | None = None, agent: bool = False, stdin: str | None = None) -> str: - result = subprocess.run((self.as_agent if agent else []) + argv, cwd=cwd or self.root, input=stdin, - capture_output=True, text=True, check=False) - if result.returncode != 0: - raise RuntimeError(f"{' '.join(argv[:3])} 失败：{result.stderr.strip()[-500:]}") - return result.stdout.strip() - - def remote_branch_exists(self, branch: str) -> bool: - return bool(self._run(["git", "ls-remote", "--heads", "origin", branch])) - - def push(self, slot: Path, branch: str) -> None: - self._run(["git", "push", "--quiet", "-u", "origin", branch], cwd=slot, agent=True) - - # 正文一律经 stdin（--body-file -），不进命令行（规范 §2：不在引号里写 Markdown）。 - def open_pr(self, slot: Path, branch: str, title: str, body: str) -> int: - url = self._run(["gh", "pr", "create", "--base", "main", "--head", branch, "--title", title, - "--body-file", "-"], cwd=slot, agent=True, stdin=body) - return int(url.rstrip("/").rsplit("/", 1)[-1]) - - def comment(self, pr: int, body: str, label: str | None = None) -> str: - """评论 PR 并返回评论 URL（供观察事件记录；标签逻辑不变）。""" - url = self._run(["gh", "pr", "comment", str(pr), "--body-file", "-"], agent=True, stdin=body) - if label: - self.add_label(pr, label) - return url - - def _ensure_label(self, label: str) -> None: - """不存在才创建；不用 --force，已有标签（含登记用标签）不被改写。""" - try: - self._run(["gh", "label", "create", label, "--color", "d93f0b"], agent=True) - except RuntimeError: - pass # 已存在 - - def add_label(self, pr: int, label: str) -> None: - self._ensure_label(label) - self._run(["gh", "pr", "edit", str(pr), "--add-label", label], agent=True) - - def create_issue(self, title: str, body: str, labels: list[str]) -> None: - for label in labels: - self._ensure_label(label) - argv = ["gh", "issue", "create", "--title", title, "--body-file", "-"] - for label in labels: - argv += ["--label", label] - self._run(argv, agent=True, stdin=body) - - def _ci_runs(self, workflow: str, branch: str) -> list[dict]: - """查一个工作流在该分支上的运行。gh 瞬时失败（如网络断开）连续 CI_QUERY_ATTEMPTS 次才抛出，避免一次瞬断中止整个派发。""" - for attempt in range(1, CI_QUERY_ATTEMPTS + 1): - try: - return json.loads(self._run(["gh", "run", "list", "--workflow", workflow, "--branch", branch, - "--json", "headSha,status,conclusion,databaseId,url", "--limit", "10"])) - except (RuntimeError, json.JSONDecodeError): - if attempt == CI_QUERY_ATTEMPTS: - raise - time.sleep(CI_QUERY_RETRY_SECONDS) - return [] - - def wait_ci(self, branch: str, sha: str, timeout: float, detail: dict | None = None) -> tuple[bool, str]: - """等 rules.toml [dispatch] ci_workflows 列出的每个工作流在该提交上跑完；全部成功才算通过。 - - detail 非 None 时写入 run_ids（本次结论对应的 Actions 运行 ID），供派发的 ci_wait 观察事件 - 使用；返回值与 gh 调用序列保持不变。 - """ - workflows = ci_workflows() - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - found = {} - for workflow in workflows: - runs = self._ci_runs(workflow, branch) - run = next((item for item in runs if item["headSha"] == sha), None) - if run and run["status"] == "completed": - found[workflow] = run - failed = next((run for run in found.values() if run["conclusion"] != "success"), None) - if failed: - log = subprocess.run(["gh", "run", "view", str(failed["databaseId"]), "--log-failed"], cwd=self.root, - capture_output=True, text=True, check=False).stdout - if detail is not None: - detail["run_ids"] = [failed["databaseId"]] - return False, f"CI 未通过：{failed['url']}\n\n```\n{log[-3000:]}\n```" - if len(found) == len(workflows): - if detail is not None: - detail["run_ids"] = sorted(run["databaseId"] for run in found.values()) - return True, "" - time.sleep(30) - if detail is not None: - detail["run_ids"] = [] - return False, f"等待 CI 超过 {int(timeout / 60)} 分钟" - - # ---- 状态与停机 ---- def status(root: Path = ROOT) -> int: @@ -761,11 +769,20 @@ def main(argv: list[str] | None = None) -> int: if (state_dir() / "stop").exists(): print("停机标记存在（bin/dispatch stop），不派发；恢复前删除该标记") return 1 + return _run_dispatch(args) + + +def _run_dispatch(args) -> int: + """run 子命令：派发并在结束路径统一归还槽位（B77）——成功、升级、预算耗尽与异常都覆盖。""" + runner = Dispatcher(ROOT, Config.load(), GitHub(), dispatch_host.PiHost(args.model)) try: - return Dispatcher(ROOT, Config.load(), GitHub(), dispatch_host.PiHost(args.model)).run(args.taskbook, args.resume) + return runner.run(args.taskbook, args.resume) except Stop as stop: print(f"未派发：{stop}") return 2 + finally: + if runner.acquired is not None: + return_slot(ROOT, runner.config, runner.acquired, runner.github.push) if __name__ == "__main__": diff --git a/.harness/engine/agents/dispatch_host.py b/.harness/engine/agents/dispatch_host.py index 2295fda..3f78891 100644 --- a/.harness/engine/agents/dispatch_host.py +++ b/.harness/engine/agents/dispatch_host.py @@ -50,6 +50,8 @@ class RunResult: # guard_allowed 前（T105 冻结断言要求它仍是末位字段），构造处用关键字传参。 missing_context: list = field(default_factory=list) missing_context_status: str = "unknown" + # T205：本轮被守卫拒绝的工具调用数（一次调用多个拒绝理由算一次），同样插在 guard_allowed 前。 + guard_denied: int = 0 guard_allowed: int = 0 # 守卫放行的工具调用数（含普通失败；被守卫拒绝的调用不计入） diff --git a/.harness/engine/agents/dispatch_observation.py b/.harness/engine/agents/dispatch_observation.py index f69f30a..77f5fc1 100644 --- a/.harness/engine/agents/dispatch_observation.py +++ b/.harness/engine/agents/dispatch_observation.py @@ -288,12 +288,15 @@ def review_materials(workspace: Path, base: str, head: str, taskbook_rel: str | """C6 materials：task/diff/ci/pr/pack 五类，recipe 用固定短 ID 与参数，不存命令正文。 无任务书时 task 材料是现有的「无」哨兵字节，ref 用 none 哨兵；单份材料读取失败只少记这一份。 + diff 按分片集合读取（T124：build/review/ 下全部 diff*.patch，与 pack 材料清单同一口径）： + 小 diff 仍是单份材料，分片时每片各记一份、引用同一比较范围。 """ folder = workspace / "build" / "review" + diffs = sorted(folder.glob("diff*.patch")) if folder.is_dir() else [] materials = [ _material("task", folder / "task.md", f"{taskbook_rel}@{head}" if taskbook_rel else "none", {"id": "task_v1"}), - _material("diff", folder / "diff.patch", f"{base}...{head}", {"id": "diff_v1"}), + *(_material("diff", path, f"{base}...{head}", {"id": "diff_v1"}) for path in diffs), _material("ci", folder / "ci.md", f"pull/{pr}/checks", {"id": "ci_v1"}), _material("pr", folder / "pr.md", f"pull/{pr}", {"id": "pr_v1"}), _material("pack", folder / "pack.md", "", {"id": "pack_v1", "engine_ref": _engine_ref(workspace), diff --git a/.harness/engine/agents/github.py b/.harness/engine/agents/github.py new file mode 100644 index 0000000..cc8162b --- /dev/null +++ b/.harness/engine/agents/github.py @@ -0,0 +1,149 @@ +"""GitHub 封装：推送与写操作以 Agent 身份经 bin/as-agent（T123 自 dispatch.py 逐字抽离，行为不变）。""" + +from __future__ import annotations + +import json +import subprocess +import time +from pathlib import Path + +from engine.core.common import ROOT, ci_workflows + +CI_QUERY_ATTEMPTS = 3 # 等 CI 时对 gh 查询的最多尝试次数 +CI_QUERY_RETRY_SECONDS = 5 + + +# ---- GitHub（推送与写操作以 Agent 身份经 bin/as-agent） ---- + +class GitHub: + def __init__(self, root: Path = ROOT): + self.root = root + self.as_agent = [str(root / "bin" / "as-agent")] + + def _run(self, argv: list[str], cwd: Path | None = None, agent: bool = False, stdin: str | None = None) -> str: + result = subprocess.run((self.as_agent if agent else []) + argv, cwd=cwd or self.root, input=stdin, + capture_output=True, text=True, check=False) + if result.returncode != 0: + raise RuntimeError(f"{' '.join(argv[:3])} 失败：{result.stderr.strip()[-500:]}") + return result.stdout.strip() + + def remote_branch_exists(self, branch: str) -> bool: + return bool(self._run(["git", "ls-remote", "--heads", "origin", branch])) + + def push(self, slot: Path, branch: str) -> None: + self._run(["git", "push", "--quiet", "-u", "origin", branch], cwd=slot, agent=True) + + # 正文一律经 stdin（--body-file -），不进命令行（规范 §2：不在引号里写 Markdown）。 + def open_pr(self, slot: Path, branch: str, title: str, body: str) -> int: + url = self._run(["gh", "pr", "create", "--base", "main", "--head", branch, "--title", title, + "--body-file", "-"], cwd=slot, agent=True, stdin=body) + return int(url.rstrip("/").rsplit("/", 1)[-1]) + + def existing_pr(self, branch: str) -> int | None: + """分支已有的开放 PR 编号，没有则 None（B70）：开新 PR 前查询，已有则复用编号不再新建。 + 查询失败（如网络断开）按「无既有 PR」处理，保留原新建路径。""" + try: + raw = self._run(["gh", "pr", "list", "--head", branch, "--state", "open", + "--json", "number", "--limit", "1"]) + return int(json.loads(raw or "[]")[0]["number"]) + except (RuntimeError, TypeError, ValueError, KeyError, IndexError): + return None + + def comment(self, pr: int, body: str, label: str | None = None) -> str: + """评论 PR 并返回评论 URL（供观察事件记录；标签逻辑不变）。""" + url = self._run(["gh", "pr", "comment", str(pr), "--body-file", "-"], agent=True, stdin=body) + if label: + self.add_label(pr, label) + return url + + def _ensure_label(self, label: str) -> None: + """不存在才创建；不用 --force，已有标签（含登记用标签）不被改写。""" + try: + self._run(["gh", "label", "create", label, "--color", "d93f0b"], agent=True) + except RuntimeError: + pass # 已存在 + + def add_label(self, pr: int, label: str) -> None: + self._ensure_label(label) + self._run(["gh", "pr", "edit", str(pr), "--add-label", label], agent=True) + + def create_issue(self, title: str, body: str, labels: list[str]) -> None: + for label in labels: + self._ensure_label(label) + argv = ["gh", "issue", "create", "--title", title, "--body-file", "-"] + for label in labels: + argv += ["--label", label] + self._run(argv, agent=True, stdin=body) + + def list_comments(self, pr: int) -> list[dict]: + """分页列出 PR 评论（id 与正文），供告警远端标记去重（B46 T205）。""" + raw = self._run(["gh", "api", f"repos/{{owner}}/{{repo}}/issues/{pr}/comments", "--paginate", "--slurp"]) + return [item for page in json.loads(raw or "[]") for item in page] + + def edit_comment(self, comment_id: int, body: str) -> None: + self._run(["gh", "api", "-X", "PATCH", f"repos/{{owner}}/{{repo}}/issues/comments/{comment_id}", + "--input", "-"], agent=True, stdin=json.dumps({"body": body})) + + def list_issues(self, label: str) -> list[dict]: + """带标签的全部议题（编号与正文，最多 200 条）：告警无 PR 时按远端标记查升级议题。""" + raw = self._run(["gh", "issue", "list", "--state", "all", "--label", label, "--limit", "200", + "--json", "number,body"]) + return json.loads(raw or "[]") + + def edit_issue(self, number: int, body: str) -> None: + self._run(["gh", "issue", "edit", str(number), "--body-file", "-"], agent=True, stdin=body) + + def _ci_runs(self, workflow: str, branch: str) -> list[dict]: + """查一个工作流在该分支上的运行。gh 瞬时失败（如网络断开）连续 CI_QUERY_ATTEMPTS 次才抛出，避免一次瞬断中止整个派发。""" + for attempt in range(1, CI_QUERY_ATTEMPTS + 1): + try: + return json.loads(self._run(["gh", "run", "list", "--workflow", workflow, "--branch", branch, + "--json", "headSha,status,conclusion,databaseId,url", "--limit", "10"])) + except (RuntimeError, json.JSONDecodeError): + if attempt == CI_QUERY_ATTEMPTS: + raise + time.sleep(CI_QUERY_RETRY_SECONDS) + return [] + + def _completed_runs(self, workflows: list[str], branch: str, sha: str) -> dict[str, dict]: + """一轮轮询：每个工作流各查一次，返回该提交上已完结的工作流 → 运行；查询失败原样抛出。""" + found = {} + for workflow in workflows: + runs = self._ci_runs(workflow, branch) + run = next((item for item in runs if item["headSha"] == sha), None) + if run and run["status"] == "completed": + found[workflow] = run + return found + + def wait_ci(self, branch: str, sha: str, timeout: float, detail: dict | None = None) -> tuple[bool, str]: + """等 rules.toml [dispatch] ci_workflows 列出的每个工作流在该提交上跑完；全部成功才算通过。 + + detail 非 None 时写入 run_ids（本次结论对应的 Actions 运行 ID），供派发的 ci_wait 观察事件 + 使用；返回值与 gh 调用序列保持不变。查询重试耗尽仍失败时按「CI 结果未知」返回，不向上抛 + （B71）：异常出 wait_ci 会崩掉整个派发进程，摘要带已推 head 与失败摘要，随派发既有升级 + 通道（PR 评论或议题）留痕，由人重跑或补查。 + """ + workflows = ci_workflows() + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + found = self._completed_runs(workflows, branch, sha) + except (RuntimeError, json.JSONDecodeError) as error: # B71：重试耗尽仍失败，CI 结果未知；不再上抛崩派发进程 + if detail is not None: + detail["run_ids"] = [] + return False, f"CI 结果未知（已推 head {sha}）：{error}" + failed = next((run for run in found.values() if run["conclusion"] != "success"), None) + if failed: + log = subprocess.run(["gh", "run", "view", str(failed["databaseId"]), "--log-failed"], cwd=self.root, + capture_output=True, text=True, check=False).stdout + if detail is not None: + detail["run_ids"] = [failed["databaseId"]] + return False, f"CI 未通过：{failed['url']}\n\n```\n{log[-3000:]}\n```" + if len(found) == len(workflows): + if detail is not None: + detail["run_ids"] = sorted(run["databaseId"] for run in found.values()) + return True, "" + time.sleep(30) + if detail is not None: + detail["run_ids"] = [] + return False, f"等待 CI 超过 {int(timeout / 60)} 分钟" diff --git a/.harness/engine/agents/plan_review.py b/.harness/engine/agents/plan_review.py new file mode 100644 index 0000000..5fbc78a --- /dev/null +++ b/.harness/engine/agents/plan_review.py @@ -0,0 +1,160 @@ +"""任务拆分评审（B54）：把设计/计划文档与其拆分材料交给独立评审方，按 docs/task-splitting.md 的拆分评审 +检查项（完整性、正确性、可验证、可完成、一致性）评审拆分能否一并提交；报告只写本地 markdown， +不评论任何 GitHub PR。 + +入口经 engine/cli.py 的 COMMANDS 注册表走既有通用路由：python3 <引擎目录>/cli.py review-plan +<设计文档路径> [--output 路径]（报告缺省 build/review/plan-verdict.md，相对仓库根解析）。 +材料以指定文档为主，自动收集它正文链接到的 docs 下本地 markdown（含追溯表、共用合同）与 +docs/plans/ 下全部任务书（含未被正文链接的）；外链、仓库外与非 docs 目标不算材料。复用 +engine/agents/review.py 的材料组装（write_materials 同格式落盘）与评审执行（run_reviewer、 +parse_output）在只读评审工作区运行；结论（含 findings）、材料清单与缺失清单写入报告。设计文档 +不存在时不评审，写缺失报告并返回 1；链接目标缺失进缺失清单并在报告中标注，不静默跳过。 +""" + +from __future__ import annotations + +import argparse +import os +import re +from pathlib import Path + +from engine.agents.review import ( + Reviewer, + Verdict, + _cell, + checkout, + make_reviewer, + review_workspace, + run_reviewer, + write_materials, +) +from engine.core.common import ROOT, git, load_rules + +PLAN_LINK = re.compile(r"\[[^\]]*\]\(([^)\s]+)\)") + + +def _plan_doc_links(plan_text: str, plan_dir: Path, root: Path) -> tuple[list[str], list[str]]: + """plan 正文里链接到的 docs 下本地 markdown（docs/plans/*.md、docs/*.md 与 docs 其余子目录）。 + 返回 (存在的仓库相对路径, 缺失的链接目标)，各自排序去重；外链、仓库外与 docs 外的目标不是材料。""" + found, missing = [], [] + for target in dict.fromkeys(PLAN_LINK.findall(plan_text)): + path = target.split("#", 1)[0] + if not path or "://" in path: + continue + try: + rel = Path(os.path.normpath(plan_dir / path)).relative_to(root).as_posix() + except ValueError: + continue # 解析到仓库外 + if rel.split("/", 1)[0] == "docs" and rel.endswith(".md"): + (found if (root / rel).exists() else missing).append(rel) + return sorted(set(found)), sorted(set(missing)) + + +def collect_plan_materials(plan_doc: Path, root: Path) -> tuple[str, str, list[tuple[str, str]], list[str]]: + """B54 拆分评审材料：以 plan 文档为主材料，收集其正文链接到的 docs 下本地 markdown，加上 docs/plans/ 下 + 全部任务书（含未被正文链接的）。返回 (plan 相对路径, plan 全文, [(仓库相对路径, 全文)], 缺失的链接目标)； + plan 文档不在仓库内或读不到时，前两项为空串、文档本身进缺失清单（不静默跳过）。""" + try: + plan_rel = plan_doc.relative_to(root).as_posix() + plan_text = plan_doc.read_text(encoding="utf-8") + except (ValueError, OSError): + try: + name = plan_doc.relative_to(root).as_posix() + except ValueError: + name = plan_doc.as_posix() + return "", "", [], [name] + linked, missing = _plan_doc_links(plan_text, plan_doc.parent, root) + taskbooks = [path.relative_to(root).as_posix() for path in sorted((root / "docs" / "plans").glob("task-*.md"))] + return plan_rel, plan_text, [(rel, (root / rel).read_text(encoding="utf-8")) + for rel in sorted(set(linked) | set(taskbooks))], missing + + +def render_plan_report(plan_rel: str, materials: list[tuple[str, str]], missing: list[str], verdict: Verdict, + reviewer: str, model: str, seconds: float) -> str: + """拆分评审报告：结论（含 findings）、材料清单与缺失清单（有缺失时在报告中标注）。""" + lines = [ + f"# 拆分评审报告：{plan_rel}", "", + f"- 评审方：{reviewer}（{model or '模型未报告'}）；用时 {seconds / 60:.1f} 分钟", + f"- 材料清单（{len(materials)} 份）：", + *(f" - `{rel}`" for rel, _ in materials), + f"- 缺失清单：{'、'.join(f'`{item}`' for item in missing) or '无'}", + "", f"## 结论：{verdict.verdict}", "", + verdict.summary or "（无）", "", + ] + if verdict.findings: + lines += ["| 严重度 | 位置 | 问题 | 修复要求 |", "|---|---|---|---|"] + lines += [f"| {item.get('severity', '')} | {item.get('location', '')} | {_cell(item.get('problem', ''))} | " + f"{_cell(item.get('fix', ''))} |" for item in verdict.findings] + elif not verdict.failure: + lines.append("没有发现。") + if verdict.failure: + lines += ["", f"评审方失败：{verdict.failure}"] + if missing: + lines += ["", "注意：缺失清单中的链接目标未参与本次评审。"] + return "\n".join(lines) + "\n" + + +def review_plan(plan_doc: Path, output: Path, *, root: Path = ROOT, reviewer: Reviewer | None = None) -> int: + """B54 任务拆分评审：以设计/计划文档为主材料，自动收集其链接的 docs 本地材料与全部任务书，复用独立评审的 + 材料组装（write_materials 同格式落盘）与评审执行（run_reviewer、parse_output）在只读评审工作区评审；结论 + （含 findings）与材料清单写入 output（markdown），不评论任何 GitHub PR。设计文档或链接目标缺失时进缺失 + 清单并在报告中标注；文档本身缺失时不评审，写缺失报告并返回 1。""" + name = load_rules().get("review", {}).get("reviewer") + runner = reviewer or (make_reviewer(name) if name else None) + if runner is None: + print("未能判定评审方：在 rules.toml [review] 配置 reviewer") + return 2 + plan_doc = plan_doc if plan_doc.is_absolute() else root / plan_doc + output = output if output.is_absolute() else root / output + plan_rel, plan_text, materials, missing = collect_plan_materials(plan_doc, root) + if not plan_rel: # 设计文档不存在或不在仓库内：写缺失报告，不评审 + report = render_plan_report(missing[0], [], missing, + Verdict("需用户验收", [], "设计文档不存在，未评审", parsed=False), + runner.name, "", 0.0) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(report, encoding="utf-8") + print(f"拆分评审：设计文档缺失（{missing[0]}），报告已写入 {output}") + return 1 + workspace = review_workspace(root) + git("fetch", "--quiet", "origin", "main", cwd=workspace) + checkout(workspace, "origin/main") + base = git("rev-parse", "origin/main", cwd=workspace) + folder = workspace / "build" / "review" / "materials" + folder.mkdir(parents=True, exist_ok=True) + for rel, text in materials: # 材料全文按仓库相对路径落盘，评审方在只读沙箱里读得到 + target = folder / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + listing = "\n".join(f"- `{rel}`" for rel, _ in materials) or "-（没有收集到任何材料）" + gone = "\n".join(f"- `{item}`" for item in missing) or "-（无）" + pr_text = ( + f"# 拆分评审：{plan_rel}\n\n" + "这不是 PR 评审，而是任务拆分评审：对照 docs/task-splitting.md 的拆分评审检查项（完整性、正确性、" + "可验证、可完成、一致性），依据下面的主材料与关联材料评审这份拆分能否一并提交。\n\n" + f"## 主材料：{plan_rel} 全文\n\n{plan_text}\n\n" + f"## 关联材料（{len(materials)} 份，全文在 build/review/materials/ 下按仓库相对路径存放）\n\n{listing}\n\n" + f"## 缺失的链接目标（未参与评审）\n\n{gone}\n" + ) + write_materials(workspace, base, pr_text, "", "（拆分评审没有 CI 运行）") + verdict, model, seconds = run_reviewer(runner, workspace, load_rules().get("review", {}).get( + "timeout_minutes", 30) * 60) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(render_plan_report(plan_rel, materials, missing, verdict, runner.name, model, seconds), + encoding="utf-8") + if verdict.failure: + print(f"拆分评审：评审方失败（{verdict.failure}），报告已写入 {output}") + return 1 + print(f"拆分评审：{verdict.verdict}（{runner.name}），报告已写入 {output}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + """review-plan 子命令（engine/cli.py COMMANDS 注册表路由）：任务拆分评审，报告写本地 markdown。""" + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("plan_doc", type=Path, help="设计/计划文档路径（相对仓库根）") + parser.add_argument("--output", type=Path, default=Path("build/review/plan-verdict.md"), + help="报告路径（相对仓库根，缺省 build/review/plan-verdict.md）") + args = parser.parse_args(argv) + doc = args.plan_doc if args.plan_doc.is_absolute() else ROOT / args.plan_doc + out = args.output if args.output.is_absolute() else ROOT / args.output + return review_plan(doc, out) diff --git a/.harness/engine/agents/review.py b/.harness/engine/agents/review.py index a72853a..eb1d093 100644 --- a/.harness/engine/agents/review.py +++ b/.harness/engine/agents/review.py @@ -38,7 +38,7 @@ from queue import Queue from typing import ClassVar -from engine.agents import dispatch, dispatch_host +from engine.agents import dispatch, dispatch_host, review_pack_io from engine.agents import dispatch_observation as observation from engine.core.common import ENGINE_DIR, ROOT, changed_files, commit_field, git, load_rules from engine.routing import run_check @@ -330,8 +330,7 @@ def write_materials(workspace: Path, base: str, pr_text: str, task_text: str, ci (folder / "pr.md").write_text(pr_text, encoding="utf-8") (folder / "task.md").write_text(task_text or "无", encoding="utf-8") (folder / "ci.md").write_text(ci_text or "（校准样本没有 CI 运行）", encoding="utf-8") - diff = git("diff", "--no-color", f"{base}...HEAD", cwd=workspace) - (folder / "diff.patch").write_text(diff[:200_000], encoding="utf-8") + review_pack_io.write_diff(workspace, base) # T124：diff 不再 200K 截断，超 2MB 按文件边界分片 # 评审材料用被评审提交自己的引擎生成；迁移前的旧布局提交里引擎在 harness/ 平铺。 entry = workspace / ".harness" / "engine" / "cli.py" command = [str(entry), "review-pack"] if entry.exists() else [str(workspace / "harness" / "review_pack.py")] diff --git a/.harness/engine/agents/review_pack.py b/.harness/engine/agents/review_pack.py index 149d02f..0d1f2fa 100644 --- a/.harness/engine/agents/review_pack.py +++ b/.harness/engine/agents/review_pack.py @@ -3,7 +3,8 @@ bin/harness review-pack --base origin/main [--head HEAD] [--output build/review/pack.md] [--no-run] 内容：改动概要、风险等级（risk.py）、修复证据（evidence.py）、本地 verify 汇总（仅当与当前 head 一致）、 -本次改动涉及的验收编号及其中的人工项、评审清单入口（docs/templates/review-checklist.md）。 +本次改动涉及的验收编号及其中的人工项、评审清单入口（docs/templates/review-checklist.md）、评审 diff +材料清单（T124：build/review/ 下的 diff 文件与行数，分片时在顶部声明无截断）。 """ from __future__ import annotations @@ -55,6 +56,24 @@ def verify_section(head_sha: str, summary_path: Path = VERIFY_SUMMARY) -> list[s return lines +def diff_materials(cwd: Path = ROOT) -> list[str]: + """评审 diff 材料清单（T124，作为 write_materials 落盘 diff 的消费方）：列出 build/review/ 下 + 全部 diff 文件与各自行数；分片时在清单顶部加一行「diff 分 N 片，无截断」。目录里没有 diff + 文件（独立运行、旧引擎组装）时不加这一节。""" + folder = cwd / "build" / "review" + files = sorted(folder.glob("diff*.patch")) if folder.is_dir() else [] + if not files: + return [] + lines = [] + for path in files: + data = path.read_bytes() + count = data.count(b"\n") + (1 if data and not data.endswith(b"\n") else 0) + lines.append(f"- `{path.name}`：{count} 行") + if len(files) > 1: + lines.insert(0, f"diff 分 {len(files)} 片，无截断。") + return [*lines, ""] + + def build(base: str, head: str = "HEAD", run_tests: bool = True, cwd: Path = ROOT) -> str: head_sha = git("rev-parse", head, cwd=cwd) commits = git("rev-list", "--count", f"{base}..{head}", cwd=cwd) @@ -62,6 +81,7 @@ def build(base: str, head: str = "HEAD", run_tests: bool = True, cwd: Path = ROO parts = [ f"# 评审证据包 `{git('rev-parse', '--short', base, cwd=cwd)}` → `{head_sha[:7]}`", "", + *diff_materials(cwd), f"{commits} 个提交；{files or '无改动'}。评审清单见 `docs/templates/review-checklist.md`。", "", risk.render_markdown(risk.classify(base, head, cwd)), diff --git a/.harness/engine/agents/review_pack_io.py b/.harness/engine/agents/review_pack_io.py new file mode 100644 index 0000000..eadd036 --- /dev/null +++ b/.harness/engine/agents/review_pack_io.py @@ -0,0 +1,66 @@ +"""评审 diff 材料的落盘（T124，待办 B74）：去掉 200K 字符静默截断，大 diff 按文件边界分片。 + +总量 ≤ SHARD_LIMIT 时维持旧形态：单个 `build/review/diff.patch`（`git diff --no-color` 加 +git() 的去结尾换行口径不变，只是不再截断）。超过时按文件边界分片为 `diff-01.patch`、 +`diff-02.patch`…：各片按序拼接与完整 diff 逐字节等价，分片只落在文件边界（不拆半个文件）。 +单个文件的 diff 本身超过 SHARD_LIMIT 时独占一片（此时该片必然超上限：文件边界优先于片大小）。 +分片清单由 review_pack 作为消费方在 pack.md 里列出（文件名与行数、分 N 片声明）。 +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from engine.core.common import git + +SHARD_LIMIT = 2 * 1024 * 1024 # 每片上限 2MB；总量以内维持单个 diff.patch +_FILE_HEADER = re.compile(rb"(?m)^diff --git ") + + +def write_diff(workspace: Path, base: str) -> list[Path]: + """把 `base...HEAD` 的评审 diff 写入 workspace 的 build/review/，返回写入的文件（按片序）。 + + 替代 write_materials 原来按 200_000 字符静默截断的 diff 段；写入前清掉目录里不再属于本次 + 结果的旧 diff 文件，保证 pack 的材料清单与磁盘一致（评审工作区跨 PR 复用时尤其如此）。 + """ + diff = git("diff", "--no-color", f"{base}...HEAD", cwd=workspace).encode("utf-8") + folder = workspace / "build" / "review" + folder.mkdir(parents=True, exist_ok=True) + shards = _shards(_file_chunks(diff)) + files = [folder / ("diff.patch" if len(shards) == 1 else f"diff-{index:02d}.patch") + for index in range(1, len(shards) + 1)] + for stale in folder.glob("diff*.patch"): + if stale not in files: + stale.unlink() + for path, shard in zip(files, shards): + path.write_bytes(shard) + return files + + +def _file_chunks(diff: bytes) -> list[bytes]: + """按文件边界切块：每个行首 `diff --git ` 开启一个文件段（diff 的正文行都有 +/空格/- 前缀， + 不会误配）；没有 diff 或只有单个文件段时原样返回一段（空 diff 也写一个空的 diff.patch，与 + 旧行为一致）。""" + starts = [match.start() for match in _FILE_HEADER.finditer(diff)] + if len(starts) <= 1: + return [diff] + if starts[0] > 0: # 理论上不出现的前导内容并入第一段 + starts.insert(0, 0) + return [diff[start:end] for start, end in zip(starts, [*starts[1:], len(diff)])] + + +def _shards(chunks: list[bytes]) -> list[bytes]: + """贪心装片：整段装入，再放下一段会超上限时先封片；单个文件段本身超 SHARD_LIMIT 时独占一片。""" + shards: list[bytes] = [] + current: list[bytes] = [] + size = 0 + for chunk in chunks: + if current and size + len(chunk) > SHARD_LIMIT: + shards.append(b"".join(current)) + current, size = [], 0 + current.append(chunk) + size += len(chunk) + if current: + shards.append(b"".join(current)) + return shards diff --git a/.harness/engine/agents/run_timeline.py b/.harness/engine/agents/run_timeline.py index c4ad6ea..f9df202 100644 --- a/.harness/engine/agents/run_timeline.py +++ b/.harness/engine/agents/run_timeline.py @@ -1,4 +1,4 @@ -"""运行记录时间线与固定链头（B46 T201）：把本机事件链组装成运行记录里的安全摘要（共用合同 C3）。 +"""运行记录时间线与固定链头（B46 T201；B77 T125 收敛为本次 attempt 窗口的安全索引）。 职责边界（任务书 F1、共用合同 C0）：事件由 dispatch_observation（T105）写入，本模块只读——从本机库 取该 trace 已发生的事件，组装 stages/anchors 摘要。dispatch.write_record 在记录落盘前调用 record_fields @@ -6,49 +6,70 @@ 尚未发生的 push_pr/ci_wait，也不为补时间线追加推送。本模块不判定退出/预算/推送，读取或组装失败一律 退化为空摘要（锚点为空），不改变派发的判定、返回码与 gh 调用序列。 -stages 覆盖任务准入至本地检查的各已发生阶段（含更早尝试已发生的 push_pr/ci_wait），每项含 stage/step/ -status/ts/duration_ms/attempt/round/inputs/outputs/decision/actor/source/head_hash；attempt/round 按链内 -步骤推导（executor_round 计轮，ci_wait/escalate 之后进入下一次尝试）。decision.reason 只用稳定结果短 ID -（事件状态或既有 Attempt 退出词），原始理由正文以 sha256 留在 outputs；字符串沿用 emit 的隐私规则 -（超长、形似本机路径、含换行一律丢弃），记录不含原始事件流、完整日志或会话。 +stages 是记录的索引而非事件流（B77 收敛）：只含本次 attempt 窗口的事件（按链内 ci_wait/escalate 边界 +推导 attempt，最后一次边界之后为本窗口），每项只保留 stage/step/status/ts/duration_ms/attempt/round +七个标量；inputs/outputs/decision/actor/source/head_hash 等细节留在事件库。更早 attempt 已发生的 +push_pr/ci_wait 各保留一条指针行（stage/step/status="prior"/attempt），标记链上有过衔接而不重复携带 +历史（resume 多轮曾膨胀至 1.1MB 被卫生守卫拦下）。anchors 不变；保底：组装后 stages 超 256KB 从尾部 +截断并标注 stages_truncated/stages_total（记录 JSON 的 512KB 自检在 dispatch.write_record）。 """ from __future__ import annotations import contextlib -import hashlib import json import re +import sys from engine.core import events, events_db +from engine.core.common import load_rules _SOURCE = "local" # 记录时间线只取本机来源链；CI/GitHub 事实由各自来源记录 _ANCHOR_STAGE = "dispatch" # 锚点在派发写记录时固定 _FIXED_IN = "run_record" -_REASON_HASH_KEY = "decision.reason.sha256" # 原始理由正文的哈希（正文本身不入记录） -_EVENT_COLUMNS = ("id", "hash", "ts", "stage", "step", "status", "duration_ms", "actor_role", - "actor_host", "model", "decision_by", "decision_rule", "decision_reason", - "outputs", "source") -_REF_KEYS = ("kind", "ref", "sha256") -_MAX_STR = 120 -_KEY_RE = re.compile(r"[A-Za-z0-9_.-]{1,64}\Z") -_FORBIDDEN_PREFIXES = ("/Users/", "/home/") -# decision.reason 的稳定结果短 ID：C0 状态之外可用既有 Attempt 退出词（C3 词表，不造同义短 token）。 -_EXIT_WORDS = frozenset(("timeout", "stall", "stopped", "error", "loop", "retries", "clarify")) +_EVENT_COLUMNS = ("hash", "ts", "stage", "step", "status", "duration_ms") +# 单条摘要瘦身（B77）：只保留七个标量字段，记录是索引，细节留在事件库。 +_POINTER_STATUS = "prior" # 更早 attempt 的 push_pr/ci_wait 指针行状态 +_STAGES_BUDGET = 256 * 1024 # 组装后 stages 的保底预算，超出从尾部截断并标注 _ROUND_STEPS = frozenset(("executor_round", "local_verify")) # 尝试边界：CI 等待与升级都意味着上一次尝试已结束，其后的事件属于下一次尝试。 _ATTEMPT_BOUNDARIES = frozenset(("ci_wait", "escalate")) +# 更早 attempt 只保留指针行的衔接步骤：其余事件不重复入记录。 +_POINTER_STEPS = frozenset(("push_pr", "ci_wait")) +# 落盘前自检（B77，dispatch.write_record 调用）：记录 JSON 上限与 CI/runner 工作区路径占位。 +RECORD_MAX_BYTES = 512 * 1024 +# 卫生规则缺失时的内置本机路径模式（与 templates 的 home_path_pattern 同款）。 +_HOME_PATH_PATTERN = (r"(/Users/|/home/|C:\\Users\\)" + r"(?!(x|you|me|user|name|example|test|someone)[/\\])[A-Za-z0-9._-]+[/\\]") def record_fields(trace_id: str) -> tuple[dict, str | None]: - """记录的新增摘要字段（trace_id/stages/anchors）与记录落盘前的本机链头。 + """记录的新增摘要字段（trace_id/stages/anchors，截断时另有标注）与记录落盘前的本机链头。 链头必须在记录落盘前读取：锚点指向已发生事件的前缀，不含尚未发生的 push_pr/ci_wait。 读取失败或尚无事件时 stages/anchors 为空列表、链头为 None（锚点为空），原派发行为不变。 """ stages, head = _read_stages(trace_id) + stages, total = fit_stages(stages, _STAGES_BUDGET) anchors = [{"source": _SOURCE, "stage": _ANCHOR_STAGE, "head_hash": head, "fixed_in": _FIXED_IN}] if head else [] - return {"trace_id": trace_id, "stages": stages, "anchors": anchors}, head + fields: dict = {"trace_id": trace_id, "stages": stages, "anchors": anchors} + if total is not None: + fields["stages_truncated"] = True + fields["stages_total"] = total + return fields, head + + +def fit_stages(stages: list[dict], budget: int) -> tuple[list[dict], int | None]: + """组装后的保底（B77）：stages 序列化超过 budget 字节时从尾部逐条截断。 + + 返回 (保留的条目, 截断前总条数)；未截断时总数为 None。stages_truncated/stages_total 标注由调用方 + 据此写入记录。 + """ + total = len(stages) + kept = stages + while kept and len(json.dumps(kept, ensure_ascii=False).encode("utf-8")) > budget: + kept = kept[:-1] + return kept, (total if len(kept) != total else None) def fix_anchor(trace_id: str, head_hash: str | None) -> None: @@ -57,122 +78,87 @@ def fix_anchor(trace_id: str, head_hash: str | None) -> None: events.set_anchor(trace_id, _ANCHOR_STAGE, head_hash, _FIXED_IN, _SOURCE) +def fit_record(record: dict) -> None: + """记录落盘前自检（B77）：JSON 超 512KB 时从 stages 尾部逐条截断并标注，再提交。 + + stages_total 保持「截断前组装总数」口径（组装侧 256KB 保底已截断时该键已在记录里，不重复累计）。 + 让卫生守卫在记录提交步杀派发的两例（stages 超 1MB）都由组装侧收敛拦在前；这里是最后保底。 + """ + if len(json.dumps(record, ensure_ascii=False, indent=2).encode("utf-8")) <= RECORD_MAX_BYTES: + return + dropped = 0 + while record.get("stages") and len( + json.dumps(record, ensure_ascii=False, indent=2).encode("utf-8")) > RECORD_MAX_BYTES: + record["stages"].pop() + dropped += 1 + if dropped: + record["stages_truncated"] = True + if "stages_total" not in record: + record["stages_total"] = len(record["stages"]) + dropped + detail = f"stages 已截断（保留 {len(record.get('stages') or [])} 条）" if dropped else "无 stages 可截断，按原样提交" + print(f"运行记录自检：记录 JSON 超过 {RECORD_MAX_BYTES // 1024} KB，{detail}。", file=sys.stderr) + + +def placeholder_workspace(text: str) -> str: + """prompt 快照自检（B77）：CI/runner 工作区本机路径按卫生规则占位为 ，守卫不再拦。""" + cleaned, hits = _home_path_re().subn("", text) + if hits: + print(f"运行记录自检：prompt 快照含 {hits} 处 CI/runner 工作区路径，已占位为 。", + file=sys.stderr) + return cleaned + + +def _home_path_re() -> re.Pattern[str]: + """卫生规则的本机路径模式（与守卫同源）；规则缺失或不可读时退回引擎内置同款模式。""" + try: + pattern = load_rules()["hygiene"]["home_path_pattern"] + except Exception: # noqa: BLE001 规则不可得时按内置模式自检，不影响派发 + pattern = _HOME_PATH_PATTERN + return re.compile(pattern) + + def _read_stages(trace_id: str) -> tuple[list[dict], str | None]: - """读取 (local, trace) 的已发生事件并按链序组装摘要；任何失败返回空摘要与 None 链头（观察旁路）。""" + """读取本机链并组装本次 attempt 窗口的安全摘要；任何失败返回空摘要与 None 链头（观察旁路）。 + + attempt 按链内边界推导（ci_wait/escalate 之后进入下一次尝试，边界事件本身属于它结束的尝试）； + 记录只含本次 attempt 的事件，更早 attempt 已发生的 push_pr/ci_wait 按链序各留一条指针行。 + 链头仍取整条链的最后一个事件（anchors 语义不变）。 + """ try: - rows, refs = _read_events(trace_id) + rows = _read_events(trace_id) except Exception: # noqa: BLE001 观察旁路：摘要组装失败不得影响派发 return [], None - items: list[dict] = [] + if not rows: + return [], None attempt, round_no = 1, 0 + numbered: list[tuple[dict, int, int]] = [] for row in rows: if row["step"] == "executor_round": round_no += 1 - outputs = _outputs(row) - items.append({ - "stage": row["stage"], "step": row["step"], "status": row["status"], "ts": row["ts"], - "duration_ms": row["duration_ms"], - "attempt": attempt, "round": round_no if row["step"] in _ROUND_STEPS else 0, - "inputs": refs.get(row["id"], []), "outputs": outputs, - "decision": _decision(row, outputs), "actor": _actor(row), - "source": row["source"], "head_hash": row["hash"], # 各阶段结束时的链头前缀 - }) + numbered.append((row, attempt, round_no if row["step"] in _ROUND_STEPS else 0)) if row["step"] in _ATTEMPT_BOUNDARIES: attempt, round_no = attempt + 1, 0 - return items, (rows[-1]["hash"] if rows else None) - - -def _read_events(trace_id: str) -> tuple[list[dict], dict[int, list[dict]]]: - """按 seq 读出本机链上的事件与输入引用；库缺失抛给调用方按空摘要处理。 - - 连接沿用 events_db 的连接参数（busy_timeout=5000）：库锁竞争或恢复窗口内等待而非立即失败（B69）。 + current = numbered[-1][1] + items: list[dict] = [] + for row, no, rnd in numbered: + if no == current: + items.append({"stage": row["stage"], "step": row["step"], "status": row["status"], + "ts": row["ts"], "duration_ms": row["duration_ms"], "attempt": no, "round": rnd}) + elif row["step"] in _POINTER_STEPS: + items.append({"stage": row["stage"], "step": row["step"], "status": _POINTER_STATUS, + "ts": row["ts"], "duration_ms": row["duration_ms"], + "attempt": no, "round": rnd}) + return items, rows[-1]["hash"] + + +def _read_events(trace_id: str) -> list[dict]: + """按 seq 读出本机链上的事件；库缺失返回空。连接沿用 events_db 的连接参数（busy_timeout=5000）： + 库锁竞争或恢复窗口内等待而非立即失败（B69）。摘要只需要索引列，引用与 outputs 留在库里按需查。 """ path = events_db.db_path() if path is None or not path.exists(): - return [], {} + return [] with contextlib.closing(events_db._connect(path)) as conn: - rows = [dict(zip(_EVENT_COLUMNS, values)) for values in conn.execute( + return [dict(zip(_EVENT_COLUMNS, values)) for values in conn.execute( f"SELECT {','.join(_EVENT_COLUMNS)} FROM events WHERE source=? AND trace_id=? ORDER BY seq", (_SOURCE, trace_id))] - refs: dict[int, list[dict]] = {} - joined = ("SELECT refs.event_id, refs.kind, refs.ref, refs.sha256, refs.size FROM refs" - " JOIN events ON refs.event_id = events.id" - " WHERE events.source=? AND events.trace_id=? AND refs.direction='in'" - " ORDER BY refs.event_id, refs.rowid") - for event_id, kind, ref, sha256, size in conn.execute(joined, (_SOURCE, trace_id)): - entry = _ref(kind, ref, sha256, size) - if entry: - refs.setdefault(event_id, []).append(entry) - return rows, refs - - -def _clean_str(value: str, limit: int) -> str | None: - """沿用 emit 的字符串隐私规则：超长、形似本机路径、含换行的丢弃，不截断、不改写。""" - if len(value) > limit or value.startswith(_FORBIDDEN_PREFIXES) or "C:\\" in value \ - or "\n" in value or "\r" in value: - return None - return value - - -def _ref(kind, ref, sha256, size) -> dict | None: - """一条输入引用：字符串键按隐私规则清洗，任一不合规整条丢弃（不保留半条引用）。""" - entry: dict = {} - for key, value in zip(_REF_KEYS, (kind, ref, sha256)): - if value is None: - continue - cleaned = _clean_str(str(value), _MAX_STR) - if cleaned is None: - return None - entry[key] = cleaned - if isinstance(size, int) and not isinstance(size, bool): - entry["size"] = size - return entry or None - - -def _outputs(row: dict) -> dict: - """outputs 列还原为扁平字典并按隐私规则重过滤；解析失败或形状不对按空处理。""" - try: - parsed = json.loads(row["outputs"] or "{}") - except (json.JSONDecodeError, TypeError): - return {} - if not isinstance(parsed, dict): - return {} - clean: dict = {} - for key, value in parsed.items(): - if not isinstance(key, str) or len(key) > _MAX_STR or not _KEY_RE.match(key): - continue - if value is None or isinstance(value, (bool, int, float)): - clean[key] = value - elif isinstance(value, str) and (cleaned := _clean_str(value, _MAX_STR)) is not None: - clean[key] = cleaned - return clean - - -def _decision(row: dict, outputs: dict) -> dict | None: - """决定摘要：by/rule 取事件原值，reason 只留稳定结果短 ID；原始理由正文以哈希留在 outputs。 - - 非成功的 executor_round 优先用 outputs 里的 Attempt 退出词（timeout/stall/…），其余用事件状态； - 都属于 C3 固定词表，不把任意理由正文塞回记录，也不重做判定。 - """ - by, rule, reason = row["decision_by"], row["decision_rule"], row["decision_reason"] - if by is None and rule is None and reason is None: - return None - short = row["status"] - if short != "ok" and row["step"] == "executor_round" and outputs.get("exit") in _EXIT_WORDS: - short = outputs["exit"] - if isinstance(reason, str) and reason and reason != short: - outputs[_REASON_HASH_KEY] = hashlib.sha256(reason.encode("utf-8")).hexdigest() - return {"by": by, "rule": rule, "reason": short} - - -def _actor(row: dict) -> dict | None: - """角色摘要：role/host/model 取事件原值并按隐私规则清洗，全空时为 None。""" - actor: dict = {} - for column, key in (("actor_role", "role"), ("actor_host", "host"), ("model", "model")): - value = row[column] - if value is None: - continue - cleaned = _clean_str(str(value), _MAX_STR) - if cleaned is not None: - actor[key] = cleaned - return actor or None diff --git a/.harness/engine/cli.py b/.harness/engine/cli.py index fd12862..22cd376 100644 --- a/.harness/engine/cli.py +++ b/.harness/engine/cli.py @@ -39,10 +39,14 @@ "dispatch": "engine.agents.dispatch", "review": "engine.agents.review", "review-pack": "engine.agents.review_pack", + "review-plan": "engine.agents.plan_review", "identity": "engine.agents.identity", # 报告 "metrics": "engine.reports.metrics", "weekly": "engine.reports.weekly", + # 可观测性查询（同一模块两入口；不追加自身事件，见 QUIET_COMMANDS） + "events": "engine.reports.trace:events_main", + "trace": "engine.reports.trace:trace_main", # 安装与升级（只能从引擎仓库的检出运行） "install": "engine.core.install", "upgrade": "engine.core.install:upgrade_main", @@ -58,6 +62,7 @@ "dispatch": "dispatch", "review": "review", "review-pack": "review", + "review-plan": "review", } # 不记通用入口事件的命令：guard 只记拒绝（T104，放行不逐条记，设计 3.6）；events/trace 查询、 # audit/alert 发布各自记事件，避免递归（C1）。 diff --git a/.harness/engine/core/alerts.py b/.harness/engine/core/alerts.py new file mode 100644 index 0000000..451cabd --- /dev/null +++ b/.harness/engine/core/alerts.py @@ -0,0 +1,181 @@ +"""共用告警 API（B46 C4，T205）：在 PR 评论或升级议题上发布带远端标记的告警。 + +渠道只有 PR/议题评论加 escalation 标签（设计 5，不新增通知渠道）。远端标记是 trace_id、NUL、 +reason 的 sha256：同键先查分页评论/带标签议题再更新，不重发；无 PR 时查/建一条升级议题。 +publish 只消费结构化既有状态，永不抛异常（C0）：发布失败返回 ok=False 并留本机 alert 观察事件， +不改调用方的判定、退出码与升级。details 只含安全阶段摘要、trace 查询方式与 PR/CI/账本链接。 + +reason 只用 C4 稳定键。派发侧两个预警由 rules.toml 的可选 [alerts] 节启用（删除整节即全部关闭）： +最后一轮 CI 在 wait 前预警（总预算 1 则首轮即预警）；守卫拒绝预警按 [alerts] guard_denials_threshold +（每轮被守卫拒绝的工具调用数，一次调用多个拒绝理由算一次）触发，未配置不启用，非正或非法值禁用 +并提示一次配置错误，不改执行预算。阈值/末轮条件与安全详情组装都在本模块，dispatch 只接线。 +""" + +from __future__ import annotations + +import hashlib +import sys + +from engine.core import events +from engine.core.common import load_rules + +LABEL = "escalation" +MARK = "" # 远端标记：trace_id、NUL、reason 的 sha256 +# C4 稳定 reason 键（新增必须先经设计方修订共用合同，不造同义短 token）。 +REASONS = frozenset(( + "integrity_failure", "review_rejected", "review_error", "dispatch_stall", "dispatch_timeout", + "dispatch_loop", "dispatch_budget", "ci_last_round", "guard_denials", "budget_exhausted", + "audit_missing_stage", "audit_anchor_mismatch", +)) +_hinted = False # 配置错误提示每进程一次（stderr 噪音控制；测试可直接复位） + + +def marker(trace_id: str, reason: str) -> str: + """远端标记值：trace_id、NUL、reason 的 sha256（C4 去重键，换机后仍靠远端标记去重）。""" + return hashlib.sha256(f"{trace_id}\0{reason}".encode()).hexdigest() + + +def dispatch_enabled(rules: dict | None = None) -> bool: + """派发预警开关：rules.toml 配置了 [alerts] 节即启用；节可选，删除整节即全部关闭。 + + 读取失败按未配置处理（观察旁路，不影响派发判定与预算）。 + """ + try: + data = rules if rules is not None else load_rules() + except Exception: # noqa: BLE001 观察旁路 + return False + return isinstance(data, dict) and "alerts" in data + + +def guard_denials_threshold(rules: dict | None = None) -> int | None: + """[alerts] guard_denials_threshold：正整数启用守卫拒绝预警；未配置返回 None（不启用）； + 非正或非法值禁用该预警并提示一次配置错误，不改执行预算。""" + global _hinted + value = (rules if rules is not None else load_rules()).get("alerts", {}).get("guard_denials_threshold") + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + if not _hinted: + _hinted = True + print(f"harness：rules.toml [alerts] guard_denials_threshold 配置非法（{value!r}），" + "守卫拒绝预警已禁用", file=sys.stderr) + return None + return value + + +def guard_denials_exceeded(denied: int, threshold: int | None) -> bool: + """阈值条件（纯函数）：轮内被拒工具调用数（绝对计数）达到阈值即触发；未配置不触发。""" + return threshold is not None and denied >= threshold + + +def is_last_ci_round(round_no: int, ci_rounds: int) -> bool: + """末轮条件（纯函数）：本轮达到 CI 预算即最后一轮（总预算 1 则首轮即预警），在 wait 前触发。""" + return ci_rounds >= 1 and round_no >= ci_rounds + + +def last_round_alert(trace_id: str, task: str, round_no: int, ci_rounds: int, *, pr, gh) -> None: + """最后一轮 CI 预警：派发在每轮 wait_ci 前调用；未启用派发预警时为空操作。""" + if dispatch_enabled() and is_last_ci_round(round_no, ci_rounds): + publish(trace_id, "ci_last_round", pr=pr, task=task, gh=gh) + + +def guard_round_alert(trace_id: str, task: str, denied: int, *, pr, gh) -> None: + """守卫拒绝预警：派发在每轮执行方结束后调用；未启用或未达阈值为空操作。""" + if dispatch_enabled() and guard_denials_exceeded(denied, guard_denials_threshold()): + publish(trace_id, "guard_denials", pr=pr, task=task, gh=gh) + + +def stage_lines(trace_id: str) -> list[str]: + """安全阶段摘要（T201 时间线，只含枚举/计数/短 ID）；读取或组装失败为空列表（观察旁路）。""" + try: + from engine.agents import run_timeline # 延迟导入：core 不在导入期反向依赖 agents + summary, _head = run_timeline.record_fields(trace_id) + except Exception: # noqa: BLE001 观察旁路：摘要失败只少记这一段 + return [] + lines = [] + for item in summary.get("stages") or []: + round_no = f"，round {item['round']}" if item.get("round") else "" + lines.append(f"- `{item['stage']}/{item['step']}` {item['status']}" + f"（attempt {item.get('attempt', 1)}{round_no}）") + return lines + + +def timeline_lines(trace_id: str, pr: int | None = None) -> list[str]: + """时间线段：trace 与时间线入口（有 PR 指向 PR/CI artifact，无 PR 指向升级议题与 trace 命令）、 + 安全阶段摘要；升级包与告警正文复用。""" + entry = ("本 PR 的 checks 与 CI artifact（事件导出，保留 90 天）" if pr is not None + else f"本升级议题与 `bin/harness trace {trace_id}`") + lines = [f"- **trace**：`{trace_id}`；时间线入口：{entry}。"] + stages = stage_lines(trace_id) + if stages: + lines += ["", "### 已发生的阶段（安全摘要）", *stages] + return lines + + +def alert_body(trace_id: str, reason: str, *, task: str | None = None, pr: int | None = None, + details: dict | None = None) -> str: + """告警正文：远端标记、标题、时间线段与 details 的附加安全行（PR/CI/账本链接由调用方给）。""" + lines = [MARK.format(marker(trace_id, reason)), f"### 告警：{task or trace_id}（{reason}）", "", + *timeline_lines(trace_id, pr)] + extra = (details or {}).get("lines") + if extra: + lines += ["", *[line for line in extra if isinstance(line, str)]] + return "\n".join(lines) + "\n" + + +def publish(trace_id: str, reason: str, *, pr: int | None = None, task: str | None = None, + details: dict | None = None, gh=None) -> dict: + """共用告警发布（C4）：有 PR 在 PR 上评论（带 escalation 标签），无 PR 查/建一条升级议题； + 同键（trace_id+reason 的远端标记）查分页评论/议题后更新，不重发。永不抛异常：失败返回 + {ok: False, target: None, updated: False, error_kind} 并留本机 alert 事件，不改调用方的 + 判定、退出码与升级。""" + result = {"ok": False, "target": None, "updated": False, "error_kind": None} + try: + if reason not in REASONS: + result["error_kind"] = "unknown_reason" + else: + body = alert_body(trace_id, reason, task=task, pr=pr, details=details) + if pr is not None: + result["target"] = "pr" + existing = _marked(gh.list_comments(pr), marker(trace_id, reason), "id") + if existing is not None: + gh.edit_comment(existing, body) + result.update(ok=True, updated=True) + else: + gh.comment(pr, body, label=LABEL) + result["ok"] = True + else: + result["target"] = "issue" + number = _marked(gh.list_issues(LABEL), marker(trace_id, reason), "number") + if number is not None: + gh.edit_issue(number, body) + result.update(ok=True, updated=True) + else: + gh.create_issue(f"告警：{task or trace_id}（{reason}）", body, [LABEL]) + result["ok"] = True + except Exception as exc: # noqa: BLE001 发布异常不传播到原判定（C0） + result.update(target=None, updated=False, error_kind=type(exc).__name__) + _record(trace_id, reason, result) + return result + + +def _marked(items, mark: str, key: str) -> int | None: + """分页列表（PR 评论或议题）里正文带该远端标记的条目 id；没有或形状不对为 None。""" + for item in items or []: + if isinstance(item, dict) and mark in str(item.get("body") or ""): + try: + return int(item[key]) + except (KeyError, TypeError, ValueError): + return None + return None + + +def _record(trace_id: str, reason: str, result: dict) -> None: + """本机 alert 观察事件（C1：alert 自己记发布结果）；emit 永不抛，组装失败也只丢弃。""" + try: + events.emit(stage="alert", step=reason, status="ok" if result["ok"] else "error", + trace_id=trace_id, + outputs={"target": result["target"], "updated": result["updated"]}, + error={"kind": result["error_kind"]} if result["error_kind"] else None) + except Exception: # noqa: BLE001 观察旁路 + return diff --git a/.harness/engine/core/events.py b/.harness/engine/core/events.py index 8d36c18..b32f798 100644 --- a/.harness/engine/core/events.py +++ b/.harness/engine/core/events.py @@ -14,6 +14,7 @@ import time from collections.abc import Callable, Iterator from contextlib import contextmanager +from datetime import UTC, datetime, timedelta from pathlib import Path from engine import __version__ as ENGINE_VERSION @@ -32,6 +33,8 @@ _DROP = object() _warned = False +_cleanup_warned = False +_days_warned = False def _warn_once() -> None: @@ -42,6 +45,53 @@ def _warn_once() -> None: print("harness：事件写入失败，已跳过（不影响本次运行）", file=sys.stderr) +def artifact_days() -> int: + """[events] artifact_days：本机产物保留天数，缺省 30。 + + 非法值（非整数、布尔、非正数）与配置读取失败提示一次并按 30 处理；不影响 emit 与原判定。 + """ + global _days_warned + try: + value = setting("events", "artifact_days", 30) + except Exception: # noqa: BLE001 配置损坏时按缺省处理 + return 30 + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + if not _days_warned: + _days_warned = True + print("harness：[events] artifact_days 配置非法，按缺省 30 处理", file=sys.stderr) + return 30 + return value + + +def _cleanup_warn_once(reasons: list[str]) -> None: + """同一进程内只向 stderr 提示一次清理未完成（不逐条刷屏，不影响业务）。""" + global _cleanup_warned + if not _cleanup_warned: + _cleanup_warned = True + print(f"harness：本机产物清理未完成，已跳过（不影响本次运行）：{len(reasons)} 项", file=sys.stderr) + + +def _maybe_cleanup() -> None: + """每天首次 emit 顺带执行一次本机保留期清理（B40）：无守护进程，跨进程以日期标志保证一天一轮。 + + 当天已有人清理则直接返回；执行与取配置的任何失败都只提示一次，不改变 emit 与业务返回。 + 清理本身不产生事件：事件/引用/锚点永不因保留期删除，也不让清理递归进 emit。 + """ + try: + now = datetime.fromisoformat(events_db._now()) + if now.tzinfo is None: + now = now.replace(tzinfo=UTC) + if not events_db.claim_cleanup_day(now.strftime("%Y-%m-%d")): + return + cutoff = now - timedelta(days=artifact_days()) + problems = events_db.run_retention_cleanup(cutoff) + except Exception as exc: # noqa: BLE001 设计要求：清理失败不得影响调用方 + _cleanup_warn_once([str(exc) or type(exc).__name__]) + return + if problems: + _cleanup_warn_once(problems) + + def enabled() -> bool: """事件开关：HARNESS_EVENTS=off 关闭；其次 checks.toml [events] enabled = false 关闭；缺省开启。""" env = os.environ.get("HARNESS_EVENTS") @@ -266,7 +316,10 @@ def emit(stage: str, step: str, status: str, *, trace_id: str | None = None, trace_id or current_trace(), duration_ms, outputs, decision, error, actor or {"role": "engine", "host": _host_class(source)}, inputs) - return events_db.insert_event(payload, clean_inputs) + event_id = events_db.insert_event(payload, clean_inputs) + # 清理挂在成功的 emit 之后：主路径失败时不叠加清理提示，保持「失败只提示一次」 + _maybe_cleanup() + return event_id except Exception: # noqa: BLE001 设计要求：事件失败不得影响调用方 _warn_once() return None diff --git a/.harness/engine/core/events_db.py b/.harness/engine/core/events_db.py index c81ac20..5a1e50d 100644 --- a/.harness/engine/core/events_db.py +++ b/.harness/engine/core/events_db.py @@ -10,7 +10,9 @@ import hashlib import json import os +import re import sqlite3 +import stat import time from contextlib import closing from datetime import UTC, datetime @@ -331,3 +333,356 @@ def add_anchor(source: str, trace_id: str, stage: str, head_hash: str, fixed_in: return conn.execute("INSERT INTO anchors (source,trace_id,stage,head_hash,fixed_in,ts) VALUES (?,?,?,?,?,?)", (source, trace_id, stage, head_hash, fixed_in, _now())) + + +# ---- 读取与批量导入（T301 events_io 的存储原语）：不改表结构，user_version 保持 1 ---- + +def read_events(source: str | None = None, trace_id: str | None = None, + stage: str | None = None, status: str | None = None) -> list[dict]: + """按可选精确过滤读取事件全列（含 id），按 (source, trace_id, seq) 排序。 + + 没有库或库版本比本代码新时返回空列表（与 chain_head 的缺省行为一致：上层按无数据处理）。 + """ + path = db_path() + if path is None or not path.exists(): + return [] + with closing(_connect(path)) as conn: + if not _schema_ready(conn): + return [] + where, params = [], [] + for column, value in (("source", source), ("trace_id", trace_id), + ("stage", stage), ("status", status)): + if value is not None: + where.append(f"{column}=?") + params.append(value) + clause = f" WHERE {' AND '.join(where)}" if where else "" + rows = conn.execute( + f"SELECT {','.join(EVENT_COLUMNS)} FROM events{clause} ORDER BY source, trace_id, seq", + params, + ).fetchall() + return [dict(zip(EVENT_COLUMNS, values)) for values in rows] + + +def read_refs(event_ids: list[int]) -> dict[int, list[dict]]: + """按事件 id 批量读取规范化输入引用（保留行序，None 值键省略），返回 {event_id: [引用]}。""" + path = db_path() + if path is None or not path.exists() or not event_ids: + return {} + with closing(_connect(path)) as conn: + if not _schema_ready(conn): + return {} + placeholders = ",".join("?" * len(event_ids)) + rows = conn.execute( + f"SELECT event_id, kind, ref, sha256, size FROM refs WHERE event_id IN ({placeholders}) ORDER BY rowid", + event_ids, + ).fetchall() + grouped: dict[int, list[dict]] = {} + for event_id, *values in rows: + grouped.setdefault(event_id, []).append(_normalize_ref(dict(zip(_REF_KEYS, values)))) + return grouped + + +def read_anchors(source: str | None = None, trace_id: str | None = None) -> list[dict]: + """按可选链过滤读取锚点（不含 id，字段与表列一致），保持表内顺序。""" + path = db_path() + if path is None or not path.exists(): + return [] + with closing(_connect(path)) as conn: + if not _schema_ready(conn): + return [] + where, params = [], [] + if source is not None: + where.append("source=?") + params.append(source) + if trace_id is not None: + where.append("trace_id=?") + params.append(trace_id) + clause = f" WHERE {' AND '.join(where)}" if where else "" + rows = conn.execute( + "SELECT source, trace_id, stage, head_hash, fixed_in, ts FROM anchors" + f"{clause} ORDER BY id", + params, + ).fetchall() + keys = ("source", "trace_id", "stage", "head_hash", "fixed_in", "ts") + return [dict(zip(keys, values)) for values in rows] + + +def present_hashes(digests: list[str]) -> set[str]: + """返回这些事件 hash 中已存在库里的部分（events_io 按事件哈希幂等去重用）。""" + path = db_path() + if path is None or not path.exists() or not digests: + return set() + with closing(_connect(path)) as conn: + if not _schema_ready(conn): + return set() + placeholders = ",".join("?" * len(digests)) + rows = conn.execute(f"SELECT hash FROM events WHERE hash IN ({placeholders})", digests).fetchall() + return {row[0] for row in rows} + + +def import_rows(events: list[tuple[dict, list[dict]]], anchors: list[dict]) -> int: + """事务写入已验证的导入事件与锚点，返回写入的事件数；任一失败整体回滚。 + + 事件按调用方给定的 seq/prev_hash/hash 原样写入（不重排、不重算哈希）；(source, trace_id, seq) + 撞上已有行时以 IntegrityError 抛出，不覆盖；锚点按整行幂等（六字段完全相同的不重复写）。 + 不在 git 仓库或库版本比本代码新时抛 RuntimeError，由调用方转为发现报告，不写半包。 + """ + path = db_path() + if path is None: + raise RuntimeError("不在 git 仓库，没有可导入的事件库") + path.parent.mkdir(parents=True, exist_ok=True) + anchor_keys = ("source", "trace_id", "stage", "head_hash", "fixed_in", "ts") + with closing(_connect(path)) as conn: + if not _schema_ready(conn): + raise RuntimeError("事件库 schema 比本代码新，不导入") + conn.execute("BEGIN IMMEDIATE") + try: + columns = [name for name in EVENT_COLUMNS if name != "id"] + for row, refs in events: + cursor = conn.execute( + f"INSERT INTO events ({','.join(columns)}) VALUES ({','.join('?' * len(columns))})", + tuple(row[name] for name in columns), + ) + for item in refs: + conn.execute( + "INSERT INTO refs (event_id,direction,kind,ref,sha256,size) VALUES (?,?,?,?,?,?)", + (cursor.lastrowid, "in", item.get("kind"), item.get("ref"), + item.get("sha256"), item.get("size")), + ) + for anchor in anchors: + values = [anchor[name] for name in anchor_keys] + conn.execute( + "INSERT INTO anchors (source,trace_id,stage,head_hash,fixed_in,ts)" + " SELECT ?,?,?,?,?,? WHERE NOT EXISTS (SELECT 1 FROM anchors" + " WHERE source=? AND trace_id=? AND stage=? AND head_hash=? AND fixed_in=? AND ts=?)", + [*values, *values], + ) + conn.execute("COMMIT") + return len(events) + except Exception: + try: + conn.execute("ROLLBACK") + except sqlite3.Error: + pass + raise + + +# ---- 保留期清理（B40）：无守护进程，每天首次 emit 顺带执行，失败只报告不改变业务 ---- + +# 只删内容寻址目录里名字为 64 位十六进制的普通文件，与派发原始流 round-N.jsonl。 +_HASH_NAME = re.compile(r"[0-9a-f]{64}\Z") +_ROUND_NAME = re.compile(r"round-\d+\.jsonl\Z") + + +def claim_cleanup_day(today: str) -> bool: + """当天首次进入返回 True：以 O_CREAT|O_EXCL 原子创建 .cleanup-<日期> 标志（兼任跨进程锁）。 + + 标志放在 git 公共目录的 harness/ 下且不删除：同日再次 emit 与并发进程都得到 False。 + 时钟回退到更早日期按回退后的日期领旗；即使因此多跑一轮，删除阈值也随当前时钟前移，不会多删。 + """ + directory = common_dir() + if directory is None: + return False + try: + harness_dir = directory / "harness" + harness_dir.mkdir(parents=True, exist_ok=True) + fd = os.open(harness_dir / f".cleanup-{today}", os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + return False + except OSError: + return False + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(f"{today}\n") + return True + + +def _parse_utc(value) -> datetime | None: + """解析库里的 UTC ISO 时间戳；损坏或缺失时返回 None（调用方按无法判断跳过）。""" + try: + parsed = datetime.fromisoformat(str(value)) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + + +def _remove_expired_artifact(directory: Path, digest: str) -> str | None: + """删除过期产物文件；返回问题描述（None 表示已删除或本就缺失）。不追随符号链接。""" + target = directory / digest + try: + info = os.lstat(target) + except FileNotFoundError: + return None # 文件已不在：过期的索引行一并清理 + except OSError as exc: + return f"产物无法确认，已跳过：{exc.strerror or exc}" + if not stat.S_ISREG(info.st_mode): + return "产物位置不是普通文件（可能是符号链接），已跳过" + try: + target.unlink() + except OSError as exc: + return f"产物删除失败，已跳过：{exc.strerror or exc}" + return None + + +def cleanup_artifacts(cutoff: datetime) -> list[str]: + """删除过期本机产物：以 UTC created 判定（早于 cutoff 才过期），文件与索引行一起清。 + + 只删内容寻址目录内名字合法的普通文件；符号链接、目录、非法名字与路径穿越一律跳过并计入 + 问题列表。events/refs/anchors 行不在此清理范围；文件已不在时仍清掉过期的索引行。 + """ + problems: list[str] = [] + path = db_path() + directory = artifacts_dir() + if path is None or directory is None or not path.exists(): + return problems + with closing(_connect(path)) as conn: + if not _schema_ready(conn): + return problems + for digest, created in conn.execute("SELECT sha256, created FROM artifacts").fetchall(): + moment = _parse_utc(created) + if moment is None or moment >= cutoff: + continue + if not _HASH_NAME.fullmatch(str(digest)): + problems.append("产物索引名字不合法，已跳过（不删除）") + continue + problem = _remove_expired_artifact(directory, str(digest)) + if problem: + problems.append(problem) + continue + conn.execute("DELETE FROM artifacts WHERE sha256=?", (digest,)) + return problems + + +def _alive(pid: int) -> bool: + """探活：系统明确 ProcessLookupError 才算不活动；权限不明等其余情形按活动处理（保守，不误删）。""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except OSError: + return True + return True + + +def _protected_tasks(slots: Path) -> tuple[set[str], list[str]]: + """收集仍活动槽位锁保护的 task 目录名，返回 (受保护集合, 问题列表)。只读锁文件，不修改派发状态。 + + 任一锁坏 JSON、缺 pid、pid 非正整数或读取失败时问题列表非空：调用方当轮整体跳过原始流清理， + 不按「读不到等于终止」删除。存活锁保护该 task 的全部 attempt 目录（F4）。 + """ + protected: set[str] = set() + problems: list[str] = [] + try: + locks = sorted(slots.glob("*.json")) + except OSError: + return set(), ["槽位锁目录无法读取"] + for lock in locks: + try: + data = json.loads(lock.read_text(encoding="utf-8") or "{}") + except (OSError, ValueError): + problems.append("槽位锁读取失败") + continue + pid = data.get("pid") if isinstance(data, dict) else None + if isinstance(pid, bool) or not isinstance(pid, int) or pid <= 0: + problems.append("槽位锁缺少有效 pid") + continue + task = data.get("task") + if not isinstance(task, str) or not task: + problems.append("槽位锁缺少 task") + continue + if _alive(pid): + protected.add(task) + return protected, problems + + +def _is_real_dir(path: Path) -> bool: + """lstat 确认是真实目录（不追随符号链接）。""" + try: + return stat.S_ISDIR(os.lstat(path).st_mode) + except OSError: + return False + + +def _delete_expired_stream(path: Path, cutoff_ts: float) -> list[str]: + """删除 mtime（UTC）早于 cutoff 的 round-N.jsonl 普通文件；删除前重新 lstat，符号链接不追随。""" + try: + info = os.lstat(path) + except FileNotFoundError: + return [] + except OSError as exc: + return [f"原始流无法确认，已跳过：{exc.strerror or exc}"] + if not stat.S_ISREG(info.st_mode): + return ["原始流不是普通文件（可能是符号链接），已跳过"] + if info.st_mtime >= cutoff_ts: + return [] + try: + path.unlink() + except OSError as exc: + return [f"原始流删除失败，已跳过：{exc.strerror or exc}"] + return [] + + +def _cleanup_attempt_streams(attempt_dir: Path, cutoff_ts: float) -> list[str]: + """删除一个 attempt 目录里的过期 round-N.jsonl；目录异常跳过并报告，符号链接不追随。""" + if not _is_real_dir(attempt_dir): + return ["attempt 目录不是普通目录，已跳过"] + try: + rounds = sorted(attempt_dir.iterdir()) + except OSError: + return ["原始流目录无法读取，已跳过"] + problems: list[str] = [] + for round_file in rounds: + if _ROUND_NAME.fullmatch(round_file.name): + problems.extend(_delete_expired_stream(round_file, cutoff_ts)) + return problems + + +def _cleanup_task_streams(task_dir: Path, protected: set[str], cutoff_ts: float) -> list[str]: + """一个任务目录的原始流清理：活动锁保护的任务连同全部 attempt 目录整目录跳过。""" + if task_dir.name in protected: + return [] + if not _is_real_dir(task_dir): + return ["原始流任务目录不是普通目录，已跳过"] + try: + attempt_dirs = sorted(task_dir.iterdir()) + except OSError: + return ["attempt 目录无法读取，已跳过"] + problems: list[str] = [] + for attempt_dir in attempt_dirs: + problems.extend(_cleanup_attempt_streams(attempt_dir, cutoff_ts)) + return problems + + +def cleanup_dispatch_streams(runs: Path, cutoff: datetime) -> list[str]: + """删除已终止运行的过期派发原始流（runs/<任务>//round-N.jsonl）。 + + runs 指向 /dispatch/runs；活动锁（pid 存活）保护该任务的全部 attempt 目录， + 无锁任务按原始流 mtime（UTC）判定。锁读取或探活不确定时当轮整体跳过并报告； + 只删名字为 round-N.jsonl 的普通文件，符号链接与目录不删、不跟随，不修改锁与派发状态。 + """ + protected, lock_problems = _protected_tasks(runs.parent / "slots") + if lock_problems: + return [*lock_problems, "原始流清理当轮跳过"] + if not _is_real_dir(runs): + return [] + cutoff_ts = cutoff.timestamp() + try: + task_dirs = sorted(runs.iterdir()) + except OSError: + return ["原始流目录无法读取"] + problems: list[str] = [] + for task_dir in task_dirs: + problems.extend(_cleanup_task_streams(task_dir, protected, cutoff_ts)) + return problems + + +def run_retention_cleanup(cutoff: datetime) -> list[str]: + """B40 保留期清理入口：过期产物文件与索引 + 终止运行的过期原始流，返回问题列表。 + + 产物按库里的 UTC created、原始流按文件 mtime（UTC）判定，都只在早于 cutoff 时删除。 + 任一异常由调用方（engine.core.events）吞掉并提示一次，不影响 emit 与业务返回。 + """ + problems = cleanup_artifacts(cutoff) + common = common_dir() + if common is not None: + problems.extend(cleanup_dispatch_streams(common / "dispatch" / "runs", cutoff)) + return problems diff --git a/.harness/engine/core/events_io.py b/.harness/engine/core/events_io.py new file mode 100644 index 0000000..999cc6c --- /dev/null +++ b/.harness/engine/core/events_io.py @@ -0,0 +1,797 @@ +"""C5 事件导出与幂等导入（可观测性 P3）：EventBundle v1、查询、导出、导入、写盘与 CI 下载。 + +查询返回设计 JSON 形状的事件字典（id 是本机定位号，不入哈希，导入可重映射）；导出按 +(source, trace) 整链导出完整前缀与锚点——since/stage/status 属展示过滤，筛选后的断链子集不是可 +导入包；manifest 只收录通过安全扫描的规范化 UTF-8 JSON 内容产物，原始命令输出、verify 完整日志 +与 Pi 流绝不外发，SQLite/WAL/SHM 更不在其中。导入验证 schema、来源、隐私、规范化哈希与链完整性 +后事务提交：按事件哈希去重，同 source/trace/seq 不同 hash 为冲突，坏包整体不写、逐项发现以 +findings 返回。load_ci 下载时再按 GitHub API 核对仓库、工作流路径、run/head 与包 origin（不能只 +信包自报 ci），内容只解析不执行；manifest 内容产物文件不随导入恢复（缺失由再次导出如实报告）。 +""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import re +import sqlite3 +import subprocess +import urllib.parse +import zipfile +from datetime import UTC, datetime +from pathlib import Path + +from engine.core import events, events_db +from engine.core.common import git + +BUNDLE_SCHEMA_VERSION = 1 +_HASH_RE = re.compile(r"[0-9a-f]{64}\Z") +_SHA_RE = re.compile(r"[0-9a-f]{40}\Z") +_KEY_RE = re.compile(r"[A-Za-z0-9_.-]{1,64}\Z") + +_LIMIT = 120 # 通用字符串上限（同 T101 过滤口径） +_REASON_LIMIT = 200 +_REF_LIMIT = 200 +_ORIGIN_LIMIT = 300 + +_BUNDLE_KEYS = ("schema_version", "origin", "events", "anchors", "artifacts", "chains", "findings") +_ORIGIN_KEYS = ("repository", "run_id", "run_attempt", "job", "workflow_ref", "head_sha", "head_branch") +_EVENT_REQUIRED = ("source", "trace_id", "seq", "prev_hash", "hash", "ts", "stage", "step", + "status", "engine_version", "redacted") +_EVENT_OPTIONAL = ("id", "duration_ms", "inputs", "outputs", "decision", "error", "actor") +_REF_KEYS = ("kind", "ref", "sha256", "size") +_ANCHOR_KEYS = ("source", "trace_id", "stage", "head_hash", "fixed_in", "ts") +_NESTED_LIMITS = { + "decision": {"by": _LIMIT, "rule": _LIMIT, "reason": _REASON_LIMIT}, + "error": {"kind": _LIMIT, "signature": _LIMIT}, + "actor": {"role": _LIMIT, "host": _LIMIT, "model": _LIMIT}, +} +_DECISION_COLUMNS = {"by": "decision_by", "rule": "decision_rule", "reason": "decision_reason"} +_ERROR_COLUMNS = {"kind": "error_kind", "signature": "error_signature"} +_ACTOR_COLUMNS = {"role": "actor_role", "host": "actor_host", "model": "model"} + +# manifest 安全扫描的禁项：换行与本机路径、会话标记（C4 词表）、shell 结构痕迹。 +_TEXT_BANS = ("\n", "\r", "/Users/", "/home/", "C:\\") +_SESSION_BANS = ("user:", "assistant:", "system:", "human:", "用户:", "助手:") +_SHELL_BANS = ("|", ";", "`", "$(", "&&", "||", ">", "<") + + +def _finding(code: str, detail: str) -> dict: + return {"code": code, "detail": detail} + + +def _is_int(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _parse_utc(value) -> datetime | None: + """解析 UTC ISO 时间戳（Z 或带时区，naive 按 UTC）；不合法返回 None。""" + try: + parsed = datetime.fromisoformat(value) + except (TypeError, ValueError): + return None + if parsed.tzinfo is None: + return parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def _parse_since(value) -> datetime: + """query 的 since：datetime 或 ISO 字符串，统一为 aware UTC；其余类型与坏字符串抛 ValueError。""" + if isinstance(value, datetime): + return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC) + if isinstance(value, str): + parsed = _parse_utc(value) + if parsed is not None: + return parsed + raise ValueError(f"since 无法解析为 UTC 时间（收到 {type(value).__name__}）") + + +def _source_matches(candidate: str, wanted: str) -> bool: + """来源匹配：精确链（ci:::）或 local/ci/github 类型前缀。""" + return candidate == wanted or candidate.startswith(f"{wanted}:") + + +# ---- 事件形状：存储行 ↔ 设计 JSON ---- + +def _unflatten(row: dict, columns: dict[str, str]) -> dict: + """actor/decision/error 三列组还原为嵌套字典（只含非 None 键）。""" + return {key: row[name] for key, name in columns.items() if row.get(name) is not None} + + +def _row_to_event(row: dict, refs: list[dict]) -> dict: + """存储行 → 设计 JSON 形状：outputs 恢复为原紧凑 JSON 的对象，id 保留为本机定位号。""" + outputs = json.loads(row["outputs"]) if row["outputs"] else {} + return { + "id": row["id"], "ts": row["ts"], "source": row["source"], "trace_id": row["trace_id"], + "seq": row["seq"], "prev_hash": row["prev_hash"], "hash": row["hash"], + "stage": row["stage"], "step": row["step"], "status": row["status"], + "duration_ms": row["duration_ms"], "inputs": [dict(item) for item in refs], "outputs": outputs, + "decision": _unflatten(row, _DECISION_COLUMNS), "error": _unflatten(row, _ERROR_COLUMNS), + "actor": _unflatten(row, _ACTOR_COLUMNS), + "engine_version": row["engine_version"], "redacted": row["redacted"], + } + + +def _event_to_row(event: dict) -> tuple[dict, list[dict]]: + """已验证的 bundle 事件 → 存储行与规范化引用（outputs 重新落成原紧凑 JSON，None 沿用原规则）。""" + decision = event.get("decision") or {} + error = event.get("error") or {} + actor = event.get("actor") or {} + row = { + "ts": event["ts"], "source": event["source"], "trace_id": event["trace_id"], + "seq": event["seq"], "prev_hash": event["prev_hash"], "hash": event["hash"], + "stage": event["stage"], "step": event["step"], "status": event["status"], + "duration_ms": event.get("duration_ms"), + "actor_role": actor.get("role"), "actor_host": actor.get("host"), "model": actor.get("model"), + "decision_by": decision.get("by"), "decision_rule": decision.get("rule"), + "decision_reason": decision.get("reason"), + "error_kind": error.get("kind"), "error_signature": error.get("signature"), + "outputs": json.dumps(event.get("outputs") or {}, sort_keys=True, separators=(",", ":"), + ensure_ascii=False), + "engine_version": event["engine_version"], "redacted": event["redacted"], + } + refs = [events_db._normalize_ref(dict(item)) for item in (event.get("inputs") or [])] + return row, refs + + +# ---- 查询（展示视图）---- + +def query(*, trace_id: str | None = None, source: str | None = None, since=None, + stage: str | None = None, status: str | None = None) -> list[dict]: + """查询事件（展示视图，不是可导入 bundle）：trace、来源、since、stage、status 过滤。 + + source 为精确链或 local/ci/github 前缀；since 为 UTC datetime 或 ISO（含边界），坏值抛 + ValueError 由观察命令转诊断；无库返回空列表，上层按无数据处理。 + """ + cutoff = _parse_since(since) if since is not None else None + rows = events_db.read_events(trace_id=trace_id, stage=stage, status=status) + refs = events_db.read_refs([row["id"] for row in rows]) + result = [] + for row in rows: + if source is not None and not _source_matches(row["source"], source): + continue + if cutoff is not None and ((moment := _parse_utc(row["ts"])) is None or moment < cutoff): + continue + result.append(_row_to_event(row, refs.get(row["id"], []))) + return result + + +# ---- 导出 ---- + +def _origin() -> dict: + """导出方环境快照：仓库、head 与真实 Actions 三键；local 导出不伪造 Actions 键。""" + origin: dict[str, str] = {} + remote = git("remote", "get-url", "origin", cwd=events_db.ROOT, check=False, isolate=True) + if remote and events._clean_str(remote, _ORIGIN_LIMIT) is not None: + origin["repository"] = remote # 本机路径形式的 remote 不外发（也是导入隐私口径） + head_sha = git("rev-parse", "HEAD", cwd=events_db.ROOT, check=False, isolate=True) + if head_sha and _SHA_RE.fullmatch(head_sha): + origin["head_sha"] = head_sha + if os.environ.get("CI") == "true": + branch = os.environ.get("GITHUB_HEAD_REF") or os.environ.get("GITHUB_REF_NAME") or "" + run_id = os.environ.get("GITHUB_RUN_ID") + run_attempt, job = os.environ.get("GITHUB_RUN_ATTEMPT"), os.environ.get("GITHUB_JOB") + if run_id and run_attempt and job: + origin["run_id"], origin["run_attempt"], origin["job"] = run_id, run_attempt, job + workflow_ref = os.environ.get("GITHUB_WORKFLOW_REF") + if workflow_ref: + origin["workflow_ref"] = workflow_ref + else: + branch = git("rev-parse", "--abbrev-ref", "HEAD", cwd=events_db.ROOT, check=False, isolate=True) + if branch: + origin["head_branch"] = branch + return origin + + +def export_bundle(*, trace_id: str | None = None, source: str | None = None) -> dict: + """导出 EventBundle v1：每条 (source, trace) 的完整链前缀、锚点与安全 manifest。 + + source 同 query 的匹配规则；不含展示过滤；findings 记被排除产物、不中断导出；无库各列表为空。 + """ + rows = events_db.read_events(trace_id=trace_id) + if source is not None: + rows = [row for row in rows if _source_matches(row["source"], source)] + refs = events_db.read_refs([row["id"] for row in rows]) + events_out = [_row_to_event(row, refs.get(row["id"], [])) for row in rows] + chains: list[dict] = [] + chain_keys: set[tuple[str, str]] = set() + for row in rows: + key = (row["source"], row["trace_id"]) + if key in chain_keys: + chains[-1]["head_hash"] = row["hash"] + continue + chain_keys.add(key) + chains.append({"source": row["source"], "trace_id": row["trace_id"], "head_hash": row["hash"]}) + anchors = [a for a in events_db.read_anchors() if (a["source"], a["trace_id"]) in chain_keys] + artifacts, findings = _build_manifest(events_out) + return {"schema_version": BUNDLE_SCHEMA_VERSION, "origin": _origin(), "events": events_out, + "anchors": anchors, "artifacts": artifacts, "chains": chains, "findings": findings} + + +def _harvest_digests(events_out: list[dict]) -> list[str]: + """收集指向本地产物的候选哈希：inputs 的 artifact 引用与 outputs 中的 64 位十六进制值。""" + digests: list[str] = [] + for event in events_out: + candidates = [item.get("sha256") for item in event["inputs"] if item.get("kind") == "artifact"] + candidates.extend(event["outputs"].values()) + fresh = [d for d in dict.fromkeys(candidates) if isinstance(d, str) and _HASH_RE.fullmatch(d)] + digests += [d for d in fresh if d not in digests] + return digests + + +def _artifact_bytes(path: Path) -> bytes | None: + try: + return path.read_bytes() + except OSError: + return None + + +def _safe_json_document(content: bytes) -> bool: + """内容是否为安全可发布的规范化 UTF-8 JSON 对象：可解析、字节即规范化形式、逐字段无禁项。""" + try: + parsed = json.loads(text := content.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + return False + if not isinstance(parsed, dict): + return False + canonical = json.dumps(parsed, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return canonical == text and _json_strings_safe(parsed) + + +def _json_strings_safe(node) -> bool: + """递归扫描 JSON 值：字符串不得含换行/本机路径/会话标记/shell 结构，键须合规短标识。""" + if isinstance(node, str): + if any(ban in node for ban in (*_TEXT_BANS, *_SHELL_BANS)): + return False + return not any(ban in node.lower() or ban in node for ban in _SESSION_BANS) + if isinstance(node, dict): + return all(isinstance(key, str) and _KEY_RE.fullmatch(key) is not None + and _json_strings_safe(value) for key, value in node.items()) + if isinstance(node, list): + return all(_json_strings_safe(item) for item in node) + return node is None or isinstance(node, (bool, int, float)) + + +def _build_manifest(events_out: list[dict]) -> tuple[list[dict], list[dict]]: + """安全 manifest：只收录内容可核验且通过安全扫描的产物；排除项逐条写 findings（不回显内容）。""" + entries: list[dict] = [] + findings: list[dict] = [] + directory = events_db.artifacts_dir() + for digest in _harvest_digests(events_out): + label = f"产物 {digest[:12]}…" + if directory is None: + findings.append(_finding("artifact_missing", f"{label} 无本地产物目录，未列入 manifest")) + continue + content = _artifact_bytes(directory / digest) + if content is None: + findings.append(_finding("artifact_missing", f"{label} 不在本地产物目录，未列入 manifest")) + elif hashlib.sha256(content).hexdigest() != digest: + findings.append(_finding("artifact_hash", f"{label} 字节与哈希不符，未列入 manifest")) + elif not _safe_json_document(content): + findings.append(_finding("artifact_unsafe", f"{label} 不是安全的规范化 JSON，未列入 manifest")) + else: + entries.append({"sha256": digest, "size": len(content), "file": digest}) + return entries, findings + + +def write_bundle(bundle: dict, target: Path) -> None: + """把 bundle 序列化为规范化 UTF-8 JSON 文本并原子写入 target（先临时文件再替换）。 + + 序列化或写盘失败抛异常，不当成功；不校验内容（校验属于 import_bundle / 设计方验收）。 + """ + target = Path(target) + text = json.dumps(bundle, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + target.parent.mkdir(parents=True, exist_ok=True) + tmp = target.with_name(f"{target.name}.tmp-{os.getpid()}") + try: + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, target) + finally: + tmp.unlink(missing_ok=True) + + +# ---- 导入 ---- + +def _event_prefix(event, index: int) -> str: + digest = event.get("hash") if isinstance(event, dict) else None + head = f"事件[{index}] {digest[:12]}…" if isinstance(digest, str) and _HASH_RE.fullmatch(digest) else f"事件[{index}]" + return head + + +def _validate_envelope(bundle, findings: list[dict]) -> bool: + """顶层形状：七个固定键、schema_version 只认 v1（未来版本明确拒写）、origin 与列表类型。""" + if not isinstance(bundle, dict): + findings.append(_finding("schema", "bundle 不是 JSON 对象")) + return False + if set(bundle) != set(_BUNDLE_KEYS): + missing, extra = sorted(set(_BUNDLE_KEYS) - set(bundle)), sorted(set(bundle) - set(_BUNDLE_KEYS)) + findings.append(_finding("schema", f"bundle 顶层键不符（缺 {missing}，多 {extra}）")) + return False + version = bundle["schema_version"] + if not _is_int(version): + findings.append(_finding("schema", "schema_version 不是整数")) + return False + if version > BUNDLE_SCHEMA_VERSION: + findings.append(_finding("future_schema", f"bundle schema 版本较新（{version} > {BUNDLE_SCHEMA_VERSION}），不导入")) + return False + if version != BUNDLE_SCHEMA_VERSION: + findings.append(_finding("schema", f"未知 schema 版本 {version}")) + return False + if not _validate_origin(bundle["origin"], findings): + return False + for name in ("events", "anchors", "artifacts", "chains", "findings"): + if not isinstance(bundle[name], list): + findings.append(_finding("schema", f"bundle.{name} 不是列表")) + return False + return True + + +def _validate_origin(origin, findings: list[dict]) -> bool: + if not isinstance(origin, dict) or not set(origin) <= set(_ORIGIN_KEYS): + findings.append(_finding("schema", "bundle.origin 形状不符")) + return False + for key, value in origin.items(): + if not isinstance(value, str) or events._clean_str(value, _ORIGIN_LIMIT) is None: + findings.append(_finding("schema", f"bundle.origin.{key} 不是合规字符串")) + return False + if key == "head_sha" and _SHA_RE.fullmatch(value) is None: + findings.append(_finding("schema", "bundle.origin.head_sha 不是完整提交 SHA")) + return False + return True + + +def _valid_text(value, limit: int) -> bool: + return isinstance(value, str) and bool(value) and events._clean_str(value, limit) is not None + + +def _validate_core_fields(event: dict, prefix: str, findings: list[dict]) -> bool: + """枚举、序号、哈希、时间戳与计数列：只接受 emit 可能写出的值。""" + if event["stage"] not in events.STAGES or event["status"] not in events.STATUSES: + findings.append(_finding("schema", f"{prefix} stage/status 不在枚举内")) + return False + if not _is_int(event["seq"]) or event["seq"] < 1 or not _is_int(event["redacted"]) or event["redacted"] < 0: + findings.append(_finding("schema", f"{prefix} seq/redacted 不是非负整数")) + return False + if not (isinstance(event["prev_hash"], str) + and (event["prev_hash"] == "" or _HASH_RE.fullmatch(event["prev_hash"]))): + findings.append(_finding("schema", f"{prefix} prev_hash 形状不符")) + return False + if not isinstance(event["hash"], str) or _HASH_RE.fullmatch(event["hash"]) is None: + findings.append(_finding("schema", f"{prefix} hash 不是 64 位十六进制")) + return False + if not isinstance(event["ts"], str) or _parse_utc(event["ts"]) is None: + findings.append(_finding("schema", f"{prefix} ts 不是可解析的 UTC 时间")) + return False + if (duration := event.get("duration_ms")) is not None and not _is_int(duration): + findings.append(_finding("schema", f"{prefix} duration_ms 不是整数")) + return False + if "id" in event and (not _is_int(event["id"]) or event["id"] < 1): + findings.append(_finding("schema", f"{prefix} id 不是正整数")) + return False + return True + + +def _validate_texts(event: dict, prefix: str, findings: list[dict]) -> bool: + """自由文本列：沿用 T101 隐私过滤口径（超长、本机路径、换行即禁项），step 另要求原样无首尾空白。""" + for name, limit in (("source", _LIMIT), ("trace_id", _REF_LIMIT), ("step", _LIMIT), ("engine_version", _LIMIT)): + if not _valid_text(event[name], limit): + findings.append(_finding("privacy", f"{prefix} 的 {name} 含隐私禁项或超限")) + return False + if event["step"] != event["step"].strip(): + findings.append(_finding("schema", f"{prefix} 的 step 带首尾空白（emit 会先剥离）")) + return False + parts = event["source"].split(":") if event["source"].startswith("ci:") else [] + if parts and (len(parts) != 4 or any(not part for part in parts)): + findings.append(_finding("schema", f"{prefix} 的 source 不是 ci::: 形状")) + return False + return True + + +def _validate_scalar(value) -> bool: + """outputs 值：标量或合规短字符串（None 合法，沿用原过滤口径）。""" + return value is None or isinstance(value, (bool, int, float)) or _valid_text(value, _LIMIT) + + +def _validate_outputs(event: dict, prefix: str, findings: list[dict]) -> bool: + """outputs 扁平字典：键为合规短标识，值为标量或合规短字符串。""" + outputs = event.get("outputs") + if not isinstance(outputs, dict): + findings.append(_finding("schema", f"{prefix} 的 outputs 不是对象")) + return False + for key, value in outputs.items(): + if not isinstance(key, str) or _KEY_RE.fullmatch(key) is None or not _validate_scalar(value): + findings.append(_finding("schema", f"{prefix} 的 outputs 键值不合规")) + return False + return True + + +def _validate_nested_mapping(nested, name: str, limits: dict[str, int], prefix: str, findings: list[dict]) -> bool: + """decision/error/actor 之一：键集合固定，值合规（role 另受枚举约束）。""" + if nested is None: + return True + if not isinstance(nested, dict) or not set(nested) <= set(limits): + findings.append(_finding("schema", f"{prefix} 的 {name} 键集合不符")) + return False + for key, value in nested.items(): + if value is None: + continue + if key == "role" and value not in events.ACTOR_ROLES: + findings.append(_finding("schema", f"{prefix} 的 actor.role 不在枚举内")) + return False + if key != "role" and not _valid_text(value, limits[key]): + findings.append(_finding("privacy", f"{prefix} 的 {name}.{key} 含隐私禁项或超限")) + return False + return True + + +def _validate_nested(event: dict, prefix: str, findings: list[dict]) -> bool: + """outputs/decision/error/actor/inputs：键集合、值类型与隐私口径逐项核对，未知键拒绝。""" + if not _validate_outputs(event, prefix, findings): + return False + for name, limits in _NESTED_LIMITS.items(): + if not _validate_nested_mapping(event.get(name), name, limits, prefix, findings): + return False + return _validate_inputs(event, prefix, findings) + + +def _validate_inputs(event: dict, prefix: str, findings: list[dict]) -> bool: + """inputs 引用列表：四定键、字符串隐私口径、sha256 形状；过滤后为空的引用在真实事件里不存在。""" + inputs = event.get("inputs") + if inputs is None: + return True + if not isinstance(inputs, list): + findings.append(_finding("schema", f"{prefix} 的 inputs 不是列表")) + return False + for item in inputs: + if not isinstance(item, dict) or not set(item) <= set(_REF_KEYS): + findings.append(_finding("schema", f"{prefix} 的引用键集合不符")) + return False + for name, limit in (("kind", _LIMIT), ("ref", _REF_LIMIT), ("sha256", 64)): + if item.get(name) is not None and not _valid_text(item[name], limit): + findings.append(_finding("privacy", f"{prefix} 的引用 {name} 含隐私禁项或超限")) + return False + if item.get("sha256") is not None and _HASH_RE.fullmatch(item["sha256"]) is None: + findings.append(_finding("schema", f"{prefix} 的引用 sha256 形状不符")) + return False + if (size := item.get("size")) is not None and not (_is_int(size) and size >= 0): + # emit 只写非负 int（len(content)）；数字字符串/浮点经 SQLite INTEGER 亲和改型后 + # 与导入时验证的哈希口径不一致，负数则是 emit 不可能写出的形状，一律拒绝。 + findings.append(_finding("schema", f"{prefix} 的引用 size 不是非负整数")) + return False + if not any(value is not None for value in item.values()): + findings.append(_finding("schema", f"{prefix} 的引用过滤后为空（emit 会整条丢弃）")) + return False + return True + + +def _validate_event(event, index: int, findings: list[dict]) -> tuple[dict, list[dict]] | None: + """校验一个 bundle 事件并转换为（存储行, 规范化引用）；不合法记 finding 返回 None。""" + prefix = _event_prefix(event, index) + if not isinstance(event, dict): + findings.append(_finding("schema", f"{prefix} 不是对象")) + return None + unknown = set(event) - set(_EVENT_REQUIRED) - set(_EVENT_OPTIONAL) + missing = [name for name in _EVENT_REQUIRED if name not in event] + if unknown or missing: + findings.append(_finding("schema", f"{prefix} 字段不符（缺 {sorted(missing)}，多 {sorted(unknown)}）")) + return None + if not (_validate_texts(event, prefix, findings) and _validate_core_fields(event, prefix, findings) + and _validate_nested(event, prefix, findings)): + return None + row, refs = _event_to_row(event) + if events_db._hash(row, refs) != row["hash"]: + findings.append(_finding("hash_mismatch", f"{prefix} 内容与 hash 不符（不重算掩盖）")) + return None + return row, refs + + +def _chain_heads(entries: list, findings: list[dict]) -> dict[tuple[str, str], str]: + """校验 bundle.chains 项形状并建 (source, trace) → head_hash 索引。""" + heads: dict[tuple[str, str], str] = {} + for entry in entries: + ok = (isinstance(entry, dict) and set(entry) == {"source", "trace_id", "head_hash"} + and all(_valid_text(entry[name], _REF_LIMIT) for name in ("source", "trace_id")) + and _HASH_RE.fullmatch(str(entry["head_hash"])) is not None) + if not ok: + findings.append(_finding("schema", "chains 项形状不符")) + continue + heads[(entry["source"], entry["trace_id"])] = entry["head_hash"] + return heads + + +def _check_merged_chain(key: tuple[str, str], existing: dict[int, dict], + fresh: list[tuple[dict, list[dict]]], expected_head: str, + findings: list[dict]) -> set[str] | None: + """合并已有行与新增行后核对链完整性：seq 从 1 连续、prev_hash 相接、新增时链头与 chains 一致。""" + fresh_map = {row["seq"]: row for row, _ in fresh} + merged = {seq: row["hash"] for seq, row in existing.items()} + merged.update({seq: row["hash"] for seq, row in fresh_map.items()}) + numbers = sorted(merged) + if numbers != list(range(1, len(numbers) + 1)): + findings.append(_finding("chain_gap", f"{key[0]}/{key[1]} 合并后 seq 有缺口或未从 1 开始")) + return None + previous = "" + for seq in numbers: + row = existing.get(seq) or fresh_map[seq] + if row["prev_hash"] != previous: + findings.append(_finding("chain_link", f"{key[0]}/{key[1]} seq={seq} 的 prev_hash 不相接")) + return None + previous = row["hash"] + if fresh_map and expected_head != previous: + findings.append(_finding("chain_head_mismatch", f"{key[0]}/{key[1]} 链头与 bundle.chains 不符")) + return None + return set(merged.values()) + + +def _plan_chains(bundle: dict, validated: list[tuple[dict, list[dict]]], findings: list[dict]): + """按链规划导入：包内去重、与库去重（按哈希）、seq 冲突与合并链完整性。 + + 返回 (新增 (row, refs) 列表, skipped 数, 每链合并哈希集合)；有发现时不写任何行。 + """ + heads = _chain_heads(bundle["chains"], findings) + grouped: dict[tuple[str, str], list[tuple[dict, list[dict]]]] = {} + for triple in validated: + row = triple[0] + grouped.setdefault((row["source"], row["trace_id"]), []).append(triple) + if set(grouped) != set(heads): + findings.append(_finding("schema", "chains 与事件链不互相覆盖")) + return [], 0, {} + fresh_all: list[tuple[dict, list[dict]]] = [] + skipped = 0 + chain_hashes: dict[tuple[str, str], set[str]] = {} + for key in sorted(grouped): + triples = sorted(grouped[key], key=lambda item: item[0]["seq"]) + seqs = [row["seq"] for row, _ in triples] + if len(set(seqs)) != len(seqs): + findings.append(_finding("schema", f"{key[0]}/{key[1]} 包内 seq 重复")) + continue + existing = {row["seq"]: row for row in events_db.read_events(source=key[0], trace_id=key[1])} + present = events_db.present_hashes([row["hash"] for row, _ in triples]) + fresh = [triple for triple in triples if triple[0]["hash"] not in present] + skipped += len(triples) - len(fresh) + for row, _ in fresh: + hit = existing.get(row["seq"]) + if hit is not None and hit["hash"] != row["hash"]: + findings.append(_finding("seq_conflict", f"{key[0]}/{key[1]} seq={row['seq']} 已有不同 hash，不覆盖")) + hashes = _check_merged_chain(key, existing, fresh, heads[key], findings) + if hashes is not None: + chain_hashes[key] = hashes + fresh_all.extend(fresh) + return fresh_all, skipped, chain_hashes + + +def _valid_anchor(anchor, chain_hashes: dict[tuple[str, str], set[str]], findings: list[dict]) -> bool: + """锚点形状与链归属：六字段、head_hash 必须是所在链某个已合并前缀的链头。""" + if not isinstance(anchor, dict) or set(anchor) != set(_ANCHOR_KEYS): + findings.append(_finding("anchor", "锚点字段不符")) + return False + for name, limit in (("source", _REF_LIMIT), ("trace_id", _REF_LIMIT), ("stage", _LIMIT), ("fixed_in", _LIMIT)): + if not _valid_text(anchor[name], limit): + findings.append(_finding("anchor", f"锚点的 {name} 含隐私禁项、超限或为空")) + return False + if not isinstance(anchor["head_hash"], str) or _HASH_RE.fullmatch(anchor["head_hash"]) is None: + findings.append(_finding("anchor", "锚点 head_hash 不是 64 位十六进制")) + return False + if _parse_utc(anchor["ts"]) is None: + findings.append(_finding("anchor", "锚点 ts 不是可解析的 UTC 时间")) + return False + hashes = chain_hashes.get((anchor["source"], anchor["trace_id"])) + if hashes is None: + findings.append(_finding("anchor", "锚点所属链不在本包事件中")) + return False + if anchor["head_hash"] not in hashes: + findings.append(_finding("anchor_mismatch", "锚点 head_hash 不在对应链的前缀内")) + return False + return True + + +def _plan_anchors(bundle: dict, chain_hashes: dict[tuple[str, str], set[str]], findings: list[dict]) -> list[dict]: + """校验并按整行幂等挑选要写的新锚点（与库内及包内完全相同的不再写）。""" + columns = ("source", "trace_id", "stage", "head_hash", "fixed_in", "ts") + existing = {tuple(row[name] for name in columns) for row in events_db.read_anchors()} + planned: list[dict] = [] + seen: set[tuple] = set() + for anchor in bundle["anchors"]: + if not _valid_anchor(anchor, chain_hashes, findings): + continue + key = tuple(anchor[name] for name in columns) + if key in existing or key in seen: + continue + seen.add(key) + planned.append({name: anchor[name] for name in columns}) + return planned + + +def import_bundle(bundle: dict) -> dict: + """幂等导入 EventBundle，返回 {imported, skipped, findings}；坏包整体不写。 + + 验证（schema、来源、隐私、哈希、链完整性）全过后才在单事务里写事件、引用与锚点；任何发现都 + 不写，findings 逐项给出 code 与不含内容的 detail。 + """ + findings: list[dict] = [] + if not _validate_envelope(bundle, findings): + return {"imported": 0, "skipped": 0, "findings": findings} + validated = [triple for index, event in enumerate(bundle["events"]) + if (triple := _validate_event(event, index, findings)) is not None] + fresh, skipped, chain_hashes = _plan_chains(bundle, validated, findings) + anchors = [] if findings else _plan_anchors(bundle, chain_hashes, findings) + if findings: + return {"imported": 0, "skipped": 0, "findings": findings} + try: + imported = events_db.import_rows(fresh, anchors) + except sqlite3.IntegrityError: + return {"imported": 0, "skipped": 0, + "findings": [_finding("seq_conflict", "写入时撞上已有事件（并发），整体未写")]} + except RuntimeError as exc: + return {"imported": 0, "skipped": 0, "findings": [_finding("storage", str(exc))]} + return {"imported": imported, "skipped": skipped, "findings": []} + + +# ---- CI 事件包下载（C5 load_ci，T302）：只信 API，不信包自报；下载内容只解析、不执行 ---- + +_TRUSTED_PATHS = frozenset(f".github/workflows/{name}{ext}" + for name in ("harness", "auto-merge") for ext in (".yml", ".yaml")) +_PACKAGE_RE = re.compile(r"harness-events-(\d+)-(\d+)-(.+)\Z") + + +class GhClient: + """load_ci 缺省 gh 客户端：只读查询与 artifact 下载，失败抛 RuntimeError。""" + + def _run(self, argv: list[str]) -> bytes: + result = subprocess.run(["gh", *argv], capture_output=True, check=False) + if result.returncode != 0: + raise RuntimeError(result.stderr.decode("utf-8", "replace").strip() or f"gh 退出码 {result.returncode}") + return result.stdout + + def pr(self, pr: int) -> dict: + data = json.loads(self._run(["pr", "view", str(pr), "--json", "headRefName,headRefOid,baseRepository"])) + base = (data.get("baseRepository") or {}).get("name") + owner = ((data.get("baseRepository") or {}).get("owner") or {}).get("login") + return {"headRefName": data.get("headRefName"), "headRefOid": data.get("headRefOid"), + "repository": f"{owner}/{base}" if owner and base else None} + + def api(self, route: str): + return json.loads(self._run(["api", route])) + + def download(self, url: str) -> bytes: + return self._run(["api", url]) + + +def _repo_identity(value: str) -> str: + """GitHub 仓库身份（owner/repo）：兼容 https/ssh 远程 URL；无 URL 形状时原样参与比较。""" + match = re.search(r"[:/]([A-Za-z0-9_.-]+)/([A-Za-z0-9_.-]+?)(?:\.git)?/?\Z", value.strip()) + return f"{match[1]}/{match[2]}" if match else value + + +def _list_all(client, route: str, key: str, findings: list[dict]) -> list[dict] | None: + """按 total_count 分页取全列表；API 失败、形状不符或 50 页不收敛记 api 发现并放弃。""" + items: list[dict] = [] + for page in range(1, 51): + try: + data = client.api(f"{route}&page={page}") + except (RuntimeError, ValueError) as exc: + findings.append(_finding("api", f"列出 {key} 第 {page} 页失败：{exc}")) + return None + batch = data.get(key) if isinstance(data, dict) else None + if not isinstance(batch, list): + findings.append(_finding("api", f"{key} 响应形状不符（第 {page} 页）")) + return None + items += [item for item in batch if isinstance(item, dict)] + total = data.get("total_count") + if not batch or (isinstance(total, int) and len(items) >= total): + return items + findings.append(_finding("api", f"{key} 分页 50 页未收敛，已放弃")) + return None + + +def _package_member(members: dict[str, bytes]) -> dict | None: + """zip 内唯一的 harness-events JSON 包成员（按形状初筛）；没有或多个都视为坏包。""" + found: list[dict] = [] + for name, content in sorted(members.items()): + if name.lower().endswith(".json"): + try: + parsed = json.loads(content) + except ValueError: + parsed = None + if (isinstance(parsed, dict) and parsed.get("schema_version") == BUNDLE_SCHEMA_VERSION + and "origin" in parsed and isinstance(parsed.get("events"), list)): + found.append(parsed) + return found[0] if len(found) == 1 else None + + +def _import_package(data: bytes, match: re.Match, branch: str, head: str, repo: str, + findings: list[dict]) -> tuple[int, int] | None: + """一个下载完成的包：zip→bundle→origin/来源核对→幂等导入；坏包记发现返回 None。""" + run_id, attempt, job = match.groups() + label = f"包 {run_id}/{attempt}/{job}" + try: + archive = zipfile.ZipFile(io.BytesIO(data)) + members = {name: archive.read(name) for name in archive.namelist() if not name.endswith("/")} + except zipfile.BadZipFile: + findings.append(_finding("package_corrupt", f"{label} 不是有效的 zip")) + return None + bundle = _package_member(members) + if bundle is None: + findings.append(_finding("package_corrupt", f"{label} 找不到唯一的 harness-events JSON")) + return None + origin = bundle.get("origin") if isinstance(bundle.get("origin"), dict) else {} + expected = {"run_id": run_id, "run_attempt": attempt, "job": job, "head_sha": head, "head_branch": branch} + wrong = [key for key, want in expected.items() if str(origin.get(key)) != want] + if "repository" not in origin or _repo_identity(str(origin["repository"])) != _repo_identity(repo): + wrong.append("repository") + if wrong: + findings.append(_finding("origin_mismatch", f"{label} origin 与 API 不符（{','.join(wrong)}）")) + return None + if any(event.get("source") != f"ci:{run_id}:{attempt}:{job}" for event in bundle["events"]): + findings.append(_finding("source_mismatch", f"{label} 事件来源不是 ci:{run_id}:{attempt}:{job}")) + return None + result = import_bundle(bundle) + findings.extend(result["findings"]) + return result["imported"], result["skipped"] + + +def _download_run(client, run: dict, packages: list[dict], branch: str, head: str, repo: str, + findings: list[dict]) -> tuple[int, int]: + """下载并导入一个运行的事件包，返回（新增，跳过）；过期/名字不符/下载失败逐项记发现。""" + imported = skipped = 0 + for item in packages: + name = str(item.get("name")) + if item.get("expired"): + findings.append(_finding("artifact_expired", f"artifact {name} 已过期，未下载")) + continue + match = _PACKAGE_RE.fullmatch(name) + if match[1] != str(run.get("id")) or match[2] != str(run.get("run_attempt")): + findings.append(_finding("artifact_name", f"artifact {name} 物理名与运行不符，未下载")) + continue + try: + data = client.download(str(item.get("archive_download_url"))) + except (RuntimeError, ValueError) as exc: + findings.append(_finding("api", f"下载 {name} 失败：{exc}")) + continue + if result := _import_package(data, match, branch, head, repo, findings): + imported, skipped = imported + result[0], skipped + result[1] + return imported, skipped + + +def load_ci(pr: int, *, head: str | None = None, gh=None) -> dict: + """下载 PR 关联的 CI 事件包（harness 与 auto-merge 多 job/attempt）并幂等导入（C5）。 + + 只信 API：仓库、工作流路径、run/head 与物理名逐项核对包 origin 与事件 source；过期/缺失/ + 坏包/其他 head 的运行逐项记 findings；分页不收敛明确报告。 + """ + findings: list[dict] = [] + client = GhClient() if gh is None else gh + try: + info = client.pr(pr) + branch, resolved, repo = info.get("headRefName"), head or info.get("headRefOid"), info.get("repository") + except (RuntimeError, ValueError, AttributeError) as exc: + return {"imported": 0, "skipped": 0, "findings": [_finding("api", f"查询 PR {pr} 失败：{exc}")]} + if not (isinstance(branch, str) and branch and isinstance(resolved, str) + and _SHA_RE.fullmatch(resolved) and isinstance(repo, str) and repo): + return {"imported": 0, "skipped": 0, + "findings": [_finding("api", f"PR {pr} 的 head/仓库信息缺失或形状不符")]} + route = f"repos/{repo}/actions/runs?branch={urllib.parse.quote(branch, safe='')}&per_page=100" + runs = _list_all(client, route, "workflow_runs", findings) + if runs is None: + return {"imported": 0, "skipped": 0, "findings": findings} + trusted = [run for run in runs if run.get("path") in _TRUSTED_PATHS] + stale = sorted({str(run.get("headSha")) for run in trusted if run.get("headSha") != resolved}) + if stale: + findings.append(_finding("head_mismatch", f"{len(stale)} 个其他 head 的运行未导入" + f"（只导入 head {resolved[:7]}…）")) + imported = skipped = 0 + for run in sorted(trusted, key=lambda item: str(item.get("id"))): + if run.get("headSha") != resolved: + continue + artifacts = _list_all(client, f"repos/{repo}/actions/runs/{run.get('id')}/artifacts?per_page=100", + "artifacts", findings) + if artifacts is None: + continue + packages = [item for item in artifacts if _PACKAGE_RE.fullmatch(str(item.get("name") or ""))] + if not packages: + findings.append(_finding("artifact_missing", f"运行 {run.get('id')} 没有 harness-events 事件包")) + got = _download_run(client, run, packages, branch, resolved, repo, findings) + imported, skipped = imported + got[0], skipped + got[1] + return {"imported": imported, "skipped": skipped, "findings": findings} diff --git a/.harness/engine/prompts/review_prompt.md b/.harness/engine/prompts/review_prompt.md index d1acfae..8bf3f8a 100644 --- a/.harness/engine/prompts/review_prompt.md +++ b/.harness/engine/prompts/review_prompt.md @@ -4,7 +4,7 @@ - `build/review/pr.md`：PR 标题与描述（作者的自述，不能当作证据） - `build/review/task.md`：任务书（没有任务书时写明「无」） -- `build/review/diff.patch`：相对合并基点的完整改动 +- `build/review/diff.patch`：相对合并基点的完整改动；超 2MB 时按文件分片为 `diff-01.patch`、`diff-02.patch`…（全部分片，无截断），以 `pack.md` 顶部的材料清单为准 - `build/review/pack.md`：机器生成的证据包（风险等级、修复证据的提交与测试映射、涉及的验收编号） - `build/review/ci.md`：当前 head 的 CI 检查结论与链接（核对 PR 描述中「CI 通过」一类说法以它为准） - 材料为空或与 PR 描述明显不符时，结论为「需用户验收」并在发现里写明，不要在空材料上给出通过 diff --git a/.harness/engine/reports/ci_events.py b/.harness/engine/reports/ci_events.py new file mode 100644 index 0000000..746eabb --- /dev/null +++ b/.harness/engine/reports/ci_events.py @@ -0,0 +1,191 @@ +"""CI 工作流的事件导出与 summary（B46 T303）：只被 harness/auto-merge 工作流模板调用，不注册进 +cli.py（无 CLI 注册冲突；导出与 summary 是观察旁路，查询与渲染不追加自身事件，共用合同 C1）。 + + python .harness/engine/reports/ci_events.py export --target <目录> + 先把当前链（default_source()）各 trace 的链头固定为 ci_artifact 锚点（设计 2.5：锚点随 + artifact 上传即被固定），再把该链的 EventBundle v1 写成 <目录>/harness-events.json，并把 + manifest 里的安全 JSON 内容产物复制进同一目录；目录先清空再写，上传集合因此精确等于 + 「bundle + 安全 manifest」，临时库 db/WAL/SHM 与本机原始日志不进入。缺库/无事件导出空包； + 被排除的产物以 findings 记进包内并向 stderr 提示，不中断导出。 + + python .harness/engine/reports/ci_events.py summary --bundle <文件> + 从导出的 bundle 渲染 verify/route 事件的 Markdown summary（检查/规则、状态、依据与理由、 + 错误）到 stdout，供工作流追加进 $GITHUB_STEP_SUMMARY。只读展示层：与原机器输出同源（同一 + 事件数据），保留既有步骤自己的 summary 输出与状态检查名，不替换、不改判定。 + +两种调用方式都须支持安装布局：直接脚本把包根（安装布局的 .harness、引擎仓库的仓库根）插入 +sys.path；`python -m engine.reports.ci_events` 在 .harness 目录（或 PYTHONPATH 含它）下运行。 +""" + +from __future__ import annotations + +import argparse +import json +import shutil +import sys +from pathlib import Path + +# 直接脚本运行时 sys.path[0] 是 reports 目录，先把包根插进去才能导入 engine 包； +# -m 方式下包根已在 sys.path，此行幂等。 +_PACKAGE_ROOT = str(Path(__file__).resolve().parents[2]) +if _PACKAGE_ROOT not in sys.path: + sys.path.insert(0, _PACKAGE_ROOT) + +from engine.core import events, events_db, events_io # 导入须在上面的 sys.path 准备之后 + +BUNDLE_NAME = "harness-events.json" # 逻辑名 harness-events 的包文件（artifact 内唯一 JSON 成员） +ANCHOR_STAGE = "ci" # 锚点在 CI 作业导出时固定，记 ci 阶段 +ANCHOR_FIXED_IN = "ci_artifact" # 链头固定处：CI 事件 artifact（events.set_anchor 的枚举值） +SUMMARY_STAGES = ("verify", "route") # summary 只渲染判定与路由事件（T303 目标终态） + + +# ---- export：链头锚点、EventBundle 与安全内容产物 ---- + +def _chains(source: str) -> list[tuple[str, str]]: + """当前来源参与导出的链（按链首次出现序）；查询只读。""" + chains: list[tuple[str, str]] = [] + for row in events_io.query(source=source): + key = (row["source"], row["trace_id"]) + if key not in chains: + chains.append(key) + return chains + + +def _fix_anchors(source: str) -> int: + """导出前把各链链头固定为 ci_artifact 锚点（set_anchor 永不抛异常）；返回固定的锚点数。""" + fixed = 0 + for chain_source, trace in _chains(source): + head = events.chain_head(trace, chain_source) # 签名是 chain_head(trace_id, source) + if head: + events.set_anchor(trace, ANCHOR_STAGE, head, ANCHOR_FIXED_IN, chain_source) + fixed += 1 + return fixed + + +def _reset_dir(target: Path) -> None: + """导出目录先清空再建：上传集合精确等于本次的 bundle + manifest，不混入上次残留。""" + if target.exists(): + shutil.rmtree(target) + target.mkdir(parents=True) + + +def _copy_manifest(bundle: dict, target: Path) -> int: + """把 manifest 里的安全内容产物复制进导出目录（文件名即 sha256）；失败抛 OSError 不当成功。""" + directory = events_db.artifacts_dir() + copied = 0 + for entry in bundle["artifacts"]: + if directory is None: + raise OSError("无本地产物目录，manifest 内容产物无法复制") + shutil.copyfile(directory / entry["file"], target / entry["file"]) + copied += 1 + return copied + + +def export_main(argv: list[str]) -> int: + """export 子命令：写 bundle 与安全内容产物；导出失败返回 1，不当成功。""" + parser = argparse.ArgumentParser(prog="ci_events export", description="导出当前链的 EventBundle 与安全内容产物") + parser.add_argument("--target", required=True, metavar="目录", + help="导出目录（存在则先清空；即 upload-artifact 的 path）") + args = parser.parse_args(argv) + source = events.default_source() + anchored = _fix_anchors(source) + bundle = events_io.export_bundle(source=source) + target = Path(args.target) + try: + _reset_dir(target) + events_io.write_bundle(bundle, target / BUNDLE_NAME) + copied = _copy_manifest(bundle, target) + except OSError as exc: + print(f"ci_events：导出失败：{exc}", file=sys.stderr) + return 1 + for finding in bundle["findings"]: + print(f"ci_events：已排除 {finding['code']}：{finding['detail']}", file=sys.stderr) + print(f"ci_events：导出 {len(bundle['events'])} 个事件、{len(bundle['anchors'])} 个锚点" + f"（本次固定 {anchored} 个）、{copied} 个内容产物 → {target / BUNDLE_NAME}") + return 0 + + +# ---- summary：从导出的 bundle 渲染 verify/route 事件 ---- + +def _sort_key(event: dict) -> tuple: + """确定性排序：时间、来源、seq（seq 缺失排前，保证任意 bundle 输出稳定）。""" + seq = event.get("seq") + return str(event.get("ts") or ""), str(event.get("source") or ""), seq if isinstance(seq, int) else 0 + + +def _decision_suffix(decision) -> str: + """依据后缀：by/rule 与理由引用；字段缺失只写有的部分。""" + if not isinstance(decision, dict): + return "" + basis = "/".join(str(decision[key]) for key in ("by", "rule") if decision.get(key) is not None) + reason = decision.get("reason") + if basis and reason is not None: + return f" · 依据 {basis}：{reason}" + if basis: + return f" · 依据 {basis}" + return f" · 依据：{reason}" if reason is not None else "" + + +def _event_line(event: dict) -> str: + """一行一个事件：检查/规则、状态、耗时、依据与理由、错误——与事件数据逐字同源。""" + duration = f"（{event['duration_ms']}ms）" if isinstance(event.get("duration_ms"), int) else "" + line = f"- {event.get('stage')}/{event.get('step')} {event.get('status')}{duration}" + line += _decision_suffix(event.get("decision")) + error = event.get("error") + if isinstance(error, dict) and error.get("kind") is not None: + line += f" · 错误 {error.get('kind')}" + return line + + +def render_summary(bundle: dict) -> str: + """verify/route 事件的确定性 Markdown：同一 bundle 输出逐字节相同；只读，不改数据。""" + rows = [event for event in bundle["events"] + if isinstance(event, dict) and event.get("stage") in SUMMARY_STAGES] + rows.sort(key=_sort_key) + chains = sorted({f"{event.get('source')} / {event.get('trace_id')}" for event in rows}) + lines = ["## harness 事件 summary", "", + f"链：{'；'.join(chains) if chains else '—'}｜verify/route 事件 {len(rows)} 条"] + lines += [_event_line(event) for event in rows] + if bundle.get("findings"): + lines.append(f"导出时排除 {len(bundle['findings'])} 项产物（见包内 findings）") + lines.append("") + return "\n".join(lines) + + +def summary_main(argv: list[str]) -> int: + """summary 子命令：只读 bundle 文件（不依赖本机库），渲染失败/形状不符返回 2。""" + parser = argparse.ArgumentParser(prog="ci_events summary", + description="从导出的 bundle 渲染 verify/route 事件 summary") + parser.add_argument("--bundle", required=True, metavar="文件", help="export 写出的 JSON 包") + args = parser.parse_args(argv) + try: + bundle = json.loads(Path(args.bundle).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError) as exc: + print(f"ci_events：读取 bundle 失败：{exc}", file=sys.stderr) + return 2 + except ValueError: + print(f"ci_events：{args.bundle} 不是有效的 JSON", file=sys.stderr) + return 2 + if not isinstance(bundle, dict) or not isinstance(bundle.get("events"), list): + print("ci_events：bundle 形状不符（应有 events 列表）", file=sys.stderr) + return 2 + print(render_summary(bundle), end="") + return 0 + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + if not argv: + print("用法：ci_events.py export --target <目录> | summary --bundle <文件>", file=sys.stderr) + return 2 + name, rest = argv[0], argv[1:] + if name == "export": + return export_main(rest) + if name == "summary": + return summary_main(rest) + print(f"未知子命令：{name}（export | summary）", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.harness/engine/reports/github_events.py b/.harness/engine/reports/github_events.py new file mode 100644 index 0000000..c06fca4 --- /dev/null +++ b/.harness/engine/reports/github_events.py @@ -0,0 +1,386 @@ +"""GitHub 事实同步（B46 T304，设计 3.5）：合并后从 GitHub API 取回可核对的事实写成事件。 + + python .harness/engine/reports/github_events.py [--pr <号>] [--head <提交>] [--since ] + +--pr 同步指定 PR；缺省从 push 合并提交的信息解析关联 PR（merge 形式以「Merge pull request #N from」 +开头，squash 形式主题为「标题 (#N)」；head 缺省取 GITHUB_SHA，本地为 HEAD）。手工合并与 App 合并都经 +push 到 main 到达这里，不靠 auto-merge job 独占触发。--since 只增量读取该 +时间后更新的 escape 议题（登记后顺带增量同步，不引入 watcher）；省略则全量读。只被合并后 main 上的 +harness 工作流模板调用，不注册进 cli.py（观察旁路；查询与渲染不追加自身事件，共用合同 C1）。 + +merge 事件记录设计 3.5 全字段：批准者 login/type、批准绑定提交是否等于合并 head、合并者、合并方式 +（merge / squash / rebase）与合并时标签；缺批准（单账号 none）记 "none"，不捏造 App；无法确定合并方式 +记 "unknown"。 +audit_sample 与 escape 按任务书用 stage=ci，不新增 STAGES 枚举。 + +每类事实各成不可变快照：source = github::<规范化事实 sha256>（键排序紧凑 JSON 的 sha256）， +读取时间不进事实摘要，同输入事实同 source（同快照幂等，重复同步不新增事件），事实变化另起新链、 +不覆盖旧快照；事件按 PR 的 headRefName 归属（trace_id）。只读查询：不做任何合并、批准、推送动作； +API 失败逐项记 findings 明确报告事实缺失（退出码 1），不伪造完整事实，也不影响原有判定（工作流步骤 +continue-on-error 隔离）。 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import urllib.parse +from pathlib import Path + +# 直接脚本运行时 sys.path[0] 是 reports 目录，先把包根插进去才能导入 engine 包； +# -m 方式下包根已在 sys.path，此行幂等。 +_PACKAGE_ROOT = str(Path(__file__).resolve().parents[2]) +if _PACKAGE_ROOT not in sys.path: + sys.path.insert(0, _PACKAGE_ROOT) + +from engine.core import events, events_db, events_io # 导入须在上面的 sys.path 准备之后 +from engine.core.common import git + +# 议题标签约定与 engine/routing/policy.py 的登记一致（escape/class:<类别>；audit 由 auto-merge 创建）。 +ESCAPE_LABEL = "escape" +AUDIT_LABEL = "audit" +_CLASS_PREFIX = "class:" +_MERGE_PR_RE = re.compile(r"Merge pull request #(\d+) from ") +_SQUASH_PR_RE = re.compile(r"\(#(\d+)\)$") +_REF_RE = re.compile(r"#(\d+)") +_PAGE_LIMIT = 50 +_NONE = "none" +_UNKNOWN = "unknown" + + +def _finding(code: str, detail: str) -> dict: + return {"code": code, "detail": detail} + + +def _text(value, limit: int = 120) -> str | None: + """规范化字符串（超长、本机路径、换行丢弃为 None）：进入事实摘要前先过隐私口径，保证摘要可复算。""" + return events._clean_str(str(value), limit) if value is not None else None + + +class GhClient: + """缺省 gh 客户端：只读 API 查询，失败抛 RuntimeError；响应只解析、不执行。""" + + def _run(self, argv: list[str]) -> bytes: + result = subprocess.run(["gh", *argv], capture_output=True, check=False) + if result.returncode != 0: + raise RuntimeError(result.stderr.decode("utf-8", "replace").strip() or f"gh 退出码 {result.returncode}") + return result.stdout + + def api(self, route: str): + return json.loads(self._run(["api", route])) + + def repo(self) -> str: + """仓库身份 owner/repo：优先 Actions/gh 环境变量，否则问 gh。""" + value = os.environ.get("GITHUB_REPOSITORY") or os.environ.get("GH_REPO") + if value and value.strip(): + return value.strip() + return self._run(["repo", "view", "--json", "nameWithOwner", "-q", ".nameWithOwner"]).decode("utf-8").strip() + + +# ---- 事实读取（只读 API，全部记录 findings、不中断同步）---- + +def _collect_pages(client, route: str, what: str, findings: list[dict]) -> list[dict] | None: + """数组型列表端点分页取全（reviews/commits/issues 返回裸数组）：空页即收，50 页不收敛放弃。""" + separator = "&" if "?" in route else "?" + items: list[dict] = [] + for page in range(1, _PAGE_LIMIT + 1): + try: + batch = client.api(f"{route}{separator}per_page=100&page={page}") + except (RuntimeError, ValueError, AttributeError) as exc: + findings.append(_finding("api", f"读取{what}第 {page} 页失败：{exc}")) + return None + if not isinstance(batch, list): + findings.append(_finding("api", f"{what}响应形状不符（第 {page} 页不是列表）")) + return None + items += [item for item in batch if isinstance(item, dict)] + if not batch: + return items + findings.append(_finding("api", f"{what}分页 {_PAGE_LIMIT} 页未收敛，已放弃")) + return None + + +def _squash_pr(message: str) -> int | None: + """squash 合并提交的主题是「 (#N)」：从首行尾注解析 PR 号，非该形式返回 None。""" + match = _SQUASH_PR_RE.search(message.split("\n", 1)[0].strip()) + return int(match[1]) if match else None + + +def _resolve_pr(client, repo: str, head: str, findings: list[dict]) -> int | None: + """从 push 合并提交的信息解析关联 PR：merge 形式以「Merge pull request #N from」开头， + squash 形式（手工 squash 合并也经 push 到 main 到达）主题以「标题 (#N)」结尾。""" + try: + commit = client.api(f"repos/{repo}/commits/{head}") + except (RuntimeError, ValueError, AttributeError) as exc: + findings.append(_finding("api", f"读取提交 {head[:12]}… 失败：{exc}")) + return None + message = commit.get("commit", {}).get("message") if isinstance(commit, dict) else None + if isinstance(message, str): + merge = _MERGE_PR_RE.match(message) + if merge is not None: + return int(merge[1]) + squash = _squash_pr(message) + if squash is not None: + return squash + findings.append(_finding("no_pr", f"提交 {head[:12]}… 不是关联 PR 的合并提交，未同步")) + return None + + +def _pull(client, repo: str, pr: int, findings: list[dict]) -> dict | None: + """PR 核心事实：未合并或读取失败都不是事实，返回 None（不产出半个 PR 的快照）。""" + try: + pull = client.api(f"repos/{repo}/pulls/{pr}") + except (RuntimeError, ValueError, AttributeError) as exc: + findings.append(_finding("api", f"读取 PR {pr} 失败：{exc}")) + return None + if not isinstance(pull, dict) or pull.get("merged") is not True: + findings.append(_finding("not_merged", f"PR {pr} 未合并或响应形状不符，跳过（事实同步只覆盖已合并 PR）")) + return None + return pull + + +def _head_ref(pull: dict) -> str | None: + head = pull.get("head") + return _text(head.get("ref")) if isinstance(head, dict) else None + + +def _approval_fact(reviews: list[dict], head_sha: str | None) -> dict: + """最近一次有效批准：批准者 login/type、绑定提交与是否等于合并的 head（被合并的 PR head 提交， + 即 auto-merge --match-head-commit 的对象）。 + + 无批准（单账号 none）记 none——合并者不是批准者，不把 owner/merged_by 当批准者，更不捏造 App。 + """ + approved = [review for review in reviews if review.get("state") == "APPROVED"] + if not approved: + return {"approver": _NONE, "approver_type": _NONE, "approval_commit": None, "approval_bound": None} + last = approved[-1] # API 按时间升序返回，最后一条即最近一次批准 + user = last.get("user") if isinstance(last.get("user"), dict) else {} + commit = _text(last.get("commit_id"), 40) + return {"approver": _text(user.get("login")), "approver_type": _text(user.get("type")), + "approval_commit": commit, + "approval_bound": (commit == head_sha) if commit and head_sha else None} + + +def _is_squash_subject(message, title: str | None, pr: int) -> bool: + """单亲提交判 squash 的核对：提交首行恰为「 (#PR号)」，标题与尾注都核对上才算， + 碰巧以「(#N)」结尾的普通提交或真 rebase 的顶提交不会误判。""" + if not isinstance(message, str) or not isinstance(title, str) or not title.strip(): + return False + return message.split("\n", 1)[0].strip() == f"{title.strip()} (#{pr})" + + +def _merge_method(client, repo: str, pr: int, title: str | None, + merge_sha: str | None, findings: list[dict]) -> str: + """合并方式：双亲为 merge；单亲且主题恰为「PR 标题 (#PR号)」判 squash；单亲且 PR 多于一个 + 提交为 rebase；其余（含无法取回）如实记 unknown。""" + if not merge_sha: + return _UNKNOWN + try: + commit = client.api(f"repos/{repo}/commits/{merge_sha}") + parents = commit.get("parents") if isinstance(commit, dict) else None + message = commit.get("commit", {}).get("message") if isinstance(commit, dict) else None + except (RuntimeError, ValueError, AttributeError) as exc: + findings.append(_finding("api", f"读取合并提交 {merge_sha[:12]}… 失败：{exc}（合并方式记 unknown）")) + return _UNKNOWN + if isinstance(parents, list) and len(parents) == 2: + return "merge" + if isinstance(parents, list) and len(parents) == 1: + if _is_squash_subject(message, title, pr): + return "squash" + commits = _collect_pages(client, f"repos/{repo}/pulls/{pr}/commits", f"PR {pr} 提交列表", findings) + if commits is not None and len(commits) > 1: + return "rebase" + return _UNKNOWN + + +def _label_names(item: dict) -> list[str]: + return [_text(label.get("name")) for label in item.get("labels") or [] + if isinstance(label, dict) and _text(label.get("name"))] + + +def _mentions(item: dict, pr: int) -> bool: + """议题是否指认该 PR（标题或正文里的 #N 精确匹配；PR 本身不算 escape/audit 议题）。""" + if item.get("pull_request"): + return False + text = " ".join(item[key] for key in ("title", "body") if isinstance(item.get(key), str)) + return str(pr) in set(_REF_RE.findall(text)) + + +# ---- 快照（不可变事实，source=github::<规范化事实 sha256>，同快照幂等）---- + +def _snapshot_source(pr: int, facts: dict) -> str: + """source = github::<规范化事实摘要>：键排序紧凑 JSON 的 sha256，读取时间不进摘要。""" + canonical = json.dumps(facts, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest() + return f"github:{pr}:{digest}" + + +def _fresh(source: str, trace: str) -> bool: + """同快照幂等：该 (source, trace) 已有事件即重复同步，不再新增相同事实事件。""" + return not events_db.read_events(source=source, trace_id=trace) + + +def _emit(stage: str, step: str, trace: str, source: str, inputs: list[dict], outputs: dict) -> bool: + return events.emit(stage, step, "ok", trace_id=trace, source=source, + actor={"role": "engine", "host": "github"}, inputs=inputs, outputs=outputs) is not None + + +def _pr_inputs(repo: str, pr: int, extra: list[tuple[str, str | None]]) -> list[dict]: + refs = [{"kind": "pr", "ref": f"{repo}#{pr}"}] + refs += [{"kind": kind, "ref": value} for kind, value in extra if value] + return refs + + +def _sync_merge(client, repo: str, pr: int, pull: dict, trace: str, + findings: list[dict], snapshots: list[dict]) -> int: + """合并事实快照：merge 事件（设计 3.5 全字段）加逐标签事件；子事实取不到记 unknown/none。""" + head = pull.get("head") if isinstance(pull.get("head"), dict) else {} + head_sha = _text(head.get("sha"), 40) + merge_sha = _text(pull.get("merge_commit_sha"), 40) + merged_by = pull.get("merged_by") if isinstance(pull.get("merged_by"), dict) else {} + labels = sorted(set(_label_names(pull))) + reviews = _collect_pages(client, f"repos/{repo}/pulls/{pr}/reviews", f"PR {pr} 评审列表", findings) + if reviews is None: + approval = {"approver": _UNKNOWN, "approver_type": _UNKNOWN, + "approval_commit": None, "approval_bound": None} + else: + approval = _approval_fact(reviews, head_sha) + method = _merge_method(client, repo, pr, pull.get("title"), merge_sha, findings) + facts = {"kind": "merge", "pr": pr, "repository": events_io._repo_identity(repo), + "head_ref": _head_ref(pull), "head_sha": head_sha, + "merge_sha": merge_sha, "merged_at": _text(pull.get("merged_at")), + "merger": _text(merged_by.get("login")), "merger_type": _text(merged_by.get("type")), + **approval, "merge_method": method, "labels": labels} + source = _snapshot_source(pr, facts) + if not _fresh(source, trace): + return 0 + inputs = _pr_inputs(repo, pr, [("merge_commit", merge_sha), + ("head", facts["head_sha"])]) + outputs = {"pr": pr, **approval, "merger": facts["merger"], "merger_type": facts["merger_type"], + "merge_method": method, "label_count": len(labels), "merged_at": facts["merged_at"]} + emitted = int(_emit("merge", "github.merge", trace, source, inputs, outputs)) + for label in labels: + emitted += int(_emit("merge", "github.merge_label", trace, source, inputs, {"pr": pr, "label": label})) + snapshots.append({"kind": "merge", "source": source}) + return emitted + + +def _sync_audit(client, repo: str, pr: int, trace: str, + findings: list[dict], snapshots: list[dict]) -> int: + """抽审事实快照：audit 议题是否指认该 PR；读不到时如实记 unknown，不伪造抽样结果。""" + issues = _collect_pages(client, f"repos/{repo}/issues?state=all&labels={AUDIT_LABEL}", + "audit 议题列表", findings) + sampled, issue_number = _UNKNOWN, None + if issues is not None: + sampled, issue_number = False, None + for item in issues: + if isinstance(item.get("number"), int) and _mentions(item, pr): + sampled, issue_number = True, item["number"] + break + source = _snapshot_source(pr, {"kind": "audit_sample", "pr": pr, + "sampled": sampled, "issue": issue_number}) + if not _fresh(source, trace): + return 0 + outputs = {"pr": pr, "sampled": sampled, "issue": issue_number} + inputs = _pr_inputs(repo, pr, [("issue", str(issue_number) if issue_number else None)]) + emitted = int(_emit("ci", "github.audit_sample", trace, source, inputs, outputs)) + snapshots.append({"kind": "audit_sample", "source": source}) + return emitted + + +def _sync_escapes(client, repo: str, pr: int, trace: str, since: str | None, + findings: list[dict], snapshots: list[dict]) -> int: + """escape 事实快照：每个指认该 PR 的 escape 议题一条，类别取 class:<类别> 标签。 + + 后登记的 escape 是新事实、另起新链，不覆盖旧快照；since 只减少读取量（增量），不影响去重。 + """ + route = f"repos/{repo}/issues?state=all&labels={ESCAPE_LABEL}" + if since: + route += f"&since={urllib.parse.quote(str(since), safe='')}" + issues = _collect_pages(client, route, "escape 议题列表", findings) + if issues is None: + return 0 + emitted = 0 + seen = 0 + for item in issues: + if not isinstance(item.get("number"), int) or not _mentions(item, pr): + continue + classes = sorted(name for name in _label_names(item) if name.startswith(_CLASS_PREFIX)) + klass = classes[0][len(_CLASS_PREFIX):] if classes else _UNKNOWN + facts = {"kind": "escape", "pr": pr, "issue": item["number"], "class": klass, + "state": _text(item.get("state")), "created_at": _text(item.get("created_at"))} + source = _snapshot_source(pr, facts) + if not _fresh(source, trace): + continue + outputs = {"pr": pr, "issue": item["number"], "class": klass, "state": facts["state"]} + emitted += int(_emit("ci", "github.escape", trace, source, + _pr_inputs(repo, pr, [("issue", str(item["number"]))]), outputs)) + seen += 1 + snapshots.append({"kind": "escape", "issues": seen}) + return emitted + + +# ---- 入口 ---- + +def _default_head() -> str | None: + """缺省 head：Actions 的 GITHUB_SHA，本地为当前 HEAD；都没有返回 None。""" + sha = os.environ.get("GITHUB_SHA") + return sha or git("rev-parse", "HEAD", cwd=events_db.ROOT, check=False, isolate=True) or None + + +def sync(pr: int | None = None, *, head: str | None = None, since: str | None = None, gh=None) -> dict: + """同步一个已合并 PR 的 GitHub 事实，返回 {pr, trace, snapshots, findings, events}。 + + pr 缺省时从 head（缺省 GITHUB_SHA/HEAD）的合并提交信息解析；gh 供测试注入桩客户端。只读查询： + 任何 API 失败都进 findings（退出码 1 由 main 报告），已取得的事实照常成快照，缺失的不伪造。 + """ + findings: list[dict] = [] + snapshots: list[dict] = [] + result = {"pr": None, "trace": None, "snapshots": snapshots, "findings": findings, "events": 0} + client = GhClient() if gh is None else gh + try: + repo = events_io._repo_identity(str(client.repo())) + except (RuntimeError, ValueError, AttributeError) as exc: + findings.append(_finding("api", f"确定仓库失败：{exc}")) + return result + if pr is None: + pr = _resolve_pr(client, repo, head or _default_head() or "", findings) + if pr is None: + return result + pull = _pull(client, repo, pr, findings) + if pull is None: + return result + trace = _head_ref(pull) + if not trace: + findings.append(_finding("shape", f"PR {pr} 的 headRefName 缺失，事件无法归属 PR 分支")) + return result + result["pr"] = pr + result["trace"] = trace + result["events"] += _sync_merge(client, repo, pr, pull, trace, findings, snapshots) + result["events"] += _sync_audit(client, repo, pr, trace, findings, snapshots) + result["events"] += _sync_escapes(client, repo, pr, trace, since, findings, snapshots) + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="github_events", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--pr", type=int, help="同步指定 PR（缺省从 push 合并提交信息解析）") + parser.add_argument("--head", metavar="提交", help="用该提交的信息解析关联 PR（缺省 GITHUB_SHA 或本地 HEAD）") + parser.add_argument("--since", metavar="UTC 时间", help="只增量读取该时间后更新的 escape 议题") + args = parser.parse_args(argv) + result = sync(args.pr, head=args.head, since=args.since) + for finding in result["findings"]: + print(f"github_events：{finding['detail']}", file=sys.stderr) + if result["pr"] is not None: + print(f"github_events：PR #{result['pr']}（{result['trace']}）：" + f"快照 {len(result['snapshots'])} 个、事件 {result['events']} 条") + return 1 if result["findings"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.harness/engine/reports/ledger.py b/.harness/engine/reports/ledger.py new file mode 100644 index 0000000..ca8a3e2 --- /dev/null +++ b/.harness/engine/reports/ledger.py @@ -0,0 +1,560 @@ +"""合并账本与 PR 锚点（B46 T305，设计 4.1，共用合同 C6）：把一个已合并 PR 的全部环节压成一条 +长期 JSON 账本，追加到 harness-audit 分支，并在 PR 上留一条可更新的锚点评论。 + + python .harness/engine/reports/ledger.py [--pr <号>] [--head <提交>] + +--head 缺省取 GITHUB_SHA（本地为 HEAD）；--pr 缺省从合并提交信息解析（同 github_events 的解析）。 +只被合并后 main 上的受信任工作流 job 调用（引擎来自默认分支检出，不执行 PR 代码），不注册进 cli.py +（观察旁路，共用合同 C1）。 + +build_ledger(pr, *, gh=None, cwd=ROOT) 只依赖三类可复取来源，不依赖任何既有本机库（runner 上的临时库 +只作导入中转）：git 里合并 head 上的运行记录（C3 摘要，标 evidence_kind=run_record_summary）、经 +T302 load_ci 幂等导入的 CI/route 事件包（C5 原始事件）、T304 github_events 同步的合并/抽审/逃逸事实 +快照，以及 PR 评论里 T105 写下的 harness-review-audit 单行 JSON（只按标记/schema v1/head/材料字段 +解析；缺标记、字段不完整、错误 head、未来版本都如实列 missing，不猜旧评论正文）。运行记录一律按 +合并 head 读取（路径@merge_sha），不用最新 head 的记录顶替。不足的环节写 missing，不捏造。 + +账本写 harness-audit/<合并 UTC 年份>/.json，仅已合并 PR（关闭未合并为 B52，不写）。原始日志、 +Pi 流、本机 SQLite 与本机内容产物一律不入账本（引用与哈希除外）。publish_ledger 用 GITHUB_TOKEN 做 +普通 fast-forward 追加：同 PR 相同字节幂等跳过；不同字节冲突不覆盖旧文件；推送竞争 fetch 后有限重试， +禁强推、禁删除。harness-audit 的 ruleset 模板只禁删除/强推、保护历史，并不阻止普通修改旧文件—— +「只追加」的边界由 writer 幂等/冲突自查保证，本模块与工作流注释都不宣称 ruleset 独自保证只追加。 +后登记 escape 由 github_events 以新 API 快照追加事件，不覆写已固定的合并账本。 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +# 直接脚本运行时 sys.path[0] 是 reports 目录，先把包根插进去才能导入 engine 包； +# -m 方式下包根已在 sys.path，此行幂等。 +_PACKAGE_ROOT = str(Path(__file__).resolve().parents[2]) +if _PACKAGE_ROOT not in sys.path: + sys.path.insert(0, _PACKAGE_ROOT) + +from engine.core import events, events_db, events_io # 导入须在上面的 sys.path 准备之后 +from engine.core.common import ROOT, clean_git_env, git +from engine.reports import github_events + +LEDGER_SCHEMA_VERSION = 1 +AUDIT_BRANCH = "harness-audit" # 账本专用分支（设计 4.1 方案 A，用户已审定） +ANCHOR_PREFIX = "harness-audit:" # PR 锚点评论的稳定标记前缀（后接 PR 号） +_REVIEW_RE = re.compile(r"") +_LEGACY_REVIEW_MARK = "", + "合并账本已固定（合并账本与链头锚点）：", + f"- 分支：`{result['branch']}`", + f"- 文件：`{result['path']}` @ {result['commit']}", + f"- 文件 sha256：`{result['sha256']}`", + f"- 链头：{chains or '（无事件链）'}", + ]) + "\n" + + +def _upsert_anchor_comment(client, ledger: dict, result: dict, findings: list[dict]) -> str | None: + """同标记评论更新而非新增：找到 harness-audit: 标记的既有评论就 PATCH，否则 POST。""" + marker = f"" + body = _anchor_body(ledger, result) + route = f"repos/{ledger['repository']}/issues/{ledger['pr']}/comments" + pages = github_events._collect_pages(client, route, "PR 评论", findings) + existing = next((item for item in pages or [] if isinstance(item, dict) and marker in str(item.get("body"))), None) + try: + if existing is not None: + client.api(f"repos/{ledger['repository']}/issues/comments/{existing.get('id')}", + method="PATCH", payload={"body": body}) + return "updated" + client.api(route, method="POST", payload={"body": body}) + return "created" + except (RuntimeError, ValueError, AttributeError) as exc: + findings.append(_finding("comment", f"锚点评论写入失败：{exc}")) + return None + + +def publish_ledger(ledger: dict, *, gh=None) -> dict: + """把账本 fast-forward 追加到 harness-audit 分支并维护 PR 锚点评论（C6）。 + + 返回 {ok, branch, path, commit, sha256, updated, conflict, comment, findings}；账本写入成功即 + ok（评论失败只记 findings）；同字节幂等 updated=False；冲突与远端故障 ok=False 且不动远端。 + """ + findings: list[dict] = [] + result = {"ok": False, "branch": AUDIT_BRANCH, "path": None, "commit": None, "sha256": None, + "updated": False, "conflict": False, "comment": None, "findings": findings} + problem = _ledger_problem(ledger) + if problem is not None: + findings.append(problem) + return result + data = canonical(ledger).encode("utf-8") + b"\n" + result["sha256"] = _digest(data) + result["path"] = f"{ledger['merged_at'][:4]}/{ledger['pr']}.json" + origin = git("remote", "get-url", "origin", cwd=events_db.ROOT, check=False, isolate=True) + if not origin: + findings.append(_finding("no_origin", "仓库没有 origin 远程，无法推送账本")) + return result + work = Path(tempfile.mkdtemp(prefix="harness-audit-")) + try: + cloned = subprocess.run(["git", "clone", "--quiet", "--no-checkout", origin, str(work)], + capture_output=True, env=clean_git_env(), check=False) + if cloned.returncode != 0: + findings.append(_finding("clone", "克隆远程失败：" + cloned.stderr.decode("utf-8", "replace").strip())) + return result + status = "retry" + for _ in range(_PUSH_ATTEMPTS): + status = _attempt_append(work, ledger["pr"], result["path"], data, result) + if status != "retry": + break + if status == "retry": + findings.append(_finding("push_exhausted", f"推送竞争 {_PUSH_ATTEMPTS} 次未成功，放弃（不强推）")) + if result["commit"] is not None: + result["ok"] = True + result["comment"] = _upsert_anchor_comment(GhClient() if gh is None else gh, ledger, result, findings) + finally: + shutil.rmtree(work, ignore_errors=True) + return result + + +def main(argv: list[str] | None = None) -> int: + """工作流入口：解析 PR → build_ledger → publish_ledger；发现与缺失都明确报告。""" + parser = argparse.ArgumentParser(prog="ledger", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--pr", type=int, help="写指定 PR 的账本（缺省从合并提交信息解析）") + parser.add_argument("--head", metavar="提交", help="用该提交解析关联 PR（缺省 GITHUB_SHA 或本地 HEAD）") + args = parser.parse_args(argv) + client = GhClient() + findings: list[dict] = [] + head = args.head or os.environ.get("GITHUB_SHA") \ + or git("rev-parse", "HEAD", cwd=events_db.ROOT, check=False, isolate=True) or "" + pr = args.pr + if pr is None: + try: + repo = events_io._repo_identity(str(client.repo())) + except (RuntimeError, ValueError, AttributeError) as exc: + print(f"ledger：确定仓库失败：{exc}", file=sys.stderr) + return 1 + pr = github_events._resolve_pr(client, repo, head, findings) + if pr is None: + for finding in findings: + print(f"ledger：{finding['detail']}", file=sys.stderr) + return 1 + try: + ledger = build_ledger(pr, gh=client) + except LedgerError as exc: + print(f"ledger：{exc}", file=sys.stderr) + return 1 + for item in ledger["missing"]: + print(f"ledger：缺 {item['item']}（{item['reason']}）{item['detail']}") + result = publish_ledger(ledger, gh=client) + for finding in result["findings"]: + print(f"ledger：{finding['detail']}", file=sys.stderr) + if result["ok"]: + print(f"ledger：账本 {result['branch']}/{result['path']}（commit {result['commit'][:12]}…，" + f"sha256 {result['sha256'][:12]}…，{'更新' if result['updated'] else '幂等命中'}，" + f"锚点评论 {result['comment'] or '未写入'}）") + return 0 if result["ok"] and not result["findings"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.harness/engine/reports/trace.py b/.harness/engine/reports/trace.py new file mode 100644 index 0000000..7f4cb0e --- /dev/null +++ b/.harness/engine/reports/trace.py @@ -0,0 +1,455 @@ +"""events 与 trace 查询命令（可观测性 P3，设计 4.2）：只读观察入口，查询不追加自身事件（共用合同 C1）。 + + python3 cli.py events [--since 1d] [--stage verify] [--status deny] [--json] + [--export 路径 | --import 路径] + 过滤展示与计数；--export/--import 走 C5 的整库导出与幂等导入（展示过滤不进导出包，筛选后 + 的断链子集不是可导入 bundle）。非法枚举/时间/组合返回 2；导入有发现返回 1。 + + python3 cli.py trace <任务编号 | PR 号 | 分支> [--ci] [--json] + 任务号经运行记录（docs/runs，task 字段须相符）与任务书文件名映射到分支；PR 号经 API + headRefName 换分支（不读检出分支，也不读 PR 正文）；完整分支名即 trace。多分支同任务、或 + 分支关联多个 PR 时列出候选（attempt/链头/时间），不随意选第一条。时间线标注输入/输出引用、 + 决定、角色、时长、来源与尝试分段（运行记录窗口优先，CI 链按来源各成一段），汇总最长阶段与 + 首个失败（deny/fail/error 区分）。--ci 经 C5 load_ci 下载 PR 关联的 CI 事件包幂等导入后再 + 展示；缺本机库或下载有发现时如实报告，不当成功。 + +退出码：0 正常（含歧义候选）；1 运行受阻（gh 失败、--ci 未完成或导入有发现）；2 参数与解析错误。 +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections import Counter +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from engine.core import events, events_db, events_io + +_TASK_RE = re.compile(r"[Tt](\d+)\Z") +_PR_RE = re.compile(r"#?(\d+)\Z") +_DURATION_RE = re.compile(r"(\d+)([dhms])\Z") +_UNITS = {"d": 86400, "h": 3600, "m": 60, "s": 1} +_FAILURES = ("fail", "deny", "error") + + +def _parse_since(text: str) -> datetime: + """--since 的取值： 相对时长或 ISO 时间戳；不合法抛 ValueError（命令转退出码 2）。""" + match = _DURATION_RE.fullmatch(text.strip()) + if match: + return datetime.now(UTC) - timedelta(seconds=int(match[1]) * _UNITS[match[2]]) + try: + parsed = datetime.fromisoformat(text.strip()) + except ValueError: + raise ValueError(f"--since 无法解析为时长或 UTC 时间：{text}") from None + return parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed.astimezone(UTC) + + +def _parse_ts(value) -> datetime | None: + """记录/事件里的时间戳 → aware UTC；缺失或损坏返回 None（按无法分段处理）。""" + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.strip()) + except ValueError: + return None + return parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed.astimezone(UTC) + + +# ---- events：过滤、计数与 C5 导出/导入 ---- + + +def _counts(rows: list[dict]) -> dict: + """过滤集的 stage/status 计数（跟随筛选结果，不跟总计）。""" + return {"stage": dict(sorted(Counter(row["stage"] for row in rows).items())), + "status": dict(sorted(Counter(row["status"] for row in rows).items()))} + + +def _export(target: Path) -> int: + try: + bundle = events_io.export_bundle() + events_io.write_bundle(bundle, target) + except OSError as exc: + print(f"导出失败：{exc}", file=sys.stderr) + return 1 + print(f"已导出 {len(bundle['events'])} 个事件、{len(bundle['chains'])} 条链、" + f"{len(bundle['anchors'])} 个锚点 → {target}") + return 0 + + +def _import(source: Path) -> int: + try: + bundle = json.loads(source.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError) as exc: + print(f"读取失败：{exc}", file=sys.stderr) + return 2 + except ValueError: + print(f"{source} 不是有效的 JSON", file=sys.stderr) + return 2 + result = events_io.import_bundle(bundle if isinstance(bundle, dict) else {}) + for finding in result["findings"]: + print(f"发现 {finding['code']}：{finding['detail']}", file=sys.stderr) + print(f"导入完成：新增 {result['imported']}，跳过 {result['skipped']}，发现 {len(result['findings'])} 项") + return 1 if result["findings"] else 0 + + +def events_main(argv: list[str] | None = None) -> int: + """events 子命令：过滤展示与计数；--export/--import 整库模式供工作流使用。""" + parser = argparse.ArgumentParser(prog="events", description="事件过滤展示、计数与 C5 导出/导入") + parser.add_argument("--since", metavar="时长|UTC", help="只看此后的事件：如 1d、12h 或 ISO 时间戳") + parser.add_argument("--stage", choices=events.STAGES, help="按环节过滤") + parser.add_argument("--status", choices=events.STATUSES, help="按状态过滤") + parser.add_argument("--json", action="store_true", help="输出 JSON（过滤集、计数与总计）") + parser.add_argument("--export", metavar="路径", help="导出整库 EventBundle 到该文件（不含展示过滤）") + parser.add_argument("--import", dest="import_path", metavar="路径", help="从该文件幂等导入 EventBundle") + args = parser.parse_args(argv) + if (args.export or args.import_path) and (args.since or args.stage or args.status): + print("--export/--import 是整库模式，不与 --since/--stage/--status 同用", file=sys.stderr) + return 2 + if args.export and args.import_path: + print("--export 与 --import 只能二选一", file=sys.stderr) + return 2 + if args.export: + return _export(Path(args.export)) + if args.import_path: + return _import(Path(args.import_path)) + try: + since = _parse_since(args.since) if args.since else None + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 + filtered = args.since is not None or args.stage is not None or args.status is not None + rows = sorted(events_io.query(since=since, stage=args.stage, status=args.status), + key=lambda row: (row["ts"], row["source"], row["seq"])) + total = len(events_io.query()) if filtered else len(rows) + if args.json: + print(json.dumps({"count": len(rows), "total": total, "counts": _counts(rows), + "events": rows}, ensure_ascii=False, indent=2)) + return 0 + print(f"事件 {len(rows)} 条 / 总计 {total} 条") + for name, counted in _counts(rows).items(): + print(f"按 {name}：" + ("、".join(f"{key} {value}" for key, value in counted.items()) or "—")) + for row in rows: + duration = f" {row['duration_ms']}ms" if row["duration_ms"] is not None else "" + print(f" #{row['seq']} {row['ts']} [{row['source']}] {row['stage']}/{row['step']} " + f"{row['status']}{duration}") + return 0 + + +# ---- trace：标识解析 ---- + + +@dataclass +class Candidate: + """一个候选：分支（run_record/taskbook 来源）或 PR（push_pr 来源），带 attempt/链头/时间。""" + + branch: str + origin: str + attempt: int | None = None + ts: str | None = None + end: str | None = None + head: str | None = None + exit_status: str | None = None + pr: int | None = None + + def row(self) -> dict: + return {"branch": self.branch, "origin": self.origin, "attempt": self.attempt, "ts": self.ts, + "head": self.head, "exit": self.exit_status, "pr": self.pr} + + +def _read_record(path: Path) -> dict | None: + try: + record = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, ValueError): + return None + return record if isinstance(record, dict) else None + + +def _record_candidate(record: dict, origin: str) -> Candidate | None: + """运行记录 → 候选：branch 必填；attempt/时间/退出与链头（锚点或 stages 末项）尽力补齐。""" + branch = record.get("branch") + if not isinstance(branch, str) or not branch: + return None + head = None + if isinstance(record.get("anchors"), list) and record["anchors"]: + last = record["anchors"][-1] + head = last.get("head_hash") if isinstance(last, dict) else None + stages = record.get("stages") + if head is None and isinstance(stages, list) and stages and isinstance(stages[-1], dict): + head = stages[-1].get("head_hash") + attempt = record.get("attempt") + return Candidate(branch, origin, attempt=attempt if isinstance(attempt, int) else None, + ts=record.get("started_at") if isinstance(record.get("started_at"), str) else None, + end=record.get("ended_at") if isinstance(record.get("ended_at"), str) else None, + head=head if isinstance(head, str) else None, + exit_status=record.get("exit") if isinstance(record.get("exit"), str) else None) + + +def _scan_records(match) -> list[Candidate]: + """扫描 docs/runs 下所有运行记录，branch/trace_id 命中的转为候选；损坏或不相符的跳过。""" + candidates: list[Candidate] = [] + runs = events_db.ROOT / "docs" / "runs" + for directory in sorted(runs.glob("*")) if runs.is_dir() else []: + for path in sorted(directory.glob("*.json")) if directory.is_dir() else []: + record = _read_record(path) + if record and match(record): + candidate = _record_candidate(record, "run_record") + if candidate is not None: + candidates.append(candidate) + return candidates + + +def _task_candidates(number: int) -> list[Candidate]: + """任务号 → 候选分支：运行记录（task 字段须为 T<编号>）加任务书文件名映射（task/）。""" + padded = f"{number:03d}" + candidates = [item for item in _scan_records( + lambda record: record.get("task") == f"T{number}") if item.branch.startswith(f"task/{padded}-")] + plans = events_db.ROOT / "docs" / "plans" + known = {item.branch for item in candidates} + for path in sorted(plans.glob(f"task-{padded}-*.md")) if plans.is_dir() else []: + branch = f"task/{path.stem.removeprefix('task-')}" + if branch not in known: + candidates.append(Candidate(branch, "taskbook")) + known.add(branch) + return candidates + + +def _pr_candidates(trace_id: str) -> list[Candidate]: + """本机 push_pr 事件记录的 PR 候选（分支重用可能先后关联多个 PR）。""" + candidates: list[Candidate] = [] + for row in events_io.query(trace_id=trace_id, stage="dispatch"): + outputs = row["outputs"] if isinstance(row["outputs"], dict) else {} + pr = outputs.get("pr") + if row["step"] == "push_pr" and isinstance(pr, int): + head = outputs.get("head") + candidates.append(Candidate(trace_id, "push_pr", ts=row["ts"], + head=head if isinstance(head, str) else None, pr=pr)) + return candidates + + +def _resolve(identifier: str) -> dict: + """标识 → 解析视图（不触网）：任务号查映射，PR 号留待 API，分支即 trace。""" + if match := _TASK_RE.fullmatch(identifier): + number = int(match[1]) + candidates = _task_candidates(number) + branches = {item.branch for item in candidates} + ambiguous = len(branches) > 1 + return {"kind": "task", "identifier": identifier, "number": number, "candidates": candidates, + "ambiguous": ambiguous, + "trace_id": None if ambiguous else (candidates[0].branch if candidates else None)} + if match := _PR_RE.fullmatch(identifier): + return {"kind": "pr", "identifier": identifier, "number": int(match[1])} + if identifier and not identifier.startswith("-"): + return {"kind": "branch", "identifier": identifier, "trace_id": identifier} + return {"kind": "unknown", "identifier": identifier} + + +# ---- trace：时间线组装与展示 ---- + + +def _ci_attempt(source: str) -> int | None: + """ci::: 来源链里的 attempt；其余来源为 None。""" + parts = source.split(":") + return int(parts[2]) if len(parts) == 4 and parts[0] == "ci" and parts[2].isdigit() else None + + +def _segment_events(rows: list[dict], records: list[Candidate]) -> tuple[list[dict], dict[int, dict]]: + """时间线分段：本机运行记录窗口（attempt/链头/起止）优先，CI 来源链各成一段，其余未分段。""" + windows = [(item, _parse_ts(item.ts), _parse_ts(item.end)) for item in records + if item.origin == "run_record"] + windows = [(item, start, end) for item, start, end in windows if start and end] + buckets: dict[str, dict] = {} + labels: dict[int, dict] = {} + for index, row in enumerate(rows): + if (source := row["source"]).startswith("ci:"): + key, attempt, head = source, _ci_attempt(source), None + else: + moment = _parse_ts(row["ts"]) + hit = next((item for item, start, end in windows + if moment is not None and start <= moment <= end), None) + key = f"attempt {hit.attempt}" if hit is not None else "本机（窗口外）" + attempt = hit.attempt if hit is not None else None + head = hit.head if hit is not None else None + labels[index] = {"attempt": attempt, "segment": key} + bucket = buckets.setdefault(key, {"segment": key, "attempt": attempt, "head": head, + "start": row["ts"], "end": row["ts"], "count": 0}) + bucket["end"], bucket["count"] = row["ts"], bucket["count"] + 1 + return sorted(buckets.values(), key=lambda item: item["start"]), labels + + +def _longest_stage(rows: list[dict]) -> dict | None: + """各环节累计时长最长的阶段（并列取字典序首个，保证确定性）；无时长数据为 None。""" + totals: dict[str, int] = {} + for row in rows: + if isinstance(row["duration_ms"], int): + totals[row["stage"]] = totals.get(row["stage"], 0) + row["duration_ms"] + if not totals: + return None + stage = max(sorted(totals), key=lambda name: totals[name]) + return {"stage": stage, "duration_ms": totals[stage]} + + +def _first_failure(rows: list[dict]) -> dict | None: + """首个失败事件（rows 已按时间排序）；deny/fail/error 按原状态区分，不合并。""" + for row in rows: + if row["status"] in _FAILURES: + return {"ts": row["ts"], "source": row["source"], "seq": row["seq"], + "stage": row["stage"], "step": row["step"], "status": row["status"]} + return None + + +def _timeline(trace_id: str, records: list[Candidate]) -> dict: + rows = sorted(events_io.query(trace_id=trace_id), key=lambda row: (row["ts"], row["source"], row["seq"])) + segments, labels = _segment_events(rows, records) + return {"trace_id": trace_id, "chains": sorted({row["source"] for row in rows}), + "segments": segments, "counts": _counts(rows), + "events": [dict(row, **labels[index]) for index, row in enumerate(rows)], + "longest_stage": _longest_stage(rows), "first_failure": _first_failure(rows)} + + +def _short(digest) -> str: + return digest[:7] if isinstance(digest, str) and digest else "—" + + +def _ref_text(item: dict) -> str: + text = f"{item.get('kind')}={item.get('ref')}" + if item.get("sha256"): + text += f"#{_short(item['sha256'])}" + if item.get("size") is not None: + text += f"({item['size']}B)" + return text + + +def _event_lines(row: dict) -> list[str]: + lines = [f" #{row['seq']} {row['ts']} [{row['source']}] {row['stage']}/{row['step']} {row['status']}" + + (f" {row['duration_ms']}ms" if row["duration_ms"] is not None else "")] + if row["inputs"]: + lines.append(" 输入 " + "；".join(_ref_text(item) for item in row["inputs"])) + if row["outputs"]: + lines.append(" 输出 " + "；".join(f"{key}={row['outputs'][key]}" + for key in sorted(row["outputs"], key=str))) + if row["decision"]: + lines.append(f" 决定 by={row['decision'].get('by')} rule={row['decision'].get('rule')} " + f"{row['decision'].get('reason')}") + if row["error"]: + lines.append(f" 错误 kind={row['error'].get('kind')} signature={row['error'].get('signature')}") + if row["actor"]: + model = f" 模型 {row['actor']['model']}" if row["actor"].get("model") else "" + lines.append(f" 角色 {row['actor'].get('role')}@{row['actor'].get('host')}{model}") + return lines + + +def _candidate_text(item: dict) -> str: + parts = [item["branch"]] + if item.get("pr") is not None: + parts.append(f"PR {item['pr']}") + if item.get("attempt") is not None: + parts.append(f"attempt {item['attempt']}") + if item.get("head"): + parts.append(f"链头 {_short(item['head'])}") + if item.get("ts"): + parts.append(item["ts"]) + return " " + "，".join(parts) + f"（来源 {item['origin']}）" + + +def _render(view: dict) -> str: + if "candidates" in view: + lines = [f"标识 {view['identifier']} 对应多个分支，未选择（用完整分支重试）："] + lines += [_candidate_text(item) for item in view["candidates"]] + return "\n".join(lines) + lines = [f"trace {view['trace_id']}（标识 {view['identifier']}，{view['kind']}）"] + for segment in view["segments"]: + head = f"链头 {_short(segment['head'])}，" if segment.get("head") else "" + attempt = f"attempt {segment['attempt']}，" if segment.get("attempt") is not None else "" + lines.append(f" {segment['segment']}：{attempt}{head}{segment['start']}→{segment['end']}" + f"（{segment['count']} 个事件）") + for row in view["events"]: + lines += _event_lines(row) + longest = view["longest_stage"] + lines.append(" 最长阶段 " + (f"{longest['stage']}（{longest['duration_ms']}ms）" if longest else "—")) + failure = view["first_failure"] + lines.append(" 首个失败 " + (f"#{failure['seq']} {failure['ts']} [{failure['source']}] " + f"{failure['stage']}/{failure['step']} {failure['status']}" + if failure else "—")) + return "\n".join(lines) + + +def _emit(view: dict, as_json: bool) -> None: + print(json.dumps(view, ensure_ascii=False, indent=2) if as_json else _render(view)) + + +def _pr_branch(number: int) -> str | None: + """PR → API headRefName（trace 权威来源）；失败打印诊断返回 None。""" + try: + branch = events_io.GhClient().pr(number).get("headRefName") + except (RuntimeError, ValueError, AttributeError) as exc: + print(f"查询 PR {number} 失败：{exc}", file=sys.stderr) + return None + if not isinstance(branch, str) or not branch: + print(f"PR {number} 的 headRefName 缺失", file=sys.stderr) + return None + return branch + + +def _with_ci(view: dict, trace_id: str, pr_number: int | None) -> int: + """--ci：确定 PR（标识给定或本机 push_pr 事件）、下载导入并写进 view；返回退出码贡献。""" + known = _pr_candidates(trace_id) + prs = [pr_number] if pr_number is not None else sorted({item.pr for item in known}) + if len(prs) > 1: + # 分支重用关联了多个 PR：列候选（pr/链头/时间），不随意选一条下载 + view["pr_candidates"] = [item.row() for item in known] + print("该分支关联多个 PR，--ci 未下载；请指定 PR 号", file=sys.stderr) + return 1 + if not prs: + print("本机没有该 trace 的 push_pr 事件，无法确定 PR；请直接给 PR 号", file=sys.stderr) + return 1 + view["pr"] = prs[0] + try: + result = events_io.load_ci(prs[0], gh=events_io.GhClient()) + except Exception as exc: # noqa: BLE001 下载失败转诊断，不替代时间线 + result = {"imported": 0, "skipped": 0, "findings": [{"code": "api", "detail": str(exc)}]} + view["ci"] = result + for finding in result["findings"]: + print(f"CI 发现 {finding['code']}：{finding['detail']}", file=sys.stderr) + return 1 if result["findings"] else 0 + + +def trace_main(argv: list[str] | None = None) -> int: + """trace 子命令：解析任务/PR/分支 → 时间线；--ci 先下载导入该 PR 的 CI 事件包。""" + parser = argparse.ArgumentParser(prog="trace", description="按任务编号、PR 号或分支复原交付时间线") + parser.add_argument("identifier", help="任务编号（T402）、PR 号（77 或 #77）或完整分支名") + parser.add_argument("--ci", action="store_true", help="先下载并导入该 PR 的 CI 事件包（C5 load_ci）") + parser.add_argument("--json", action="store_true", help="输出 JSON") + args = parser.parse_args(argv) + resolved = _resolve(args.identifier) + if resolved["kind"] == "unknown": + print(f"无法识别的标识：{args.identifier!r}（任务编号 T / PR 号 / 完整分支）", file=sys.stderr) + return 2 + if resolved["kind"] == "task" and resolved["ambiguous"]: + _emit({"identifier": resolved["identifier"], "kind": "task", "ambiguous": True, + "trace_id": None, "candidates": [item.row() for item in resolved["candidates"]]}, + args.json) + return 0 + if resolved["kind"] == "task" and resolved["trace_id"] is None: + print(f"找不到任务 T{resolved['number']:03d} 的运行记录或任务书", file=sys.stderr) + return 1 + pr_number = None + if resolved["kind"] == "pr": + branch = _pr_branch(resolved["number"]) + if branch is None: + return 1 + resolved["trace_id"], pr_number = branch, resolved["number"] + trace_id = resolved["trace_id"] + view: dict = {"identifier": args.identifier, "kind": resolved["kind"], "trace_id": trace_id, + "pr": pr_number, "ambiguous": resolved.get("ambiguous", False)} + code = _with_ci(view, trace_id, pr_number) if args.ci else 0 + # 时间线在导入之后组装：--ci 新导入的事件同屏展示 + records = _scan_records(lambda record: trace_id in (record.get("branch"), record.get("trace_id"))) + view.update({"records": [item.row() for item in records], **_timeline(trace_id, records)}) + _emit(view, args.json) + return code + + diff --git a/.harness/engine/routing/run_check.py b/.harness/engine/routing/run_check.py index d6a41e0..0399370 100644 --- a/.harness/engine/routing/run_check.py +++ b/.harness/engine/routing/run_check.py @@ -464,8 +464,10 @@ def _p_guard_denials(value, location, problems) -> None: _ACTOR_FIELDS = {"role": _enum_factory(frozenset(events.ACTOR_ROLES)), "host": _p_token, "model": _p_token} _STAGE_FIELDS = { "stage": _enum_factory(frozenset(events.STAGES)), "step": _p_token, - "status": _enum_factory(frozenset(events.STATUSES)), "ts": _p_ts, + "status": _enum_factory(frozenset(events.STATUSES) | {"prior"}), "ts": _p_ts, "duration_ms": _int_factory(0, none=True), "attempt": _int_factory(1), "round": _int_factory(0), + # B77/T125：六键在收敛后的新记录不再写入（留在事件库），但旧记录（C3 十二键形状）仍合法—— + # 校验器保留、键可选，exact_keys 语义下缺席即通过、在场则按原口径校验。 "inputs": _p_refs, "outputs": _p_outputs, "decision": _none_or( lambda value, location, problems: _scan_struct(value, location, problems, _DECISION_FIELDS, exact_keys=True)), @@ -473,6 +475,7 @@ def _p_guard_denials(value, location, problems) -> None: _ACTOR_FIELDS)), "source": _p_token, "head_hash": _p_sha, } +# status=prior 是更早 attempt 的 push_pr/ci_wait 指针行专用状态（run_timeline._POINTER_STATUS）。 _ANCHOR_FIELDS = {"source": _p_token, "stage": _enum_factory(frozenset(events.STAGES)), "head_hash": _p_sha, "fixed_in": _enum_factory(_FIXED_IN_VALUES)} _MC_FIELDS = {"category": _enum_factory(_MC_CATEGORIES), "summary": _enum_factory(_MC_SUMMARIES), @@ -489,13 +492,19 @@ def _p_guard_denials(value, location, problems) -> None: "gen_ai.usage.input_tokens": _int_factory(0, none=True), "gen_ai.usage.output_tokens": _int_factory(0, none=True), "cost": _number_factory(none=True), "escalation": _none_or(_enum_factory(_EXIT_WORDS)), - "stages": lambda value, location, problems: _scan_list(value, location, problems, _STAGE_FIELDS, - exact_keys=True), + "stages": lambda value, location, problems: _scan_list( + value, location, problems, _STAGE_FIELDS, + # B77/T125：必填七键（新形状）；旧六键在场则校验、缺席合法（旧记录不受影响） + required=("stage", "step", "status", "ts", "duration_ms", "attempt", "round")), "anchors": lambda value, location, problems: _scan_list(value, location, problems, _ANCHOR_FIELDS, exact_keys=True), "missing_context": lambda value, location, problems: _scan_list(value, location, problems, _MC_FIELDS, required=("category", "summary")), "missing_context_status": _enum_factory(_MC_STATUSES), + # B77/T125：fit_record 的截断标注（顶层可选，布尔/正整数） + "stages_truncated": lambda value, location, problems: problems.append((location, "布尔")) + if not (value is True or value is False) else None, + "stages_total": _int_factory(1), }