Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .harness/engine.lock
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"engine": "delivery-harness",
"version": "0.1.0",
"commit": "5cf0640e8560932500f8a219a0d77ff7950f94d3",
"tree": "sha256:17b18f24a64ec6e0d28f45d350d2b41451ee77da57a09d82bce76efc9fd38a9c"
"commit": "2908a9aed239cc888d7e809e58ae7a211c96e6f4",
"tree": "sha256:4ad2d218b92a138c9309115e7edfd169891f1adac0bdb8983d6b677500647076"
}
129 changes: 98 additions & 31 deletions .harness/engine/agents/dispatch.py

Large diffs are not rendered by default.

20 changes: 20 additions & 0 deletions .harness/engine/agents/dispatch_host.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ class RunResult:
model: str = ""
usage: dict = field(default_factory=dict)
guard_denials: dict[str, int] = field(default_factory=dict)
guard_allowed: int = 0 # 守卫放行的工具调用数(含普通失败;被守卫拒绝的调用不计入)


def executor_env(base: dict[str, str], identity: dict[str, str], gh_config_dir: Path) -> dict[str, str]:
Expand Down Expand Up @@ -156,3 +157,22 @@ def parse(self, events: Path) -> tuple[str, dict, dict[str, int]]:
if "cost" in result:
result["cost"] = round(result["cost"], 6)
return model, result, dict(denials)

def parse_observability(self, events: Path) -> dict:
"""观察计数(B46 T105):按每个 tool_execution_end 计一次完成——结果带守卫拒绝标记的计
guard_denied(一次调用一次,拒绝理由条数不冒充调用数),其余(成功或普通失败)计
guard_allowed(守卫放行,不代表工具成功)。missing_context 供 T202 填充,本任务恒为空列表。"""
counts = {"guard_allowed": 0, "guard_denied": 0, "missing_context": []}
if not events.exists():
return counts
for line in events.read_text(encoding="utf-8", errors="replace").splitlines():
try:
event = json.loads(line)
except json.JSONDecodeError:
continue
if event.get("type") != "tool_execution_end":
continue
content = (event.get("result") or {}).get("content") or []
text = "\n".join(part.get("text", "") for part in content if isinstance(part, dict))
counts["guard_denied" if GUARD_DENIAL in text else "guard_allowed"] += 1
return counts
337 changes: 337 additions & 0 deletions .harness/engine/agents/dispatch_observation.py

Large diffs are not rendered by default.

83 changes: 66 additions & 17 deletions .harness/engine/agents/review.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from typing import ClassVar

from engine.agents import dispatch, dispatch_host
from engine.agents import dispatch_observation as observation
from engine.core.common import ENGINE_DIR, ROOT, changed_files, commit_field, git, load_rules
from engine.routing import run_check

Expand All @@ -51,6 +52,9 @@ class Verdict:
summary: str = ""
parsed: bool = True
failure: str = "" # 评审方本身失败(报错退出、超时、额度用尽):不是结论,不计入校准,不评论为结论
error_kind: str = "" # 失败的机器类别(timeout / reviewer_exit / no_output),供评审观察事件
audit_model: str | None = None # 审计摘要用的模型(明确报告优先,其次显式请求;两者皆无为 None)
model_basis: str = "unknown" # reported / explicit_request / unknown(共用合同 C6)

@property
def flagged(self) -> bool:
Expand Down Expand Up @@ -79,13 +83,14 @@ def parse_output(text: str) -> Verdict:

class Reviewer:
name = ""
model_name: str | None = None # 显式请求的模型(审计摘要用;展示名可带档位等修饰)
env: ClassVar[dict[str, str]] = {} # 评审方专用的环境变量(如 OpenCode 的权限配置)

def argv(self, prompt: str, workspace: Path, output: Path) -> list[str]:
raise NotImplementedError

def read(self, stdout: str, output: Path) -> tuple[str, str]:
"""返回 (最终文本, 模型)。"""
def read(self, stdout: str, output: Path) -> tuple[str, str, str]:
"""返回 (最终文本, 展示用模型, model_basis:reported / explicit_request / unknown)。"""
raise NotImplementedError


Expand All @@ -94,6 +99,7 @@ class CodexReviewer(Reviewer):

def __init__(self, model: str | None, effort: str | None = None):
self.model, self.effort = model, effort
self.model_name = model

def argv(self, prompt, workspace, output):
# 没配置模型时沿用 Codex 自己的默认设置。
Expand All @@ -105,14 +111,16 @@ def argv(self, prompt, workspace, output):
def read(self, stdout, output):
base = self.model or "codex 默认模型"
label = f"{base} {self.effort}" if self.effort else base
return (output.read_text(encoding="utf-8") if output.exists() else stdout), label
basis = "explicit_request" if self.model else "unknown"
return (output.read_text(encoding="utf-8") if output.exists() else stdout), label, basis


class ClaudeReviewer(Reviewer):
name = "claude-code"

def __init__(self, model: str | None = None):
self.model = model
self.model_name = model

def argv(self, prompt, workspace, output):
command = ["claude", "-p", "--allowedTools", "Read,Grep,Glob", "--output-format", "json"]
Expand All @@ -124,16 +132,19 @@ def read(self, stdout, output):
try:
data = json.loads(stdout)
except json.JSONDecodeError:
return stdout, self.model or ""
model = next(iter(data.get("modelUsage") or {}), self.model or "")
return str(data.get("result", "")), model
return stdout, self.model or "", "explicit_request" if self.model else "unknown"
reported = next(iter(data.get("modelUsage") or {}), "")
model = reported or self.model or ""
basis = "reported" if reported else ("explicit_request" if self.model else "unknown")
return str(data.get("result", "")), model, basis


class PiReviewer(Reviewer):
name = "pi"

def __init__(self, model: str | None = None):
self.model = model
self.model_name = model

def argv(self, prompt, workspace, output):
command = ["pi", "-na", "--tools", "read,grep,find,ls", "-p", "--mode", "json", "--no-session"]
Expand All @@ -143,7 +154,7 @@ def argv(self, prompt, workspace, output):

def read(self, stdout, output):
"""取最后一条助手消息的文本与模型(pi 的 json 事件流)。"""
text, model = "", self.model or ""
text, model, reported = "", self.model or "", ""
for line in stdout.splitlines():
try:
event = json.loads(line)
Expand All @@ -154,8 +165,11 @@ def read(self, stdout, output):
parts = [part.get("text", "") for part in message.get("content", []) if part.get("type") == "text"]
if any(parts):
text = "\n".join(parts)
if message.get("model"):
reported = message["model"]
model = message.get("model", model)
return text, model
basis = "reported" if reported else ("explicit_request" if self.model else "unknown")
return text, model, basis


class OpenCodeReviewer(Reviewer):
Expand All @@ -171,6 +185,7 @@ class OpenCodeReviewer(Reviewer):

def __init__(self, model: str | None = None):
self.model = model
self.model_name = model

def argv(self, prompt, workspace, output):
command = ["opencode", "run", "--pure", "--agent", "plan", "--format", "json", "--dir", str(workspace)]
Expand All @@ -189,7 +204,7 @@ def read(self, stdout, output):
part = event.get("part") or {}
if event.get("type") == "text" and part.get("type") == "text":
texts.append(part.get("text", ""))
return "\n".join(texts), self.model or ""
return "\n".join(texts), self.model or "", "explicit_request" if self.model else "unknown"


def make_reviewer(name: str) -> Reviewer:
Expand Down Expand Up @@ -218,13 +233,20 @@ def run_reviewer(reviewer: Reviewer, workspace: Path, timeout: float) -> tuple[V
result = subprocess.run(reviewer.argv(prompt, workspace, output), cwd=workspace, env=env,
stdin=subprocess.DEVNULL, capture_output=True, text=True, timeout=timeout,
check=False)
text, model = reviewer.read(result.stdout, output)
text, model, basis = reviewer.read(result.stdout, output)
failure = "" if result.returncode == 0 else _failure_line(result.stdout + result.stderr, result.returncode)
error_kind = "" if result.returncode == 0 else "reviewer_exit"
except subprocess.TimeoutExpired:
text, model, failure = "", "", f"超过 {int(timeout / 60)} 分钟未完成"
text, model, basis = "", "", "unknown"
failure, error_kind = f"超过 {int(timeout / 60)} 分钟未完成", "timeout"
verdict = parse_output(text)
if failure or not text.strip():
verdict = Verdict("评审失败", [], failure or "评审方没有输出", parsed=False, failure=failure or "没有输出")
verdict = Verdict("评审失败", [], failure or "评审方没有输出", parsed=False, failure=failure or "没有输出",
error_kind=error_kind or "no_output")
# C6:模型明确报告优先;其次实际命令显式指定的请求模型;两者皆无不把配置默认值当已调用模型。
verdict.audit_model = model if basis == "reported" else (
reviewer.model_name if basis == "explicit_request" else None)
verdict.model_basis = basis
return verdict, model, time.monotonic() - started


Expand Down Expand Up @@ -310,7 +332,7 @@ def run_check_records(workspace: Path, base: str) -> list[dict]:


def render_comment(verdict: Verdict, reviewer: str, model: str, designer: str | None, head: str, seconds: float,
same_host: bool = False) -> str:
same_host: bool = False, audit: dict | None = None) -> str:
lines = [
f"### 独立评审(试行):{verdict.verdict}", "",
f"- 评审方:{reviewer}({model or '模型未报告'});设计方:{designer or '未能判定'};评审的提交:`{head[:12]}`;用时 {seconds / 60:.1f} 分钟",
Expand All @@ -328,16 +350,21 @@ def render_comment(verdict: Verdict, reviewer: str, model: str, designer: str |
data = {"verdict": verdict.verdict, "reviewer": reviewer, "model": model, "head": head,
"findings": len(verdict.findings), "flagged": verdict.flagged, "parsed": verdict.parsed}
lines += ["", f"<!-- independent-review {json.dumps(data, ensure_ascii=False)} -->"]
if audit is not None:
# C6 审计摘要:与旧标记并列,单行 JSON,字段与材料编码见共用合同 C6;T305 只解析这个新标记。
lines.append(f"<!-- harness-review-audit {json.dumps(audit, ensure_ascii=False)} -->")
return "\n".join(lines) + "\n"


def review_pr(number: int, reviewer_name: str | None, root: Path = ROOT, github=None) -> int:
github = github or dispatch.GitHub(root)
pr = json.loads(github._run(["gh", "pr", "view", str(number), "--json",
"title,body,headRefName,headRefOid,baseRefName,state,mergeCommit"]))
trace = pr["headRefName"] # 评审事件的显式 trace:PR 的 headRefName
head = pr["headRefOid"]
workspace = review_workspace(root)
git("fetch", "--quiet", "origin", f"pull/{number}/head", "main", cwd=workspace)
checkout(workspace, pr["headRefOid"])
checkout(workspace, head)
base = review_base(pr, workspace)
designer = designer_of(base, "HEAD", pr["headRefName"], workspace)
name = reviewer_name or load_rules().get("review", {}).get("reviewer") or OTHER.get(designer or "")
Expand All @@ -350,14 +377,36 @@ def review_pr(number: int, reviewer_name: str | None, root: Path = ROOT, github=
target = run_check.scope(base, "HEAD", pr["headRefName"], workspace)
task_text = (workspace / target.taskbook).read_text(encoding="utf-8") if target and target.taskbook else ""
write_materials(workspace, base, f"# {pr['title']}\n\n{pr['body']}", task_text, ci_summary(number, github))
materials = observation.review_materials(
workspace, base, head, target.taskbook if target and target.taskbook else None, number)
executors = {record.get("gen_ai.agent.name") for record in run_check_records(workspace, base)}
independent = None if designer is None else name != designer
verdict, model, seconds = run_reviewer(make_reviewer(name), workspace, load_rules().get("review", {}).get(
"timeout_minutes", 30) * 60)
if verdict.failure:
observation.review(trace=trace, head=head, reviewer=name, verdict=verdict.verdict,
duration_ms=int(seconds * 1000), findings=verdict.findings, materials=materials,
designer=designer, model=verdict.audit_model, model_basis=verdict.model_basis,
parsed=verdict.parsed, independent=independent, same_host=name in executors,
failure=verdict.failure, error_kind=verdict.error_kind)
print(f"PR #{number}:评审方失败({verdict.failure}),没有评论,保留待评审标签")
return 1
executors = {record.get("gen_ai.agent.name") for record in run_check_records(workspace, base)}
github.comment(number, render_comment(verdict, name, model, designer, pr["headRefOid"], seconds,
same_host=name in executors))
same_host = name in executors
audit = observation.review_audit(trace_id=trace, head=head, base=base, reviewer=name,
model=verdict.audit_model, model_basis=verdict.model_basis,
designer=designer,
implementers=sorted(item for item in executors if item),
independent=independent, same_host=same_host, parsed=verdict.parsed,
verdict=verdict.verdict, duration_ms=int(seconds * 1000),
findings=verdict.findings, materials=materials)
body = render_comment(verdict, name, model, designer, head, seconds, same_host, audit)
url = github.comment(number, body)
# 事件在评论实际发布后写:URL 与评论字节哈希只有此刻可知(C6:摘要不预写 URL、不含自身哈希)。
observation.review(trace=trace, head=head, reviewer=name, verdict=verdict.verdict,
duration_ms=int(seconds * 1000), findings=verdict.findings, materials=materials,
designer=designer, model=verdict.audit_model, model_basis=verdict.model_basis,
parsed=verdict.parsed, independent=independent, same_host=same_host,
comment=url, comment_body=body)
try:
github._run(["gh", "pr", "edit", str(number), "--remove-label", LABEL], agent=True)
except RuntimeError:
Expand Down
12 changes: 12 additions & 0 deletions .harness/engine/checks/base_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import tempfile
from pathlib import Path

from engine.core import events
from engine.core.common import ROOT, clean_git_env, git
from engine.routing import risk

Expand Down Expand Up @@ -73,13 +74,24 @@ def run(base: str, head: str = "HEAD", cwd: Path = ROOT) -> tuple[bool, bool, st
return completed.returncode == 0, not intended or completed.returncode == TAMPERED, summary


def _record(base: str, head: str, ok: bool, enforced: bool, summary: str) -> None:
"""观察旁路:一条 base_tests 事件(是否通过、是否强制、安全摘要行),失败不影响原判定。"""
try:
events.emit(stage="ci", step="base_tests", status="ok" if ok else "fail",
inputs=[events.ref("rev", base), events.ref("rev", head)],
outputs={"ok": ok, "enforced": enforced, "summary": summary or None})
except Exception: # noqa: BLE001 设计要求:事件失败不得影响调用方
return


def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--base", required=True)
parser.add_argument("--head", default="HEAD")
parser.add_argument("--repo", type=Path, default=ROOT, help="要检查的仓库(默认本仓库)")
args = parser.parse_args(argv)
ok, enforced, summary = run(args.base, args.head, cwd=args.repo)
_record(args.base, args.head, ok, enforced, summary) # 观察:结果、强制与摘要行
if ok:
print(f"✓ base 版本的已有测试在 head 代码上全部通过({summary})")
return 0
Expand Down
21 changes: 19 additions & 2 deletions .harness/engine/checks/evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from pathlib import Path

from engine.checks import verify
from engine.core import events
from engine.core.cases import load_project_cases
from engine.core.common import ROOT, clean_git_env, commit_field, git, path_matches

Expand Down Expand Up @@ -386,6 +387,21 @@ def render_markdown(evidences: dict[str, DefectEvidence], base: str, head: str,
return "\n".join(lines) + "\n"


def _record(evidences: dict[str, DefectEvidence]) -> None:
"""观察旁路:每个缺陷编号一条事件(有无提交、是否改代码、修复前后结果),不含缺陷正文。"""
try:
for evidence in evidences.values():
events.emit(stage="ci", step="evidence", status="ok" if evidence.ok else "fail",
outputs={"defect": evidence.defect, "commits": len(evidence.commits),
"doc_only": evidence.doc_only, "code_files": len(evidence.code_files),
"tests": len(evidence.python_tests) + len(evidence.swift_refs),
"before": evidence.before or None, "after": evidence.after or None,
"withdrawn": bool(evidence.withdrawn_by),
"problems": len(evidence.problems), "warnings": len(evidence.warnings)})
except Exception: # noqa: BLE001 设计要求:事件失败不得影响调用方
return


def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--base", required=True)
Expand All @@ -398,10 +414,11 @@ def main(argv: list[str] | None = None) -> int:

ids = [item for item in (args.ids or "").split(",") if item]
evidences = analyse(
args.base, args.head, ids or None, run_tests=not args.no_run, swift=args.swift
args.base, args.head, ids or None, run_tests=not args.no_run, swift=args.swift, cwd=ROOT
)
_record(evidences) # 观察:每个缺陷的提交、代码改动与修复前后结果
coverage = replay_coverage(evidences)
report = render_markdown(evidences, args.base, args.head) + "\n" + render_replay(coverage)
report = render_markdown(evidences, args.base, args.head, ROOT) + "\n" + render_replay(coverage)
print(report)
if args.markdown:
with open(args.markdown, "a") as handle:
Expand Down
Loading
Loading