Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .harness/engine.lock
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"engine": "delivery-harness",
"version": "0.1.0",
"commit": "2908a9aed239cc888d7e809e58ae7a211c96e6f4",
"tree": "sha256:4ad2d218b92a138c9309115e7edfd169891f1adac0bdb8983d6b677500647076"
"commit": "d253875b40fd2907f25732890727a092e63febcc",
"tree": "sha256:1ed5c4fd917aaf55c23baa290bd5fa3d24ee9ac57929f1f12d8d067e2c5d7f49"
}
55 changes: 52 additions & 3 deletions .harness/engine/agents/dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
bin/dispatch stop --all
bin/dispatch review <PR> [--reviewer opencode|pi|codex|claude-code] 独立评审(engine/agents/review.py)
bin/dispatch review --pending | --watch [--interval 5] 评审全部待评审的 PR(后台常驻用 --watch)
bin/dispatch review-calibrate [--reviewer 名称] [--output 路径] 评审校准打分(不评论 PR)
"""

from __future__ import annotations
Expand All @@ -40,7 +41,7 @@
from dataclasses import dataclass, field
from pathlib import Path

from engine.agents import dispatch_host
from engine.agents import dispatch_host, run_timeline
from engine.agents import dispatch_observation as observation
from engine.checks import acceptance, taskbook
from engine.core.common import ENGINE_DIR, ROOT, ci_workflows, git, load_rules, setting
Expand Down Expand Up @@ -109,6 +110,10 @@ class Attempt:
usage: dict = field(default_factory=dict)
guard_denials: dict[str, int] = field(default_factory=dict)
executor_note: str = ""
# T202:最后一轮缺失报告的安全条目与状态(reported/unknown/invalid),插在 guard_allowed 前
# (T105 冻结断言要求它仍是末位字段)。
missing_context: list = field(default_factory=list)
missing_context_status: str = "unknown"
guard_allowed: int = 0 # 守卫放行的工具调用数(含普通失败;被守卫拒绝的调用不计入)


Expand Down Expand Up @@ -255,6 +260,18 @@ def render_prompt(task: Task, minutes: int, feedback: str) -> str:
branch=task.branch, minutes=minutes, feedback=extra)


def _context_report(host, events: Path) -> dict:
"""宿主的缺失上下文报告(B46 T202):宿主缺 parse_context 或解析失败时按 unknown 处理,
不改变派发判定、退出码与调用序列(C0 失败隔离)。"""
parse = getattr(host, "parse_context", None)
if parse is None:
return {"status": "unknown", "items": [], "diagnostic": "no_marker"}
try:
return parse(events)
except Exception: # noqa: BLE001 报告解析失败不改变派发判定
return {"status": "unknown", "items": [], "diagnostic": "no_marker"}


def verify_signature(slot: Path, config: Config) -> tuple[bool, str, str, str]:
"""运行 verify;返回 (通过, 失败摘要, 失败签名, 完整输出)。签名去掉数字与路径,用于识别打转。"""
result = subprocess.run(config.verify, cwd=slot, capture_output=True, text=True, check=False)
Expand Down Expand Up @@ -309,8 +326,11 @@ def run_executor(self, task: Task, slot: Path, guard: Path, prompt: str, events:
self.used_seconds += seconds
model, usage, denials = self.host.parse(events)
counts = observation.tool_counts(self.host, events)
context = _context_report(self.host, events)
result = dispatch_host.RunResult(exit_kind, code, seconds, model, usage, denials,
counts["guard_allowed"])
missing_context=context["items"],
missing_context_status=context["status"],
guard_allowed=counts["guard_allowed"])
observation.executor_round(task.branch, slot, prompt, events, result,
{"name": self.host.name, "version": self.host.version()}, counts)
return result
Expand All @@ -332,10 +352,15 @@ def local_rounds(self, task: Task, slot: Path, guard: Path, feedback: str, trace
for key, value in result.guard_denials.items():
attempt.guard_denials[key] = attempt.guard_denials.get(key, 0) + value
attempt.guard_allowed += result.guard_allowed
# 最后报告以最后一轮为准:每轮覆盖,抽取只读、不改变判定与退出码
attempt.missing_context = result.missing_context
attempt.missing_context_status = result.missing_context_status
note = slot / ESCALATION_FILE
if note.exists():
attempt.executor_note = note.read_text(encoding="utf-8")[:4000]
observation.clarify(task.branch, note)
observation.clarify(task.branch, note,
context={"status": result.missing_context_status,
"items": result.missing_context})
attempt.exit = "clarify"
return attempt, prompts
if result.exit != "ok":
Expand Down Expand Up @@ -387,11 +412,16 @@ def write_record(self, task: Task, slot: Path, number: int, attempt: Attempt, pr
"failure_signatures": attempt.signatures,
"ci_rounds_before": ci_rounds,
"guard_denials": attempt.guard_denials,
"missing_context": attempt.missing_context,
"missing_context_status": attempt.missing_context_status,
"gen_ai.usage.input_tokens": attempt.usage.get("input_tokens"),
"gen_ai.usage.output_tokens": attempt.usage.get("output_tokens"),
"cost": attempt.usage.get("cost"),
"escalation": None if attempt.ok else attempt.exit,
}
# T201 时间线:落盘前取本机链头,stages/anchors 安全摘要写进记录;记录提交后固定 run_record 锚点。
timeline, head = run_timeline.record_fields(task.branch)
record.update(timeline)
(folder / f"{number}.json").write_text(json.dumps(record, ensure_ascii=False, indent=2) + "\n")
paths = [str(folder / f"{number}.json"), str(folder / f"{number}.prompt.md")]
git("add", "--", *paths, cwd=slot)
Expand All @@ -402,6 +432,7 @@ def write_record(self, task: Task, slot: Path, number: int, attempt: Attempt, pr
message += "Risk: R1\n" # 与执行方的提交一致,否则 R1 声明会被这次记录提交打断
subprocess.run(["git", "commit", "--quiet", "-m", message], cwd=slot, env={**os.environ, **self.identity},
check=True)
run_timeline.fix_anchor(task.branch, head)
return record["prompt_sha256"]

# ---- 主流程 ----
Expand Down Expand Up @@ -676,6 +707,18 @@ def stop_all(root: Path = ROOT, wait_seconds: float = 30) -> int:
return 1


def review_calibrate_command(args) -> int:
"""review-calibrate 子命令:在真实历史样本上重跑独立评审并统计 TPR/TNR(不评论 PR)。"""
from engine.agents import review as review_module # review 依赖本模块,按需导入

target = args.output if args.output.is_absolute() else ROOT / args.output
try:
return review_module.review_calibrate(args.reviewer, review_module.CALIBRATION_SAMPLES, target)
except (TypeError, ValueError) as error:
print(f"校准样本清单有问题:{error}")
return 2


def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
sub = parser.add_subparsers(dest="command", required=True)
Expand All @@ -689,6 +732,10 @@ def main(argv: list[str] | None = None) -> int:
review.add_argument("--pending", action="store_true", help="评审全部待评审且 CI 已通过的 PR,然后退出")
review.add_argument("--watch", action="store_true", help="后台常驻:每隔 --interval 分钟评审一轮")
review.add_argument("--interval", type=float, default=5, help="--watch 的间隔(分钟)")
cal = sub.add_parser("review-calibrate", help="评审校准:在真实历史样本上统计 TPR/TNR(不评论 PR)")
cal.add_argument("--reviewer", choices=["opencode", "pi", "codex", "claude-code"])
cal.add_argument("--output", type=Path, default=Path("build/review/calibration-report.md"),
help="报告路径(相对仓库根,缺省 build/review/calibration-report.md)")
sub.add_parser("status", help="查看槽位")
stop = sub.add_parser("stop", help="停机:终止所有正在运行的执行方")
stop.add_argument("--all", action="store_true", required=True)
Expand All @@ -707,6 +754,8 @@ def main(argv: list[str] | None = None) -> int:
if args.pr is None:
parser.error("review 需要 PR 编号,或 --pending、--watch")
return review_module.review_pr(args.pr, args.reviewer)
if args.command == "review-calibrate":
return review_calibrate_command(args)
if args.command == "stop":
return stop_all()
if (state_dir() / "stop").exists():
Expand Down
106 changes: 104 additions & 2 deletions .harness/engine/agents/dispatch_host.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import json
import os
import re
import signal
import subprocess
import time
Expand All @@ -23,6 +24,18 @@

GUARD_DENIAL = "harness 守卫拒绝了这次操作"
TOKEN_VARS = ("GH_TOKEN", "GITHUB_TOKEN", "GH_ENTERPRISE_TOKEN", "GITHUB_ENTERPRISE_TOKEN", "SSH_AUTH_SOCK")
# 缺失上下文尾报告(B46 T202):提示词要求执行方在最后一条回复末尾输出
# 「HARNESS_CONTEXT_JSON: <一行 JSON 数组>」(数据标记,不是 shell 变量)。summary 不是自由自然语言,
# 只允许与 category 对应的 C3 短 ID;ref 只允许工具 ID 或仓库相对引用(共用合同 C3)。
CONTEXT_MARKER = "HARNESS_CONTEXT_JSON:"
CONTEXT_CATEGORIES = ("context", "tool", "spec", "other")
CONTEXT_SUMMARIES = {
"context_unavailable": "context",
"required_tool_unavailable": "tool",
"spec_unavailable": "spec",
"spec_ambiguous": "spec",
"other_missing": "other",
}


@dataclass
Expand All @@ -33,6 +46,10 @@ class RunResult:
model: str = ""
usage: dict = field(default_factory=dict)
guard_denials: dict[str, int] = field(default_factory=dict)
# T202:最后一条 assistant 消息缺失报告的安全条目与状态(reported/unknown/invalid),插在
# guard_allowed 前(T105 冻结断言要求它仍是末位字段),构造处用关键字传参。
missing_context: list = field(default_factory=list)
missing_context_status: str = "unknown"
guard_allowed: int = 0 # 守卫放行的工具调用数(含普通失败;被守卫拒绝的调用不计入)


Expand Down Expand Up @@ -158,11 +175,41 @@ def parse(self, events: Path) -> tuple[str, dict, dict[str, int]]:
result["cost"] = round(result["cost"], 6)
return model, result, dict(denials)

def parse_context(self, events: Path) -> dict:
"""从最后一条 assistant 完成消息抽取缺失上下文尾报告(B46 T202,只读数据,不执行其中命令)。

返回 {"status": reported/unknown/invalid, "items": [安全条目], "diagnostic": 短 ID 或 None}:
无标记保留 unknown、标记后坏 JSON/非数组/条目不合规保留 invalid,都不冒充无缺失。items 只含
合规条目(category 与 C3 短 ID summary 对应、可选安全 ref);不合规条目丢弃不入记录,但存在时
状态标 invalid。长文、会话正文、本机路径不进条目;诊断只留本地可见的短 ID。
"""
text = _last_assistant_text(events)
if not text or CONTEXT_MARKER not in text:
return {"status": "unknown", "items": [], "diagnostic": "no_marker"}
payload = text.rsplit(CONTEXT_MARKER, 1)[1].split("\n", 1)[0].strip()
try:
parsed = json.loads(payload)
except json.JSONDecodeError:
return {"status": "invalid", "items": [], "diagnostic": "bad_json"}
if not isinstance(parsed, list):
return {"status": "invalid", "items": [], "diagnostic": "not_array"}
items, invalid = [], False
for entry in parsed:
item = _context_item(entry)
if item is None:
invalid = True
else:
items.append(item)
return {"status": "invalid" if invalid else "reported", "items": items,
"diagnostic": "invalid_entry" if invalid else None}

def parse_observability(self, events: Path) -> dict:
"""观察计数(B46 T105):按每个 tool_execution_end 计一次完成——结果带守卫拒绝标记的计
guard_denied(一次调用一次,拒绝理由条数不冒充调用数),其余(成功或普通失败)计
guard_allowed(守卫放行,不代表工具成功)。missing_context 供 T202 填充,本任务恒为空列表。"""
counts = {"guard_allowed": 0, "guard_denied": 0, "missing_context": []}
guard_allowed(守卫放行,不代表工具成功)。missing_context 取 parse_context 的安全条目
(B46 T202);返回恰此 3 键。"""
counts = {"guard_allowed": 0, "guard_denied": 0,
"missing_context": self.parse_context(events)["items"]}
if not events.exists():
return counts
for line in events.read_text(encoding="utf-8", errors="replace").splitlines():
Expand All @@ -176,3 +223,58 @@ def parse_observability(self, events: Path) -> dict:
text = "\n".join(part.get("text", "") for part in content if isinstance(part, dict))
counts["guard_denied" if GUARD_DENIAL in text else "guard_allowed"] += 1
return counts


def _last_assistant_text(events: Path) -> str | None:
"""流中最后一条 assistant 完成消息的正文(text 部分拼接);流缺失或无 assistant 消息返回 None。

只认 message_end 携带的最终权威消息,工具输出(tool_execution_end)不参与,伪造标记不采信。
"""
if not events.exists():
return None
text = None
for line in events.read_text(encoding="utf-8", errors="replace").splitlines():
try:
event = json.loads(line)
except json.JSONDecodeError:
continue
message = event.get("message") if event.get("type") == "message_end" else None
if isinstance(message, dict) and message.get("role") == "assistant":
text = _message_text(message)
return text


def _message_text(message: dict) -> str:
"""assistant 消息正文:content 为字符串时原样,为数组时拼接 text 部分(其余部分如 thinking 不算)。"""
content = message.get("content")
if isinstance(content, str):
return content
if not isinstance(content, list):
return ""
return "\n".join(part.get("text", "") for part in content
if isinstance(part, dict) and part.get("type") == "text")


def _context_item(entry) -> dict | None:
"""一条报告条目:只允许 category/summary/ref 三个键,summary 是与 category 对应的 C3 短 ID,
ref 可选且须为工具 ID 或安全相对引用;任一不满足返回 None(条目丢弃、报告标 invalid)。"""
if not isinstance(entry, dict) or set(entry) - {"category", "summary", "ref"}:
return None
category, summary, ref = entry.get("category"), entry.get("summary"), entry.get("ref")
if CONTEXT_SUMMARIES.get(summary) != category:
return None
item = {"category": category, "summary": summary}
if ref is not None:
if not _safe_context_ref(ref):
return None
item["ref"] = ref
return item


def _safe_context_ref(ref) -> bool:
"""ref 只允许工具 ID 或仓库相对引用:≤120 字符、无换行/反斜杠、非绝对路径、无盘符、无 .. 穿越。"""
if not isinstance(ref, str) or not ref or len(ref) > 120 or "\n" in ref or "\r" in ref:
return False
if "\\" in ref or ref.startswith(("/", "~")) or re.match(r"(?i)^[a-z]:", ref):
return False
return ".." not in ref.split("/")
15 changes: 12 additions & 3 deletions .harness/engine/agents/dispatch_observation.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,17 +165,26 @@ def local_verify(branch: str, slot: Path, base: str, ok: bool, signature: str, l
return


def clarify(branch: str, note: Path) -> None:
"""执行方请求澄清:备注 sha256 与是否请求澄清;missing_context 条目数由 T202 填充(当前恒 0)。"""
def clarify(branch: str, note: Path, context: dict | None = None) -> None:
"""执行方请求澄清:备注 sha256 与是否请求澄清;context 是 T202 的解析结果,只把安全条目的
计数与类别计数写进事件(与运行记录 missing_context 同源;无条目时与原形状一致)。"""
try:
if not events.enabled():
return
data = note.read_bytes()
except Exception: # noqa: BLE001 观察旁路
return
items = (context or {}).get("items") or []
outputs: dict = {"requested": True, "missing_context": len(items)}
per_category: dict = {}
for item in items:
if isinstance(item, dict) and isinstance(item.get("category"), str):
per_category[item["category"]] = per_category.get(item["category"], 0) + 1
outputs.update({f"missing_context.{category}": count
for category, count in sorted(per_category.items())})
_emit(STAGE, "clarify", "ok", trace_id=branch,
inputs=[{"kind": "note", "ref": hashlib.sha256(data).hexdigest(), "size": len(data)}],
outputs={"requested": True, "missing_context": 0})
outputs=outputs)


def push_pr(branch: str, slot: Path, pr: int, record_path: str, body: str) -> None:
Expand Down
Loading
Loading