PR-API channel: optional collect-pr command for review comments (#5)

CHANGELOG.md

@@ -27,3 +27,7 @@
   an unpaired line with no target-script content routes to `code-or-markup` instead of
   `addition`/`deletion`, mirroring the paired-line rule, so the omission taxonomy counts prose only.
   The total pair count is unchanged — only categories shift.
+- PR-API channel ([#5](https://github.com/QuantEcon/textstrata/issues/5)): a new optional `collect-pr`
+  command (the package's only network-touching code) records review comments and suggestion fences per
+  document into `pr_channel.jsonl`/`pr_channel.json` — the human signal squash merges and closed-unmerged
+  PRs hide from git. `scan` neither reads nor writes the channel; nothing is blended into blame.

configs/quantecon/intro-zh-cn.yml

@@ -29,3 +29,4 @@ people:
 review_state:
   stale_after_syncs: 3
 overrides: overrides/intro-zh-cn.yml
+pr_channel: {repo: QuantEcon/lecture-intro.zh-cn}

configs/quantecon/programming-zh-cn.yml

@@ -21,3 +21,4 @@ disclosure: {trailers: [AI-Assisted], authors: ['copilot-swe-agent']}
 people:
   roster: ../../../project-translation/team/reviewers.yml
   roles: {editor: human-editor, translator: human-translator}
+pr_channel: {repo: QuantEcon/lecture-python-programming.zh-cn}

configs/quantecon/python-zh-cn.yml

@@ -14,3 +14,4 @@ people:
   roster: ../../../project-translation/team/reviewers.yml
   roles: {editor: human-editor, translator: human-translator}
 overrides: overrides/python-zh-cn.yml
+pr_channel: {repo: QuantEcon/lecture-python.zh-cn}

docs/configuration.md

@@ -33,6 +33,9 @@ people:
 review_state:
   stale_after_syncs: 3             # machine syncs since the last human touch -> audit-stale
   min_prose_lines: 1               # prose lines a roster commit must change to count as a touch
+pr_channel:                        # optional: enables `collect-pr` (network; gh auth required)
+  repo: QuantEcon/lecture-intro.zh-cn   # owner/name slug — the checkout's remote may be a fork
+  # since: 2026-03-01              # skip review comments created before this date
 overrides: overrides/intro-zh-cn.yml   # per-commit tier overrides (reviewed data file)
 ```
 

docs/method.md

@@ -62,6 +62,16 @@ A sync with no state record — pre-engine history, or a document the engine doe
 `unrecorded`; a state file that itself says `model: unknown` keeps that literal value, so the two cases
 stay distinguishable.
 
+## The PR-API channel
+
+Squash merges hide in-PR human cleanup and closed-unmerged PRs carry review comments no commit records.
+The optional `collect-pr` command (the package's only network-touching code; requires an authenticated
+`gh`) records review comments and suggestion fences per document into `pr_channel.jsonl` — author
+resolved to a roster role by GitHub login, bot reviewers flagged rather than dropped, suggestion fences
+carrying an extracted before/after pair — with a summary in `pr_channel.json`. The channel is a separate
+human-signal series: `scan` neither reads nor writes it, so scan determinism is untouched, and nothing is
+blended into blame; joins happen downstream on document path. Design of record: [#5](https://github.com/QuantEcon/textstrata/issues/5).
+
 ## Known limits
 
 - **Squash merges** hide human work done inside a machine-drafted PR. `ai-initial` means *as landed*; human effort is a lower bound.

src/textstrata/cli.py

@@ -23,20 +23,39 @@ def main(argv: list[str] | None = None) -> int:
     s.add_argument("-q", "--quiet", action="store_true")
     r = sub.add_parser("summary", help="print the corpus summary of a finished scan")
     r.add_argument("out_dir")
+    p = sub.add_parser("collect-pr",
+                       help="collect the PR-API review-comment channel (needs gh; network)")
+    p.add_argument("config", help="path to a textstrata config YAML with a pr_channel block")
+    p.add_argument("-o", "--out", default=None, help="output directory (default: ./out/<name>)")
+    p.add_argument("-q", "--quiet", action="store_true")
     a = ap.parse_args(argv)
-    if a.cmd == "scan":
+    if a.cmd in ("scan", "collect-pr"):
         try:
             cfg = load_config(a.config)
         except ConfigError as e:
             print(f"config error: {e}", file=sys.stderr)
             return 2
         out = Path(a.out) if a.out else Path("out") / cfg.name
+    if a.cmd == "scan":
         with open(os.devnull, "w") as devnull:
             run = scan(cfg, out, log=devnull if a.quiet else sys.stderr)
         print(json.dumps({k: run[k] for k in ("config", "head", "translated", "prose_lines",
                                                 "composition_pct", "review_state", "pairs")},
                          ensure_ascii=False))
         return 0
+    if a.cmd == "collect-pr":
+        from .pr_channel import GhError, collect
+        try:
+            with open(os.devnull, "w") as devnull:
+                summary = collect(cfg, out, log=devnull if a.quiet else sys.stderr)
+        except ConfigError as e:
+            print(f"config error: {e}", file=sys.stderr)
+            return 2
+        except GhError as e:
+            print(str(e), file=sys.stderr)
+            return 1
+        print(json.dumps(summary, ensure_ascii=False))
+        return 0
     if a.cmd == "summary":
         run = json.loads((Path(a.out_dir) / "run.json").read_text(encoding="utf-8"))
         print(f"{run['config']} @ {run['head'][:7]} — {run['translated']}/{run['documents']} documents translated, "

src/textstrata/config.py

@@ -82,6 +82,12 @@ class PeopleConfig:
         "editor": "human-editor", "translator": "human-translator"})
 
 
+@dataclass
+class PrChannelConfig:
+    repo: str | None = None             # owner/name slug (the checkout's remote may be a fork)
+    since: str | None = None            # ISO date; earlier review comments are skipped
+
+
 @dataclass
 class ReviewStateConfig:
     stale_after_syncs: int = 3          # ai-sync commits since last human touch -> audit-stale
@@ -106,6 +112,7 @@ class Config:
     disclosure: DisclosureConfig = field(default_factory=DisclosureConfig)
     people: PeopleConfig = field(default_factory=PeopleConfig)
     review_state: ReviewStateConfig = field(default_factory=ReviewStateConfig)
+    pr_channel: PrChannelConfig = field(default_factory=PrChannelConfig)
     overrides_file: str | None = None   # per-commit tier overrides (sha prefix -> tier)
     base_dir: Path = field(default_factory=Path)
 
@@ -134,7 +141,7 @@ def load_config(path: str | Path) -> Config:
         if key not in raw:
             raise ConfigError(f"{path}: missing required key {key!r}")
     known = {"name", "repo", "files", "source", "prose", "baseline", "machine",
-             "disclosure", "people", "review_state", "overrides"}
+             "disclosure", "people", "review_state", "pr_channel", "overrides"}
     unknown = set(raw) - known
     if unknown:
         raise ConfigError(f"{path}: unknown keys {sorted(unknown)}")
@@ -162,6 +169,7 @@ def load_config(path: str | Path) -> Config:
         disclosure=_sub(DisclosureConfig, raw.get("disclosure"), "disclosure"),
         people=_sub(PeopleConfig, raw.get("people"), "people"),
         review_state=_sub(ReviewStateConfig, raw.get("review_state"), "review_state"),
+        pr_channel=_sub(PrChannelConfig, raw.get("pr_channel"), "pr_channel"),
         overrides_file=raw.get("overrides"),
         base_dir=path.parent,
     )

src/textstrata/pr_channel.py

@@ -0,0 +1,125 @@
+"""Optional PR-API channel — the only network-touching module in the package.
+
+Records review comments and suggestion fences per document as a separate
+human-signal channel (design: issue #5). Squash merges and closed-unmerged
+PRs hide this signal from git entirely; the channel recovers it without ever
+blending it into blame. The scan neither reads nor writes these artefacts,
+so its determinism contract is untouched.
+
+    pr_channel.jsonl   one record per review comment anchored to a configured document
+    pr_channel.json    collection summary (volumes by kind, role and PR state)
+"""
+from __future__ import annotations
+
+import fnmatch
+import json
+import re
+import subprocess
+import sys
+from collections import Counter
+from datetime import UTC, datetime
+from pathlib import Path
+
+from .config import Config, ConfigError
+from .roster import Roster
+
+SUGGESTION = re.compile(r"```suggestion[^\n]*\n(.*?)```", re.DOTALL)
+
+
+class GhError(RuntimeError):
+    pass
+
+
+def _concat(text: str):
+    """`gh api --paginate` concatenates JSON documents; merge arrays, pass dicts through."""
+    dec = json.JSONDecoder()
+    idx, out, text = 0, None, text.strip()
+    while idx < len(text):
+        obj, idx = dec.raw_decode(text, idx)
+        if isinstance(obj, list):
+            out = out if isinstance(out, list) else []
+            out.extend(obj)
+        else:
+            out = obj
+        while idx < len(text) and text[idx] in " \n\r\t":
+            idx += 1
+    return [] if out is None else out
+
+
+def gh_api(path: str):
+    try:
+        r = subprocess.run(["gh", "api", "--paginate", path], capture_output=True, check=False)
+    except FileNotFoundError as e:
+        raise GhError("gh is not installed — collect-pr requires an authenticated GitHub CLI") from e
+    if r.returncode != 0:
+        raise GhError(f"gh api {path}: {r.stderr.decode(errors='replace')[:400]}")
+    return _concat(r.stdout.decode("utf-8", errors="replace"))
+
+
+def _hunk_tail(diff_hunk: str) -> str:
+    """The commented line: the last content line of the comment's diff hunk."""
+    lines = [ln for ln in (diff_hunk or "").splitlines() if not ln.startswith("@@")]
+    return lines[-1][1:] if lines else ""
+
+
+def collect(cfg: Config, out_dir: Path, log=sys.stderr, api=gh_api) -> dict:
+    pc = cfg.pr_channel
+    if not pc.repo:
+        raise ConfigError("collect-pr needs pr_channel.repo (an owner/name slug) in the config")
+    roster = Roster.load(cfg.resolve(cfg.people.roster))
+    bots = [re.compile(p, re.IGNORECASE) for p in cfg.machine.bots + cfg.disclosure.authors]
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    comments = api(f"repos/{pc.repo}/pulls/comments?per_page=100&sort=created&direction=asc")
+    pr_state: dict[int, str] = {}
+    records: list[dict] = []
+    for c in comments:
+        path = c.get("path") or ""
+        if not fnmatch.fnmatch(path, cfg.files):
+            continue
+        if pc.since and (c.get("created_at") or "") < pc.since:
+            continue
+        pr = int(c["pull_request_url"].rstrip("/").rsplit("/", 1)[-1])
+        if pr not in pr_state:
+            p = api(f"repos/{pc.repo}/pulls/{pr}")
+            # self-describing states: "closed" without a merge is the channel's key case
+            pr_state[pr] = ("merged" if p.get("merged_at")
+                            else "closed-unmerged" if p.get("state") == "closed"
+                            else p.get("state", "unknown"))
+        login = (c.get("user") or {}).get("login") or ""
+        person = roster.by_handle.get(login.lower())
+        # the comments endpoint reports Copilot's reviewer as a bare "Copilot" login
+        bot = login.lower() == "copilot" or any(p.search(login) for p in bots)
+        body = c.get("body") or ""
+        m = SUGGESTION.search(body)
+        records.append({"pr": pr, "pr_state": pr_state[pr], "comment_id": c["id"],
+                        "document": path, "line": c.get("line") or c.get("original_line"),
+                        "author_login": login, "roster_id": person.id if person else None,
+                        "role": person.role if person else None, "bot": bot,
+                        "kind": "suggestion" if m else "comment",
+                        "created_at": c.get("created_at"), "body_chars": len(body),
+                        "in_reply_to": c.get("in_reply_to_id"),
+                        "before": _hunk_tail(c.get("diff_hunk", "")) if m else None,
+                        "after": m.group(1).rstrip("\n") if m else None})
+    records.sort(key=lambda r: (r["pr"], r["comment_id"]))
+
+    with (out_dir / "pr_channel.jsonl").open("w", encoding="utf-8") as fh:
+        for r in records:
+            fh.write(json.dumps(r, ensure_ascii=False) + "\n")
+    summary = {
+        "repo": pc.repo,
+        "collected_at": datetime.now(UTC).isoformat(timespec="seconds"),
+        "comments": len(records),
+        "suggestions": sum(1 for r in records if r["kind"] == "suggestion"),
+        "human": sum(1 for r in records if not r["bot"]),
+        "bot": sum(1 for r in records if r["bot"]),
+        "documents": len({r["document"] for r in records}),
+        "by_role": dict(Counter(r["role"] or ("bot" if r["bot"] else "unresolved") for r in records)),
+        "pr_states": dict(Counter(r["pr_state"] for r in records)),
+    }
+    (out_dir / "pr_channel.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1),
+                                             encoding="utf-8")
+    print(f"{pc.repo}: {summary['comments']} review comments on {summary['documents']} documents "
+          f"({summary['human']} human, {summary['bot']} bot; {summary['suggestions']} suggestions)",
+          file=log)
+    return summary

tests/test_pr_channel.py

@@ -0,0 +1,92 @@
+"""The PR-API channel collector against a recorded, stubbed API — no network."""
+import json
+import re
+import sys
+
+import pytest
+
+from textstrata.config import Config, ConfigError
+from textstrata.pr_channel import _concat, _hunk_tail, collect
+
+COMMENTS = [
+    {"id": 11, "path": "lectures/a.md", "line": 12, "user": {"login": "HumphreyYang"},
+     "body": "This is reverting code comments back to English",
+     "pull_request_url": "https://api.github.com/repos/o/r/pulls/7",
+     "created_at": "2026-04-14T01:00:00Z",
+     "diff_hunk": "@@ -10,2 +10,2 @@\n 上下文行\n+# Update wealth"},
+    {"id": 12, "path": "lectures/a.md", "line": 13, "user": {"login": "HumphreyYang"},
+     "body": "请修正：\n```suggestion\n# 更新财富\n```\n如上。",
+     "pull_request_url": "https://api.github.com/repos/o/r/pulls/7",
+     "created_at": "2026-04-14T01:05:00Z",
+     "diff_hunk": "@@ -13 +13 @@\n+# Update wealth"},
+    {"id": 13, "path": "lectures/b.md", "line": 2, "user": {"login": "Copilot"},
+     "body": "Consider handling the empty case.",
+     "pull_request_url": "https://api.github.com/repos/o/r/pulls/9",
+     "created_at": "2026-05-01T00:00:00Z", "diff_hunk": ""},
+    # outside the files glob: never collected
+    {"id": 14, "path": "README.md", "line": 1, "user": {"login": "HumphreyYang"},
+     "body": "readme note", "pull_request_url": "https://api.github.com/repos/o/r/pulls/7",
+     "created_at": "2026-04-14T02:00:00Z", "diff_hunk": ""},
+    # before `since`: skipped
+    {"id": 15, "path": "lectures/a.md", "line": 3, "user": {"login": "someoneelse"},
+     "body": "early note", "pull_request_url": "https://api.github.com/repos/o/r/pulls/7",
+     "created_at": "2026-01-01T00:00:00Z", "diff_hunk": ""},
+]
+PRS = {7: {"merged_at": None, "state": "closed"},
+       9: {"merged_at": "2026-05-02T00:00:00Z", "state": "closed"}}
+
+
+def fake_api(path):
+    if "pulls/comments" in path:
+        return COMMENTS
+    m = re.search(r"/pulls/(\d+)$", path)
+    return PRS[int(m.group(1))]
+
+
+def make_cfg(tmp_path):
+    (tmp_path / "roster.yml").write_text(
+        "people:\n  - id: HumphreyYang\n    role: editor\n    emails: [x@example.org]\n",
+        encoding="utf-8")
+    cfg = Config(name="t", repo=tmp_path, base_dir=tmp_path)
+    cfg.pr_channel.repo = "o/r"
+    cfg.pr_channel.since = "2026-02-01"
+    cfg.people.roster = "roster.yml"
+    return cfg
+
+
+def test_collect(tmp_path, capsys):
+    cfg = make_cfg(tmp_path)
+    summary = collect(cfg, tmp_path / "out", log=sys.stderr, api=fake_api)
+    recs = [json.loads(ln) for ln in (tmp_path / "out" / "pr_channel.jsonl").open(encoding="utf-8")]
+    assert [r["comment_id"] for r in recs] == [11, 12, 13]   # glob and since filters applied, sorted
+    by_id = {r["comment_id"]: r for r in recs}
+    # roster resolution by login, and the closed-unmerged PR is the point of the channel
+    assert by_id[11]["roster_id"] == "HumphreyYang" and by_id[11]["role"] == "editor"
+    assert by_id[11]["pr_state"] == "closed-unmerged" and by_id[13]["pr_state"] == "merged"
+    # suggestion fences carry the before (hunk tail) / after (fence body) pair
+    assert by_id[12]["kind"] == "suggestion"
+    assert by_id[12]["before"] == "# Update wealth" and by_id[12]["after"] == "# 更新财富"
+    assert by_id[11]["kind"] == "comment" and by_id[11]["before"] is None
+    # the Copilot reviewer is collected as a bot, not dropped
+    assert by_id[13]["bot"] is True and by_id[11]["bot"] is False
+    assert summary["comments"] == 3 and summary["suggestions"] == 1
+    assert summary["human"] == 2 and summary["bot"] == 1
+    assert summary["by_role"] == {"editor": 2, "bot": 1}
+    assert summary["pr_states"] == {"closed-unmerged": 2, "merged": 1}
+    assert "3 review comments" in capsys.readouterr().err
+
+
+def test_collect_needs_repo(tmp_path):
+    cfg = make_cfg(tmp_path)
+    cfg.pr_channel.repo = None
+    with pytest.raises(ConfigError, match="pr_channel.repo"):
+        collect(cfg, tmp_path / "out", api=fake_api)
+
+
+def test_concat_and_hunk_tail():
+    # gh --paginate concatenates array pages; single documents pass through
+    assert _concat('[{"a": 1}]\n[{"b": 2}]') == [{"a": 1}, {"b": 2}]
+    assert _concat('{"state": "closed"}') == {"state": "closed"}
+    assert _concat("") == []
+    assert _hunk_tail("@@ -1,2 +1,2 @@\n context\n+new line") == "new line"
+    assert _hunk_tail("") == ""