diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e5e176..82d32f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,3 +19,7 @@ [#1](https://github.com/QuantEcon/textstrata/issues/1)): the probable whole-file AI passes inside the 2025 hand-translation campaign, derived by rule and adjudicated from their before/after pairs, so the first report's human churn is not inflated by regenerated files committed under roster names. +- Engine-version strata ([#2](https://github.com/QuantEcon/textstrata/issues/2)): each `ai-sync` commit is + stamped with `engine_model`/`engine_tool_version` from the state-file history (the record written by or + immediately after the commit), `run.json` carries per-version totals (`engine_strata`), and + `overwrites.json` entries are stamped the same way; syncs with no record read `unrecorded`. diff --git a/docs/method.md b/docs/method.md index ef915e7..1340441 100644 --- a/docs/method.md +++ b/docs/method.md @@ -43,20 +43,32 @@ All stock metrics count **lines containing the target script**. On raw lines mos | S2 | Baseline survival | S1's `ai-initial` share; plus `difflib` similarity of the initial translation to HEAD | | S3 | Derived review state | `machine-only` (no prose-changing roster commit since translation) → `human-touched` → `audit-stale` (≥ `review_state.stale_after_syncs` machine syncs since the last touch) | | S4 | Freshness | source commits since the state file's `source-sha` (only with a source repo and state directory) | -| F1 | Human churn | prose lines added + deleted by roster-tier commits, per document — also in changed characters; normalise by machine-delivered lines or characters and stratify by engine version downstream | +| F1 | Human churn | prose lines added + deleted by roster-tier commits, per document — also in changed characters; normalise by machine-delivered lines or characters, against the engine strata below | | F2 | Overwrites | for each `ai-sync` commit, the prose lines it deleted, blamed at the parent, counted by prior tier | | F3 | Edit categories | pair counts by category and taxonomy bucket | | F4 | Recurring substitutions | short `(before, after)` replacements inside terminology / fluency / width pairs, counted across the corpus | | F5 | Time to first human touch | days from the translation moment to the first prose-changing roster commit | | F6 | Human-review coverage | documents with a human touch / translated documents | +## Engine-version strata + +Each `ai-sync` commit is stamped (`engine_model`, `engine_tool_version` in `commits.jsonl`) from the +document's state-file history: the record written by the commit or its wave (the first revision +at-or-after the commit, accepted within two days — exact for the engine's commit modes, +state-with-document and state-in-an-adjacent-commit), otherwise the record last in force. `run.json` aggregates +the machine's work per version (`engine_strata`: sync commits, prose and character churn, overwritten +prose by prior tier), in order of first appearance, and `overwrites.json` entries carry the same stamp. +A sync with no state record — pre-engine history, or a document the engine does not track — reads +`unrecorded`; a state file that itself says `model: unknown` keeps that literal value, so the two cases +stay distinguishable. + ## Known limits - **Squash merges** hide human work done inside a machine-drafted PR. `ai-initial` means *as landed*; human effort is a lower bound. - **Last-toucher blame** credits a whole line to whoever changed one character of it. Human shares are an upper bound at line granularity; churn is therefore also reported in changed characters (`chars_changed` per pair, `prose_chars_added`/`prose_chars_deleted` per commit, `prose_char_churn_by_tier` per document), where a one-character fix counts as one character. Counts come from `SequenceMatcher` opcodes over the paired lines' raw text; unpaired additions and deletions count the full line. - **Line pairing** inside rewritten paragraphs is heuristic (similarity-matched within a hunk). Category counts are indicative. - **Identity** is resolved by e-mail and GitHub noreply handle only; display names are ignored. Unresolved authors fall to `ai-assisted` and should be reviewed in `commits.jsonl`. -- **Pre-engine history** has no recorded engine version. Stratify flow metrics by version downstream and label the pre-engine stratum as such; do not read its rates as the shipping engine's. +- **Pre-engine history** has no recorded engine version and lands in the `unrecorded` stratum; do not read its rates as the shipping engine's. ## Provenance of the method diff --git a/src/textstrata/scan.py b/src/textstrata/scan.py index eb99be5..80e4f3e 100644 --- a/src/textstrata/scan.py +++ b/src/textstrata/scan.py @@ -16,7 +16,7 @@ import sys from collections import Counter from dataclasses import asdict, dataclass, field -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from pathlib import Path from . import __version__ @@ -70,21 +70,52 @@ def _days(a: str, b: str) -> int: return (datetime.fromisoformat(b) - datetime.fromisoformat(a)).days -def read_state(repo: Path, state_dir: str | None, doc: str) -> dict[str, str]: +def parse_state(text: str) -> dict[str, str]: """action-translation style per-document state file: flat `key: value` YAML.""" - if not state_dir: - return {} - p = repo / state_dir / (Path(doc).name + ".yml") - if not p.exists(): - return {} out: dict[str, str] = {} - for line in p.read_text(encoding="utf-8").splitlines(): + for line in text.splitlines(): if ":" in line and not line.startswith((" ", "\t", "#")): k, v = line.split(":", 1) out[k.strip()] = v.strip().strip('"').strip("'") return out +def read_state(repo: Path, state_dir: str | None, doc: str) -> dict[str, str]: + if not state_dir: + return {} + p = repo / state_dir / (Path(doc).name + ".yml") + if not p.exists(): + return {} + return parse_state(p.read_text(encoding="utf-8")) + + +def state_timeline(repo: Path, state_dir: str | None, doc: str) -> list[tuple[str, dict[str, str]]]: + """(date, fields) for every revision of the document's state file, oldest first.""" + if not state_dir: + return [] + path = state_dir.rstrip("/") + "/" + Path(doc).name + ".yml" + return [(c.date, parse_state(show(repo, c.sha, c.path))) for c in file_history(repo, path)] + + +def engine_version_at(revs: list[tuple[str, dict[str, str]]], date: str) -> dict[str, str]: + """The state record for the sync commit at `date`. + + A sync writes its record in the same commit as the document, an adjacent + commit seconds later, or its wave's PR — so the first revision at-or-after + the commit is that sync's record when it lands within two days. Otherwise + the sync wrote no state and the record last in force applies; a sync before + the state file existed at all has no record ({} -> "unrecorded"). + """ + when = datetime.fromisoformat(date) + prior: dict[str, str] = {} + for d, fields in revs: + dd = datetime.fromisoformat(d) + if dd >= when: + return fields if dd - when <= timedelta(days=2) else prior + prior = fields + return prior + + def translation_moment(cfg: Config, repo: Path, prose: Prose, f: str, hist: list[Commit], log=sys.stderr) -> tuple[str | None, str | None]: """The document's translation moment under the configured baseline strategy. @@ -151,6 +182,7 @@ def scan(cfg: Config, out_dir: Path, log=sys.stderr) -> dict: hist = file_history(repo, f) histories[f] = hist t_sha, t_date = translation_moment(cfg, repo, prose, f, hist, log) + revs: list[tuple[str, dict[str, str]]] | None = None # state timeline, read on first sync before = True for c in hist: if c.sha == t_sha: @@ -159,10 +191,19 @@ def scan(cfg: Config, out_dir: Path, log=sys.stderr) -> dict: c.trailers = commit_meta(repo, c.sha).trailers tier = ctx.classify(c, t_sha, before and c.sha != t_sha) tier_of[(f, c.sha)] = tier - commit_rows.append({"document": f, "sha": c.sha, "author": c.author, "email": c.email, - "date": c.date, "subject": c.subject, "tier": tier, - "adds": c.adds, "dels": c.dels, "prose_adds": 0, "prose_dels": 0, - "prose_chars_added": 0, "prose_chars_deleted": 0}) + row = {"document": f, "sha": c.sha, "author": c.author, "email": c.email, + "date": c.date, "subject": c.subject, "tier": tier, + "adds": c.adds, "dels": c.dels, "prose_adds": 0, "prose_dels": 0, + "prose_chars_added": 0, "prose_chars_deleted": 0, + "engine_model": None, "engine_tool_version": None} + if tier == "ai-sync": + if revs is None: + revs = state_timeline(repo, cfg.machine.state_dir, f) + if revs: + st = engine_version_at(revs, c.date) + row["engine_model"] = st.get("model") + row["engine_tool_version"] = st.get("tool-version") + commit_rows.append(row) d = DocResult(path=f, translated=t_sha is not None, translation_sha=t_sha, translation_date=t_date, n_commits=len(hist)) docs[f] = d @@ -326,6 +367,37 @@ def tier_for(f: str, sha: str) -> str: except GitError: pass + # ---- engine-version strata over the machine's sync work ----------------------------- + # keyed by the state-file record each sync wrote; None -> "unrecorded" (pre-engine + # history, or no state file for the document) + strata: dict[tuple, dict] = {} + for r in commit_rows: + if r["tier"] != "ai-sync": + continue + key = (r["engine_model"], r["engine_tool_version"]) + s = strata.setdefault(key, {"model": key[0] or "unrecorded", + "tool_version": key[1] or "unrecorded", + "_shas": set(), "_first": r["date"], + "prose_churn": 0, "prose_char_churn": 0, + "overwrote_prose_by_prior_tier": Counter()}) + s["_shas"].add(r["sha"]) + s["_first"] = min(s["_first"], r["date"], key=datetime.fromisoformat) + s["prose_churn"] += r["prose_adds"] + r["prose_dels"] + s["prose_char_churn"] += r["prose_chars_added"] + r["prose_chars_deleted"] + for sha, rec in overwrites.items(): + row = rows_by_key[(next(iter(rec["documents"])), sha)] + rec["engine_model"], rec["engine_tool_version"] = row["engine_model"], row["engine_tool_version"] + key = (row["engine_model"], row["engine_tool_version"]) + if key in strata: + for dd in rec["documents"].values(): + strata[key]["overwrote_prose_by_prior_tier"].update(dd["prose_deleted_by_prior_tier"]) + engine_strata = [{"model": s["model"], "tool_version": s["tool_version"], + "sync_commits": len(s["_shas"]), "prose_churn": s["prose_churn"], + "prose_char_churn": s["prose_char_churn"], + "overwrote_prose_by_prior_tier": dict(s["overwrote_prose_by_prior_tier"])} + for _key, s in sorted(strata.items(), + key=lambda kv: datetime.fromisoformat(kv[1]["_first"]))] + # ---- write artefacts ---------------------------------------------------------------- translated = [d for d in docs.values() if d.translated] corpus: Counter = Counter() @@ -347,6 +419,7 @@ def tier_for(f: str, sha: str) -> str: "pairs": len(pairs_out), "pair_categories": dict(Counter(p["category"] for p in pairs_out)), "recurring_substitutions": sum(1 for c in subs.values() if c >= 2), + "engine_strata": engine_strata, "tiers": list(TIERS), } (out_dir / "run.json").write_text(json.dumps(run, ensure_ascii=False, indent=1), encoding="utf-8") diff --git a/tests/test_baseline.py b/tests/test_baseline.py index cb64316..f98ed7e 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -5,6 +5,7 @@ the pair landing as adjacent single-file commits (the autodiff case), and a document with no state file at all. """ +import json import os import subprocess import sys @@ -15,7 +16,7 @@ from textstrata.config import Config, ProseConfig from textstrata.git import file_history from textstrata.prose import Prose -from textstrata.scan import translation_moment +from textstrata.scan import engine_version_at, scan, state_timeline, translation_moment def git(repo, *args, date=None): @@ -44,19 +45,22 @@ def engine_repo(tmp_path): git(repo, "init", "-q") # a.md: document and state file in one commit (repo / "lectures" / "a.md").write_text("# 讲座甲\n\n这是机器翻译的第一稿。\n", encoding="utf-8") - (repo / ".translate" / "state" / "a.md.yml").write_text("mode: NEW\n", encoding="utf-8") + (repo / ".translate" / "state" / "a.md.yml").write_text( + "mode: NEW\nmodel: alpha-1\ntool-version: 0.1.0\n", encoding="utf-8") shas = {"init": commit(repo, "Initial translation via translate init", "2026-03-20T10:00:00Z")} # b.md: document first, state file one second later (the autodiff shape) (repo / "lectures" / "b.md").write_text("# 讲座乙\n\n另一篇机器初稿。\n", encoding="utf-8") shas["b_doc"] = commit(repo, "Update translation: lectures/b.md", "2026-04-09T04:40:27+00:00") - (repo / ".translate" / "state" / "b.md.yml").write_text("mode: NEW\n", encoding="utf-8") + (repo / ".translate" / "state" / "b.md.yml").write_text( + "mode: NEW\nmodel: beta-1\ntool-version: 0.1.5\n", encoding="utf-8") shas["b_state"] = commit(repo, "Update translation: .translate/state/b.md.yml", "2026-04-09T04:40:28+00:00") # c.md: no state file (repo / "lectures" / "c.md").write_text("# 讲座丙\n\n没有状态文件的文稿。\n", encoding="utf-8") shas["c_doc"] = commit(repo, "Add c.md by hand", "2026-05-01T09:00:00Z") # a later sync touches a.md and its state file: must not move a.md's moment (repo / "lectures" / "a.md").write_text("# 讲座甲\n\n这是机器重新同步的稿子。\n", encoding="utf-8") - (repo / ".translate" / "state" / "a.md.yml").write_text("mode: UPDATE\n", encoding="utf-8") + (repo / ".translate" / "state" / "a.md.yml").write_text( + "mode: UPDATE\nmodel: alpha-2\ntool-version: 0.2.0\n", encoding="utf-8") shas["sync"] = commit(repo, "[translation-sync] resync a.md", "2026-06-01T09:00:00Z") return repo, shas @@ -105,3 +109,42 @@ def test_override_without_match_warns(engine_repo, capsys): cfg = make_cfg(repo, overrides={"lectures/a.md": "deadbeef"}) assert moment(cfg, repo, "lectures/a.md") == (None, None) assert "matches no commit" in capsys.readouterr().err + + +def test_engine_version_at(engine_repo): + repo, _shas = engine_repo + revs = state_timeline(repo, ".translate/state", "lectures/a.md") + assert [r[1]["model"] for r in revs] == ["alpha-1", "alpha-2"] + # a record written seconds after the commit is that sync's own record + assert engine_version_at(revs, "2026-06-01T08:59:59Z")["model"] == "alpha-2" + # a commit long before the next record wrote no state: the record last in force applies + assert engine_version_at(revs, "2026-05-01T00:00:00Z")["model"] == "alpha-1" + # a commit after the last revision likewise + assert engine_version_at(revs, "2026-07-01T00:00:00Z")["model"] == "alpha-2" + # a commit before the state file existed has no record + assert engine_version_at(revs, "2026-01-01T00:00:00Z") == {} + assert engine_version_at([], "2026-07-01T00:00:00Z") == {} + assert state_timeline(repo, ".translate/state", "lectures/c.md") == [] + + +def test_engine_strata_in_scan(engine_repo, tmp_path): + repo, shas = engine_repo + cfg = make_cfg(repo) + cfg.machine.sync = [r"\[translation-sync\]", "^Update translation: "] + with open(os.devnull, "w") as devnull: + run = scan(cfg, tmp_path / "out", log=devnull) + rows = [json.loads(ln) for ln in (tmp_path / "out" / "commits.jsonl").open(encoding="utf-8")] + by = {(r["document"], r["sha"]): r for r in rows} + # same-commit mode: the sync is stamped with the record it wrote + sync = by[("lectures/a.md", shas["sync"])] + assert sync["tier"] == "ai-sync" + assert (sync["engine_model"], sync["engine_tool_version"]) == ("alpha-2", "0.2.0") + # adjacent-commit mode: the doc commit is stamped from the state written seconds later + bdoc = by[("lectures/b.md", shas["b_doc"])] + assert bdoc["tier"] == "ai-sync" + assert (bdoc["engine_model"], bdoc["engine_tool_version"]) == ("beta-1", "0.1.5") + # non-sync rows carry no stamp + assert by[("lectures/a.md", shas["init"])]["engine_model"] is None + # per-version totals, ordered by first appearance + assert [(s["model"], s["tool_version"], s["sync_commits"]) for s in run["engine_strata"]] == [ + ("beta-1", "0.1.5", 1), ("alpha-2", "0.2.0", 1)]