feat: heal series grouping and run the audit nightly

README.md

@@ -37,8 +37,10 @@ lock pins the edition, so a healed book *stays* healed (set-once, no oscillation
   the metadata provider for candidates and asks an LLM to pick the *same work* — but only
   from the retrieved set, validated against it (the model can never invent an id), and
   only auto-applies above a confidence threshold. Below it, or on doubt: left flagged.
-- **Series audit** — compares each book's series number against the provider's
-  authoritative position and heals genuine mismatches (read-only by default).
+- **Series audit** — compares each book's series number *and* grouping against the provider's
+  authoritative series, and heals genuine mismatches: a wrong/missing number, an ungrouped book (no
+  series name), or a variant series name where the title still matches the provider. A true mis-seed
+  (name *and* title disagree) is left for the resolver. Read-only by default; runs nightly.
 - **Dedup** — collapses duplicate records, moving the loser's file onto the keeper as an
   *alternative format* (nothing deleted from disk) and removing the empty record.
 - **Oversight** — a weekly changelog review that flags drift (a book healed more than
@@ -100,7 +102,7 @@ python -m colophon.cli backfill --apply         # heal broken ISBNs
 python -m colophon.cli enrich --apply           # seed bare no-id imports; remember the unresolvable
 python -m colophon.cli resolve --apply          # LLM-resolve mis-identified books (>= 0.9 conf)
 python -m colophon.cli resolve --force          # re-query the cached-unresolvable mis-seeds
-python -m colophon.cli series-audit             # series-number report (read-only)
+python -m colophon.cli series-audit             # series numbering + grouping report (read-only)
 python -m colophon.cli maintain --apply --email # backfill + resolve in one run + summary email
 python -m colophon.cli oversight --days 7       # weekly changelog review + verdict
 python -m colophon.cli log                      # the change history

colophon/cli.py

@@ -298,9 +298,10 @@ def main(argv=None):
     rs.add_argument("--min-conf", type=float, default=0.9)
     rs.add_argument("--force", action="store_true", help="re-query books on the cached-unresolvable skip-list")
     rs.add_argument("--clear-skips", action="store_true", help="forget all cached-unresolvable entries, then exit")
-    sn = sub.add_parser("series-audit", help="Phase 3 — series-numbering audit (read-only unless --apply)")
+    sn = sub.add_parser("series-audit", help="Phase 3 — series numbering + grouping audit (read-only unless --apply)")
     sn.add_argument("--limit", type=int, default=None)
-    sn.add_argument("--apply", action="store_true", help="heal clean number mismatches (reuses the heal path)")
+    sn.add_argument("--apply", action="store_true",
+                    help="heal clean number-mismatch / number-missing / series-name-missing (reuses the heal path)")
     ov = sub.add_parser("oversight", help="Phase 4b — weekly changelog oversight + verdict (emails only if flagged)")
     ov.add_argument("--days", type=int, default=7)
     ov.add_argument("--email", action="store_true", help="email the digest only when flagged (DRIFT/REVIEW)")

colophon/maintain.py

@@ -12,14 +12,14 @@
 """
 import time
 
-from . import backfill, grimmory
+from . import backfill, grimmory, series_audit
 from .heal import assert_preconditions
 from .resolver import run_resolve
 
 
 def run_maintain(g, store, limit=20, min_conf=0.9, apply=False, force=False):
     res = {"ts": time.strftime("%Y-%m-%d %H:%M"), "apply": apply, "ok": True,
-           "aborted": False, "backfill": None, "resolve": None, "errors": []}
+           "aborted": False, "backfill": None, "resolve": None, "series": None, "errors": []}
     if apply:
         try:
             assert_preconditions(g)
@@ -42,6 +42,15 @@ def run_maintain(g, store, limit=20, min_conf=0.9, apply=False, force=False):
     except Exception as e:  # noqa: BLE001
         res["ok"] = False
         res["errors"].append(f"resolve: {str(e)[:200]}")
+    # Series: numbering + grouping. Auto-heals clean number-mismatch/number-missing
+    # and the ungrouped series-name-missing cases; series-mismatch stays for resolve.
+    try:
+        res["series"] = series_audit.run(apply=apply, g=g, store=store)
+        if res["series"]["errors"]:
+            res["ok"] = False
+    except Exception as e:  # noqa: BLE001
+        res["ok"] = False
+        res["errors"].append(f"series: {str(e)[:200]}")
     # Informational only — surfacing the enrich stuck-list must never fail the run.
     res["stuck"] = []
     try:
@@ -76,7 +85,8 @@ def verdict(res):
 def _healed_count(res):
     bf = (res["backfill"] or {}).get("healed", 0)
     rs = sum(1 for p in (res["resolve"] or {}).get("proposals", []) if p.get("applied"))
-    return bf + rs
+    sa = (res.get("series") or {}).get("healed", 0)
+    return bf + rs + sa
 
 
 def subject(res):
@@ -132,6 +142,21 @@ def render_summary(res):
     else:
         L.append("Resolve: did not run")
 
+    sa = res.get("series")
+    if sa:
+        cats = sa["categories"]
+        smis = len(cats.get("series-mismatch", []))
+        L.append(f"Series (numbering + grouping): {sa['total']} scanned · {sa['healed']} healed · "
+                 f"{smis} series-mismatch → resolver · {sa['errors']} errors"
+                 + ("  ABORTED (circuit-breaker)" if sa.get("errors", 0) >= series_audit.ABORT_ERRORS else ""))
+        for k in ("number-mismatch", "number-missing", "series-name-missing", "series-name-variant"):
+            for rec in cats.get(k, []):
+                if rec.get("applied"):
+                    b = rec["book"]
+                    L.append(f"  + book {b['book_id']} {b['title']!r} — {rec['reason']}")
+    else:
+        L.append("Series: did not run")
+
     stuck = res.get("stuck") or []
     if stuck:
         L.append("")

colophon/series_audit.py

@@ -7,46 +7,93 @@
 and grimmory repopulates series_number from Hardcover's position (verified live on book
 590 "Zero Hour": German-edition ISBN → number 2; heal → canonical ISBN → number 5).
 
-Deliberately NOT wired into the daily sweep (numbering is the least-tested surface and
-overlaps dedup + mis-seed resolution). Read-only `series-audit`; `--apply` heals only
-the clean number mismatches, gated, dry-run by default.
+Runs as a guarded phase of the nightly sweep (`maintain`) and standalone via
+`series-audit` (read-only unless `--apply`). `--apply` heals the clean cases —
+number-mismatch / number-missing and the grouping repairs series-name-missing /
+series-name-variant — gated, dry-run by default. Only a *true* mis-seed
+(series-mismatch: the name AND the title disagree with the hcid) stays deferred
+to the resolver, which validates the identity against candidates before locking.
 
 Categories:
-  number-mismatch : series matches, grimmory number != Hardcover position  → FIX (heal)
-  number-missing  : series matches, grimmory number null, Hardcover has one → FIX (heal)
-  series-mismatch : grimmory series != hcid's series → likely a mis-seed     → resolver
-  dup-overlap     : hcid shared with another owned book                      → plan 21
-  no-position     : Hardcover has no position for this id                    → leave
-  no-hcid         : in a series but unidentified                            → leave
-  number-ok       : grimmory number == Hardcover position
+  number-mismatch     : series matches, grimmory number != Hardcover position  → FIX (heal)
+  number-missing      : series matches, grimmory number null, Hardcover has one → FIX (heal)
+  series-name-missing : grimmory has no series_name, Hardcover puts it in one   → FIX (heal)
+  series-name-variant : name differs but the book TITLE matches the hcid        → FIX (heal)
+  series-mismatch     : name AND title differ → the hcid itself is suspect       → resolver
+  dup-overlap         : hcid shared with another owned book                      → plan 21
+  no-position         : Hardcover has no position for this id                    → leave
+  no-series           : no series in grimmory or Hardcover (standalone)          → leave
+  no-hcid             : in a series but unidentified                            → leave
+  number-ok           : grimmory number == Hardcover position
+
+The `series-name-missing` path (Symptom 1 — owned books that fall out of their
+series because grimmory never derived a `series_name`) reuses the exact heal
+recipe: set the canonical ISBN + hcid, lock, refresh, and grimmory repopulates
+BOTH series_name and series_number. It heals only books on a *non-canonical*
+13-char edition (current ISBN != Hardcover's canonical) — books already on the
+canonical edition with a null name are a grimmory-side derivation gap, not ours
+to churn on. Broken-ISBN books are left to the backfill phase. A *disagreeing*
+name (series-mismatch) is still deferred to the resolver: the disagreement is
+itself evidence the hcid may be wrong, so it needs adjudication before a lock.
 """
+import re
 import time
 from collections import defaultdict
 
 from . import audit, grimmory, hardcover, matcher
 from .heal import assert_preconditions, heal_book
 
 ABORT_ERRORS = 3
+UNGROUPED_LIMIT = 60  # per-run cap on the ungrouped (null series_name) survey
 
 _SQL = (
     "SELECT bm.book_id, IFNULL(bm.title,''), IFNULL(bm.hardcover_book_id,''), "
     "IFNULL(bm.series_name,''), IFNULL(bm.series_number,''), IFNULL(bm.isbn_13,''), "
-    "IFNULL(bm.series_number_locked,0) "
+    "IFNULL(bm.series_number_locked,0), IFNULL(bm.series_name_locked,0) "
     "FROM book_metadata bm JOIN book b ON b.id=bm.book_id "
     "WHERE (b.deleted IS NULL OR b.deleted=0) "
     "AND bm.series_name IS NOT NULL AND bm.series_name<>'';"
 )
 
 
+_UNGROUPED_SQL = (
+    "SELECT bm.book_id, IFNULL(bm.title,''), IFNULL(bm.hardcover_book_id,''), "
+    "IFNULL(bm.isbn_13,''), IFNULL(bm.series_name_locked,0) "
+    "FROM book_metadata bm JOIN book b ON b.id=bm.book_id "
+    "WHERE (b.deleted IS NULL OR b.deleted=0) "
+    "AND bm.hardcover_book_id IS NOT NULL AND bm.hardcover_book_id<>'' "
+    "AND (bm.series_name IS NULL OR bm.series_name='') "
+    "AND bm.isbn_13 IS NOT NULL AND LENGTH(REPLACE(bm.isbn_13,'-',''))=13 "
+    "ORDER BY bm.book_id LIMIT {limit};"
+)
+
+
 def _series_books():
     out, rows = grimmory._db(_SQL), []
     for line in out.splitlines():
         c = line.split("\t")
-        if len(c) < 7:
+        if len(c) < 8:
             continue
         rows.append({"book_id": int(c[0]), "title": c[1], "hcid": c[2].strip(),
                      "series_name": c[3], "series_number": c[4].strip(),
-                     "isbn": c[5].strip(), "num_locked": c[6] == "1"})
+                     "isbn": c[5].strip(), "num_locked": c[6] == "1", "name_locked": c[7] == "1"})
+    return rows
+
+
+def _ungrouped_candidates(limit):
+    """hcid'd books with a clean 13-char ISBN but no series_name — possible
+    ungrouped series members (Symptom 1). Bounded: the cap keeps the nightly
+    Hardcover lookups cheap; healed books gain a name and drop out next run."""
+    if not limit:
+        return []
+    out, rows = grimmory._db(_UNGROUPED_SQL.format(limit=int(limit))), []
+    for line in out.splitlines():
+        c = line.split("\t")
+        if len(c) < 5:
+            continue
+        rows.append({"book_id": int(c[0]), "title": c[1], "hcid": c[2].strip(),
+                     "series_name": "", "series_number": "", "isbn": c[3].strip(),
+                     "num_locked": False, "name_locked": c[4] == "1"})
     return rows
 
 
@@ -57,6 +104,10 @@ def _as_float(s):
         return None
 
 
+def _isbn_digits(s):
+    return re.sub(r"\D", "", s or "")
+
+
 def audit_one(b, hcid_counts):
     """Return (category, reason, fix|None). fix = (isbn, hcid) to heal, when applicable."""
     hcid = b["hcid"]
@@ -70,10 +121,40 @@ def audit_one(b, hcid_counts):
     if not cand:
         return "series-mismatch", f"hcid {hcid} not found in Hardcover", None
     hc_series, hc_pos, hc_isbn = cand.get("series"), cand.get("position"), cand.get("isbn")
+    if not b["series_name"]:
+        if not hc_series:
+            return "no-series", "standalone — no series in grimmory or Hardcover", None
+        if b.get("name_locked"):
+            return "series-name-missing", f"ungrouped from {hc_series!r} (series_name LOCKED — manual)", None
+        if not hc_isbn:
+            return "series-name-missing", f"ungrouped from {hc_series!r} — no canonical ISBN to heal", None
+        if _isbn_digits(b["isbn"]) == _isbn_digits(hc_isbn):
+            return ("series-name-missing",
+                    f"ungrouped from {hc_series!r}; already on canonical ISBN — grimmory derived no series (leave)",
+                    None)
+        pos = "" if hc_pos is None else f" #{float(hc_pos):g}"
+        return "series-name-missing", f"ungrouped → series {hc_series!r}{pos}", (hc_isbn, hcid)
     if hc_series and not matcher._title_match(b["series_name"], hc_series):
-        return ("series-mismatch",
-                f"grimmory series {b['series_name']!r} != hcid series {hc_series!r} → mis-seed (resolver)",
-                None)
+        # Series name disagrees. If the book TITLE corroborates the hcid, it's a
+        # stale/variant name — heal to canonical so grimmory re-derives the right
+        # one (same heal as number-mismatch). If the title ALSO disagrees, the
+        # hcid itself is suspect (a real mis-seed): leave it for the resolver,
+        # which validates the identity against candidates before locking.
+        if not (cand.get("title") and matcher._title_match(b["title"], cand["title"])):
+            return ("series-mismatch",
+                    f"grimmory series {b['series_name']!r} != hcid series {hc_series!r} → mis-seed (resolver)",
+                    None)
+        if b["name_locked"]:
+            return "series-name-variant", f"variant name {b['series_name']!r} → {hc_series!r} (series_name LOCKED — manual)", None
+        if not hc_isbn:
+            return "series-name-variant", f"variant name {b['series_name']!r} → {hc_series!r} — no canonical ISBN to heal", None
+        if _isbn_digits(b["isbn"]) == _isbn_digits(hc_isbn):
+            return ("series-name-variant",
+                    f"variant name {b['series_name']!r} → {hc_series!r}; already on canonical ISBN (leave)",
+                    None)
+        return ("series-name-variant",
+                f"grimmory series {b['series_name']!r} → {hc_series!r} (title matches hcid)",
+                (hc_isbn, hcid))
     if hc_pos is None:
         return "no-position", "Hardcover has no series position for this id", None
     gn = _as_float(b["series_number"])
@@ -95,10 +176,13 @@ def audit_one(b, hcid_counts):
     return cat, reason, fix
 
 
-def run(limit=None, apply=False, g=None, store=None):
+def run(limit=None, apply=False, g=None, store=None, ungrouped_limit=None):
     books = _series_books()
     if limit:
         books = books[:limit]
+    # Default cap walks the ungrouped set nightly; a targeted --limit mirrors it.
+    ug_cap = ungrouped_limit if ungrouped_limit is not None else (limit or UNGROUPED_LIMIT)
+    books += _ungrouped_candidates(ug_cap)
     hcid_counts = defaultdict(int)
     for b in books:
         if b["hcid"]:
@@ -111,7 +195,8 @@ def run(limit=None, apply=False, g=None, store=None):
     for b in sorted(books, key=lambda x: (x["series_name"], _as_float(x["series_number"]) or 0)):
         cat, reason, fix = audit_one(b, hcid_counts)
         rec = {"book": b, "reason": reason, "fix": fix, "applied": False}
-        if apply and fix and cat in ("number-mismatch", "number-missing"):
+        if apply and fix and cat in ("number-mismatch", "number-missing",
+                                     "series-name-missing", "series-name-variant"):
             try:
                 heal_book(g, store, run_id, b["book_id"], fix[0], fix[1], None, dry_run=False)
                 rec["applied"] = True
@@ -128,13 +213,16 @@ def run(limit=None, apply=False, g=None, store=None):
             "healed": healed, "errors": errors, "apply": apply}
 
 
-_ORDER = ["number-mismatch", "number-missing", "series-mismatch", "dup-overlap",
-          "no-position", "no-hcid", "error", "number-ok"]
+_ORDER = ["number-mismatch", "number-missing", "series-name-missing", "series-name-variant",
+          "series-mismatch", "dup-overlap", "no-position", "no-series", "no-hcid", "error", "number-ok"]
 _LABEL = {"number-mismatch": "Wrong number (heal-fixable)",
           "number-missing": "Missing number (heal-fixable)",
+          "series-name-missing": "Ungrouped — missing series name (heal-fixable)",
+          "series-name-variant": "Variant series name (heal-fixable)",
           "series-mismatch": "Series mismatch → mis-seed (resolver)",
           "dup-overlap": "Duplicate hcid → dedup (plan 21)",
           "no-position": "Hardcover has no position (leave)",
+          "no-series": "Standalone — not in any series (leave)",
           "no-hcid": "Unidentified in a series (leave)",
           "error": "Provider lookup error", "number-ok": "Correct"}
 
@@ -150,7 +238,7 @@ def render(res):
         if cats.get(k):
             L.append(f"- **{len(cats[k])}** {_LABEL[k]}")
     for k in _ORDER:
-        if k == "number-ok" or not cats.get(k):
+        if k in ("number-ok", "no-series") or not cats.get(k):
             continue
         L.append(f"\n## {_LABEL[k]} ({len(cats[k])})\n")
         for rec in sorted(cats[k], key=lambda r: (r["book"]["series_name"], r["book"]["book_id"])):