feat: daily summary email + skip re-querying unresolvable mis-seeds

CHANGELOG.md

@@ -6,6 +6,25 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [0.2.0] — 2026-06-04
+
+### Added
+- **`maintain` command** — runs backfill + resolve in a single process (each phase
+  guarded so one failing doesn't skip the other), writes a one-page summary, and with
+  `--email` always sends it as a daily heartbeat — including on an aborted run. Exits
+  non-zero on a phase failure (so a supervising timer is marked failed and retries); an
+  SMTP failure does not change the exit code.
+- **Resolve skip-list** — mis-seeds that resolve to no-match or a below-threshold match
+  are remembered (`resolve_skip` table) and not re-queried on later runs, cutting the
+  repeated Hardcover + LLM calls for permanently-unresolvable books. The skip key is the
+  normalized title+author, so a book is re-queried automatically once its metadata
+  changes. `resolve --force` re-checks everything; `resolve --clear-skips` forgets the
+  list; `COLOPHON_RESOLVE_RETRY_DAYS` (default `0` = never) sets an optional re-check TTL.
+
+### Changed
+- `resolve` reports now show the cached-unresolvable set, surfacing standing
+  below-threshold matches with a one-line manual-apply command.
+
 ## [0.1.0] — 2026-06-03
 
 Initial public release.
@@ -26,5 +45,6 @@ Initial public release.
   per-run limits, and a circuit-breaker.
 - Standard-library-only; configurable entirely via environment variables.
 
-[Unreleased]: https://github.com/vidaks/colophon/compare/v0.1.0...HEAD
+[Unreleased]: https://github.com/vidaks/colophon/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/vidaks/colophon/releases/tag/v0.2.0
 [0.1.0]: https://github.com/vidaks/colophon/releases/tag/v0.1.0

README.md

@@ -81,7 +81,8 @@ Copy `.env.example` to `.env` and fill it in (or export the variables):
 | `COLOPHON_HARDCOVER_KEY` | Hardcover API key (or `COLOPHON_HARDCOVER_QUERY_CMD`) |
 | `ANTHROPIC_API_KEY` | LLM adjudication (omit to use the `claude` CLI) |
 | `COLOPHON_DB`, `COLOPHON_REPORTS` | where to write the changelog + reports |
-| `SMTP_*`, `OVERSIGHT_TO` | weekly oversight email (only sent when flagged) |
+| `COLOPHON_RESOLVE_RETRY_DAYS` | re-query a cached-unresolvable mis-seed after N days (default `0` = never) |
+| `SMTP_*`, `OVERSIGHT_TO` | oversight + `maintain --email` summaries |
 
 ## Usage
 
@@ -90,7 +91,9 @@ python -m colophon.cli precheck                 # assert the files-never-touched
 python -m colophon.cli backfill                 # survey + propose (dry-run)
 python -m colophon.cli backfill --apply         # heal broken ISBNs
 python -m colophon.cli resolve --apply          # LLM-resolve mis-identified books (>= 0.9 conf)
+python -m colophon.cli resolve --force          # re-query the cached-unresolvable mis-seeds
 python -m colophon.cli series-audit             # series-number report (read-only)
+python -m colophon.cli maintain --apply --email # backfill + resolve in one run + summary email
 python -m colophon.cli oversight --days 7       # weekly changelog review + verdict
 python -m colophon.cli log                      # the change history
 python -m colophon.cli revert <run_id> --apply  # undo a metadata run

colophon/cli.py

@@ -140,11 +140,15 @@ def cmd_audit(args, g, store):
 def cmd_resolve(args, g, store):
     from . import anthropic
     from .resolver import render, run_resolve
+    if args.clear_skips:
+        n = store.skip_clear()
+        print(f"cleared {n} cached-unresolvable entr{'y' if n == 1 else 'ies'}")
+        return 0
     if not anthropic.have_key():
         print("(no ANTHROPIC_API_KEY — using the `claude` CLI; production needs the vaulted key)")
     try:
-        res = run_resolve(limit=args.limit, book_ids=args.book or None,
-                          apply=args.apply, min_conf=args.min_conf, g=g, store=store)
+        res = run_resolve(limit=args.limit, book_ids=args.book or None, apply=args.apply,
+                          min_conf=args.min_conf, g=g, store=store, force=args.force)
     except PreconditionError as e:
         print(e)
         return 2
@@ -194,6 +198,35 @@ def cmd_oversight(args, g, store):
     return 0
 
 
+def cmd_maintain(args, g, store):
+    """Run the nightly sweep (backfill + resolve) and report. With --email, always
+    send the summary (a daily heartbeat) — even on an aborted run. Exits non-zero
+    when a phase failed so the systemd unit fails + the next timer fire retries; an
+    SMTP failure does NOT change the exit code (the heal work still happened)."""
+    from . import maintain as M
+    res = body = None
+    try:
+        res = M.run_maintain(g, store, limit=args.limit, min_conf=args.min_conf,
+                             apply=args.apply, force=args.force)
+        body = M.render_summary(res)
+        reports_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "reports"))
+        os.makedirs(reports_dir, exist_ok=True)
+        path = os.path.join(reports_dir, f"maintain-{time.strftime('%Y%m%dT%H%M%S')}.md")
+        with open(path, "w") as f:
+            f.write(body)
+        print(body)
+        print(f"(report written to {path})")
+    finally:
+        if args.email:
+            from . import oversight
+            subject = M.subject(res) if res else "Colophon daily [CRASH] — see host journal"
+            mail = body or ("Colophon maintain crashed before producing a summary.\n"
+                            "Check `journalctl -u colophon.service` on the host.\n")
+            ok, detail = oversight.send_email(subject, mail)
+            print(f"email: {detail}")
+    return 0 if (res and res["ok"]) else 1
+
+
 def main(argv=None):
     p = argparse.ArgumentParser(prog="colophon", description="autonomous library metadata healer (Phase 0)")
     sub = p.add_subparsers(dest="cmd", required=True)
@@ -219,19 +252,27 @@ def main(argv=None):
     rs.add_argument("--limit", type=int, default=None)
     rs.add_argument("--apply", action="store_true", help="auto-apply re-seeds at conf >= --min-conf")
     rs.add_argument("--min-conf", type=float, default=0.9)
+    rs.add_argument("--force", action="store_true", help="re-query books on the cached-unresolvable skip-list")
+    rs.add_argument("--clear-skips", action="store_true", help="forget all cached-unresolvable entries, then exit")
     sn = sub.add_parser("series-audit", help="Phase 3 — series-numbering audit (read-only unless --apply)")
     sn.add_argument("--limit", type=int, default=None)
     sn.add_argument("--apply", action="store_true", help="heal clean number mismatches (reuses the heal path)")
     ov = sub.add_parser("oversight", help="Phase 4b — weekly changelog oversight + verdict (emails only if flagged)")
     ov.add_argument("--days", type=int, default=7)
     ov.add_argument("--email", action="store_true", help="email the digest only when flagged (DRIFT/REVIEW)")
+    mt = sub.add_parser("maintain", help="nightly sweep: backfill + resolve in one run, with a summary (dry-run unless --apply)")
+    mt.add_argument("--limit", type=int, default=20, help="max books for the backfill survey")
+    mt.add_argument("--min-conf", type=float, default=0.9, help="auto-apply gate for resolve re-seeds")
+    mt.add_argument("--apply", action="store_true", help="actually write (default: dry-run)")
+    mt.add_argument("--force", action="store_true", help="re-query cached-unresolvable mis-seeds this run")
+    mt.add_argument("--email", action="store_true", help="always email the summary (a daily heartbeat)")
     args = p.parse_args(argv)
 
     g, store = Grimmory(), Store()
     fn = {"precheck": cmd_precheck, "heal": cmd_heal, "log": cmd_log,
           "revert": cmd_revert, "backfill": cmd_backfill, "audit": cmd_audit,
           "resolve": cmd_resolve, "series-audit": cmd_series_audit,
-          "oversight": cmd_oversight}[args.cmd]
+          "oversight": cmd_oversight, "maintain": cmd_maintain}[args.cmd]
     try:
         return fn(args, g, store)
     except (GrimmoryError, PreconditionError) as e:

colophon/maintain.py

@@ -0,0 +1,114 @@
+"""Daily maintenance run — backfill + resolve in one process, with a summary.
+
+Runs the two write phases the timer fires nightly, each guarded so a failure in
+one does not skip the other, then composes a tight summary (counts + what changed
++ standing manual items + a one-line health verdict). The CLI emails it — always,
+including on an aborted run — so the summary doubles as a heartbeat. The process
+still exits non-zero when a phase failed, so the systemd unit is marked failed and
+the next timer fire retries.
+
+This module never raises for a phase or precondition failure: it captures the
+failure into the result so the caller can always render + send a report.
+"""
+import time
+
+from . import backfill
+from .heal import assert_preconditions
+from .resolver import run_resolve
+
+
+def run_maintain(g, store, limit=20, min_conf=0.9, apply=False, force=False):
+    res = {"ts": time.strftime("%Y-%m-%d %H:%M"), "apply": apply, "ok": True,
+           "aborted": False, "backfill": None, "resolve": None, "errors": []}
+    if apply:
+        try:
+            assert_preconditions(g)
+        except Exception as e:  # noqa: BLE001 — precondition/network failure: abort cleanly, still report
+            res["ok"], res["aborted"] = False, True
+            res["errors"].append(f"preconditions: {str(e)[:200]}")
+            return res
+    try:
+        res["backfill"] = backfill.run(g, store, limit=limit, apply=apply)
+        if res["backfill"]["errors"] or res["backfill"]["aborted"]:
+            res["ok"] = False
+    except Exception as e:  # noqa: BLE001
+        res["ok"] = False
+        res["errors"].append(f"backfill: {str(e)[:200]}")
+    try:
+        res["resolve"] = run_resolve(apply=apply, min_conf=min_conf, g=g, store=store, force=force)
+        props = res["resolve"]["proposals"]
+        if any(p.get("apply_error") for p in props) or any(p.get("aborted") for p in props):
+            res["ok"] = False
+    except Exception as e:  # noqa: BLE001
+        res["ok"] = False
+        res["errors"].append(f"resolve: {str(e)[:200]}")
+    return res
+
+
+def verdict(res):
+    if res["aborted"]:
+        return "ABORTED"
+    return "OK" if res["ok"] else "ERRORS"
+
+
+def _healed_count(res):
+    bf = (res["backfill"] or {}).get("healed", 0)
+    rs = sum(1 for p in (res["resolve"] or {}).get("proposals", []) if p.get("applied"))
+    return bf + rs
+
+
+def subject(res):
+    n = _healed_count(res)
+    mode = "" if res["apply"] else " [dry-run]"
+    return f"Colophon daily [{verdict(res)}]{mode} — {n} healed ({res['ts']})"
+
+
+def render_summary(res):
+    L = [f"Colophon daily maintenance — {res['ts']}",
+         f"STATUS: {verdict(res)}" + ("" if res["apply"] else "  (dry-run — nothing written)"), ""]
+    if res.get("errors"):
+        L.append("Errors:")
+        L += [f"  - {e}" for e in res["errors"]]
+        L.append("")
+
+    bf = res["backfill"]
+    if bf:
+        heals = [p for p in bf["proposals"] if p["action"] == "heal"]
+        flags = [p for p in bf["proposals"] if p["action"].startswith("review")]
+        L.append(f"Backfill (broken ISBN → canonical): {len(bf['proposals'])} surveyed · "
+                 f"{bf['healed']} healed · {len(flags)} flagged · {bf['errors']} errors"
+                 + ("  ABORTED (circuit-breaker)" if bf["aborted"] else ""))
+        for p in heals:
+            L.append(f"  + book {p['book_id']} → hcid {p.get('hcid')} {p.get('hc_title')!r}")
+    else:
+        L.append("Backfill: did not run")
+
+    rs = res["resolve"]
+    if rs:
+        props = rs["proposals"]
+        applied = [p for p in props if p.get("applied")]
+        proposed = [p for p in props if p["action"] == "propose" and not p.get("applied")]
+        none = [p for p in props if p["action"] == "none"]
+        err = [p for p in props if p["action"] == "error"]
+        skipped = rs.get("skipped", [])
+        L.append(f"Resolve (mis-seed → correct identity): {len(props)} queried · "
+                 f"{len(applied)} auto-healed · {len(proposed)} below-threshold · "
+                 f"{len(none)} no-match · {len(err)} error · {len(skipped)} cached-skipped")
+        for p in applied:
+            L.append(f"  + book {p['book_id']} {p['title']!r} → {p.get('chosen_title')!r} "
+                     f"(conf {p.get('confidence')})")
+        # New + standing below-threshold matches are the actionable items (human nudge).
+        standing = [(p["book_id"], p.get("title"), p.get("chosen_title"), p.get("confidence"))
+                    for p in proposed]
+        standing += [(s["book_id"], s.get("title"), s.get("chosen_title"), s.get("conf"))
+                     for s in skipped if s.get("action") == "propose-below"]
+        if standing:
+            L.append("  Below-threshold matches (apply manually if right — "
+                     "`colophon resolve --book <id> --apply --min-conf 0`):")
+            for bid, title, chosen, conf in standing:
+                L.append(f"    ? book {bid} {title!r} → {chosen!r} (conf {conf})")
+    else:
+        L.append("Resolve: did not run")
+
+    L += ["", "Full reports under reports/ on the host."]
+    return "\n".join(L) + "\n"