feat(analytics): add route reliability and latency analytics

devlog/_plan/260804_router_intelligence/001_pr_stack_status.md

@@ -0,0 +1,135 @@
+# 001 - PR stack status ledger
+
+Continuously updated during the programme. Every branch records: base SHA,
+head SHA, PR number/URL, verification result, and review state.
+
+## Programme facts
+
+- Stack base (dev): `e44d234f08e03dd4dbf0c4aa13af43046d86b0a6` (`upstream/dev`)
+- `origin/dev` (fork, stale ancestor): `be177ea501e5007f4a56d19d069ef5cd76ea24b9`
+- Bun: `1.3.14`; package version: `2.10.0`
+- Worktree: `D:\codex-worktrees\ocx-router-intelligence`
+- Push remote: `origin` (Wibias/opencodex); PR target: `lidge-jun/opencodex:dev`
+- All PRs opened as DRAFT; nothing merged by this programme.
+
+## Related in-flight PRs (not superseded by this stack)
+
+| PR | Branch | Note |
+|---|---|---|
+| #922 | `fix/914-account-neutral-network` | #914 alternative; consumed as health evidence input by RI-06 |
+| #966 | `codex/260804-issue914-transport-attribution` | #914 alternative; consumed as health evidence input by RI-06 |
+| #715 | `feat/priority-levels` | Pool selection order; out of scope |
+| #988 | `codex/providers-copy-doctor` | GUI providers/combos; conflict-checked at RI-10 |
+| #998 | `codex/260803-integration-switches` | Write substrate; rebase watch on request-log.ts |
+
+No open PR found that implements the same vertical as any PR in this stack,
+so no stale PR is closed by this programme. Both #914 drafts overlap each
+other; closing one is a maintainer decision and neither is stale.
+
+## Baseline
+
+- Full-suite baseline on clean `upstream/dev` (worktree
+  `D:\codex-worktrees\ocx-typecheck-base`, head `e44d234f0`): running in
+  background; exact pass/fail counts appended here when done.
+- `bun x tsc --noEmit` on clean `upstream/dev`: **PASSED** (0 errors, verified
+  in the pristine base worktree).
+- `bun run privacy:scan`: passed per-PR (see RI-01 below).
+
+## Stack status
+
+| RI | Branch | Base | Head SHA | PR | URL | Status |
+|---|---|---|---|---|---|---|
+| RI-01 | `feat/ri-01-route-decision-traces` | `e44d234f0` | `b5a8e7c4c` | #1003 | https://github.com/lidge-jun/opencodex/pull/1003 | DRAFT OPEN |
+| RI-02 | `feat/ri-02-request-history-index` | `b5a8e7c4c` (RI-01 head) | pending | pending | pending | in progress |
+| RI-03 | `feat/ri-03-routing-analytics` | `7efb6e842` (RI-02 head) | pending | pending | pending | in progress |
+| RI-04 | `feat/ri-04-policy-profile-core` | `feat/ri-03` head | pending | pending | pending | queued |
+| RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued |
+| RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued |
+| RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued |
+| RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued |
+| RI-09 | `feat/ri-09-route-explainability-api` | `feat/ri-08` head | pending | pending | pending | queued |
+| RI-10 | `feat/ri-10-routing-intelligence-ui` | `feat/ri-09` head | pending | pending | pending | queued |
+
+## Per-PR acceptance log
+
+### RI-01 - feat/ri-01-route-decision-traces
+
+- Base SHA: `e44d234f08e03dd4dbf0c4aa13af43046d86b0a6`
+- Reviewed commit: same as final (single implementation commit; independent
+  review pass performed by the author before push)
+- Findings (self-review): 3 test failures caught pre-push - (1) missing value
+  import for `normalizeRouteDecisionTrace` in request-log hydration,
+  (2) selected combo target marked ineligible because `ComboPick.attempted`
+  includes the winner, (3) account-namespace fixture missing the canonical
+  ChatGPT forward `baseUrl` (test-fixture bug, not product code).
+- Fixes: import fixed; `comboRouteCandidates` now excludes the selected target
+  from `already-attempted`; fixture uses `https://chatgpt.com/backend-api/codex`.
+- Regression tests: all three cases are covered by the final
+  `tests/route-decision-trace.test.ts` (14 tests, 75 assertions).
+- Final commit: `b5a8e7c4cd25dc3b83726e377899f4c49fca7753`
+  (2 commits: plan+ledger `97681a9e5`, implementation `b5a8e7c4c`)
+- PR: #1003 (DRAFT) https://github.com/lidge-jun/opencodex/pull/1003
+  - base: `dev`, head: `Wibias:feat/ri-01-route-decision-traces`
+  - local head == remote head: verified (`b5a8e7c4c`)
+- Review state: awaiting review; no external review comments yet
+- Verification:
+  - `bun x tsc --noEmit`: PASSED (0 errors)
+  - `bun run test tests/route-decision-trace.test.ts`: 14/14 pass
+  - Focused regression suites: 253/253 pass across combos, codex-routing,
+    usage-log, request-log, combo-management-api, codex-account-namespaces
+  - `tests/server-combo-failover-e2e.test.ts`: 44/44 pass
+  - `bun run privacy:scan`: passed
+- Remaining Low findings: none
+
+### RI-02..RI-10
+
+### RI-02 - feat/ri-02-request-history-index
+
+- Base SHA: `b5a8e7c4cd25dc3b83726e377899f4c49fca7753` (RI-01 head)
+- Reviewed commit: same as final (author self-review before push)
+- Findings (self-review): 4 defects caught pre-push -
+  1. `destroyAndRecreate` never reassigned the fresh handle to module `db`
+     (first-open rebuild crashed);
+  2. bun:sqlite named-parameter objects silently failed to bind for
+     `LIMIT $x` and INSERT statements (datatype mismatch / silent no-op) -
+     query and insert paths switched to positional parameters;
+  3. Windows file locking: an unfinalized prepared statement kept the DB
+     locked after close (EBUSY in tests) - insert statement now finalizes;
+     a partially-opened handle on a corrupt file is closed before recreate;
+  4. duplicate-replay accounting counted ignored rows in `indexedRows` -
+     now counts real `INSERT` changes.
+- Fixes: all four above; tests cover every one.
+- Final commit: pending (recorded after commit)
+- PR: pending
+- Verification:
+  - `bun x tsc --noEmit`: PASSED (0 errors)
+  - `bun run test tests/request-history-index.test.ts`: 16/16 pass
+    (1574 assertions) covering the mandatory matrix: empty/missing/corrupt/
+    old-schema/partial-line/replacement/truncation/duplicate-replay/cursor
+    stability/invalid-cursor/page-bounds/rebuild-equivalence/filters/row-by-id
+  - Focused regression suites: 269/269 pass across 8 files (incl. RI-01
+    tests, request-log, usage-log, combos, combo-management-api,
+    codex-routing, codex-account-namespaces)
+  - `bun run privacy:scan`: passed
+- Remaining Low findings: none
+
+### RI-03 - feat/ri-03-routing-analytics
+
+- Base SHA: `7efb6e84284c070c155c2e5254f1400917df31a1` (RI-02 head)
+- Reviewed commit: same as final (author self-review before push)
+- Findings (self-review): 3 fixed pre-push - (1) `requestHistoryDb` accessor
+  missing from the indexer (analytics needs the handle after open);
+  (2) SQL column names are snake_case - analytics SELECT now aliases to
+  camelCase; (3) cost field is `estimate.cost.total` (CostBreakdown), not
+  `costUsd`; plus the row-cap is injectable for truncation tests.
+- Final commit: pending (recorded after commit)
+- PR: pending
+- Verification:
+  - `bun x tsc --noEmit`: PASSED (0 errors)
+  - `bun run test tests/routing-analytics.test.ts`: 8/8 pass (32 assertions):
+    classification (success/failure/cancel/incomplete), percentiles +
+    coverage, fallback rate, provider/model/account + profile breakdown,
+    unknown-price honesty, filters, truncation flag, API payload
+  - Focused regression suites: 144/144 pass across 6 files
+  - `bun run privacy:scan`: passed
+- Remaining Low findings: none

src/cli/observe.ts

@@ -14,6 +14,8 @@ import {
 const USAGE = `Usage:
   ocx observe logs [--provider <name>] [--model <id>] [--status <code>]
       [--limit <n>] [--follow] [--json|--jsonl]
+  ocx logs rebuild-index
+  ocx logs index-status
   ocx observe usage [--range <7d|30d|all>] [--surface <all|codex|claude|grok>] [--json]
   ocx observe storage [--json]
   ocx observe memory [--json]
@@ -79,6 +81,39 @@ async function logs(argv: string[], deps: RuntimeApiDeps): Promise<void> {
   } while (true);
 }
 
+async function rebuildIndex(argv: string[], deps: RuntimeApiDeps): Promise<void> {
+  const args = [...argv];
+  const wantsJson = takeFlag(args, "--json");
+  rejectArgs(args, USAGE);
+  const { rebuildRequestHistoryIndex } = await import("../routing/history/indexer");
+  const meta = await rebuildRequestHistoryIndex();
+  if (wantsJson) printData(meta, true);
+  else {
+    console.log(`Request-history index rebuilt (${meta.dbPath})`);
+    console.log(`  schema version: ${meta.schemaVersion}`);
+    console.log(`  indexed rows:   ${meta.indexedRows}`);
+    console.log(`  source size:    ${meta.sourceSize} bytes`);
+    console.log(`  last error:     ${meta.lastError ?? "none"}`);
+  }
+}
+
+async function indexStatus(argv: string[], deps: RuntimeApiDeps): Promise<void> {
+  const args = [...argv];
+  const wantsJson = takeFlag(args, "--json");
+  rejectArgs(args, USAGE);
+  const { requestHistoryIndexStatus } = await import("../routing/history/indexer");
+  const meta = await requestHistoryIndexStatus();
+  if (wantsJson) printData(meta, true);
+  else {
+    console.log(`Request-history index (${meta.dbPath})`);
+    console.log(`  schema version: ${meta.schemaVersion}`);
+    console.log(`  indexed rows:   ${meta.indexedRows}`);
+    console.log(`  source size:    ${meta.sourceSize} bytes`);
+    console.log(`  indexed offset: ${meta.indexedOffset} bytes`);
+    console.log(`  last error:     ${meta.lastError ?? "none"}`);
+  }
+}
+
 async function usage(argv: string[], deps: RuntimeApiDeps): Promise<void> {
   const args = [...argv];
   const wantsJson = takeFlag(args, "--json");
@@ -103,7 +138,12 @@ async function simple(path: string, argv: string[], deps: RuntimeApiDeps): Promi
 export async function handleObserveCommand(argv: string[], deps: RuntimeApiDeps = {}): Promise<number> {
   return runCliAction(async () => {
     const [sub = "logs", ...rest] = argv;
-    if (sub === "logs") await logs(rest, deps);
+    if (sub === "logs") {
+      const action = rest[0];
+      if (action === "rebuild-index") await rebuildIndex(rest.slice(1), deps);
+      else if (action === "index-status") await indexStatus(rest.slice(1), deps);
+      else await logs(rest, deps);
+    }
     else if (sub === "usage") await usage(rest, deps);
     else if (sub === "storage") await simple("/api/storage", rest, deps);
     else if (sub === "memory") await simple("/api/system/memory", rest, deps);