feat(debug): warn on observed quota ratio mismatch

Author: liyanbowne

src/commands/inspection.ts

@@ -22,6 +22,7 @@ import {
 import { writeJson } from "../cli/output.js";
 import {
   computeWatchHistoryEta,
+  computeWatchObservedRatioDiagnostics,
   createWatchHistoryStore,
   type WatchHistoryEtaContext,
 } from "../watch/history.js";
@@ -639,6 +640,7 @@ export async function handleListCommand(options: {
   store: AccountStore;
   stdout: NodeJS.WriteStream;
   debugLog: DebugLogger;
+  debug: boolean;
   json: boolean;
   targetName?: string;
   verbose: boolean;
@@ -658,6 +660,23 @@ export async function handleListCommand(options: {
   options.debugLog(
     `list: target=${options.targetName ?? "all"} successes=${result.successes.length} failures=${result.failures.length} current_matches=${current.matched_accounts.length} watch_history_samples=${watchHistory.length}`,
   );
+  if (options.debug) {
+    const ratioDiagnostics = computeWatchObservedRatioDiagnostics(watchHistory, now);
+    if (ratioDiagnostics.length === 0) {
+      options.debugLog("list: observed_ratio window=24h insufficient_samples");
+    } else {
+      for (const diagnostic of ratioDiagnostics) {
+        options.debugLog(
+          `list: observed_ratio window=24h dimension=${diagnostic.dimension} key=${diagnostic.key} samples=${diagnostic.sample_count} expected=${diagnostic.expected_raw_ratio ?? "n/a"} observed_weighted=${diagnostic.observed_weighted_raw_ratio} mean=${diagnostic.observed_mean_raw_ratio} variance=${diagnostic.variance}`,
+        );
+        if (diagnostic.warning) {
+          options.debugLog(
+            `warning: list observed_ratio_mismatch dimension=${diagnostic.dimension} key=${diagnostic.key} expected=${diagnostic.expected_raw_ratio ?? "n/a"} observed_weighted=${diagnostic.observed_weighted_raw_ratio} relative_delta=${diagnostic.relative_delta ?? "n/a"} samples=${diagnostic.sample_count}`,
+          );
+        }
+      }
+    }
+  }
   if (options.json) {
     writeJson(options.stdout, {
       ...toCliQuotaRefreshResult(result),

src/main.ts

@@ -198,6 +198,7 @@ export async function runCli(
           store,
           stdout: streams.stdout,
           debugLog,
+          debug,
           json,
           targetName: parsed.positionals[0],
           verbose: parsed.flags.has("--verbose"),

src/watch/history.ts

@@ -4,13 +4,17 @@ import { dirname, join } from "node:path";
 import {
   convertFiveHourPercentToPlusWeeklyUnits,
   convertOneWeekPercentToPlusWeeklyUnits,
+  resolveFiveHourWindowsPerWeek,
 } from "../plan-quota-profile.js";
 
 const WATCH_HISTORY_FILE_NAME = "watch-quota-history.jsonl";
 const WATCH_HISTORY_MAX_AGE_MS = 14 * 24 * 60 * 60 * 1000;
 const WATCH_HISTORY_KEEPALIVE_MS = 60 * 1000;
 const WATCH_HISTORY_WINDOW_MS = 60 * 60 * 1000;
 const WATCH_HISTORY_RESET_AT_TOLERANCE_MS = 60 * 1000;
+const WATCH_HISTORY_RATIO_DIAGNOSTIC_WINDOW_MS = 24 * 60 * 60 * 1000;
+const WATCH_HISTORY_RATIO_WARNING_RELATIVE_DELTA = 0.2;
+const WATCH_HISTORY_RATIO_WARNING_MIN_SAMPLES = 3;
 
 export interface WatchHistoryWindowSnapshot {
   used_percent: number;
@@ -69,6 +73,18 @@ export interface WatchHistoryStore {
 export type WatchQuotaHistoryRecord = WatchHistoryRecord;
 export type WatchEtaContext = WatchHistoryEtaContext;
 
+export interface WatchHistoryObservedRatioDiagnostic {
+  dimension: "bucket" | "plan";
+  key: string;
+  sample_count: number;
+  observed_mean_raw_ratio: number;
+  observed_weighted_raw_ratio: number;
+  variance: number;
+  expected_raw_ratio: number | null;
+  relative_delta: number | null;
+  warning: boolean;
+}
+
 function roundToTwo(value: number): number {
   return Number(value.toFixed(2));
 }
@@ -192,6 +208,11 @@ function isInsideRateWindow(recordedAt: string, now: Date): boolean {
   return now.getTime() - recordedAtMs <= WATCH_HISTORY_WINDOW_MS;
 }
 
+function isInsideObservedRatioWindow(recordedAt: string, now: Date): boolean {
+  const recordedAtMs = Date.parse(recordedAt);
+  return now.getTime() - recordedAtMs <= WATCH_HISTORY_RATIO_DIAGNOSTIC_WINDOW_MS;
+}
+
 function formatRecord(record: WatchHistoryRecord): string {
   return `${JSON.stringify(record)}\n`;
 }
@@ -450,6 +471,184 @@ function computeSegmentDelta(
   return scoreDeltaForSegment(start, end, end.plan_type ?? start.plan_type);
 }
 
+interface WatchHistorySegment {
+  start: WatchHistoryRecord;
+  end: WatchHistoryRecord;
+  elapsedHours: number;
+}
+
+interface WatchHistorySegmentMetrics extends WatchHistorySegment {
+  delta_5h_raw: number | null;
+  delta_1w_raw: number | null;
+  delta_5h_eq_1w: number | null;
+  delta_1w_eq: number | null;
+  segment_delta_1w_units: number | null;
+}
+
+function collectContinuousSegments(records: WatchHistoryRecord[]): WatchHistorySegment[] {
+  if (records.length < 2) {
+    return [];
+  }
+
+  const segments: WatchHistorySegment[] = [];
+  let segmentStart = records[0] ?? null;
+  let segmentEnd = segmentStart;
+
+  const appendSegment = (start: WatchHistoryRecord | null, end: WatchHistoryRecord | null) => {
+    if (!start || !end || start === end) {
+      return;
+    }
+
+    const elapsedHours = (Date.parse(end.recorded_at) - Date.parse(start.recorded_at)) / 3_600_000;
+    if (elapsedHours <= 0) {
+      return;
+    }
+
+    segments.push({ start, end, elapsedHours });
+  };
+
+  for (let index = 1; index < records.length; index += 1) {
+    const right = records[index];
+    const left = segmentEnd;
+    if (!left || !right) {
+      continue;
+    }
+
+    const ageMs = Date.parse(right.recorded_at) - Date.parse(left.recorded_at);
+    if (
+      ageMs > 0 &&
+      ageMs <= WATCH_HISTORY_WINDOW_MS &&
+      isSameRateSegment(left, right)
+    ) {
+      segmentEnd = right;
+      continue;
+    }
+
+    appendSegment(segmentStart, segmentEnd);
+    segmentStart = right;
+    segmentEnd = right;
+  }
+
+  appendSegment(segmentStart, segmentEnd);
+  return segments;
+}
+
+function toSegmentMetrics(segment: WatchHistorySegment): WatchHistorySegmentMetrics {
+  const planType = segment.end.plan_type ?? segment.start.plan_type;
+  const delta5hRaw = deltaForContinuousWindow(segment.start.five_hour, segment.end.five_hour);
+  const delta1wRaw = deltaForContinuousWindow(segment.start.one_week, segment.end.one_week);
+  const delta5hEq1w =
+    delta5hRaw === null ? null : convertFiveHourPercentToWeeklyEquivalent(delta5hRaw, planType);
+  const delta1wEq =
+    delta1wRaw === null ? null : convertOneWeekPercentToPlusWeeklyUnits(delta1wRaw, planType);
+  const validDeltas = [delta5hEq1w, delta1wEq].filter(
+    (value): value is number => typeof value === "number",
+  );
+
+  return {
+    ...segment,
+    delta_5h_raw: delta5hRaw,
+    delta_1w_raw: delta1wRaw,
+    delta_5h_eq_1w: delta5hEq1w,
+    delta_1w_eq: delta1wEq,
+    segment_delta_1w_units: validDeltas.length === 0 ? null : Math.max(...validDeltas),
+  };
+}
+
+function resolveObservedRatioBucket(accountName: string): string | null {
+  const normalized = accountName.trim().toLowerCase();
+  if (normalized.startsWith("plus")) {
+    return "plus";
+  }
+  if (normalized.startsWith("team")) {
+    return "team";
+  }
+  return null;
+}
+
+function buildObservedRatioDiagnostic(
+  dimension: "bucket" | "plan",
+  key: string,
+  metrics: WatchHistorySegmentMetrics[],
+): WatchHistoryObservedRatioDiagnostic {
+  const ratios = metrics.map((segment) => (segment.delta_5h_raw ?? 0) / (segment.delta_1w_raw ?? 1));
+  const observedMean = ratios.reduce((sum, value) => sum + value, 0) / ratios.length;
+  const variance =
+    ratios.reduce((sum, value) => sum + (value - observedMean) ** 2, 0) / ratios.length;
+  const total5h = metrics.reduce((sum, segment) => sum + (segment.delta_5h_raw ?? 0), 0);
+  const total1w = metrics.reduce((sum, segment) => sum + (segment.delta_1w_raw ?? 0), 0);
+  const expected = Number.isFinite(resolveFiveHourWindowsPerWeek(key))
+    ? resolveFiveHourWindowsPerWeek(key)
+    : null;
+  const weighted = total1w > 0 ? total5h / total1w : observedMean;
+  const relativeDelta = expected && expected > 0 ? (weighted - expected) / expected : null;
+
+  return {
+    dimension,
+    key,
+    sample_count: metrics.length,
+    observed_mean_raw_ratio: roundToTwo(observedMean),
+    observed_weighted_raw_ratio: roundToTwo(weighted),
+    variance: roundToTwo(variance),
+    expected_raw_ratio: expected === null ? null : roundToTwo(expected),
+    relative_delta: relativeDelta === null ? null : roundToTwo(relativeDelta),
+    warning:
+      relativeDelta !== null &&
+      metrics.length >= WATCH_HISTORY_RATIO_WARNING_MIN_SAMPLES &&
+      Math.abs(relativeDelta) >= WATCH_HISTORY_RATIO_WARNING_RELATIVE_DELTA,
+  };
+}
+
+export function computeWatchObservedRatioDiagnostics(
+  history: WatchHistoryRecord[],
+  now = new Date(),
+): WatchHistoryObservedRatioDiagnostic[] {
+  const recentHistory = history
+    .filter((record) => isInsideObservedRatioWindow(record.recorded_at, now))
+    .sort((left, right) => Date.parse(left.recorded_at) - Date.parse(right.recorded_at));
+
+  const metrics = collectContinuousSegments(recentHistory)
+    .map(toSegmentMetrics)
+    .filter(
+      (segment) =>
+        typeof segment.delta_5h_raw === "number" &&
+        typeof segment.delta_1w_raw === "number" &&
+        segment.delta_1w_raw > 0,
+    );
+
+  const groups = new Map<string, WatchHistorySegmentMetrics[]>();
+  const pushGroup = (dimension: "bucket" | "plan", key: string, segment: WatchHistorySegmentMetrics) => {
+    const groupKey = `${dimension}:${key}`;
+    const existing = groups.get(groupKey);
+    if (existing) {
+      existing.push(segment);
+      return;
+    }
+    groups.set(groupKey, [segment]);
+  };
+
+  for (const segment of metrics) {
+    const bucket = resolveObservedRatioBucket(segment.end.account_name);
+    if (bucket) {
+      pushGroup("bucket", bucket, segment);
+    }
+    if (segment.end.plan_type) {
+      pushGroup("plan", segment.end.plan_type, segment);
+    }
+  }
+
+  return [...groups.entries()]
+    .map(([groupKey, groupMetrics]) => {
+      const [dimension, key] = groupKey.split(":", 2) as ["bucket" | "plan", string];
+      return buildObservedRatioDiagnostic(dimension, key, groupMetrics);
+    })
+    .sort((left, right) =>
+      left.dimension === right.dimension
+        ? left.key.localeCompare(right.key)
+        : left.dimension.localeCompare(right.dimension),
+    );
+}
+
 function computeRemainingPercent(window: WatchHistoryWindowSnapshot | null): number | null {
   if (!window) {
     return null;
@@ -662,61 +861,22 @@ function computeRateFromHistory(records: WatchHistoryRecord[]): number | null {
     return null;
   }
 
-  let totalDelta = 0;
-  let totalHours = 0;
-  let sawValidSegment = false;
-  let segmentStart = records[0] ?? null;
-  let segmentEnd = segmentStart;
-
-  const appendSegmentRate = (start: WatchHistoryRecord | null, end: WatchHistoryRecord | null) => {
-    if (!start || !end || start === end) {
-      return;
-    }
-
-    const elapsedHours = (Date.parse(end.recorded_at) - Date.parse(start.recorded_at)) / 3_600_000;
-    if (elapsedHours <= 0) {
-      return;
-    }
-
-    const delta = computeSegmentDelta(start, end);
-    if (delta === null) {
-      return;
-    }
-
-    sawValidSegment = true;
-    totalDelta += delta;
-    totalHours += elapsedHours;
-  };
-
-  for (let index = 1; index < records.length; index += 1) {
-    const right = records[index];
-    const left = segmentEnd;
-    if (!left || !right) {
-      continue;
-    }
-
-    const ageMs = Date.parse(right.recorded_at) - Date.parse(left.recorded_at);
-    if (
-      ageMs > 0 &&
-      ageMs <= WATCH_HISTORY_WINDOW_MS &&
-      isSameRateSegment(left, right)
-    ) {
-      segmentEnd = right;
-      continue;
-    }
-
-    appendSegmentRate(segmentStart, segmentEnd);
-    segmentStart = right;
-    segmentEnd = right;
-  }
-
-  appendSegmentRate(segmentStart, segmentEnd);
+  const segments = collectContinuousSegments(records).map(toSegmentMetrics);
+  const validSegments = segments.filter(
+    (segment) => typeof segment.segment_delta_1w_units === "number" && segment.elapsedHours > 0,
+  );
 
-  if (!sawValidSegment || totalHours <= 0) {
+  if (validSegments.length === 0) {
     return null;
   }
 
-  return roundToTwo(totalDelta / totalHours);
+  const totalDelta = validSegments.reduce(
+    (sum, segment) => sum + (segment.segment_delta_1w_units ?? 0),
+    0,
+  );
+  const totalHours = validSegments.reduce((sum, segment) => sum + segment.elapsedHours, 0);
+
+  return totalHours <= 0 ? null : roundToTwo(totalDelta / totalHours);
 }
 
 export function computeWatchHistoryEta(