From afa0a61af8796104a0b8f8247663baac181dd48f Mon Sep 17 00:00:00 2001 From: dhgoal <153369624+dhgoal@users.noreply.github.com> Date: Sun, 12 Jul 2026 02:59:17 +0900 Subject: [PATCH] feat(analytics): surface org-wide slop-band calibration + add the dashboard card (#2196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the existing, unit-tested buildSlopOutcomeCalibration over listAllPullRequests into the operator-dashboard payload as slopCalibration: org-wide per-band merge/close rates over resolved PRs carrying a persisted slop band, plus whether the deterministic slop score discriminates. The read is wrapped in a buildOrgSlopCalibration() helper that fails safe to an empty calibration on any read error (listAllPullRequests can throw, unlike the sibling reads) so one DB hiccup never fails the whole dashboard — with both the success and fail-safe branches unit-tested. Adds SlopBandCalibrationCard (per-band merge-rate bars + predictive/inverted/insufficient verdict, bands only, never raw scores) across all-bands, empty-band, inverted, no-data, and absent-field branches. --- .../slop-band-calibration-card.test.tsx | 76 +++++++++++++ .../app-panels/slop-band-calibration-card.tsx | 106 ++++++++++++++++++ .../src/routes/app.analytics.tsx | 7 ++ src/services/operator-dashboard.ts | 21 +++- test/unit/operator-dashboard.test.ts | 26 +++++ 5 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.test.tsx create mode 100644 apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.tsx diff --git a/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.test.tsx b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.test.tsx new file mode 100644 index 000000000..2dc5881f9 --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.test.tsx @@ -0,0 +1,76 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; + +import { SlopBandCalibrationCard } from "@/components/site/app-panels/slop-band-calibration-card"; + +describe("SlopBandCalibrationCard", () => { + it("renders a row per band with merge rate + counts and the discrimination verdict when all bands have data", () => { + render( + , + ); + expect(screen.getByText("clean")).toBeTruthy(); + expect(screen.getByText("high")).toBeTruthy(); + expect(screen.getByText("90% merged")).toBeTruthy(); + expect(screen.getByText("predictive")).toBeTruthy(); + expect(screen.getByText(/40 resolved · 60% merged overall/)).toBeTruthy(); + }); + + it("shows the '— no samples' state for an empty band and marks insufficient data", () => { + render( + , + ); + expect(screen.getByText("— no samples")).toBeTruthy(); + expect(screen.getByText("insufficient data")).toBeTruthy(); + }); + + it("flags an inverted (non-predictive) score", () => { + render( + , + ); + expect(screen.getByText("inverted")).toBeTruthy(); + }); + + it("shows the 'no resolved PRs' empty state when the calibration has no resolved data", () => { + render( + , + ); + expect(screen.getByText("No resolved PRs with a slop band")).toBeTruthy(); + expect(screen.queryByText("predictive")).toBeNull(); + }); + + it("shows the 'not yet available' empty state when the calibration field is absent", () => { + render(); + expect(screen.getByText("Not yet available")).toBeTruthy(); + }); +}); diff --git a/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.tsx b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.tsx new file mode 100644 index 000000000..bb7bfbed3 --- /dev/null +++ b/apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.tsx @@ -0,0 +1,106 @@ +import { AnalyticsCardShell } from "@/components/site/app-panels/analytics-card-shell"; +import { StatusPill, type Status } from "@/components/site/control-primitives"; +import { cn } from "@/lib/utils"; + +/** Slop-band calibration (#2196): per-band merge/close rates over resolved PRs that carry a persisted slop band — + * is the deterministic slop score predictive (do higher-slop bands merge less)? Display slice over the + * operator-dashboard's `slopCalibration` payload. Bands only, never raw scores (public/private boundary). */ +export type SlopBand = "clean" | "low" | "elevated" | "high"; + +export type SlopBandCalibration = { + band: SlopBand; + sampleSize: number; + merged: number; + closed: number; + mergeRate: number; +}; + +export type SlopOutcomeCalibration = { + totalResolved: number; + bands: SlopBandCalibration[]; + overallMergeRate: number | null; + discriminates: boolean | null; +}; + +/** clean → high reads low-to-high severity; the bar color tracks band severity. */ +const BAND_BAR: Record = { + clean: "bg-success", + low: "bg-mint", + elevated: "bg-warning", + high: "bg-danger", +}; + +function discriminationPill(discriminates: boolean | null): { status: Status; label: string } { + if (discriminates === true) return { status: "ready", label: "predictive" }; + if (discriminates === false) return { status: "degraded", label: "inverted" }; + return { status: "info", label: "insufficient data" }; +} + +function BandRow({ row }: { row: SlopBandCalibration }) { + const hasSamples = row.sampleSize > 0; + const pct = hasSamples ? Math.round(row.mergeRate * 100) : null; + return ( +
+
+ {row.band} + + {pct === null ? "— no samples" : `${pct}% merged`} + +
+
+ {hasSamples ? ( +
+ ) : null} +
+
+ {row.sampleSize} resolved · {row.merged} merged · {row.closed} closed +
+
+ ); +} + +export function SlopBandCalibrationCard({ calibration }: { calibration?: SlopOutcomeCalibration }) { + const hasData = + calibration != null && calibration.totalResolved > 0 && calibration.bands.length > 0; + + if (!hasData) { + return ( + + ); + } + + const pill = discriminationPill(calibration.discriminates); + const overall = + calibration.overallMergeRate === null ? null : Math.round(calibration.overallMergeRate * 100); + + return ( + +
+ + {calibration.totalResolved} resolved + {overall === null ? "" : ` · ${overall}% merged overall`} + + {pill.label} +
+
+ {calibration.bands.map((row) => ( + + ))} +
+ + ); +} diff --git a/apps/gittensory-ui/src/routes/app.analytics.tsx b/apps/gittensory-ui/src/routes/app.analytics.tsx index 7a26005d6..845bdd8fc 100644 --- a/apps/gittensory-ui/src/routes/app.analytics.tsx +++ b/apps/gittensory-ui/src/routes/app.analytics.tsx @@ -28,6 +28,10 @@ import { FindingsBreakdownCard, type FindingsBreakdown, } from "@/components/site/app-panels/findings-breakdown-card"; +import { + SlopBandCalibrationCard, + type SlopOutcomeCalibration, +} from "@/components/site/app-panels/slop-band-calibration-card"; import { useApiResource } from "@/lib/api/use-api-resource"; import { exportOperatorDashboardCsv } from "@/lib/csv-export"; @@ -124,6 +128,7 @@ type OperatorDashboard = { agentHealth?: ReversalHealth; acceptance?: FindingAcceptance; findingsBreakdown?: FindingsBreakdown; + slopCalibration?: SlopOutcomeCalibration; }; function ProductAnalytics() { @@ -232,6 +237,8 @@ function ProductAnalytics() { + + {data.usageSummary ? ( { + try { + return buildSlopOutcomeCalibration(await listAllPullRequests(env)); + } catch { + return buildSlopOutcomeCalibration([]); + } +} + function operatorAgentConfig(env: Env): { slug: string; secrets: Record } { const slug = typeof env.GITHUB_APP_SLUG === "string" && env.GITHUB_APP_SLUG.trim() @@ -235,7 +254,7 @@ function operatorAgentConfig(env: Env): { slug: string; secrets: Record { recentAutoActions: 0, reversedTargets: [], }); + // #2196: org-wide slop-band calibration fails safe to an empty calibration when no resolved PR carries a band. + expect(payload.slopCalibration).toMatchObject({ + totalResolved: 0, + overallMergeRate: null, + discriminates: null, + }); // Empty fleet → instanceCount 0, null precision card ("—"), no-outlier delta. expect(payload.fleetMetrics.instanceCount).toBe(0); expect(payload.metrics).toEqual( @@ -103,6 +109,26 @@ describe("operator dashboard payload", () => { }); }); + it("buildOrgSlopCalibration (#2196) degrades to an empty calibration when the PR read throws", async () => { + const { buildOrgSlopCalibration } = __operatorDashboardInternals; + // A DB whose every access throws forces listAllPullRequests to reject; the fail-safe must swallow it. + const brokenDb = new Proxy( + {}, + { + get() { + throw new Error("DB unavailable"); + }, + }, + ); + const brokenEnv = { ...createTestEnv(), DB: brokenDb as unknown as D1Database }; + const calibration = await buildOrgSlopCalibration(brokenEnv); + expect(calibration).toMatchObject({ + totalResolved: 0, + overallMergeRate: null, + discriminates: null, + }); + }); + it("surfaces populated fleet metrics + outliers from orb_signals", async () => { const env = createTestEnv(); let n = 0;