feat(analytics): surface org-wide slop-band calibration + dashboard card (#2196)

apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.test.tsx

@@ -0,0 +1,76 @@
+import { render, screen } from "@testing-library/react";
+import { describe, expect, it } from "vitest";
+
+import { SlopBandCalibrationCard } from "@/components/site/app-panels/slop-band-calibration-card";
+
+describe("SlopBandCalibrationCard", () => {
+  it("renders a row per band with merge rate + counts and the discrimination verdict when all bands have data", () => {
+    render(
+      <SlopBandCalibrationCard
+        calibration={{
+          totalResolved: 40,
+          overallMergeRate: 0.6,
+          discriminates: true,
+          bands: [
+            { band: "clean", sampleSize: 20, merged: 18, closed: 2, mergeRate: 0.9 },
+            { band: "low", sampleSize: 10, merged: 6, closed: 4, mergeRate: 0.6 },
+            { band: "elevated", sampleSize: 6, merged: 2, closed: 4, mergeRate: 0.333 },
+            { band: "high", sampleSize: 4, merged: 0, closed: 4, mergeRate: 0 },
+          ],
+        }}
+      />,
+    );
+    expect(screen.getByText("clean")).toBeTruthy();
+    expect(screen.getByText("high")).toBeTruthy();
+    expect(screen.getByText("90% merged")).toBeTruthy();
+    expect(screen.getByText("predictive")).toBeTruthy();
+    expect(screen.getByText(/40 resolved · 60% merged overall/)).toBeTruthy();
+  });
+
+  it("shows the '— no samples' state for an empty band and marks insufficient data", () => {
+    render(
+      <SlopBandCalibrationCard
+        calibration={{
+          totalResolved: 3,
+          overallMergeRate: 1,
+          discriminates: null,
+          bands: [
+            { band: "clean", sampleSize: 3, merged: 3, closed: 0, mergeRate: 1 },
+            { band: "high", sampleSize: 0, merged: 0, closed: 0, mergeRate: 0 },
+          ],
+        }}
+      />,
+    );
+    expect(screen.getByText("— no samples")).toBeTruthy();
+    expect(screen.getByText("insufficient data")).toBeTruthy();
+  });
+
+  it("flags an inverted (non-predictive) score", () => {
+    render(
+      <SlopBandCalibrationCard
+        calibration={{
+          totalResolved: 20,
+          overallMergeRate: 0.5,
+          discriminates: false,
+          bands: [{ band: "clean", sampleSize: 20, merged: 10, closed: 10, mergeRate: 0.5 }],
+        }}
+      />,
+    );
+    expect(screen.getByText("inverted")).toBeTruthy();
+  });
+
+  it("shows the 'no resolved PRs' empty state when the calibration has no resolved data", () => {
+    render(
+      <SlopBandCalibrationCard
+        calibration={{ totalResolved: 0, overallMergeRate: null, discriminates: null, bands: [] }}
+      />,
+    );
+    expect(screen.getByText("No resolved PRs with a slop band")).toBeTruthy();
+    expect(screen.queryByText("predictive")).toBeNull();
+  });
+
+  it("shows the 'not yet available' empty state when the calibration field is absent", () => {
+    render(<SlopBandCalibrationCard />);
+    expect(screen.getByText("Not yet available")).toBeTruthy();
+  });
+});

apps/gittensory-ui/src/components/site/app-panels/slop-band-calibration-card.tsx

@@ -0,0 +1,106 @@
+import { AnalyticsCardShell } from "@/components/site/app-panels/analytics-card-shell";
+import { StatusPill, type Status } from "@/components/site/control-primitives";
+import { cn } from "@/lib/utils";
+
+/** Slop-band calibration (#2196): per-band merge/close rates over resolved PRs that carry a persisted slop band —
+ *  is the deterministic slop score predictive (do higher-slop bands merge less)? Display slice over the
+ *  operator-dashboard's `slopCalibration` payload. Bands only, never raw scores (public/private boundary). */
+export type SlopBand = "clean" | "low" | "elevated" | "high";
+
+export type SlopBandCalibration = {
+  band: SlopBand;
+  sampleSize: number;
+  merged: number;
+  closed: number;
+  mergeRate: number;
+};
+
+export type SlopOutcomeCalibration = {
+  totalResolved: number;
+  bands: SlopBandCalibration[];
+  overallMergeRate: number | null;
+  discriminates: boolean | null;
+};
+
+/** clean → high reads low-to-high severity; the bar color tracks band severity. */
+const BAND_BAR: Record<SlopBand, string> = {
+  clean: "bg-success",
+  low: "bg-mint",
+  elevated: "bg-warning",
+  high: "bg-danger",
+};
+
+function discriminationPill(discriminates: boolean | null): { status: Status; label: string } {
+  if (discriminates === true) return { status: "ready", label: "predictive" };
+  if (discriminates === false) return { status: "degraded", label: "inverted" };
+  return { status: "info", label: "insufficient data" };
+}
+
+function BandRow({ row }: { row: SlopBandCalibration }) {
+  const hasSamples = row.sampleSize > 0;
+  const pct = hasSamples ? Math.round(row.mergeRate * 100) : null;
+  return (
+    <div className="space-y-1.5">
+      <div className="flex items-center justify-between gap-3 text-token-sm">
+        <span className="font-medium capitalize text-foreground">{row.band}</span>
+        <span className="font-mono text-token-xs text-muted-foreground">
+          {pct === null ? "— no samples" : `${pct}% merged`}
+        </span>
+      </div>
+      <div className="h-2 overflow-hidden rounded-full bg-border" aria-hidden>
+        {hasSamples ? (
+          <div className={cn("h-full", BAND_BAR[row.band])} style={{ width: `${pct}%` }} />
+        ) : null}
+      </div>
+      <div className="font-mono text-token-2xs text-muted-foreground">
+        {row.sampleSize} resolved · {row.merged} merged · {row.closed} closed
+      </div>
+    </div>
+  );
+}
+
+export function SlopBandCalibrationCard({ calibration }: { calibration?: SlopOutcomeCalibration }) {
+  const hasData =
+    calibration != null && calibration.totalResolved > 0 && calibration.bands.length > 0;
+
+  if (!hasData) {
+    return (
+      <AnalyticsCardShell
+        title="Slop-band calibration"
+        description="Predicted slop band vs realized merge/close outcome, across resolved PRs."
+        state="empty"
+        emptyTitle={calibration ? "No resolved PRs with a slop band" : "Not yet available"}
+        emptyHint={
+          calibration
+            ? "Calibration appears once resolved PRs carry a persisted slop band in the window."
+            : "Slop-band calibration appears once the payload includes resolved-PR slop bands."
+        }
+      />
+    );
+  }
+
+  const pill = discriminationPill(calibration.discriminates);
+  const overall =
+    calibration.overallMergeRate === null ? null : Math.round(calibration.overallMergeRate * 100);
+
+  return (
+    <AnalyticsCardShell
+      title="Slop-band calibration"
+      description="Predicted slop band vs realized merge/close outcome, across resolved PRs."
+      state="ready"
+    >
+      <div className="flex flex-wrap items-center justify-between gap-2">
+        <span className="font-mono text-token-2xs text-muted-foreground">
+          {calibration.totalResolved} resolved
+          {overall === null ? "" : ` · ${overall}% merged overall`}
+        </span>
+        <StatusPill status={pill.status}>{pill.label}</StatusPill>
+      </div>
+      <div className="mt-3 space-y-4">
+        {calibration.bands.map((row) => (
+          <BandRow key={row.band} row={row} />
+        ))}
+      </div>
+    </AnalyticsCardShell>
+  );
+}

apps/gittensory-ui/src/routes/app.analytics.tsx

@@ -28,6 +28,10 @@ import {
   FindingsBreakdownCard,
   type FindingsBreakdown,
 } from "@/components/site/app-panels/findings-breakdown-card";
+import {
+  SlopBandCalibrationCard,
+  type SlopOutcomeCalibration,
+} from "@/components/site/app-panels/slop-band-calibration-card";
 import { useApiResource } from "@/lib/api/use-api-resource";
 import { exportOperatorDashboardCsv } from "@/lib/csv-export";
 
@@ -124,6 +128,7 @@ type OperatorDashboard = {
   agentHealth?: ReversalHealth;
   acceptance?: FindingAcceptance;
   findingsBreakdown?: FindingsBreakdown;
+  slopCalibration?: SlopOutcomeCalibration;
 };
 
 function ProductAnalytics() {
@@ -232,6 +237,8 @@ function ProductAnalytics() {
 
           <FindingsBreakdownCard findings={data.findingsBreakdown} />
 
+          <SlopBandCalibrationCard calibration={data.slopCalibration} />
+
           {data.usageSummary ? (
             <ProductUsageBreakdownPanel
               byEvent={data.usageSummary.byEvent}

src/services/operator-dashboard.ts

@@ -4,6 +4,7 @@ import {
   getCommandUsefulnessSummary,
   getLatestScoringModelSnapshot,
   getProductUsageRollupStatus,
+  listAllPullRequests,
   listInstallationHealth,
   listInstallations,
   listLatestGitHubRateLimitObservations,
@@ -32,6 +33,7 @@ import { computeCycleTimeAggregate, type CycleTimeAggregate } from "../review/st
 import { loadUpstreamStatus, type UpstreamStatus } from "../upstream/ruleset";
 import { nowIso } from "../utils/json";
 import { buildRecommendationQualityReport, type RecommendationQualityReport } from "./recommendation-quality-report";
+import { buildSlopOutcomeCalibration, type SlopOutcomeCalibration } from "./outcome-calibration";
 import { buildWeeklyValueReport } from "./weekly-value-report";
 
 export type OperatorDashboardMetric = {
@@ -71,6 +73,9 @@ export type OperatorDashboardPayload = {
   calibration: Calibration;
   // Agent reversal health (#2193): how often humans reopened/reverted bot auto-actions (ops.ts AgentHealth).
   agentHealth: AgentHealth;
+  // Slop-band calibration (#2196): org-wide per-band merge/close rates over resolved PRs carrying a persisted
+  // slop band — is the deterministic slop score predictive? Bands only, never raw scores. Fails safe to empty.
+  slopCalibration: SlopOutcomeCalibration;
 };
 
 const USAGE_WINDOW_DAYS = 7;
@@ -98,6 +103,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
     cycleTime,
     calibration,
     agentHealth,
+    slopCalibration,
   ] = await Promise.all([
     listRepositories(env),
     listInstallations(env),
@@ -121,6 +127,11 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
     computeCycleTimeAggregate(env, { days: 90, nowMs: Date.now() }),
     computeCalibration(env, operatorAgentConfig(env)),
     computeAgentHealth(env, operatorAgentConfig(env)),
+    // #2196: org-wide slop-band calibration from persisted slop bands on resolved PRs; fails safe to an empty
+    // calibration on any read error (mirrors the sibling reads) so one DB hiccup never fails the whole dashboard.
+    listAllPullRequests(env)
+      .then(buildSlopOutcomeCalibration)
+      .catch(() => buildSlopOutcomeCalibration([])),
   ]);
   const weeklyValueReport = buildWeeklyValueReport({
     generatedAt: nowIso(),
@@ -224,6 +235,7 @@ export async function buildOperatorDashboardPayload(env: Env): Promise<OperatorD
     cycleTime,
     calibration,
     agentHealth,
+    slopCalibration,
   };
 }
 

test/unit/operator-dashboard.test.ts

@@ -50,6 +50,12 @@ describe("operator dashboard payload", () => {
       recentAutoActions: 0,
       reversedTargets: [],
     });
+    // #2196: org-wide slop-band calibration fails safe to an empty calibration when no resolved PR carries a band.
+    expect(payload.slopCalibration).toMatchObject({
+      totalResolved: 0,
+      overallMergeRate: null,
+      discriminates: null,
+    });
     // Empty fleet → instanceCount 0, null precision card ("—"), no-outlier delta.
     expect(payload.fleetMetrics.instanceCount).toBe(0);
     expect(payload.metrics).toEqual(