diff --git a/apps/worker/src/incidents/intake.ts b/apps/worker/src/incidents/intake.ts index 381dbe15..2c024aa4 100644 --- a/apps/worker/src/incidents/intake.ts +++ b/apps/worker/src/incidents/intake.ts @@ -501,17 +501,35 @@ async function findMatchingIncident( if (heuristic) { return { match: heuristic, standaloneSource: null, standaloneReason: null, failedReason: null }; } + // Load the service-unfiltered candidates once and share them between the + // same-trace check and the LLM call. Both paths need identical data; + // fetching it twice per transition was costing one extra + // findOpenIncidentCandidates + loadLinkedIncidentIssues round-trip for every + // new issue that falls through heuristic matching, which compounds under the + // perProject mutex when many new issues arrive in the same project at once. + const candidates = await deps.repo.findOpenIncidentCandidates(issue, { filterService: false }); + if (candidates.length === 0) { + return { + match: null, + standaloneSource: "heuristic", + standaloneReason: "No open incidents in this project.", + failedReason: null, + }; + } + const linked = await deps.repo.loadLinkedIncidentIssues(candidates); // Same request = same incident: join an open incident that shares this // issue's trace id (a span exception and its own log line, or several logs // from one failed request) before spending an LLM call. Cross-service, since // the span and its Vercel-drained log arrive under different service names. - const sameTrace = await findSameTraceMatchingIncident(issue, deps); - if (sameTrace) { - return { match: sameTrace, standaloneSource: null, standaloneReason: null, failedReason: null }; + if (issueSample(issue)?.traceId) { + const sameTrace = findSameTraceIncidentMatch(issue, candidates, linked); + if (sameTrace) { + return { match: sameTrace, standaloneSource: null, standaloneReason: null, failedReason: null }; + } } // Alert-episode issues go through LLM grouping like errors do: a breach can // be another manifestation of an incident opened by errors (or vice versa). - return findLlmMatchingIncident(issue, deps, opts); + return findLlmMatchingIncident(issue, deps, opts, candidates, linked); } async function findHeuristicMatchingIncident( @@ -542,18 +560,9 @@ async function findLlmMatchingIncident( issue: schema.Issue, deps: IntakeDeps, opts: { includeLifetimeOccurrenceEvidence?: boolean } = {}, + candidates: schema.Incident[], + linked: LinkedIncidentIssue[], ): Promise { - const candidates = await deps.repo.findOpenIncidentCandidates(issue, { filterService: false }); - if (candidates.length === 0) { - return { - match: null, - standaloneSource: "heuristic", - standaloneReason: "No open incidents in this project.", - failedReason: null, - }; - } - - const linked = await deps.repo.loadLinkedIncidentIssues(candidates); const groupingCandidates = candidates .map((incident) => buildGroupingCandidate(incident, linked)) .filter((candidate): candidate is GroupingCandidateIncident => candidate !== null); diff --git a/apps/worker/src/issues/repository.ts b/apps/worker/src/issues/repository.ts index 889ded45..da82bb73 100644 --- a/apps/worker/src/issues/repository.ts +++ b/apps/worker/src/issues/repository.ts @@ -6,7 +6,7 @@ import { db, schema, } from "@superlog/db"; -import { and, desc, eq, inArray, isNull, or, sql } from "drizzle-orm"; +import { and, asc, desc, eq, inArray, isNull, or, sql } from "drizzle-orm"; import type { IssueGroupingSource, IssueGroupingState, LinkedIncidentIssue } from "./domain.js"; const INCIDENT_GROUPING_CANDIDATE_LIMIT = parsePositiveInt( @@ -15,6 +15,13 @@ const INCIDENT_GROUPING_CANDIDATE_LIMIT = parsePositiveInt( 1000, ); +// Must match GROUPING_CANDIDATE_ISSUE_LIMIT in domain.ts. Capping here +// prevents an incident with thousands of linked issues (e.g., after a bulk +// backfill event) from returning an unbounded result set to every caller of +// loadLinkedIncidentIssues, since the grouping logic only uses up to this +// many issues per incident. +const LINKED_ISSUES_PER_INCIDENT_CAP = 5; + const incidentLifecycle = createIncidentLifecycle(db); function parsePositiveInt(value: string | undefined, fallback: number, max: number): number { @@ -94,7 +101,7 @@ export async function loadLinkedIncidentIssues( incidents: schema.Incident[], ): Promise { if (incidents.length === 0) return []; - return db + const rows = await db .select({ incidentId: schema.incidentIssues.incidentId, issueId: schema.issues.id, @@ -116,7 +123,19 @@ export async function loadLinkedIncidentIssues( schema.incidentIssues.incidentId, incidents.map((incident) => incident.id), ), - ); + ) + .orderBy(asc(schema.incidentIssues.incidentId), desc(schema.issues.lastSeen)); + // Cap to the most-recently-seen issues per incident. Incidents with thousands + // of linked issues (e.g., after a bulk-issue backfill event) would otherwise + // return an unbounded result set even though the grouping logic only consumes + // the first LINKED_ISSUES_PER_INCIDENT_CAP issues per incident. + const countByIncident = new Map(); + return rows.filter((row) => { + const count = countByIncident.get(row.incidentId) ?? 0; + if (count >= LINKED_ISSUES_PER_INCIDENT_CAP) return false; + countByIncident.set(row.incidentId, count + 1); + return true; + }); } export async function findProject(projectId: string): Promise {