diff --git a/api-snapshots/do.api.md b/api-snapshots/do.api.md index feff217927..f3fc5702f2 100644 --- a/api-snapshots/do.api.md +++ b/api-snapshots/do.api.md @@ -367,13 +367,14 @@ abstract class ShardDO { protected resolveShape(_name: string, _args: Record, _identity?: SubscriptionIdentity): ResolvedShape | undefined; protected isShapeRelayUniform(name: string, args: Record): boolean; protected readGlobalShapeRows(_resolved: ResolvedShape, _identity?: SubscriptionIdentity): Promise; - protected pollExternalSources(): Promise; + protected pollExternalSources(_trace?: TraceRefLike): Promise; protected scheduleSourcePoll(): Promise; protected ttlSweeps(): ReadonlyArray; - protected pollTtlSweeps(): Promise; + protected pollTtlSweeps(trace?: TraceRefLike): Promise; protected scheduleTtlSweep(): Promise; protected currentShardKey(): string; - protected recordExternalSourceError(table: string, error: unknown): void; + protected recordExternalSourceError(table: string, error: unknown, trace?: TraceRefLike): void; + protected recordExternalSourceWarning(table: string, message: string, trace?: TraceRefLike): void; protected executeStream(_functionPath: string, _args: Record): null | { iterator: (signal: AbortSignal) => AsyncIterable; }; @@ -509,6 +510,14 @@ interface TelemetrySink { } ``` +### `TraceRefLike` (interface) + +```ts +interface TraceRefLike { + traceId: string; +} +``` + ### `TransactionHeadroomTracker` (class) Re-exported from `@lunora/shard-engine` — signature tracked at its source. diff --git a/api-snapshots/lunora.api.md b/api-snapshots/lunora.api.md index ce0d727f5f..8a33ff8dac 100644 --- a/api-snapshots/lunora.api.md +++ b/api-snapshots/lunora.api.md @@ -2237,6 +2237,10 @@ Re-exported from `@lunora/shard-engine` — signature tracked at its source. Re-exported from `@lunora/do` — signature tracked at its source. +### `TraceRefLike` (interface) + +Re-exported from `@lunora/do` — signature tracked at its source. + ### `TransactionHeadroomTracker` (class) Re-exported from `@lunora/shard-engine` — signature tracked at its source. diff --git a/api-snapshots/observability.api.md b/api-snapshots/observability.api.md index b36a9a18aa..be22a52ca2 100644 --- a/api-snapshots/observability.api.md +++ b/api-snapshots/observability.api.md @@ -56,6 +56,7 @@ interface AppendRequestLogEntry { subscriptionsReRun?: number; tablesRead?: string[]; tablesWritten?: string[]; + traceId?: string; ts: number; userId?: string; } @@ -425,6 +426,7 @@ interface LogEntry { level: LogLevel; message: string; timestamp: number; + traceId?: string; } ``` @@ -680,6 +682,7 @@ interface RequestLogEntry { subscriptionsReRun: number; tablesRead: string[]; tablesWritten: string[]; + traceId?: string; ts: number; userId?: string; } @@ -870,6 +873,8 @@ interface TraceSpan { message: string; type: string; }; + events?: SpanEventPoint[]; + kind?: OtlpSpanKind; name: string; offsetMs: number; ok: boolean; diff --git a/api-snapshots/studio.api.md b/api-snapshots/studio.api.md index c71095f909..b025f89b55 100644 --- a/api-snapshots/studio.api.md +++ b/api-snapshots/studio.api.md @@ -650,6 +650,7 @@ interface LogEntry { level: LogLevel; message: string; timestamp: number; + traceId?: string; } ``` diff --git a/apps/docs/src/content/docs/concepts/observability.mdx b/apps/docs/src/content/docs/concepts/observability.mdx index ae10cd3ad3..a73a1b17dd 100644 --- a/apps/docs/src/content/docs/concepts/observability.mdx +++ b/apps/docs/src/content/docs/concepts/observability.mdx @@ -395,6 +395,25 @@ hibernation. It is a development readout, not a trace store: for retention and cross-instance search, point [`otlpSink`](#otlpsink) at a real collector, where each span is exported as an OTLP `INTERNAL` span carrying its `parentSpanId`. +Expanding a trace gives an elapsed-time ruler over the bars, a per-span detail +block (span id, kind, the full attribute bag, the error, and any +`span.addEvent` / `span.recordException` events), and the `ctx.log` lines the +same dispatch emitted, joined by trace id. **Errors only** narrows the list to +traces where something threw; the search box matches a span name or span id as +well as the trace's own identifiers, so an id pasted from an error report finds +the trace containing it. Both **Logs** views carry a **Trace** link on rows +emitted inside a dispatch, opening the waterfall on the shard it came from. + +:::caution[The Requests view's Trace link is best-effort] +The durable request log records each dispatch's trace id, but it **outlives** the +span ring it points at: the log survives hibernation and the ring does not. A +Trace link on a row older than the current DO instance therefore lands on an +empty Traces panel — the trace has aged out locally. The id itself stays valid +wherever [`otlpSink`](#otlpsink) ships spans, which is where a deployed app's +traces are retained; it is also emitted on the `Logpush` console event, so a SIEM +can join a request to its spans. +::: + ### Generation spans (AI) An AI model call is a span with the OpenTelemetry diff --git a/apps/studio/src/mock/dev-client.ts b/apps/studio/src/mock/dev-client.ts index c227b4a6ec..d4aa2bf032 100644 --- a/apps/studio/src/mock/dev-client.ts +++ b/apps/studio/src/mock/dev-client.ts @@ -127,6 +127,11 @@ const TRACES = [ attributes: { method: "POST", url: "api.stripe.com/v1/charges" }, depth: 2, durationMs: 200, + // A retried-then-succeeded outbound call: `kind` and the recorded + // events are what the span-detail block exists to show, and a + // handled retry is invisible anywhere else in the Studio. + events: [{ attributes: { "retry.attempt": 1 }, name: "http.retry", ts: now - 3900 }], + kind: "client", name: "http.post", offsetMs: 28, ok: true, @@ -170,6 +175,8 @@ const TRACES = [ depth: 1, durationMs: 165, error: { message: "card_declined: Your card was declined.", type: "StripeError" }, + events: [{ attributes: { "exception.type": "StripeError" }, name: "exception", ts: now - 14_900 }], + kind: "client", name: "stripe.charge", offsetMs: 10, ok: false, @@ -379,10 +386,20 @@ const dataFor = (reference: string, args: unknown): unknown => { } case ADMIN_FUNCTIONS.getLogs: { return { + // `traceId` matches the `TRACES` fixtures above, so the Traces + // panel's correlated-log section and the Logs panel's Trace link + // both have something to resolve. The last line deliberately has + // none — a container-lifecycle entry, emitted outside any dispatch. entries: [ - { functionPath: "posts:publish", level: "error", message: "Rate limited (429) from upstream", timestamp: now - 4000 }, - { functionPath: "messages:list", level: "info", message: "Served 128 rows in 12ms", timestamp: now - 9000 }, - { functionPath: "messages:send", level: "warn", message: "Slow write: 28ms", timestamp: now - 15_000 }, + { + functionPath: "posts:publish", + level: "error", + message: "Rate limited (429) from upstream", + timestamp: now - 4000, + traceId: "trace-publish", + }, + { functionPath: "messages:list", level: "info", message: "Served 128 rows in 12ms", timestamp: now - 9000, traceId: "trace-list" }, + { functionPath: "messages:send", level: "warn", message: "Slow write: 28ms", timestamp: now - 15_000, traceId: "trace-send" }, { level: "debug", message: "Reactive cache warm (312 entries)", timestamp: now - 22_000 }, ], }; @@ -420,6 +437,7 @@ const dataFor = (reference: string, args: unknown): unknown => { subscriptionsReRun: 0, tablesRead: ["messages"], tablesWritten: [], + traceId: "trace-list", ts: now - 4000, }, { @@ -431,8 +449,22 @@ const dataFor = (reference: string, args: unknown): unknown => { subscriptionsReRun: 0, tablesRead: ["posts"], tablesWritten: [], + traceId: "trace-publish", ts: now - 9000, }, + // No `traceId`: a row written before the column existed. The + // Requests view must render it with no Trace link rather than + // an empty button. + { + durationMs: 8, + functionPath: "messages:send", + outcome: "ok", + seq: 8999, + subscriptionsReRun: 2, + tablesRead: [], + tablesWritten: ["messages"], + ts: now - 14_000, + }, ], }; } diff --git a/packages/codegen/__tests__/emit-external-source.test.ts b/packages/codegen/__tests__/emit-external-source.test.ts index d1cae6ccdd..0a87a73e40 100644 --- a/packages/codegen/__tests__/emit-external-source.test.ts +++ b/packages/codegen/__tests__/emit-external-source.test.ts @@ -45,7 +45,7 @@ describe("emitShard — external-source ingest", () => { const shard = emitShard({ schema: discover(SOURCED) }); - expect(shard).toContain("protected override async pollExternalSources()"); + expect(shard).toContain("protected override async pollExternalSources(trace?: TraceRefLike)"); // The per-table work delegates to the tested @lunora/do helper, not an inline projection. expect(shard).toContain("pullExternalSourceTick(this.sql as SqlExec, writer, client"); // Incremental mode (plan 136) branches to the durable-watermark helper. @@ -62,7 +62,7 @@ describe("emitShard — external-source ingest", () => { // Plan 148: the override reports the earliest NEXT-DUE timestamp (not a // bare active count), so the shared alarm can sleep until a source is // actually due instead of spinning at the 2 s global-shape floor. - expect(shard).toContain("protected override async pollExternalSources(): Promise"); + expect(shard).toContain("protected override async pollExternalSources(trace?: TraceRefLike): Promise"); expect(shard).toContain("nextDueAt = nextDueAt === undefined ? sourceNextDueAt : Math.min(nextDueAt, sourceNextDueAt);"); // Plan 207 step 2: each table's writer gets its OWN fresh per-work-item // meter — an alarm tick has no `/rpc` dispatch to fall back to. @@ -71,7 +71,49 @@ describe("emitShard — external-source ingest", () => { // warns (not `recordExternalSourceError`) and leaves the table due so the // shared alarm re-arms promptly instead of throttling to `refresh.everyMs`. expect(shard).toContain('error instanceof LunoraError && error.code === "TRANSACTION_LIMIT_EXCEEDED"'); - expect(shard).toContain('level: "warn"'); + // The `warn` level itself is the base class's (`recordExternalSourceWarning`); + // what the emitter owns is routing the back-off there rather than to + // `recordExternalSourceError`, which would group it as a real Issue. + expect(shard).toContain("this.recordExternalSourceWarning("); + }); + + it("forwards the alarm's trace to every contained ingest failure it records", () => { + expect.assertions(4); + + const shard = emitShard({ schema: discover(SOURCED) }); + + // The parameter is only worth having if all three log sites in the + // generated poll loop actually pass it on — a signature that accepts a + // trace and drops it would leave these lines uncorrelated exactly as + // before, and the type-checker would not notice. + expect(shard).toContain("this.recordExternalSourceError(table, error, trace);"); + // A regex so the emitted `${source.binding}` interpolation stays out of a + // string literal (`no-template-curly-in-string`). + expect(shard).toMatch(/no sourceClient resolved for binding[^\n]*, trace\);/u); + expect(shard).toContain("this.recordExternalSourceWarning("); + // Structural projection from the base package, so a generated app never + // takes on an `@lunora/observability` dependency for this. + expect(shard).toContain("import type { ExternalSourceLike, SourceClientLike, TraceRefLike }"); + }); + + it("only touches base-class members the compile-time contract covers", () => { + expect.assertions(1); + + // The emitted shard is a string, so nothing type-checks it — which is how + // `this.logs.push(...)` (private on `ShardDO`) shipped in the poll loop for + // the life of this feature. `emitted-shard-contract.ts` compiles a subclass + // using each member below, so `lint:types` fails if one changes visibility + // or signature; this asserts the emitter still restricts itself to that set. + // + // Adding a member here means adding it to the contract file too — that + // pairing is the whole guard. Members read off `this` inside the emitted + // poll loop, ignoring locals and the `schema`/`config` module closures. + const membersOf = (shard: string): Set => new Set([...shard.matchAll(/\bthis\.([A-Za-z_]\w*)/gu)].map((match) => match[1] as string)); + + const plain = membersOf(emitShard({ schema: discover(PLAIN) })); + const added = [...membersOf(emitShard({ schema: discover(SOURCED) }))].filter((member) => !plain.has(member)).toSorted((a, b) => a.localeCompare(b)); + + expect(added).toStrictEqual(["alarmHeadroom", "currentShardKey", "recordExternalSourceError", "recordExternalSourceWarning", "scheduleSourcePoll"]); }); it("stays byte-identical (none of the ingest surface) for a non-sourced schema", () => { diff --git a/packages/codegen/__tests__/emitted-shard-contract.ts b/packages/codegen/__tests__/emitted-shard-contract.ts new file mode 100644 index 0000000000..6dba542a7c --- /dev/null +++ b/packages/codegen/__tests__/emitted-shard-contract.ts @@ -0,0 +1,59 @@ +/** + * A compile-time contract for the shard subclass `emitShard` generates. + * + * **Why this file exists.** The emitter produces TypeScript as a *string*, and + * every test around it asserts on substrings of that string. Nothing compiles + * it. So the generated code can reference a base-class member that a subclass + * cannot legally touch, and the whole suite stays green — which is exactly what + * happened: the external-source poll loop emitted `this.logs.push(...)` against + * a `private logs`, so every project declaring a `.source()` table emitted a + * shard that failed `tsc`. No fixture or example declares one, so nobody ran + * into it. + * + * Golden fixtures cannot close that gap: `tsconfig.json` deliberately excludes + * `__tests__/fixtures/**`, because generated output only type-checks inside a + * whole app (its schema, its `_generated` siblings, its bindings). This file is + * the cheap half of the job — it mirrors the base-class surface the emitted + * shard depends on, in a real subclass, so `lint:types` fails HERE, in the + * package that owns the emitter, the moment one of those members changes + * visibility or signature. + * + * It is not a test and is never executed. Keeping it honest is a two-part + * contract with `emit-external-source.test.ts`, which asserts the emitted text + * really does call these members (and really does not touch `this.logs`): + * this file proves the calls compile, that one proves they are the calls made. + * + * When the emitter starts using a new base member, add it here. + */ +import type { TraceRefLike } from "@lunora/do"; +import { ShardDO } from "@lunora/do"; + +class EmittedShardContract extends ShardDO { + // eslint-disable-next-line class-methods-use-this -- required abstract override; this file exercises the base surface, it never dispatches + public override handleRpc(): Promise { + return Promise.resolve(undefined); + } + + /** + * Mirrors the `hasSourcedTables` branch of the generated shard: the override + * signature, and every base member its poll loop touches. + */ + protected override async pollExternalSources(trace?: TraceRefLike): Promise { + await Promise.resolve(); + + const shardKey = this.currentShardKey(); + + this.alarmHeadroom(); + this.recordChangedTable("documents"); + + // The two contained-failure seams. `recordExternalSourceWarning` exists + // precisely so this loop never has to reach for the private log ring. + this.recordExternalSourceError("documents", new Error(shardKey), trace); + this.recordExternalSourceWarning("documents", "hit the transaction limit mid-batch", trace); + + await this.scheduleSourcePoll(); + + return undefined; + } +} +export default EmittedShardContract; diff --git a/packages/codegen/src/emit.ts b/packages/codegen/src/emit.ts index 100cb08a1a..ba76b02728 100644 --- a/packages/codegen/src/emit.ts +++ b/packages/codegen/src/emit.ts @@ -4297,7 +4297,11 @@ const LUNORA_RLS_READ_REGISTRY = buildRlsReadRegistry(Object.values(LUNORA_FUNCT const importLines = [ `import type { ${doTypeImports.join(", ")} } from "${base.do}";`, `import { applyCdcChanges, buildReprojectionMigration, ${shapeGuardImport}createReadFootprint, createShardCtxDb, exportShardRows, importShardRows, ${hasSourcedTables ? "isSourceDue, pullExternalSourceIncrementalTick, pullExternalSourceTick, " : ""}${hasShardedVectors ? "ROOT_SHARD_NAME, " : ""}runDataMigration, runShardMigrations, ${relationFanout.importFragment}ShardDO as ShardDOBase } from "${base.do}";`, - ...(hasSourcedTables ? [`import type { ExternalSourceLike, SourceClientLike } from "${base.do}";`] : []), + // `TraceRefLike` rides along with the source imports: the poll override + // takes the alarm's trace so its contained failures are correlated, and + // `@lunora/do` projects it structurally rather than re-exporting the + // observability type. + ...(hasSourcedTables ? [`import type { ExternalSourceLike, SourceClientLike, TraceRefLike } from "${base.do}";`] : []), // `asBucketStorage` (the bucket-aware `ctx.storage` wrapper) and // `createSecrets` (the `ctx.secrets` core built-in) live in // `@lunora/server`, the single source — imported here rather than stamped @@ -4598,7 +4602,7 @@ const sourcePollAtCache = new WeakMap>(); /* eslint-disable no-secrets/no-secrets -- the emitted `pullExternalSourceIncrementalTick(this.sql …)` poll-loop call is a dense generated identifier, not a credential */ const externalSourceOverride = hasSourcedTables ? ` - protected override async pollExternalSources(): Promise { + protected override async pollExternalSources(trace?: TraceRefLike): Promise { const env = (this.env ?? {}) as Record; const sourced = Object.entries((schema as unknown as SchemaLike).tables) .map(([table, definition]) => [table, (definition as { externalSource?: ExternalSourceLike }).externalSource] as const) @@ -4655,7 +4659,7 @@ const sourcePollAtCache = new WeakMap>(); // \`config.sourceClient\`, or wired it wrong). Surface it in the // Logs panel and stamp \`polledAt\` so a persistent misconfig backs // off to \`refresh.everyMs\` instead of retrying every alarm tick. - this.recordExternalSourceError(table, new Error(\`external-source: no sourceClient resolved for binding "\${source.binding}"\`)); + this.recordExternalSourceError(table, new Error(\`external-source: no sourceClient resolved for binding "\${source.binding}"\`), trace); } else { // A FRESH tracker per TABLE, not one shared across the whole // loop: one table's runaway pull must not spend a budget a @@ -4703,19 +4707,18 @@ const sourcePollAtCache = new WeakMap>(); // table stays "due" and \`nextPollAlarmTarget\`'s existing // due-now floor re-arms the shared alarm promptly — not a // fresh \`setAlarm(now)\`. - this.logs.push({ - functionPath: \`source:\${table}\`, - level: "warn", - message: \`external-source poll for "\${table}" hit the transaction limit mid-batch; resuming next tick: \${error.message}\`, - timestamp: Date.now(), - }); + this.recordExternalSourceWarning( + table, + \`external-source poll for "\${table}" hit the transaction limit mid-batch; resuming next tick: \${error.message}\`, + trace, + ); nextDueAt = nextDueAt === undefined ? now : Math.min(nextDueAt, now); continue; } - this.recordExternalSourceError(table, error); + this.recordExternalSourceError(table, error, trace); // Stamp on failure too, so a persistently failing source throttles // to \`refresh.everyMs\` rather than being hammered every tick. polledAt.set(table, Date.now()); diff --git a/packages/do/__tests__/shard-do.admin.test.ts b/packages/do/__tests__/shard-do.admin.test.ts index da5dca75cb..4002fec8e5 100644 --- a/packages/do/__tests__/shard-do.admin.test.ts +++ b/packages/do/__tests__/shard-do.admin.test.ts @@ -90,7 +90,7 @@ const QUEUE_METADATA_KEY_GUARD: KeysMatch = true; @@ -2111,6 +2111,37 @@ describe("shardDO admin data migrations", () => { expect(body.result.entries[0]).toMatchObject({ functionPath: "boom:explode", level: "error", message: "boom" }); }); + it("correlates an alarm-path failure to the trigger's own trace", async () => { + expect.assertions(3); + + // The alarm's ingest tier throws, so `pollTier` contains it and records a + // `source:poll` line — an alarm-path log site, which used to be recorded + // with no trace at all. + class FailingSourceShard extends AdminShard { + // eslint-disable-next-line class-methods-use-this -- test stub mirroring the codegen override + protected override pollExternalSources(): Promise { + return Promise.reject(new Error("source ingest exploded")); + } + } + + const shard = new FailingSourceShard(state, { LUNORA_ADMIN_TOKEN: ADMIN_TOKEN }); + + await shard.alarm(); + await shard.alarm(); + + const response = await shard.fetch(adminRequest(ADMIN_FUNCTIONS.getLogs, {})); + const body = await response.json<{ result: { entries: { functionPath?: string; message: string; traceId?: string }[] } }>(); + const lines = body.result.entries.filter((entry) => entry.functionPath === "source:poll"); + + expect(lines[0]?.message).toBe("source ingest exploded"); + // The alarm publishes an anchor through `withTriggerTrace`, so the line is + // attributable rather than orphaned — a 32-hex W3C trace id. + expect(lines[0]?.traceId).toMatch(/^[\da-f]{32}$/u); + // Each alarm tick mints its own anchor: a shared id would mean the second + // tick's failure was filed under the first tick's trace. + expect(lines[0]?.traceId).not.toBe(lines[1]?.traceId); + }); + it("getFunctionStats reports per-function call and error counts", async () => { expect.assertions(6); diff --git a/packages/do/__tests__/shard-do.sampling.test.ts b/packages/do/__tests__/shard-do.sampling.test.ts index 9030e9e3f7..f59eba7518 100644 --- a/packages/do/__tests__/shard-do.sampling.test.ts +++ b/packages/do/__tests__/shard-do.sampling.test.ts @@ -1,4 +1,5 @@ import type { ContextTracer } from "@lunora/observability"; +import { readRequestLog } from "@lunora/observability"; import { describe, expect, it } from "vitest"; import type { SpanEvent } from "../../../shared/span-event"; @@ -229,6 +230,55 @@ describe("shardDO trace sampling", () => { } }); + it("stamps the durable request-log row with its OWN trace when a sibling dispatch interleaves", async () => { + expect.assertions(2); + + const database = createSqliteExec(); + + try { + const shard = new SamplingShard(makeState(database), {}); + + let markStarted!: () => void; + const slowStarted = new Promise((resolve) => { + markStarted = resolve; + }); + let releaseSlow!: () => void; + const slowGate = new Promise((resolve) => { + releaseSlow = resolve; + }); + + shard.plan = async (_trace, functionPath) => { + if (functionPath === "slow:ok") { + markStarted(); + // Park inside the handler, so the sibling below runs its whole + // dispatch — including the `finally` that CLEARS the shared + // `currentRequestTrace` — before this one records its row. + await slowGate; + } + }; + + const slow = shard.fetch(request("slow:ok", { sampled: true, traceId: TRACE_ID })); + + await slowStarted; + await shard.fetch(request("fast:ok", { sampled: true, traceId: TRACE_ID_B })); + + releaseSlow(); + await slow; + + const rows = readRequestLog(database.sql); + const byPath = new Map(rows.map((row) => [row.functionPath, row.traceId])); + + // The regression: `recordRequestLog` runs after the handler's awaits, so + // reading the shared anchor there stamped the sibling's trace — or + // `undefined`, once the sibling's `finally` had cleared it. The anchor is + // threaded by value precisely so each row carries its own dispatch's id. + expect(byPath.get("slow:ok")).toBe(TRACE_ID); + expect(byPath.get("fast:ok")).toBe(TRACE_ID_B); + } finally { + database.close(); + } + }); + it("flushes a sampled-out trace's held error span even after the ring evicted it (eviction race closed)", async () => { expect.assertions(1); diff --git a/packages/do/src/index.ts b/packages/do/src/index.ts index 2244d1fb71..93000fabc9 100644 --- a/packages/do/src/index.ts +++ b/packages/do/src/index.ts @@ -28,6 +28,7 @@ export type { ShardDOState, SubscriptionOutcome, TelemetrySink, + TraceRefLike, } from "./shard-do"; export { ROOT_DO_SIZE_WARN_BYTES, ROOT_SHARD_NAME, ShardDO, subscriptionListDeltas } from "./shard-do"; export { SHARD_REGISTRY_DO_NAME, ShardRegistryDO } from "./shard-registry-do"; diff --git a/packages/do/src/shard-do.ts b/packages/do/src/shard-do.ts index 46c0756fa0..270fecbf7e 100644 --- a/packages/do/src/shard-do.ts +++ b/packages/do/src/shard-do.ts @@ -495,6 +495,21 @@ interface ContextLogger { with: (fields: LogFields) => ContextLogger; } +/** + * The only part of a trace anchor an alarm-path log site needs: the id it files + * the line under. + * + * Declared here as a structural projection rather than re-exporting + * `@lunora/observability`'s `TraceAnchor`, because `@lunora/do` deliberately + * does not re-export observability (see this package's index) and the generated + * shard — which forwards this value into `recordExternalSourceError` — must be + * able to name the type without taking on that dependency. A real `TraceAnchor` + * satisfies it, so internal callers pass theirs unchanged. + */ +interface TraceRefLike { + traceId: string; +} + /** * Minimal projection of `DurableObjectState` that the ShardDO base requires. * Declared structurally so unit tests can pass in plain object doubles @@ -1005,6 +1020,20 @@ abstract class ShardDO { */ private currentRequestTrace: { rootSpanId: string; traceId: string } | undefined; + /** + * Anchor of the in-flight trigger (`withTriggerTrace`), handed across the + * `runner.handleAlarm()` boundary that stops the alarm handler from just + * taking it as an argument. + * + * A field only because of that indirection, and read under one rule: capture + * it into a local SYNCHRONOUSLY at the top of the handler, before any `await`. + * Read later it is no safer than `currentRequestTrace` — a socket frame + * interleaving at an await point would pick up the alarm's trace and file its + * own failure under it. Everything downstream takes the captured value as a + * parameter for exactly that reason. + */ + private currentTriggerTrace: TraceAnchor | undefined; + /** * Per-trace head-sampling state, keyed by `traceId` so concurrent dispatches on * the same DO instance can't clobber each other's decision. A DO interleaves @@ -3170,8 +3199,8 @@ abstract class ShardDO { * `GLOBAL_SHAPE_POLL_INTERVAL_MS` floor for a source whose `refresh.everyMs` * is, say, an hour away. */ - // eslint-disable-next-line class-methods-use-this -- base-class override hook: the codegen subclass implements the real Hyperdrive-backed poll - protected pollExternalSources(): Promise { + // eslint-disable-next-line class-methods-use-this -- base-class override hook: the codegen subclass implements the real Hyperdrive-backed poll, and is the only consumer of `_trace` + protected pollExternalSources(_trace?: TraceRefLike): Promise { return Promise.resolve(undefined); } @@ -3223,7 +3252,7 @@ abstract class ShardDO { * alarm re-arms promptly via `nextPollAlarmTarget`'s existing due-now floor, * rather than waiting out the full `TTL_SWEEP_INTERVAL_MS` cadence. */ - protected async pollTtlSweeps(): Promise { + protected async pollTtlSweeps(trace?: TraceRefLike): Promise { const specs = this.ttlSweeps(); if (specs.length === 0) { @@ -3243,7 +3272,7 @@ abstract class ShardDO { for (const id of page.ids) { // eslint-disable-next-line no-await-in-loop -- per-row writer deletes must serialise to avoid OCC contention on the shard DO (same reasoning as `runShardBulkDelete`) - const limitHit = await this.deleteExpiredTtlRow(spec.table, id, headroom); + const limitHit = await this.deleteExpiredTtlRow(spec.table, id, headroom, trace); if (limitHit) { return Date.now(); @@ -3273,9 +3302,39 @@ abstract class ShardDO { return this.runner.shardKey ?? ROOT_SHARD_NAME; } - /** Record a contained external-source ingest failure (one sourced table's poll) into the log ring without aborting the others. */ - protected recordExternalSourceError(table: string, error: unknown): void { - this.recordShapeError(`source:${table}`, error); + /** + * Record a contained external-source ingest failure (one sourced table's + * poll) into the log ring without aborting the others. + * + * `trace` is the alarm's anchor, forwarded by the generated + * `pollExternalSources` override from the value it was handed. Optional so a + * subclass generated before this parameter existed still compiles and simply + * records an uncorrelated line. + */ + protected recordExternalSourceError(table: string, error: unknown, trace?: TraceRefLike): void { + this.recordShapeError(`source:${table}`, error, trace); + } + + /** + * Record a contained external-source BACK-OFF — a transaction-limit hit + * mid-batch, which is "batch full" rather than a failure, so it lands at + * `warn` and does NOT group as an Issue the way + * {@link ShardDO.recordExternalSourceError} does. + * + * Exists because the generated poll loop needs to write this line and the log + * ring is private: emitting `this.logs.push(...)` into the subclass does not + * compile, which went unnoticed only because no fixture or example declares a + * `.source()` table. A protected seam keeps the buffer encapsulated and gives + * the line the same trace correlation as its sibling above. + */ + protected recordExternalSourceWarning(table: string, message: string, trace?: TraceRefLike): void { + this.logs.push({ + functionPath: `source:${table}`, + level: "warn", + message, + timestamp: Date.now(), + traceId: trace?.traceId, + }); } /* eslint-disable no-secrets/no-secrets -- JSDoc names the `AsyncIterable` type, not a credential */ @@ -3654,7 +3713,7 @@ abstract class ShardDO { userId: this.getCurrentUserId(), }; - this.logs.push({ fields, functionPath, level, message, timestamp: event.ts }); + this.logs.push({ fields, functionPath, level, message, timestamp: event.ts, traceId: event.traceId }); try { emitLogEvent(event); @@ -4464,7 +4523,7 @@ abstract class ShardDO { // so the request log would record an empty write set. const tablesWritten = [...(this.pendingChangedTables ?? [])]; - this.recordRequestLog(payload.functionPath, payload.args ?? {}, durationMs, "ok", tablesWritten); + this.recordRequestLog(payload.functionPath, payload.args ?? {}, durationMs, "ok", tablesWritten, dispatchTrace); // Inspect the post-write size before responding. SQLite-in-DO // exposes `databaseSize` as a real getter; reading it is a @@ -4523,12 +4582,23 @@ abstract class ShardDO { // Flush statement samples even on error paths — partial sampling // is better than losing the timing signal entirely. this.flushStmtSamples(); - this.recordRequestLog(payload.functionPath, payload.args ?? {}, durationMs, "error", [...(this.pendingChangedTables ?? [])], message); + this.recordRequestLog( + payload.functionPath, + payload.args ?? {}, + durationMs, + "error", + [...(this.pendingChangedTables ?? [])], + dispatchTrace, + message, + ); this.logs.push({ functionPath: payload.functionPath, level: "error", message, timestamp: Date.now(), + // The local, not the shared field: this runs after the handler's + // awaits, where an interleaved dispatch may have re-set it. + traceId: dispatchTrace.traceId, }); // A fresh error row landed, but the failed dispatch's own writes (if @@ -4610,18 +4680,24 @@ abstract class ShardDO { * as the host-specific handler while the engine is progressively extracted. */ protected async handleAlarmCloudflare(): Promise { + // Captured before the first `await`, then passed by value everywhere below: + // this handler is the alarm's own scope, so nothing has interleaved yet. + // `undefined` when the alarm ran outside `withTriggerTrace` (a direct call + // in a test), which simply leaves these log lines uncorrelated. + const trace = this.currentTriggerTrace; + this.globalPollScheduled = false; let globalShapesRemaining: number; try { - globalShapesRemaining = await this.pollGlobalShapes(); + globalShapesRemaining = await this.pollGlobalShapes(trace); } catch (error) { // `pollGlobalShapes` already contains per-socket/per-shape failures; // this guards a catastrophic failure (e.g. `getWebSockets` throwing) // so the poll heartbeat re-arms and retries next tick instead of // dying permanently and silently dropping every global subscriber. - this.recordShapeError("shape:poll", error); + this.recordShapeError("shape:poll", error, trace); globalShapesRemaining = 1; } @@ -4631,7 +4707,7 @@ abstract class ShardDO { try { return await run(); } catch (error) { - this.recordShapeError(tag, error); + this.recordShapeError(tag, error, trace); return Date.now() + ShardDO.GLOBAL_SHAPE_POLL_INTERVAL_MS; } @@ -4641,13 +4717,13 @@ abstract class ShardDO { // base hook returns `undefined` (dormant); the codegen subclass overrides // it to materialize each sourced table and report the earliest NEXT-DUE // timestamp across every non-manual source. - const nextSourceDueAt = await pollTier("source:poll", async () => this.pollExternalSources()); + const nextSourceDueAt = await pollTier("source:poll", async () => this.pollExternalSources(trace)); // Declarative TTL expiry (`.ttl(...)`) shares this alarm too. The base hook // returns `undefined` (no TTL tables); the codegen subclass overrides // `ttlSweeps()` from the schema so the sweep pages + removes expired rows and // reports its next-due. - const nextTtlDueAt = await pollTier("ttl:sweep", async () => this.pollTtlSweeps()); + const nextTtlDueAt = await pollTier("ttl:sweep", async () => this.pollTtlSweeps(trace)); // Drain the tables the ingest poll just wrote: a sourced table is local, so // its `defineShape` subscribers are poked through the standard @@ -4729,6 +4805,13 @@ abstract class ShardDO { this.currentRequestTrace = anchor; } + // Published unconditionally, unlike the shared dispatch field above: this + // one is written only here, and a trigger is the only thing that reads it + // (synchronously, at entry — see the field's docstring). + const previousTrigger = this.currentTriggerTrace; + + this.currentTriggerTrace = anchor; + let failure: { thrown: unknown } | undefined; try { @@ -4738,6 +4821,8 @@ abstract class ShardDO { throw error; } finally { + this.currentTriggerTrace = previousTrigger; + if (claimed && this.currentRequestTrace === anchor) { this.currentRequestTrace = undefined; } @@ -6411,6 +6496,7 @@ abstract class ShardDO { durationMs: number, outcome: "error" | "ok", tablesWritten: string[], + trace: TraceAnchor, errorMessage?: string, ): void { const config = this.requestLogConfig(); @@ -6439,6 +6525,13 @@ abstract class ShardDO { shardKey: this.runner.shardKey, tablesRead: this.currentRequestReadTables === undefined ? [] : [...this.currentRequestReadTables], tablesWritten, + // Passed BY VALUE, never read off `currentRequestTrace` here: this + // method runs after the handler's awaits, by which point an + // interleaved dispatch may have re-set (or cleared) that shared + // field — which would file this row, and the Logpush event carrying + // it, under another request's trace. Same hazard, and same fix, as + // the dispatch root span's `dispatchTrace` capture. + traceId: trace.traceId, ts: Date.now(), userId: this.getCurrentUserId(), }; @@ -8441,7 +8534,7 @@ abstract class ShardDO { * on an ordinary successful delete. Any OTHER thrown error still propagates — * only the meter's own signal is contained here. */ - private async deleteExpiredTtlRow(table: string, id: string, headroom: TransactionHeadroomTracker): Promise { + private async deleteExpiredTtlRow(table: string, id: string, headroom: TransactionHeadroomTracker, trace?: TraceRefLike): Promise { try { await this.deleteRowThroughWriter(table, id, headroom); @@ -8453,6 +8546,7 @@ abstract class ShardDO { level: "warn", message: `TTL sweep for "${table}" hit the transaction limit mid-batch; resuming next tick: ${error.message}`, timestamp: Date.now(), + traceId: trace?.traceId, }); return true; @@ -8468,13 +8562,20 @@ abstract class ShardDO { * best-effort fan-out: one socket's read or one shape's resolve failing must * never take down the others — so callers swallow the throw and surface it * here for diagnosis. `context` is a synthetic `shape:phase:subId` path. + * + * `trace` is passed by the alarm path, which has an anchor to attribute the + * failure to; the socket-frame callers omit it because their path is + * deliberately untraced (see `webSocketMessage`). It is a parameter rather + * than a field read so an alarm interleaving with a socket frame cannot file + * one path's failure under the other's trace. */ - private recordShapeError(context: string, error: unknown): void { + private recordShapeError(context: string, error: unknown, trace?: TraceRefLike): void { this.logs.push({ functionPath: context, level: "error", message: error instanceof Error ? error.message : String(error), timestamp: Date.now(), + traceId: trace?.traceId, }); } @@ -8508,7 +8609,7 @@ abstract class ShardDO { * subscribed so {@link ShardDO.alarm} knows whether to re-arm. Expired sockets * are dropped in passing (mirrors {@link ShardDO.pokeShapeSubscribers}). */ - private async pollGlobalShapes(): Promise { + private async pollGlobalShapes(trace?: TraceRefLike): Promise { const sockets = [...this.runner.sockets()]; let remaining = 0; @@ -8529,7 +8630,7 @@ abstract class ShardDO { const identity: SubscriptionIdentity = { identity: attachment.identity, userId: attachment.userId }; // eslint-disable-next-line no-await-in-loop -- per-socket reads are intentionally serialized to bound concurrent global reads per tick - remaining += await this.pollSocketGlobalShapes(ws, shapes, identity, attachment.connectionId ?? ""); + remaining += await this.pollSocketGlobalShapes(ws, shapes, identity, attachment.connectionId ?? "", trace); } return remaining; @@ -8547,6 +8648,7 @@ abstract class ShardDO { shapes: Record, identity: SubscriptionIdentity, connectionId: string, + trace?: TraceRefLike, ): Promise { let count = 0; @@ -8557,7 +8659,7 @@ abstract class ShardDO { resolved = this.resolveShape(shape.name, shape.args ?? {}, identity); } catch (error) { count += 1; - this.recordShapeError(`shape:poll:${subId}`, error); + this.recordShapeError(`shape:poll:${subId}`, error, trace); continue; } @@ -8572,7 +8674,7 @@ abstract class ShardDO { // eslint-disable-next-line no-await-in-loop -- per-shape D1 reads serialized within a socket to bound concurrency await this.refreshGlobalShape(ws, subId, resolved, identity, connectionId); } catch (error) { - this.recordShapeError(`shape:poll:${subId}`, error); + this.recordShapeError(`shape:poll:${subId}`, error, trace); } } @@ -9207,4 +9309,4 @@ export type { // canonical home is `./subscription-delivery`. export { subscriptionListDeltas } from "@lunora/shard-engine"; -export type { HibernatableWebSocket, LogSink, ShardDOOptions, ShardDOState, SubscriptionOutcome, TelemetrySink }; +export type { HibernatableWebSocket, LogSink, ShardDOOptions, ShardDOState, SubscriptionOutcome, TelemetrySink, TraceRefLike }; diff --git a/packages/observability/__tests__/request-log.test.ts b/packages/observability/__tests__/request-log.test.ts index 7c1d6adeb2..82a971c80a 100644 --- a/packages/observability/__tests__/request-log.test.ts +++ b/packages/observability/__tests__/request-log.test.ts @@ -15,6 +15,14 @@ import { } from "../src/request-log"; import createSqliteExec from "./_helpers/node-sqlite"; +/** + * 32-hex trace ids, built by `repeat` rather than written as literals: a real + * trace id is high-entropy, and a literal one trips the repo's `no-secrets` + * lint. The tests only need the right shape and length. + */ +const TRACE_ID = "ab".repeat(16); +const OTHER_TRACE_ID = "cd".repeat(16); + /** One minimal `ok` entry, overridable per case. */ const entry = (overrides: Partial[1]> = {}): Parameters[1] => { return { @@ -83,6 +91,92 @@ describe("request-log module", () => { } }); + it("round-trips the dispatch's trace id, leaving it undefined when there was none", () => { + expect.assertions(2); + + const database = createSqliteExec(); + + try { + appendRequestLogEntry(database.sql, entry({ traceId: TRACE_ID })); + appendRequestLogEntry(database.sql, entry({ functionPath: "cron:sweep" })); + + const rows = readRequestLog(database.sql); + + // Newest first, so the trace-less cron row leads. + expect(rows[0]!.traceId).toBeUndefined(); + expect(rows[1]!.traceId).toBe(TRACE_ID); + } finally { + database.close(); + } + }); + + it("keeps the trace id intact on a row whose args were redacted", () => { + expect.assertions(2); + + const database = createSqliteExec(); + + try { + appendRequestLogEntry(database.sql, entry({ redactedArgs: { password: "hunter2" }, traceId: TRACE_ID })); + + const [row] = readRequestLog(database.sql); + + // The masked arg proves redaction ran on THIS row, so the intact id + // below is a real exemption and not just an unredacted code path. The + // guard is against widening redaction to the whole entry: a masked + // correlation key joins to nothing, which is its entire purpose. + expect((row!.redactedArgs as Record).password).not.toBe("hunter2"); + expect(row!.traceId).toBe(TRACE_ID); + } finally { + database.close(); + } + }); + + it("adds trace_id to a shard whose table predates the column, without losing its rows", () => { + expect.assertions(4); + + const database = createSqliteExec(); + + try { + // The pre-column schema, verbatim: a shard that logged before this + // change has exactly this table and must gain the column on the next + // ensure — the guarded ALTER path, which a fresh CREATE never exercises. + database.raw( + `CREATE TABLE "${REQUEST_LOG_TABLE}" ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + ts REAL NOT NULL, + function_path TEXT NOT NULL, + shard_key TEXT, + user_id TEXT, + identity TEXT, + args TEXT, + outcome TEXT NOT NULL, + error_message TEXT, + error_fingerprint TEXT, + duration_ms REAL NOT NULL, + tables_read TEXT NOT NULL DEFAULT '[]', + tables_written TEXT NOT NULL DEFAULT '[]', + cache_hit INTEGER, + subscriptions_rerun INTEGER NOT NULL DEFAULT 0 + )`, + ); + database.raw(`INSERT INTO "${REQUEST_LOG_TABLE}" (ts, function_path, outcome, duration_ms) VALUES (?, ?, ?, ?)`, 900, "legacy:call", "ok", 3); + + ensureRequestLogTable(database.sql); + appendRequestLogEntry(database.sql, entry({ traceId: OTHER_TRACE_ID })); + + const rows = readRequestLog(database.sql); + + expect(rows).toHaveLength(2); + expect(rows[0]!.traceId).toBe(OTHER_TRACE_ID); + // The pre-existing row survives and simply has no trace id — the key + // is omitted rather than set to `undefined`, so assert both. + expect(rows[1]!.functionPath).toBe("legacy:call"); + expect(rows[1]!.traceId).toBeUndefined(); + } finally { + database.close(); + } + }); + it("records error outcome with the failure message", () => { expect.assertions(2); @@ -582,6 +676,18 @@ describe("emitRequestLogEvent (PLAN3 §3.3 Logpush emit)", () => { expect(JSON.stringify(event)).not.toContain("alice@example.com"); }); + it("carries the trace id so a SIEM can join the event to the collector's spans", () => { + expect.assertions(1); + + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + emitRequestLogEvent(entry({ traceId: TRACE_ID })); + + const event = JSON.parse(log.mock.calls.at(0)?.at(0) as string) as Record; + + expect(event.traceId).toBe(TRACE_ID); + }); + it("captureRaw emits un-redacted args/identity (the dev escape hatch)", () => { expect.assertions(2); diff --git a/packages/observability/__tests__/span-buffer.test.ts b/packages/observability/__tests__/span-buffer.test.ts index 9b3f7a536c..a63d80f114 100644 --- a/packages/observability/__tests__/span-buffer.test.ts +++ b/packages/observability/__tests__/span-buffer.test.ts @@ -123,6 +123,34 @@ describe("foldTraces", () => { expect(DEFAULT_TRACE_LIMIT).toBeGreaterThan(0); }); + it("carries a span's kind and recorded events through to the folded row", () => { + expect.assertions(2); + + // Both are recorded by `ctx.trace` and were dropped by the fold, so a + // handled exception (`span.recordException`) reached the buffer and then + // vanished before anything could render it. + const { traces } = foldTraces([ + span({ + events: [{ attributes: { "exception.type": "TimeoutError" }, name: "exception", ts: 1002 }], + kind: "client", + }), + ]); + + expect(traces[0]?.spans[0]?.kind).toBe("client"); + expect(traces[0]?.spans[0]?.events).toStrictEqual([{ attributes: { "exception.type": "TimeoutError" }, name: "exception", ts: 1002 }]); + }); + + it("omits kind and events entirely for a span that recorded neither", () => { + expect.assertions(2); + + // Absent rather than `undefined`-valued: these ride every admin `getTraces` + // payload, and the common `ctx.trace` span has neither. + const [row] = foldTraces([span()]).traces[0]?.spans ?? []; + + expect(row).not.toHaveProperty("kind"); + expect(row).not.toHaveProperty("events"); + }); + it("still folds a trace whose root span is missing", () => { expect.assertions(2); diff --git a/packages/observability/src/log-buffer.ts b/packages/observability/src/log-buffer.ts index 815cb16a81..a37a9f53ef 100644 --- a/packages/observability/src/log-buffer.ts +++ b/packages/observability/src/log-buffer.ts @@ -26,6 +26,9 @@ interface LogEntry { level: LogLevel; message: string; timestamp: number; + + /** Trace the line was emitted under; absent outside a dispatch (container lifecycle, hibernation-path errors). What the Studio joins a log line to its waterfall on. */ + traceId?: string; } const DEFAULT_CAPACITY = 500; diff --git a/packages/observability/src/request-log.ts b/packages/observability/src/request-log.ts index b926dbeb93..958f67445b 100644 --- a/packages/observability/src/request-log.ts +++ b/packages/observability/src/request-log.ts @@ -12,6 +12,16 @@ * masking PII in the message can't change which Issue a row groups into — see * {@link appendRequestLogEntry} and {@link readErrorIssues}. * + * Each row also carries the dispatch's `trace_id`. Note the deliberate + * retention asymmetry: this table is DURABLE and bounded by row count, while + * the `ctx.trace` span ring it correlates to is in-memory and resets on + * hibernation. A trace id on a row older than the current DO instance therefore + * resolves in an external collector but NOT against the local waterfall, and + * the Studio treats a local miss as ordinary rather than as an error. Recording + * it is still worth it: the id is what joins this row to the collector's spans + * and to the `Logpush`-shipped console event, which is where a deployed app's + * traces actually live. + * * Modelled exactly on `audit-log.ts` (the CDC-log helpers in `ctx-db.ts` * `migrateCdcLog`/`appendCdcChange`/`readCdcChanges`/`trimCdcChanges` and the * reserved-table pattern in `data-migration.ts` `ensureStateTable`). Unlike the @@ -45,6 +55,14 @@ const REQUEST_LOG_RETENTION = 1000; /** Stable tag on every console-emitted event so a Logpush/SIEM consumer can filter lunora request events out of the raw Workers-trace firehose. */ const REQUEST_LOG_EVENT_SOURCE = "lunora"; +/** + * Columns added after the table's first release, as ` ` — each one + * `ALTER`ed in on its own so a shard created before it gains it on the next + * {@link ensureRequestLogTable}. A new column is one entry here plus the matching + * line in the `CREATE`; nothing else needs a migration. + */ +const ADDED_COLUMNS = ["error_fingerprint TEXT", "trace_id TEXT"]; + /** Outcome of one dispatch — `ok` for a returned result, `error` for a thrown handler. */ type RequestOutcome = "error" | "ok"; @@ -74,6 +92,9 @@ interface RequestLogEntry { tablesRead: string[]; /** Tables the handler wrote (from the change tracker); empty for a read-only dispatch. */ tablesWritten: string[]; + + /** W3C trace id (32-hex) of the dispatch; `undefined` on a row appended before this column existed. See the module docstring on the retention asymmetry. */ + traceId?: string; /** Wall-clock millis when the dispatch completed. */ ts: number; /** Acting userId forwarded by the runtime, or `undefined` when anonymous. */ @@ -93,6 +114,7 @@ interface AppendRequestLogEntry { subscriptionsReRun?: number; tablesRead?: string[]; tablesWritten?: string[]; + traceId?: string; ts: number; userId?: string; } @@ -245,12 +267,17 @@ const redactArgs = (value: unknown, captureRaw = false): unknown => { * * `error_fingerprint` is the {@link fingerprintError} grouping hash captured * from the RAW `error_message` at write time, before {@link appendRequestLogEntry} - * redacts it — see that function's docstring. It is added via a guarded - * `ALTER TABLE` rather than baked into the `CREATE`, mirroring + * redacts it — see that function's docstring. `trace_id` is the dispatch's W3C + * trace id, the correlation key to the span ring and to whatever collector + * `otlpSink` ships to. + * + * Both are also added via a guarded `ALTER TABLE`, mirroring * `function-metrics.ts`'s `ensureFunctionMetricsTables`, so a shard whose - * `__lunora_reqlog__` predates this column gains it on the next call without a + * `__lunora_reqlog__` predates a column gains it on the next call without a * migration. SQLite has no `ADD COLUMN IF NOT EXISTS`, so the duplicate-column - * error from a re-run (or the freshly-created schema above) is swallowed. + * error from a re-run (or from the freshly-created schema above) is swallowed + * per column — the loop is what keeps one column's duplicate from skipping the + * next column's add. */ const ensureRequestLogTable = (sql: SqlExec): void => { runSql( @@ -266,6 +293,7 @@ const ensureRequestLogTable = (sql: SqlExec): void => { outcome TEXT NOT NULL, error_message TEXT, error_fingerprint TEXT, + trace_id TEXT, duration_ms REAL NOT NULL, tables_read TEXT NOT NULL DEFAULT '[]', tables_written TEXT NOT NULL DEFAULT '[]', @@ -274,10 +302,12 @@ const ensureRequestLogTable = (sql: SqlExec): void => { )`, ); - try { - runSql(sql, `ALTER TABLE "${REQUEST_LOG_TABLE}" ADD COLUMN error_fingerprint TEXT`); - } catch { - // Column already exists — no-op. + for (const column of ADDED_COLUMNS) { + try { + runSql(sql, `ALTER TABLE "${REQUEST_LOG_TABLE}" ADD COLUMN ${column}`); + } catch { + // Column already exists — no-op. + } } }; @@ -331,8 +361,8 @@ const appendRequestLogEntry = (sql: SqlExec, entry: AppendRequestLogEntry, optio runSql( sql, `INSERT INTO "${REQUEST_LOG_TABLE}" - (ts, function_path, shard_key, user_id, identity, args, outcome, error_message, error_fingerprint, duration_ms, tables_read, tables_written, cache_hit, subscriptions_rerun) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + (ts, function_path, shard_key, user_id, identity, args, outcome, error_message, error_fingerprint, trace_id, duration_ms, tables_read, tables_written, cache_hit, subscriptions_rerun) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, entry.ts, entry.functionPath, // eslint-disable-next-line unicorn/no-null -- SQL NULL is the correct value for a request with no shard key / anonymous caller / absent field. @@ -353,6 +383,10 @@ const appendRequestLogEntry = (sql: SqlExec, entry: AppendRequestLogEntry, optio entry.errorMessage === undefined ? null : redactArgs(entry.errorMessage, captureRaw), // eslint-disable-next-line unicorn/no-null -- success path, or a legacy row appended before this column existed. errorFingerprint ?? null, + // Never redacted: a trace id is a random 32-hex identifier, not user + // data, and redacting it would destroy the only thing it is for. + // eslint-disable-next-line unicorn/no-null -- dispatch with no ambient trace, or a legacy row appended before this column existed. + entry.traceId ?? null, entry.durationMs, encodeTables(entry.tablesRead), encodeTables(entry.tablesWritten), @@ -395,6 +429,11 @@ const emitRequestLogEvent = (entry: AppendRequestLogEntry, options: RequestLogWr source: REQUEST_LOG_EVENT_SOURCE, tablesRead: entry.tablesRead ?? [], tablesWritten: entry.tablesWritten ?? [], + // The join key on the far side: a SIEM receiving this event and an OTLP + // collector receiving the same dispatch's spans can only be correlated + // through the trace id, so emitting it here is most of the point of + // recording it at all. + traceId: entry.traceId, ts: entry.ts, type: "request", userId: entry.userId, @@ -588,6 +627,7 @@ interface RequestLogRow { subscriptions_rerun: number; tables_read: string; tables_written: string; + trace_id: null | string; ts: number; user_id: null | string; } @@ -640,7 +680,7 @@ const readRequestLog = (sql: SqlExec, options: ReadRequestLogOptions = {}): Requ const rows = runSql( sql, - `SELECT seq, ts, function_path, shard_key, user_id, identity, args, outcome, error_message, duration_ms, tables_read, tables_written, cache_hit, subscriptions_rerun + `SELECT seq, ts, function_path, shard_key, user_id, identity, args, outcome, error_message, trace_id, duration_ms, tables_read, tables_written, cache_hit, subscriptions_rerun FROM "${REQUEST_LOG_TABLE}" WHERE ${conjuncts.join(" AND ")} ORDER BY seq DESC LIMIT ?`, ...parameters, ).toArray(); @@ -681,6 +721,10 @@ const readRequestLog = (sql: SqlExec, options: ReadRequestLogOptions = {}): Requ base.cacheHit = row.cache_hit === 1; } + if (row.trace_id !== null) { + base.traceId = row.trace_id; + } + return base; }); }; diff --git a/packages/observability/src/span-buffer.ts b/packages/observability/src/span-buffer.ts index 22af597149..613ff87fb4 100644 --- a/packages/observability/src/span-buffer.ts +++ b/packages/observability/src/span-buffer.ts @@ -9,7 +9,7 @@ * a real collector that can retain and index them across instances. */ import type { LogFields } from "../../../shared/log-fields"; -import type { SpanEvent } from "../../../shared/span-event"; +import type { SpanEvent, SpanEventPoint, SpanKind } from "../../../shared/span-event"; /* eslint-disable import/exports-last -- a data + types module: the public TraceSpan/TraceSummary shapes are declared next to the ring buffer and fold that produce them; grouping all exports at the end would scatter the contract. */ @@ -30,6 +30,16 @@ export interface TraceSpan { message: string; type: string; }; + + /** + * Timestamped occurrences inside the span — `span.addEvent(...)` and + * `span.recordException(...)`. Carried through so a handled retry or a + * swallowed exception is visible on the span it happened in, which is the + * only place it is interpretable. Absent when the body recorded none. + */ + events?: SpanEventPoint[]; + /** OTel `SpanKind`; absent means `"internal"`. */ + kind?: SpanKind; name: string; /** Start of this span relative to the trace's start, in ms. */ offsetMs: number; @@ -271,6 +281,8 @@ export const foldTraces = (spans: ReadonlyArray, limit: number = DEFA depth: depthOf(span), durationMs: span.durationMs, ...(span.error === undefined ? {} : { error: span.error }), + ...(span.events === undefined ? {} : { events: span.events }), + ...(span.kind === undefined ? {} : { kind: span.kind }), name: span.name, // Clamped at 0: a child whose parent was evicted can predate the // anchor, and a negative offset would render as a bar off-canvas. diff --git a/packages/studio/__tests__/app/studio.test.tsx b/packages/studio/__tests__/app/studio.test.tsx index ed0d7a2d9b..931b939c33 100644 --- a/packages/studio/__tests__/app/studio.test.tsx +++ b/packages/studio/__tests__/app/studio.test.tsx @@ -132,7 +132,7 @@ const QUEUE_METADATA_KEY_GUARD: KeysMatch = true; diff --git a/packages/studio/__tests__/features/logs/logs-panel.test.tsx b/packages/studio/__tests__/features/logs/logs-panel.test.tsx index 36be41c493..64bc245049 100644 --- a/packages/studio/__tests__/features/logs/logs-panel.test.tsx +++ b/packages/studio/__tests__/features/logs/logs-panel.test.tsx @@ -1,13 +1,15 @@ import { LunoraProvider } from "@lunora/react"; import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import type { LogFilterCriteria } from "../../../src/features/logs/logs-panel"; import { filterLogs, LogsPanel, summarizeLogs } from "../../../src/features/logs/logs-panel"; import type { LogEntry, RequestLogEntry } from "../../../src/lib/admin"; import { ADMIN_FUNCTIONS } from "../../../src/lib/admin"; +import { peekPendingTraceFilter } from "../../../src/lib/trace-handoff"; import type { MockClientHooks } from "../../mock-client"; import { createMockClient } from "../../mock-client"; +import wrapInRouter from "../../render-with-router"; const WINDOWED_ENTRY = /entry-\d+/; @@ -68,15 +70,23 @@ const createClient = (entries: LogEntry[] = ENTRIES, requests: RequestLogEntry[] }, }); -const renderPanel = (mock: MockClientHooks) => ( - - - -); - -/** Switch to the in-memory Errors view (the default view is the durable request log). */ -const switchToErrors = (): void => { - fireEvent.click(screen.getByTestId("lg-view-errors")); +// Under a router: a log row's "Trace" link navigates to `/traces`, so the panel +// now calls `useNavigate` and must mount inside a `RouterProvider`. +const renderPanel = (mock: MockClientHooks) => + wrapInRouter( + + + , + ); + +/** + * Switch to the in-memory Errors view (the default view is the durable request + * log). Awaits the toggle: the panel mounts under a `RouterProvider`, which + * resolves its route before rendering children, so nothing is in the DOM on the + * synchronous tick after `render`. + */ +const switchToErrors = async (): Promise => { + fireEvent.click(await screen.findByTestId("lg-view-errors")); }; describe("logsPanel — requests view (default)", () => { @@ -196,6 +206,54 @@ describe("logsPanel — requests view (default)", () => { }); }); +describe("logsPanel — trace drill-down", () => { + afterEach(() => { + sessionStorage.clear(); + }); + + it("hands a request row's trace to the Traces panel, scoped to the browsed shard", async () => { + expect.assertions(2); + + const traced: RequestLogEntry[] = [{ ...(REQUESTS[0] as RequestLogEntry), traceId: "trace-req" }]; + + render(renderPanel(createClient(ENTRIES, traced))); + + fireEvent.click(await screen.findByTestId("lg-trace-link")); + + // The hand-off, not the navigation, is the contract: Traces reads it on + // mount and pre-filters. Shard "" is the root DO the panel defaults to. + expect(peekPendingTraceFilter()?.traceId).toBe("trace-req"); + expect(peekPendingTraceFilter()?.shardKey).toBe(""); + }); + + it("shows no Trace link on a request row written before the trace_id column existed", async () => { + expect.assertions(2); + + render(renderPanel(createClient())); + + // The shared REQUESTS fixtures carry no `traceId` — the legacy-row shape. + await screen.findAllByTestId("lg-req-row"); + + expect(screen.queryByTestId("lg-trace-link")).toBeNull(); + expect(screen.getAllByTestId("lg-req-row")).toHaveLength(2); + }); + + it("hands an errors-view log line's trace over the same way", async () => { + expect.assertions(1); + + const traced: LogEntry[] = [{ ...(ENTRIES[0] as LogEntry), traceId: "trace-log" }]; + + render(renderPanel(createClient(traced))); + + await screen.findByTestId("lg-table"); + await switchToErrors(); + + fireEvent.click(await screen.findByTestId("lg-trace-link")); + + expect(peekPendingTraceFilter()?.traceId).toBe("trace-log"); + }); +}); + describe("logsPanel — errors view", () => { it("renders a row per captured log after switching to Errors", async () => { expect.assertions(3); @@ -203,7 +261,7 @@ describe("logsPanel — errors view", () => { render(renderPanel(createClient())); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); const rows = await screen.findAllByTestId("lg-row"); @@ -223,7 +281,7 @@ describe("logsPanel — errors view", () => { render(renderPanel(createClient(withFields))); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); const fields = await screen.findByTestId("lg-fields"); @@ -243,7 +301,7 @@ describe("logsPanel — errors view", () => { render(renderPanel(createClient([], []))); - switchToErrors(); + await switchToErrors(); const empty = await screen.findByTestId("lg-empty"); @@ -272,7 +330,7 @@ describe("logsPanel — errors view", () => { render(renderPanel(createClient(MIXED_ENTRIES))); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); await screen.findAllByTestId("lg-row"); fireEvent.change(screen.getByTestId("lg-search"), { target: { value: "BOOM" } }); @@ -290,7 +348,7 @@ describe("logsPanel — errors view", () => { render(renderPanel(createClient(MIXED_ENTRIES))); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); await screen.findAllByTestId("lg-row"); fireEvent.change(screen.getByTestId("lg-search"), { target: { value: "no-such-message" } }); @@ -306,7 +364,7 @@ describe("logsPanel — errors view", () => { render(renderPanel(createClient(MIXED_ENTRIES))); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); await screen.findAllByTestId("lg-row"); fireEvent.click(screen.getByTestId("logs-level-info")); @@ -323,7 +381,7 @@ describe("logsPanel — errors view", () => { render(renderPanel(createClient(MIXED_ENTRIES))); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); await screen.findAllByTestId("lg-row"); fireEvent.change(screen.getByTestId("lg-search"), { target: { value: "boom" } }); @@ -354,7 +412,7 @@ describe("logsPanel — errors view", () => { render(renderPanel(createClient(big))); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); const rows = await screen.findAllByTestId("lg-row"); @@ -373,7 +431,7 @@ describe("logsPanel — errors view", () => { render(renderPanel(mock)); await screen.findByTestId("lg-empty"); - switchToErrors(); + await switchToErrors(); // No Live toggle: switching to the Errors view subscribes to getLogs. await waitFor(() => { @@ -532,7 +590,7 @@ describe("logsPanel — advanced explorer (errors view)", () => { render(renderPanel(createClient(MIXED_ENTRIES))); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); await screen.findAllByTestId("lg-row"); fireEvent.click(screen.getByTestId("logs-level-info")); @@ -549,7 +607,7 @@ describe("logsPanel — advanced explorer (errors view)", () => { render(renderPanel(createClient(MIXED_ENTRIES))); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); await screen.findAllByTestId("lg-row"); fireEvent.change(screen.getByTestId("logs-path-filter"), { target: { value: "auth:" } }); @@ -566,7 +624,7 @@ describe("logsPanel — advanced explorer (errors view)", () => { render(renderPanel(createClient(MIXED_ENTRIES))); await screen.findByTestId("lg-table"); - switchToErrors(); + await switchToErrors(); await screen.findAllByTestId("lg-row"); fireEvent.click(screen.getByTestId("logs-summary-toggle")); diff --git a/packages/studio/__tests__/features/traces/trace-geometry.test.ts b/packages/studio/__tests__/features/traces/trace-geometry.test.ts index 35e54025f1..2213507957 100644 --- a/packages/studio/__tests__/features/traces/trace-geometry.test.ts +++ b/packages/studio/__tests__/features/traces/trace-geometry.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { filterTraces, formatSpanDuration, spanBar } from "../../../src/features/traces/trace-geometry"; +import { filterTraces, formatSpanDuration, spanBar, traceTicks } from "../../../src/features/traces/trace-geometry"; import type { TraceSpan, TraceSummary } from "../../../src/lib/admin"; const span = (overrides: Partial = {}): TraceSpan => { @@ -102,6 +102,55 @@ describe("filterTraces", () => { expect(all).not.toBe(traces); expect(all).toEqual(traces); }); + + it("matches on a nested span's name and id, not just the trace's own identifiers", () => { + expect.assertions(2); + + // A span id pasted from an error report or a traceparent header should find + // the trace CONTAINING it — the trace is rarely named after the span you have. + const withSpans = [trace({ spans: [span(), span({ name: "stripe.charge", spanId: "ffff9999" })], traceId: "ccc333" })]; + + expect(filterTraces(withSpans, "stripe.charge")).toHaveLength(1); + expect(filterTraces(withSpans, "ffff9999")).toHaveLength(1); + }); + + it("keeps only failed traces when onlyErrors is set", () => { + expect.assertions(2); + + const mixed = [trace({ ok: true, traceId: "ok1" }), trace({ ok: false, traceId: "bad1" })]; + + expect(filterTraces(mixed, "", true).map((t) => t.traceId)).toEqual(["bad1"]); + expect(filterTraces(mixed, "", false)).toHaveLength(2); + }); + + it("composes the error filter with the search term rather than replacing it", () => { + expect.assertions(1); + + const mixed = [trace({ ok: false, rootName: "signIn", traceId: "bad1" }), trace({ ok: false, rootName: "listMessages", traceId: "bad2" })]; + + expect(filterTraces(mixed, "signin", true).map((t) => t.traceId)).toEqual(["bad1"]); + }); +}); + +describe("traceTicks", () => { + it("spreads labelled gridlines across the trace, ending at its full duration", () => { + expect.assertions(2); + + const ticks = traceTicks(100); + + expect(ticks.map((tick) => tick.percent)).toEqual([25, 50, 75, 100]); + expect(ticks.map((tick) => tick.label)).toEqual(["25ms", "50ms", "75ms", "100ms"]); + }); + + it("draws no ruler for a duration spanBar would lay out full-width", () => { + expect.assertions(3); + + // Same guard as `spanBar`: with no timeline, there is nothing to annotate, + // and evenly-spaced "0ms" labels would imply a resolution the trace lacks. + expect(traceTicks(0)).toEqual([]); + expect(traceTicks(-5)).toEqual([]); + expect(traceTicks(Number.NaN)).toEqual([]); + }); }); describe("formatSpanDuration", () => { diff --git a/packages/studio/__tests__/features/traces/traces-panel.test.tsx b/packages/studio/__tests__/features/traces/traces-panel.test.tsx index 0a50ed8490..a2095e30d1 100644 --- a/packages/studio/__tests__/features/traces/traces-panel.test.tsx +++ b/packages/studio/__tests__/features/traces/traces-panel.test.tsx @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { filterTraces, formatSpanDuration, spanBar } from "../../../src/features/traces/trace-geometry"; import TracesPanel from "../../../src/features/traces/traces-panel"; -import type { TraceSpan, TraceSummary } from "../../../src/lib/admin"; +import type { LogEntry, TraceSpan, TraceSummary } from "../../../src/lib/admin"; import { ADMIN_FUNCTIONS } from "../../../src/lib/admin"; import { clearPendingTraceFilter, peekPendingTraceFilter, writePendingTraceFilter } from "../../../src/lib/trace-handoff"; import type { MockClientHooks } from "../../mock-client"; @@ -92,13 +92,31 @@ const ZERO_TRACE: TraceSummary = { traceId: "trace-zero", }; -const createClient = (traces: TraceSummary[] = TRACES): MockClientHooks => +/** + * Log lines the panel joins to a waterfall by `traceId`. `trace-orphan` has no + * trace at all — the shape of a container-lifecycle line, which must never be + * bucketed under some unrelated waterfall. + */ +const LOG_ENTRIES: LogEntry[] = [ + { functionPath: "messages:send", level: "info", message: "charged card", timestamp: 1_700_000_002_050, traceId: "trace-send" }, + { fields: { orderId: "o-1" }, functionPath: "messages:send", level: "warn", message: "retrying", timestamp: 1_700_000_002_010, traceId: "trace-send" }, + { functionPath: "messages:list", level: "info", message: "listed", timestamp: 1_700_000_001_000, traceId: "trace-list" }, + { level: "info", message: "container started", timestamp: 1_700_000_000_500 }, +]; + +const createClient = (traces: TraceSummary[] = TRACES, entries: LogEntry[] = LOG_ENTRIES): MockClientHooks => createMockClient({ query: (reference): unknown => { if (reference === ADMIN_FUNCTIONS.getTraces) { return { traces }; } + // The panel joins the live log ring to the waterfalls; `getLogs` is + // newest-first, as the real admin read returns it. + if (reference === ADMIN_FUNCTIONS.getLogs) { + return { entries }; + } + throw new Error(`unexpected ${reference}`); }, }); @@ -235,6 +253,146 @@ describe("tracesPanel", () => { expect(message.textContent).toContain("StripeError"); }); + it("keeps a span's detail collapsed until its row is clicked, then shows the full attribute bag", async () => { + expect.assertions(3); + + render(renderPanel(createClient())); + + fireEvent.click(await screen.findByTestId("tr-toggle-trace-send")); + await screen.findAllByTestId("tr-span-row"); + + expect(screen.queryByTestId("tr-span-detail")).toBeNull(); + + fireEvent.click(screen.getByTestId("tr-span-toggle-s1")); + + const detail = await screen.findByTestId("tr-span-detail"); + + // The row's chip is truncated to a fixed-width cell; the detail is the + // only place the id and the whole attribute bag are legible. + expect(detail.textContent).toContain("s1"); + expect(screen.getByTestId("tr-detail-attributes").textContent).toContain("o-1"); + }); + + it("shows a span's kind and recorded events in its detail", async () => { + expect.assertions(2); + + // Cast the indexed reads, not the whole literal: casting the object would + // switch off excess-property and missing-field checking on the fixture. + const traced: TraceSummary = { + ...(TRACES[1] as TraceSummary), + spans: [ + { + ...(LIST_SPANS[0] as TraceSpan), + events: [{ attributes: { "exception.type": "TimeoutError" }, name: "exception", ts: 1_700_000_001_004 }], + kind: "client", + }, + ], + }; + + render(renderPanel(createClient([traced]))); + + fireEvent.click(await screen.findByTestId("tr-toggle-trace-list")); + fireEvent.click(await screen.findByTestId("tr-span-toggle-l0")); + + const events = await screen.findByTestId("tr-detail-events"); + + expect(events.textContent).toContain("exception"); + expect(events.textContent).toContain("TimeoutError"); + }); + + it("shows the log lines the same dispatch emitted under the expanded trace", async () => { + expect.assertions(3); + + render(renderPanel(createClient())); + + fireEvent.click(await screen.findByTestId("tr-toggle-trace-send")); + + const logs = await screen.findByTestId("tr-logs"); + const rows = screen.getAllByTestId("tr-log-row"); + + // Only this trace's two lines: the other trace's line and the untraced + // container line must not be bucketed here. + expect(rows).toHaveLength(2); + expect(logs.textContent).toContain("charged card"); + // `getLogs` is newest-first; a trace's own lines read as a narrative. + expect(rows[0]?.textContent).toContain("retrying"); + }); + + it("does not read the log ring until a trace is actually expanded", async () => { + expect.assertions(2); + + const mock = createClient(); + const readsLogs = (): boolean => mock.query.mock.calls.some((call) => (call[0] as { __lunoraRef: string }).__lunoraRef === ADMIN_FUNCTIONS.getLogs); + + render(renderPanel(mock)); + + await screen.findByTestId("tr-row-trace-send"); + + // The page starts fully collapsed, so the correlated-log section is not on + // screen; reading it unconditionally would open a second live subscription + // and stream the whole ring on every visit for nothing. + expect(readsLogs()).toBe(false); + + fireEvent.click(screen.getByTestId("tr-toggle-trace-send")); + + await waitFor(() => { + expect(readsLogs()).toBe(true); + }); + }); + + it("omits the log section for a trace whose dispatch logged nothing", async () => { + expect.assertions(1); + + render(renderPanel(createClient(TRACES, []))); + + fireEvent.click(await screen.findByTestId("tr-toggle-trace-send")); + await screen.findAllByTestId("tr-span-row"); + + expect(screen.queryByTestId("tr-logs")).toBeNull(); + }); + + it("narrows the list to failed traces when 'Errors only' is checked", async () => { + expect.assertions(2); + + render(renderPanel(createClient([TRACES[0] as TraceSummary, ERROR_TRACE]))); + + await screen.findByTestId("tr-row-trace-send"); + + fireEvent.click(screen.getByTestId("tr-only-errors")); + + await waitFor(() => { + expect(screen.queryByTestId("tr-row-trace-send")).toBeNull(); + }); + + expect(screen.getByTestId("tr-row-trace-error")).toBeTruthy(); + }); + + it("draws an elapsed-time ruler across the expanded waterfall", async () => { + expect.assertions(2); + + render(renderPanel(createClient())); + + fireEvent.click(await screen.findByTestId("tr-toggle-trace-send")); + + const ruler = await screen.findByTestId("tr-ruler"); + + expect(ruler.textContent).toContain("25ms"); + expect(ruler.textContent).toContain("100ms"); + }); + + it("draws no ruler for a zero-duration trace, whose spans are all laid out full-width", async () => { + expect.assertions(1); + + // Its own render: asserting absence against a DOM that still holds a + // previous render's ruler would pass for the wrong reason. + render(renderPanel(createClient([ZERO_TRACE]))); + + fireEvent.click(await screen.findByTestId("tr-toggle-trace-zero")); + await screen.findAllByTestId("tr-span-bar"); + + expect(screen.queryByTestId("tr-ruler")).toBeNull(); + }); + it("renders visible bars for a zero-duration trace rather than NaN", async () => { expect.assertions(3); diff --git a/packages/studio/src/features/logs/log-line.tsx b/packages/studio/src/features/logs/log-line.tsx new file mode 100644 index 0000000000..58469874e3 --- /dev/null +++ b/packages/studio/src/features/logs/log-line.tsx @@ -0,0 +1,53 @@ +import type { ReactElement } from "react"; + +// Bundler-inlined, zero-dep `key=value` field renderer shared with the runtime +// sinks and the dev-terminal formatter (see CLAUDE.md `shared/` rules). +import { formatLogFields } from "../../../../../shared/log-fields"; +import { Badge } from "../../components/ui/badge"; +import type { LogEntry } from "../../lib/admin"; +import { formatTimestamp } from "../../lib/internal"; +import { LEVEL_VARIANT } from "./log-level-variant"; + +interface LogLineProps { + readonly entry: LogEntry; +} + +/** + * The cells of one `ctx.log` line — timestamp, severity, function path, and the + * message with its structured fields. + * + * A fragment of `gridcell`s rather than a row element, so each caller supplies + * its own row container: the Logs panel needs an absolutely-positioned, + * measured one for react-virtual, and the Traces panel's per-trace log section + * needs a plain one. Both must render the SAME line — they did not when this was + * duplicated, having already drifted on the level column's width and on whether + * the timestamp went through {@link formatTimestamp}. + */ +const LogLine = ({ entry }: LogLineProps): ReactElement => { + // Rendered once; `""` (no fields, or an empty bag from a worker predating + // field normalization) skips the chip entirely rather than showing a blank span. + const fields = formatLogFields(entry.fields); + + return ( + <> + + {formatTimestamp(entry.timestamp)} + + + {entry.level} + + + {entry.functionPath ?? "—"} + + + {entry.message} + {fields === "" ? null : ( + + {fields} + + )} + + + ); +}; +export default LogLine; diff --git a/packages/studio/src/features/logs/logs-panel.tsx b/packages/studio/src/features/logs/logs-panel.tsx index e737d64e5d..1488f24f7b 100644 --- a/packages/studio/src/features/logs/logs-panel.tsx +++ b/packages/studio/src/features/logs/logs-panel.tsx @@ -11,6 +11,7 @@ import ErrorAlert from "../../components/error-alert"; import { Badge } from "../../components/ui/badge"; import { EmptyState } from "../../components/ui/empty-state"; import { useAdminQuery } from "../../hooks/use-admin-query"; +import useOpenTrace from "../../hooks/use-open-trace"; import { useShardKey } from "../../hooks/use-shard-key"; import { useT } from "../../i18n/i18n-context"; import type { LogEntry, LogLevel, RequestLogEntry, RequestLogQuery, RequestOutcome } from "../../lib/admin"; @@ -19,7 +20,7 @@ import { recordShard } from "../../lib/shard-history"; import flooredRectObserver from "../../lib/virtual-rect"; import { ArchiveFeed } from "./archive-feed"; import type { BadgeVariant } from "./log-level-variant"; -import { LEVEL_VARIANT } from "./log-level-variant"; +import LogLine from "./log-line"; import LogsErrorFilters from "./logs-error-filters"; import LogsRequestFilters from "./logs-request-filters"; import type { LogSummary, SummaryBucket } from "./logs-summary"; @@ -50,11 +51,49 @@ const OUTCOME_VARIANT: Record = { ok: "secondary", }; +interface TraceLinkCellProps { + /** Open the Traces panel filtered to this row's trace. */ + readonly onOpenTrace: (traceId: string) => void; + /** Absent for a row with no ambient trace, or one predating the field. */ + readonly traceId?: string; +} + +/** + * The trailing "Trace" cell shared by both log views — the drill-down from a + * line to the waterfall it came from, instead of matching timestamps by eye. + * + * Renders an empty cell rather than nothing when there is no trace, so the + * column width stays constant down the virtualized list. The two views are + * mutually exclusive, so both use one test id. + */ +const TraceLinkCell = ({ onOpenTrace, traceId }: TraceLinkCellProps): ReactElement => { + const t = useT(); + + return ( + + {traceId === undefined ? null : ( + + )} + + ); +}; + interface LogRowProps { readonly entry: LogEntry; readonly index: number; /** react-virtual's ref-callback that measures the rendered row. */ readonly measureRef: (node: HTMLDivElement | null) => void; + /** Open the Traces panel filtered to this line's trace. */ + readonly onOpenTrace: (traceId: string) => void; /** Pixel offset of this row from the top of the virtualized list. */ readonly start: number; } @@ -65,11 +104,8 @@ interface LogRowProps { * offset) is a `useMemo`-stable reference — keeping the hot map body free of * fresh inline objects. */ -const LogRow = ({ entry, index, measureRef, start }: LogRowProps): ReactElement => { +const LogRow = ({ entry, index, measureRef, onOpenTrace, start }: LogRowProps): ReactElement => { const style = { ...ROW_BASE_STYLE, transform: `translateY(${String(start)}px)` }; - // Rendered once; `""` (no fields, or an empty bag from a worker predating - // field normalization) skips the chip entirely rather than showing a blank span. - const fields = formatLogFields(entry.fields); return (
- - {new Date(entry.timestamp).toLocaleString()} - - - {entry.level} - - - {entry.functionPath ?? "—"} - - - {entry.message} - {fields === "" ? null : ( - - {fields} - - )} - + +
); }; @@ -105,6 +126,8 @@ interface RequestRowProps { readonly entry: RequestLogEntry; readonly index: number; readonly measureRef: (node: HTMLDivElement | null) => void; + /** Open the Traces panel filtered to this dispatch's trace. */ + readonly onOpenTrace: (traceId: string) => void; readonly start: number; } @@ -116,7 +139,7 @@ const tablesLabel = (tables: string[]): string => (tables.length === 0 ? "—" : * log adds over the in-memory error buffer: function path, shard, acting user, * outcome, duration, and the read/written table sets. */ -const RequestRow = ({ entry, index, measureRef, start }: RequestRowProps): ReactElement => { +const RequestRow = ({ entry, index, measureRef, onOpenTrace, start }: RequestRowProps): ReactElement => { const style = { ...ROW_BASE_STYLE, transform: `translateY(${String(start)}px)` }; return ( @@ -151,6 +174,7 @@ const RequestRow = ({ entry, index, measureRef, start }: RequestRowProps): React ? entry.errorMessage : `r:${tablesLabel(entry.tablesRead)} · w:${tablesLabel(entry.tablesWritten)}`} + ); }; @@ -364,6 +388,8 @@ export const LogsPanel = ({ initialShardKey }: LogsPanelProps): ReactElement => const activeQuery = view === "requests" ? requestsQuery : errorsQuery; const { error, errorSource, isLoading: activeLoading, liveError } = activeQuery; + const openTrace = useOpenTrace(queryShardKey); + // Coerce each view's resolved payload into its entries array (a one-shot read // or live push without an `entries` array yields `[]`); an unloaded gated // query's `undefined` data also yields `[]`. @@ -582,6 +608,7 @@ export const LogsPanel = ({ initialShardKey }: LogsPanelProps): ReactElement => index={virtualRow.index} key={virtualRow.key} measureRef={virtualizer.measureElement} + onOpenTrace={openTrace} start={virtualRow.start} /> ) : ( @@ -590,6 +617,7 @@ export const LogsPanel = ({ initialShardKey }: LogsPanelProps): ReactElement => index={virtualRow.index} key={virtualRow.key} measureRef={virtualizer.measureElement} + onOpenTrace={openTrace} start={virtualRow.start} /> ), diff --git a/packages/studio/src/features/reports/metrics-panel.tsx b/packages/studio/src/features/reports/metrics-panel.tsx index e4cb39aae3..c8dcd70eb8 100644 --- a/packages/studio/src/features/reports/metrics-panel.tsx +++ b/packages/studio/src/features/reports/metrics-panel.tsx @@ -1,5 +1,4 @@ import { useLunora } from "@lunora/react"; -import { useNavigate } from "@tanstack/react-router"; import type { ReactElement } from "react"; import { useEffect, useMemo, useRef, useState } from "react"; @@ -7,6 +6,7 @@ import { LiveError } from "../../components/live-status"; import { ShardInput } from "../../components/shard-input"; import { Button } from "../../components/ui/button"; import { useAdminQuery } from "../../hooks/use-admin-query"; +import useOpenTrace from "../../hooks/use-open-trace"; import { useShardKey } from "../../hooks/use-shard-key"; import { useT } from "../../i18n/i18n-context"; import type { MetricsSnapshot, ShardMetrics } from "../../lib/admin"; @@ -14,7 +14,6 @@ import { ADMIN_FUNCTIONS } from "../../lib/admin"; import { CLOUDFLARE_DURABLE_OBJECTS_URL } from "../../lib/cf-links"; import { adminRef, callOptions, errorMessage, fireAndForget } from "../../lib/internal"; import { loadRecentShards, recordShard } from "../../lib/shard-history"; -import { writePendingTraceFilter } from "../../lib/trace-handoff"; import { InstrumentsTable } from "./instruments-table"; import type { ShardMetricsResult } from "./metrics-aggregate"; import { aggregateMetrics, computeLatencyPercentiles, enrichQueryStats, shardsToAggregate } from "./metrics-aggregate"; @@ -59,7 +58,6 @@ const resolveTab = (hasQueryStats: boolean, activeTab: "overview" | "query-insig export const MetricsPanel = ({ initialShardKey }: MetricsPanelProps): ReactElement => { const client = useLunora(); const t = useT(); - const navigate = useNavigate(); const { queryShardKey, setShardKey, shardKey } = useShardKey(initialShardKey); /** Active panel tab: "overview" (default) or "query-insights" (shown when queryStats present). */ @@ -83,14 +81,9 @@ export const MetricsPanel = ({ initialShardKey }: MetricsPanelProps): ReactEleme }; }, []); - // Exemplar drill-down: stash the trace id — and the shard the series was queried - // on, so the Traces panel searches the right ring rather than the root — for the - // Traces panel to pick up and pre-filter on mount, then navigate there. A - // one-shot handoff keeps the two panels decoupled from the router's search schema. - const openTrace = (traceId: string): void => { - writePendingTraceFilter({ shardKey: queryShardKey, traceId }); - fireAndForget(navigate({ to: "/traces" })); - }; + // Exemplar drill-down: open the trace a metric point sampled, on the shard the + // series was queried from. + const openTrace = useOpenTrace(queryShardKey); // One-shot read + always-on live subscription for the committed shard. Each // server push folds in like a refresh; `liveError` holds a rejection message diff --git a/packages/studio/src/features/traces/span-detail.tsx b/packages/studio/src/features/traces/span-detail.tsx new file mode 100644 index 0000000000..a145852953 --- /dev/null +++ b/packages/studio/src/features/traces/span-detail.tsx @@ -0,0 +1,110 @@ +import type { ReactElement } from "react"; + +import { Badge } from "../../components/ui/badge"; +import { useT } from "../../i18n/i18n-context"; +import type { TraceSpan } from "../../lib/admin"; +import { formatTimestamp } from "../../lib/internal"; +import { formatSpanDuration } from "./trace-geometry"; + +/** + * Render one attribute value for the detail list. Strings print as-is; anything + * structured is JSON so a nested bag stays inspectable rather than collapsing to + * `[object Object]`. A value that cannot be encoded (a cycle surviving + * normalization) falls back to `String(value)` — the row is worth keeping even + * when its value is unprintable. + */ +const formatValue = (value: unknown): string => { + if (typeof value === "string") { + return value; + } + + try { + return JSON.stringify(value); + } catch { + return String(value); + } +}; + +interface DetailFieldProps { + readonly label: string; + readonly value: string; +} + +/** One `label / value` pair of the detail grid, wrapping rather than truncating. */ +const DetailField = ({ label, value }: DetailFieldProps): ReactElement => ( +
+
{label}
+
{value}
+
+); + +interface SpanDetailProps { + readonly span: TraceSpan; +} + +/** + * The expanded detail for one waterfall row: identity, timing, kind, the span's + * FULL attribute bag, its error, and its recorded span events. + * + * The waterfall row itself can only afford a truncated one-line attribute chip, + * which is enough to notice an attribute exists and not enough to read it — the + * `{ orderId, amount, retryCount }` you opened the panel for is exactly what gets + * cut. Everything here is already in the `getTraces` payload; this only stops + * throwing it away at render time. + */ +const SpanDetail = ({ span }: SpanDetailProps): ReactElement => { + const t = useT(); + const attributes = Object.entries(span.attributes ?? {}); + const events = span.events ?? []; + + return ( +
+
+ + + + + +
+ + {span.error !== undefined && ( +
+ {span.error.type} + {/* Pre-wrapped: a stack-ish message keeps its line breaks here, which is the only place in the panel with room for them. */} +

{span.error.message}

+
+ )} + + {attributes.length > 0 && ( +
+

{t("Attributes")}

+
+ {attributes.map(([key, value]) => ( + + ))} +
+
+ )} + + {events.length > 0 && ( +
+

{t("Events")}

+
    + {events.map((event) => ( +
  • + {formatTimestamp(event.ts)} + {event.name} + + {Object.entries(event.attributes ?? {}) + .map(([key, value]) => `${key}=${formatValue(value)}`) + .join(" ")} + +
  • + ))} +
+
+ )} +
+ ); +}; +export default SpanDetail; diff --git a/packages/studio/src/features/traces/trace-geometry.ts b/packages/studio/src/features/traces/trace-geometry.ts index a1d716f521..1a75bf5ee1 100644 --- a/packages/studio/src/features/traces/trace-geometry.ts +++ b/packages/studio/src/features/traces/trace-geometry.ts @@ -20,6 +20,9 @@ import type { TraceSpan, TraceSummary } from "../../lib/admin"; */ const MIN_BAR_PERCENT = 0.5; +/** Gridlines drawn across a waterfall, excluding the 0% edge (the track's own border reads as that). */ +const TICK_COUNT = 4; + /** Clamp `value` into the inclusive `[0, 100]` percentage range. */ const clampPercent = (value: number): number => Math.min(100, Math.max(0, value)); @@ -61,26 +64,40 @@ export const spanBar = (span: TraceSpan, traceDurationMs: number): SpanBar => { /** * Narrow the trace list by a case-insensitive substring over the trace's root - * span name, function path, and full trace id — the identifiers the row shows, - * plus the id so a metric exemplar can filter straight to its trace. An empty or - * whitespace-only term matches everything, so an untouched control never hides a - * trace. + * span name, function path, full trace id, and every span's name and id — so + * pasting a span id from an error report or a `traceparent` header finds the + * trace that contains it, not just the trace it happens to be named after. An + * empty or whitespace-only term matches everything, so an untouched control never + * hides a trace. * @param traces The loaded traces, newest first. * @param search The raw search-box value. + * @param onlyErrors Keep only traces where the root or a descendant span threw. */ -export const filterTraces = (traces: ReadonlyArray, search: string): TraceSummary[] => { +export const filterTraces = (traces: ReadonlyArray, search: string, onlyErrors = false): TraceSummary[] => { const needle = search.trim().toLowerCase(); + const byStatus = onlyErrors ? traces.filter((trace) => !trace.ok) : traces; if (needle === "") { - return [...traces]; + return [...byStatus]; } - return traces.filter( + return byStatus.filter( (trace) => - trace.rootName.toLowerCase().includes(needle) || trace.functionPath.toLowerCase().includes(needle) || trace.traceId.toLowerCase().includes(needle), + trace.rootName.toLowerCase().includes(needle) || + trace.functionPath.toLowerCase().includes(needle) || + trace.traceId.toLowerCase().includes(needle) || + trace.spans.some((span) => span.name.toLowerCase().includes(needle) || span.spanId.toLowerCase().includes(needle)), ); }; +/** One labelled gridline of the waterfall's time ruler. */ +export interface TraceTick { + /** Elapsed time at this gridline, pre-formatted. */ + readonly label: string; + /** Distance from the left edge of the timeline track, in percent. */ + readonly percent: number; +} + /** * Render a millisecond duration for a waterfall row. Sub-millisecond spans are * shown to two decimals so an instant span reads as `0.4ms` rather than @@ -94,3 +111,29 @@ export const formatSpanDuration = (durationMs: number): string => { return durationMs > 0 && durationMs < 1 ? `${durationMs.toFixed(2)}ms` : `${String(Math.round(durationMs))}ms`; }; + +/** + * Evenly-spaced elapsed-time gridlines for a trace's timeline. + * + * A waterfall without a ruler shows the *shape* of a request but not where in it + * a span sits — you can see one bar starts later than another and have to guess + * by how much. Even spacing rather than "nice" round intervals keeps this to + * arithmetic: the labels are derived from the duration, so they land on whatever + * values the trace actually has. + * + * A non-positive or non-finite duration yields no ticks, matching {@link spanBar}, + * which lays every span out full-width in that case — there is no timeline to + * annotate. + * @param traceDurationMs The enclosing trace's total duration, in ms. + */ +export const traceTicks = (traceDurationMs: number): TraceTick[] => { + if (!Number.isFinite(traceDurationMs) || traceDurationMs <= 0) { + return []; + } + + return Array.from({ length: TICK_COUNT }, (_unused, index) => { + const fraction = (index + 1) / TICK_COUNT; + + return { label: formatSpanDuration(traceDurationMs * fraction), percent: fraction * 100 }; + }); +}; diff --git a/packages/studio/src/features/traces/trace-waterfall.tsx b/packages/studio/src/features/traces/trace-waterfall.tsx new file mode 100644 index 0000000000..fa160e22f2 --- /dev/null +++ b/packages/studio/src/features/traces/trace-waterfall.tsx @@ -0,0 +1,187 @@ +import type { CSSProperties, ReactElement } from "react"; +import { useState } from "react"; + +// Bundler-inlined, zero-dep `key=value` field renderer shared with the runtime +// sinks and the dev-terminal formatter (see CLAUDE.md `shared/` rules). +import { formatLogFields } from "../../../../../shared/log-fields"; +import { useT } from "../../i18n/i18n-context"; +import type { TraceSpan } from "../../lib/admin"; +import { cn } from "../../lib/utils"; +import SpanDetail from "./span-detail"; +import { formatSpanDuration, spanBar, traceTicks } from "./trace-geometry"; + +/** Pixels of indent per nesting level of a span row. */ +const INDENT_PER_DEPTH = 14; + +/** + * The waterfall's column track, shared by the ruler and every span row. + * + * One definition rather than matching `w-*` classes in two components: the ruler + * annotates the bar column, so if the two drift the gridlines label the wrong + * position — a silent, purely visual failure no test would catch. Columns are + * name / bar / duration / error / attributes. + */ +const WATERFALL_GRID = "grid grid-cols-[14rem_1fr_4rem_16rem_16rem] items-center gap-2 px-3"; + +interface TimeRulerProps { + /** Total duration the ruler annotates, in ms. */ + readonly traceDurationMs: number; +} + +/** + * Elapsed-time gridlines across the waterfall's bar column. Renders nothing for + * a zero-duration trace, where {@link spanBar} lays every span out full-width + * and there is no timeline to annotate. + */ +const TimeRuler = ({ traceDurationMs }: TimeRulerProps): ReactElement | null => { + const ticks = traceTicks(traceDurationMs); + + if (ticks.length === 0) { + return null; + } + + return ( +
+ + + {ticks.map((tick) => ( + + {tick.label} + + ))} + +
+ ); +}; + +interface SpanRowProps { + readonly span: TraceSpan; + /** The enclosing trace's total duration, the denominator for the bar geometry. */ + readonly traceDurationMs: number; +} + +/** + * One waterfall row: the span's name indented by its server-computed `depth`, a + * bar positioned by `offsetMs` and sized by `durationMs`, its duration, its + * structured attributes, and — when the body threw — an error chip plus message. + * + * Clicking the row opens {@link SpanDetail} beneath it. The row's attribute and + * error cells are hard-truncated to keep the waterfall scannable, so the detail + * block is the only place the full values are legible. + */ +const SpanRow = ({ span, traceDurationMs }: SpanRowProps): ReactElement => { + const t = useT(); + const [expanded, setExpanded] = useState(false); + const { leftPercent, widthPercent } = spanBar(span, traceDurationMs); + + // Rendered once; `""` (no attributes) skips the chip entirely rather than + // showing a blank span — the same convention as the Logs panel's fields. + const attributes = formatLogFields(span.attributes); + + const nameStyle: CSSProperties = { paddingLeft: span.depth * INDENT_PER_DEPTH }; + const barStyle: CSSProperties = { left: `${String(leftPercent)}%`, width: `${String(widthPercent)}%` }; + + const onClick = (): void => { + setExpanded((open) => !open); + }; + + return ( +
  • + {/* + * One explicit label rather than a name computed from the row's five + * cells: as a button, every child's text (and the bar's own label) + * would concatenate into a run-on string re-read on each arrow-key + * move. The cells stay visible; only the announcement is curated. + */} + + + {expanded && ( +
    + +
    + )} +
  • + ); +}; + +interface TraceWaterfallProps { + /** Total duration of the trace, the denominator for every bar. */ + readonly durationMs: number; + /** Spans pre-ordered by `(offsetMs, depth)` — a valid pre-order of the span tree. */ + readonly spans: ReadonlyArray; +} + +/** A trace's time ruler plus one expandable row per span. */ +const TraceWaterfall = ({ durationMs, spans }: TraceWaterfallProps): ReactElement => { + const t = useT(); + + return ( + <> + +
      + {spans.map((span) => ( + + ))} +
    + + ); +}; +export default TraceWaterfall; diff --git a/packages/studio/src/features/traces/traces-panel.tsx b/packages/studio/src/features/traces/traces-panel.tsx index 0aead10d5c..147b80dceb 100644 --- a/packages/studio/src/features/traces/traces-panel.tsx +++ b/packages/studio/src/features/traces/traces-panel.tsx @@ -1,29 +1,25 @@ -import type { ChangeEvent, CSSProperties, ReactElement } from "react"; +import type { ChangeEvent, ReactElement } from "react"; import { useEffect, useState } from "react"; -// Bundler-inlined, zero-dep `key=value` field renderer shared with the runtime -// sinks and the dev-terminal formatter (see CLAUDE.md `shared/` rules). -import { formatLogFields } from "../../../../../shared/log-fields"; import ErrorAlert from "../../components/error-alert"; import { LiveError } from "../../components/live-status"; import { ShardInput } from "../../components/shard-input"; import { Badge } from "../../components/ui/badge"; +import { Checkbox } from "../../components/ui/checkbox"; import { EmptyState } from "../../components/ui/empty-state"; import { Input } from "../../components/ui/input"; import { useAdminQuery } from "../../hooks/use-admin-query"; import { useShardKey } from "../../hooks/use-shard-key"; import { useT } from "../../i18n/i18n-context"; -import type { TraceSpan, TracesResult, TraceSummary } from "../../lib/admin"; +import type { LogEntry, LogsResult, TracesResult, TraceSummary } from "../../lib/admin"; import { ADMIN_FUNCTIONS } from "../../lib/admin"; import { formatTimestamp } from "../../lib/internal"; import { recordShard } from "../../lib/shard-history"; import type { PendingTraceFilter } from "../../lib/trace-handoff"; import { clearPendingTraceFilter, peekPendingTraceFilter } from "../../lib/trace-handoff"; -import { cn } from "../../lib/utils"; -import { filterTraces, formatSpanDuration, spanBar } from "./trace-geometry"; - -/** Pixels of indent per nesting level of a span row. */ -const INDENT_PER_DEPTH = 14; +import LogLine from "../logs/log-line"; +import { filterTraces, formatSpanDuration } from "./trace-geometry"; +import TraceWaterfall from "./trace-waterfall"; /** * Coerce a (possibly partial or malformed) `getTraces` result into its `traces` @@ -32,87 +28,43 @@ const INDENT_PER_DEPTH = 14; */ const tracesOf = (result: TracesResult | undefined): TraceSummary[] => (Array.isArray(result?.traces) ? result.traces : []); -interface SpanRowProps { - readonly span: TraceSpan; - /** The enclosing trace's total duration, the denominator for the bar geometry. */ - readonly traceDurationMs: number; +interface TraceLogsProps { + /** Lines emitted under this trace, oldest first. */ + readonly entries: ReadonlyArray; } /** - * One waterfall row: the span's name indented by its server-computed `depth`, a - * bar positioned by `offsetMs` and sized by `durationMs`, its duration, its - * structured attributes, and — when the body threw — an error chip plus message. + * The `ctx.log` lines emitted under one trace, shown beneath its waterfall — the + * waterfall says WHICH span was slow or threw, these say what the code thought + * it was doing at the time. + * + * Read from the same live `getLogs` ring the Logs panel uses, so no extra RPC and + * no extra retention: a line evicted from that ring disappears here too. */ -const SpanRow = ({ span, traceDurationMs }: SpanRowProps): ReactElement => { +const TraceLogs = ({ entries }: TraceLogsProps): ReactElement => { const t = useT(); - const { leftPercent, widthPercent } = spanBar(span, traceDurationMs); - - // Rendered once; `""` (no attributes) skips the chip entirely rather than - // showing a blank span — the same convention as the Logs panel's fields. - const attributes = formatLogFields(span.attributes); - - const nameStyle: CSSProperties = { paddingLeft: span.depth * INDENT_PER_DEPTH }; - const barStyle: CSSProperties = { left: `${String(leftPercent)}%`, width: `${String(widthPercent)}%` }; return ( -
    - - {span.ok ? null : ( - - ● - - )} - {span.name} - - {/* - * The bar cell carries no text, so a screen reader would announce an - * empty gridcell. Label it with the waterfall position it encodes — - * where in the trace the span starts and how long it ran — which is - * otherwise available only visually. - */} - - - - - {formatSpanDuration(span.durationMs)} - - {/* - * Error and attributes get their own cells rather than sharing one. - * Sharing meant a failed span dropped its attributes — hiding - * `{ orderId }` on exactly the span you opened the panel to debug. - */} - - {span.error === undefined ? null : ( - - {`${span.error.type}: ${span.error.message}`} - - )} - - - {attributes === "" ? null : ( - - {attributes} - - )} - +
    +

    {t("Logs ({count})", { count: entries.length })}

    +
    + {/* + * Keyed by position, not by content: two identical messages logged + * in the same millisecond are ordinary (a retry loop) and would + * collide on any content-derived key. The bucket is rebuilt whole + * from an append-only ring per render, so an index is stable. + */} + {entries.map((entry, index) => ( +
    + +
    + ))} +
    ); }; @@ -120,6 +72,8 @@ const SpanRow = ({ span, traceDurationMs }: SpanRowProps): ReactElement => { interface TraceRowProps { /** Whether this trace's waterfall is currently expanded. */ readonly expanded: boolean; + /** `ctx.log` lines emitted under this trace, oldest first. Empty when none were captured. */ + readonly logs: ReadonlyArray; /** Lifts the per-row click out of the map so the row carries no inline closure. */ readonly onToggle: (traceId: string) => void; readonly trace: TraceSummary; @@ -127,9 +81,10 @@ interface TraceRowProps { /** * One trace in the list: a clickable summary header (root name, function path, - * total duration, span count, ok/error badge) that expands into the waterfall. + * total duration, span count, ok/error badge) that expands into the waterfall + * and the log lines the same dispatch emitted. */ -const TraceRow = ({ expanded, onToggle, trace }: TraceRowProps): ReactElement => { +const TraceRow = ({ expanded, logs, onToggle, trace }: TraceRowProps): ReactElement => { const t = useT(); const onClick = (): void => { @@ -166,16 +121,9 @@ const TraceRow = ({ expanded, onToggle, trace }: TraceRowProps): ReactElement => {expanded && ( -
    - {trace.spans.map((span) => ( - - ))} +
    + + {logs.length > 0 && }
    )}
    @@ -196,8 +144,10 @@ interface TracesPanelProps { * Traces are the drill-down from a log line: a dispatch's synthetic root span * plus every `ctx.trace(name, fn)` span recorded beneath it. The server folds * them (`@lunora/do`'s pure `foldTraces`) and stamps each span with a `depth` - * and an `offsetMs`, so this panel does no tree math — it indents by `depth` and - * sizes each bar from `offsetMs`/`durationMs` via the pure {@link spanBar} helper. + * and an `offsetMs`, so nothing here does tree math — see `trace-waterfall.tsx`. + * + * Expanding a trace also brings up the `ctx.log` lines the same dispatch emitted, + * joined by `traceId` from the live log ring. * * The backing span ring is bounded and in-memory, so it resets on hibernation or * restart and a trace can legitimately arrive partial. This is a local-development @@ -219,6 +169,7 @@ const TracesPanel = ({ initialShardKey }: TracesPanelProps): ReactElement => { pending?.shardKey !== undefined && pending.shardKey !== "" ? pending.shardKey : (initialShardKey ?? ""), ); const [search, setSearch] = useState(pending?.traceId ?? ""); + const [onlyErrors, setOnlyErrors] = useState(false); const [expanded, setExpanded] = useState>(() => new Set()); // Consume the hand-off on mount (a committed boundary): clear the store so a @@ -235,6 +186,17 @@ const TracesPanel = ({ initialShardKey }: TracesPanelProps): ReactElement => { { live: true, shardKey: queryShardKey }, ); + // The same live log ring the Logs panel reads, joined to the waterfalls by + // `traceId` — a client-side join of two reads already pushed over this socket, + // rather than a bespoke server-side join that would only ever serve this panel. + // + // Gated on an open trace: the log section renders only inside an expanded row, + // so an unconditional read would open a second live subscription and stream the + // whole ring on every visit to a page that starts fully collapsed. Its failure + // is deliberately not surfaced — losing the log ring must degrade the trace + // detail, never blank the waterfalls the panel exists to show. + const { data: logsData } = useAdminQuery(ADMIN_FUNCTIONS.getLogs, {}, { enabled: expanded.size > 0, live: true, shardKey: queryShardKey }); + // Record the browsed shard into recent-shards history once the read resolves, // so a shard visited here shows up in every other panel's autocomplete — // matching the nine other shard-scoped panels. @@ -252,7 +214,12 @@ const TracesPanel = ({ initialShardKey }: TracesPanelProps): ReactElement => { const truncated = total > traces.length; // The span ring is not a queryable store, so the search runs client-side over // the loaded traces (the same model as the Logs panel's Errors view). - const filtered = filterTraces(traces, search); + const filtered = filterTraces(traces, search, onlyErrors); + + // `getLogs` is newest-first; a trace's own lines read as a narrative, so + // reverse into chronological order before bucketing. Untraced lines land in an + // `undefined` bucket that no lookup can reach, which is the correct home. + const logsByTrace = Map.groupBy((logsData?.entries ?? []).toReversed(), (entry) => entry.traceId); const onSearchChange = (event: ChangeEvent): void => { setSearch(event.target.value); @@ -286,9 +253,13 @@ const TracesPanel = ({ initialShardKey }: TracesPanelProps): ReactElement => { className="h-8 w-56" data-testid="tr-search" onChange={onSearchChange} - placeholder={t("search trace or function")} + placeholder={t("search trace, span, or function")} value={search} /> +
    @@ -326,7 +297,13 @@ const TracesPanel = ({ initialShardKey }: TracesPanelProps): ReactElement => { {error === null && filtered.length > 0 && (
    {filtered.map((trace) => ( - + ))}
    )} diff --git a/packages/studio/src/hooks/use-open-trace.ts b/packages/studio/src/hooks/use-open-trace.ts new file mode 100644 index 0000000000..2817e8d335 --- /dev/null +++ b/packages/studio/src/hooks/use-open-trace.ts @@ -0,0 +1,26 @@ +import { useNavigate } from "@tanstack/react-router"; + +import { fireAndForget } from "../lib/internal"; +import { writePendingTraceFilter } from "../lib/trace-handoff"; + +/** + * The trace drill-down every panel with a trace id shares: stash the one-shot + * hand-off — with the shard the row was read from, so Traces searches that ring + * rather than the root's — then navigate to the Traces page, which consumes the + * hand-off on mount and pre-filters. + * + * Bundling the navigate with the write is the point: a caller that stashes but + * forgets to navigate leaves a filter that silently applies to the user's NEXT + * manual visit to Traces. The Metrics exemplar link and both Logs views drive + * the same two steps, so they drive them through here. + * @param shardKey Shard the calling panel is reading, carried into the hand-off. + */ +const useOpenTrace = (shardKey: string): ((traceId: string) => void) => { + const navigate = useNavigate(); + + return (traceId: string): void => { + writePendingTraceFilter({ shardKey, traceId }); + fireAndForget(navigate({ to: "/traces" })); + }; +}; +export default useOpenTrace; diff --git a/packages/studio/src/lib/admin.ts b/packages/studio/src/lib/admin.ts index 220ad58026..eef1039b13 100644 --- a/packages/studio/src/lib/admin.ts +++ b/packages/studio/src/lib/admin.ts @@ -762,6 +762,9 @@ export interface LogEntry { level: LogLevel; message: string; timestamp: number; + + /** Trace the line was emitted under; absent outside a dispatch (container lifecycle, hibernation-path errors) and on any worker predating the field. The key the Logs panel's Trace link and the Traces panel's correlated logs both join on. */ + traceId?: string; } /** Payload of a `__lunora_admin__:getLogs` call: the buffered entries, newest first. */ @@ -942,6 +945,9 @@ export interface RequestLogEntry { tablesRead: string[]; /** Tables the handler wrote; empty for a read-only dispatch. */ tablesWritten: string[]; + + /** W3C trace id (32-hex) of the dispatch; absent on a row written before the column existed. The Trace link is best-effort — see `@lunora/observability`'s `request-log.ts` on the retention asymmetry. */ + traceId?: string; /** Epoch-ms the dispatch completed. */ ts: number; /** Acting userId forwarded by the runtime; absent when anonymous. */ @@ -1042,6 +1048,20 @@ export interface TraceSpan { message: string; type: string; }; + + /** + * Timestamped occurrences inside the span — `span.addEvent(...)` / + * `span.recordException(...)`. A handled retry or a swallowed exception, which + * only makes sense against the span it happened in. Absent when none. + */ + events?: { + attributes?: Record; + name: string; + /** Epoch-ms the event was recorded. */ + ts: number; + }[]; + /** OTel `SpanKind`. Absent means `"internal"` — the overwhelming majority of `ctx.trace` spans. */ + kind?: "client" | "consumer" | "internal" | "producer" | "server"; /** Caller-supplied span name, e.g. `"stripe.charge"`. */ name: string; /** Start of this span relative to the trace's start, in ms. Clamped at 0 server-side. */ diff --git a/packages/studio/src/locales/en.ts b/packages/studio/src/locales/en.ts index 33a3799753..7151b97ba1 100644 --- a/packages/studio/src/locales/en.ts +++ b/packages/studio/src/locales/en.ts @@ -1254,14 +1254,24 @@ const MESSAGE_IDS = [ "Traces", "Recent ctx.trace waterfalls for this shard — the drill-down from a log line.", "Search traces", - "search trace or function", + "search trace, span, or function", + "Errors only", "Recent traces", "Trace waterfall", - "Errored span", "{count} spans", - "starts {offset} in, runs {duration}", + "{name}, {duration}, starts {offset} in{failure}", + ", failed: {message}", "Showing the {shown} most recent of {total} traces in the buffer.", "No traces", + // Observability — expanded span detail + the dispatch's correlated log lines + "Span ID", + "Parent span ID", + "— (trace root)", + "Started at", + "+{offset}", + "Duration", + "Attributes", + "Logs ({count})", "Traces recorded with ctx.trace show up here as your app handles traffic. The span buffer resets when the shard hibernates.", // Observability — grouped error Issues "Issues",