diff --git a/api-snapshots/lunora.api.md b/api-snapshots/lunora.api.md index eb18269776..23d9c39a8f 100644 --- a/api-snapshots/lunora.api.md +++ b/api-snapshots/lunora.api.md @@ -4061,6 +4061,10 @@ Re-exported from `@lunora/runtime` — signature tracked at its source. Re-exported from `@lunora/runtime` — signature tracked at its source. +### `TraceSamplingConfig` (interface) + +Re-exported from `@lunora/runtime` — signature tracked at its source. + ### `VERSION` (const) Re-exported from `@lunora/runtime` — signature tracked at its source. diff --git a/api-snapshots/runtime.api.md b/api-snapshots/runtime.api.md index c1ce5626ce..27c8a4ccd2 100644 --- a/api-snapshots/runtime.api.md +++ b/api-snapshots/runtime.api.md @@ -1883,6 +1883,15 @@ interface StorageObject { } ``` +### `TraceSamplingConfig` (interface) + +```ts +interface TraceSamplingConfig { + alwaysSampleErrors?: boolean; + headRate?: number; +} +``` + ### `VERSION` (const) ```ts @@ -1987,6 +1996,7 @@ interface WorkerOptions { resolveTableSharding?: AdminTableResolver; restRateLimit?: RestRateLimit; routes?: Record; + sampling?: TraceSamplingConfig; schedulerDO?: ShardNamespaceLike; schedulerInstanceName?: string; security?: SecurityOptions; @@ -2173,7 +2183,7 @@ const emitLogEvent: (sink: ObservabilitySink | undefined, event: LogEvent, conte ### `emitRpcEvent` (const) ```ts -const emitRpcEvent: (sink: ObservabilitySink | undefined, event: ObservabilityEvent, context?: ObservabilitySinkContext) => void; +const emitRpcEvent: (sink: ObservabilitySink | undefined, event: ObservabilityEvent, context?: ObservabilitySinkContext, sampling?: TraceSamplingConfig) => void; ``` ### `enforceOrigin` (const) diff --git a/packages/do/__tests__/shard-do.sampling.test.ts b/packages/do/__tests__/shard-do.sampling.test.ts new file mode 100644 index 0000000000..fcfcb4bf22 --- /dev/null +++ b/packages/do/__tests__/shard-do.sampling.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, it } from "vitest"; + +import type { SpanEvent } from "../../../shared/span-event"; +import type { ContextTracer } from "../src/context-telemetry"; +import type { ShardDOState } from "../src/shard-do"; +import { ShardDO } from "../src/shard-do"; +import createSqliteExec from "./_helpers/node-sqlite"; + +// A fixed W3C trace id + parent span id used to build the inbound `traceparent`. +const TRACE_ID = "0af7651916cd43dd8448eb211c80319c"; +// A second, distinct trace id for the interleaved-dispatch race test. +const TRACE_ID_B = "1bb7652917de54ee9559fc322d91420d"; +const PARENT_SPAN_ID = "b7ad6b7169203331"; + +/** + * A shard whose real `handleRpc` runs a caller-supplied trace `plan` against a + * `ctx.trace` factory wired to a span-collecting sink, so the full base-class + * dispatch machinery — inbound `traceparent` sampled flag, `recordSpan` gating, + * and the `finally` error-keep flush — decides what reaches `onSpan`. + */ +class SamplingShard extends ShardDO { + public readonly exportedSpans: SpanEvent[] = []; + + // Assigned by each test before `fetch`; a definite-assignment field rather than + // an arrow default (which trips `class-methods-use-this`). `functionPath` lets a + // plan branch per dispatch (used by the interleaved-dispatch race test). + public plan!: (trace: ContextTracer, functionPath: string) => Promise; + + public override async handleRpc(functionPath: string): Promise { + const trace = this.makeTracer( + functionPath, + { + onSpan: (span: SpanEvent) => { + this.exportedSpans.push(span); + }, + }, + this.getCurrentTrace(), + ); + + await this.plan(trace, functionPath); + + return { ok: true }; + } +} + +const makeState = (database: ReturnType): ShardDOState => { + return { + acceptWebSocket() {}, + getWebSockets() { + return []; + }, + storage: { sql: database.sql as unknown as ShardDOState["storage"]["sql"] }, + }; +}; + +/** + * A `/rpc` request carrying the runtime's sampling signals: the `traceparent` + * sampled flag (`01` in / `00` out) and the `x-lunora-sample-errors` tail-bias + * header. Omitting `keepErrors` leaves the header off (defaulting to keep). + */ +const request = (functionPath: string, options: { keepErrors?: boolean; sampled: boolean; traceId?: string }): Request => { + const headers: Record = { + "content-type": "application/json", + traceparent: `00-${options.traceId ?? TRACE_ID}-${PARENT_SPAN_ID}-${options.sampled ? "01" : "00"}`, + }; + + if (options.keepErrors !== undefined) { + headers["x-lunora-sample-errors"] = options.keepErrors ? "1" : "0"; + } + + return new Request("https://shard.internal/rpc", { + body: JSON.stringify({ args: {}, functionPath }), + headers, + method: "POST", + }); +}; + +describe("shardDO trace sampling", () => { + it("streams spans live when the trace is head-sampled", async () => { + expect.assertions(2); + + const database = createSqliteExec(); + + try { + const shard = new SamplingShard(makeState(database), {}); + + shard.plan = async (trace) => { + await trace("work", () => undefined); + }; + + await shard.fetch(request("a:b", { sampled: true })); + + expect(shard.exportedSpans).toHaveLength(1); + expect(shard.exportedSpans[0]?.name).toBe("work"); + } finally { + database.close(); + } + }); + + it("drops a sampled-out trace that produced no error span", async () => { + expect.assertions(1); + + const database = createSqliteExec(); + + try { + const shard = new SamplingShard(makeState(database), {}); + + shard.plan = async (trace) => { + await trace("first", () => undefined); + await trace("second", () => undefined); + }; + + await shard.fetch(request("a:b", { keepErrors: true, sampled: false })); + + // Head-sampled out, no error → nothing exported (still buffered locally + // for the Studio panel, but never handed to the sink). + expect(shard.exportedSpans).toHaveLength(0); + } finally { + database.close(); + } + }); + + it("keeps a sampled-out trace whole when a child span errored (tail bias)", async () => { + expect.assertions(3); + + const database = createSqliteExec(); + + try { + const shard = new SamplingShard(makeState(database), {}); + + shard.plan = async (trace) => { + await trace("ok-child", () => undefined); + // The handler catches so it returns ok, but the child span settled + // `ok: false` — enough to keep the whole trace on export. + await trace("bad-child", () => { + throw new Error("kaboom"); + }).catch(() => undefined); + }; + + await shard.fetch(request("a:b", { keepErrors: true, sampled: false })); + + const names = shard.exportedSpans.map((span) => span.name); + + expect(shard.exportedSpans).toHaveLength(2); + // The whole trace is kept, including the sibling that settled BEFORE the error. + expect(names).toContain("ok-child"); + expect(names).toContain("bad-child"); + } finally { + database.close(); + } + }); + + it("drops a sampled-out error trace when alwaysSampleErrors is off", async () => { + expect.assertions(1); + + const database = createSqliteExec(); + + try { + const shard = new SamplingShard(makeState(database), {}); + + shard.plan = async (trace) => { + await trace("bad-child", () => { + throw new Error("kaboom"); + }).catch(() => undefined); + }; + + await shard.fetch(request("a:b", { keepErrors: false, sampled: false })); + + expect(shard.exportedSpans).toHaveLength(0); + } finally { + database.close(); + } + }); + + it("keeps a slow sampled-out error trace even when a sibling dispatch's finally interleaves", async () => { + expect.assertions(2); + + const database = createSqliteExec(); + + try { + const shard = new SamplingShard(makeState(database), {}); + + let markRecorded!: () => void; + const slowRecorded = new Promise((resolve) => { + markRecorded = resolve; + }); + let releaseSlow!: () => void; + const slowGate = new Promise((resolve) => { + releaseSlow = resolve; + }); + + shard.plan = async (trace, functionPath) => { + if (functionPath === "slow:out") { + // Sampled out + errors: the span is held and the sink captured. + await trace("slow-error", () => { + throw new Error("boom"); + }).catch(() => undefined); + markRecorded(); + // Park before returning, so this dispatch's `finally` (which + // flushes the held error span) runs AFTER the sibling below. + await slowGate; + + return; + } + + await trace("fast-ok", () => undefined); + }; + + // Start the slow sampled-out+error dispatch (its own traceId) and let it + // hold its error span, then park on the gate. + const slow = shard.fetch(request("slow:out", { keepErrors: true, sampled: false, traceId: TRACE_ID })); + + await slowRecorded; + + // Run a sibling sampled-in dispatch (a DIFFERENT traceId) to completion. + // Its `finally` must not wipe the slow trace's held sink/verdict — the + // race the per-instance fields used to lose. It streams its own span live. + await shard.fetch(request("fast:in", { sampled: true, traceId: TRACE_ID_B })); + + // Now let the slow dispatch finish; its `finally` must still flush its + // own held error span (keyed by its traceId), not silently drop it. + releaseSlow(); + await slow; + + expect(shard.exportedSpans.filter((span) => span.traceId === TRACE_ID).map((span) => span.name)).toContain("slow-error"); + expect(shard.exportedSpans.filter((span) => span.traceId === TRACE_ID_B).map((span) => span.name)).toContain("fast-ok"); + } finally { + database.close(); + } + }); +}); diff --git a/packages/do/src/shard-do.ts b/packages/do/src/shard-do.ts index 4d41287df4..68774597f1 100644 --- a/packages/do/src/shard-do.ts +++ b/packages/do/src/shard-do.ts @@ -10,6 +10,7 @@ import { jsonResponse } from "../../../shared/json-response"; import type { LogSinkContext } from "../../../shared/log-event"; import type { LogFields } from "../../../shared/log-fields"; import type { MetricEvent } from "../../../shared/metric-event"; +import { parseTraceparent } from "../../../shared/otlp"; import type { SpanEvent } from "../../../shared/span-event"; import { decodeWire, encodeWire } from "../../../shared/wire-codec"; import { verifyWsAdminToken } from "../../../shared/ws-admin-token"; @@ -1809,6 +1810,25 @@ abstract class ShardDO { */ private currentRequestTrace: { rootSpanId: string; traceId: string } | undefined; + /** + * Per-trace head-sampling state, keyed by `traceId` so concurrent dispatches on + * the same DO instance can't clobber each other's decision. A DO interleaves + * dispatches across `await` points, and the span-recording and error-flush + * choke points run after a span body (or the whole dispatch) settles — by which + * point a flat per-instance field could have been overwritten by a sibling + * dispatch (the same hazard `dispatchTrace` guards against for the trace anchor). + * Each entry holds `sampled` (the inbound `traceparent` flag; absent means keep, + * so alarms, subscription re-runs and non-Lunora callers export exactly as + * before — when false, `ctx.trace` INTERNAL spans are held out of the live export + * and re-decided in the dispatch `finally` as a tail bias), `keepErrors` (the + * runtime's `x-lunora-sample-errors` / `alwaysSampleErrors` toggle — a sampled-out + * trace that errored is still exported whole unless off), and `sink` (captured + * when a sampled-out span is first held, so the `finally` can flush the trace's + * held spans). Registered at dispatch entry by the dispatch's own `traceId`, read + * by `span.traceId`, and deleted in the `finally`. + */ + private traceSampling = new Map(); + /** * Client-issued idempotency key for the in-flight mutation, forwarded via the * `x-lunora-mutation-id` header. When set, the dispatch path dedups the call @@ -2224,6 +2244,16 @@ abstract class ShardDO { // re-set the shared field — reading it there would file this dispatch's // root span under another request's trace (and leave that one rootless). const dispatchTrace = this.currentRequestTrace; + // Trace-sampling verdict propagated by the runtime: the `traceparent` + // sampled flag carries the head decision (absent → keep, so this path is + // unchanged for alarms / subscription re-runs / non-Lunora callers) and + // `x-lunora-sample-errors` carries the tail-bias toggle. Registered keyed by + // THIS dispatch's `traceId` (not a flat field) so a concurrent dispatch's + // `recordSpan` / `finally` reads its own verdict — see `traceSampling`. + this.traceSampling.set(dispatchTrace.traceId, { + keepErrors: request.headers.get("x-lunora-sample-errors") !== "0", + sampled: parseTraceparent(this.currentRequestTraceparent)?.sampled ?? true, + }); // Reset the per-request read/cache capture (filled by `runCachedQuery` // for cached query paths) so a previous dispatch can't leak into this // entry's logged read set / cache-hit flag. @@ -2417,6 +2447,12 @@ abstract class ShardDO { if (this.spans.hasTrace(dispatchTrace.traceId)) { this.recordDispatchRootSpan(payload.functionPath, dispatchStartedAt, dispatchError, dispatchTrace); } + // Export boundary for a sampled-out trace: now that the dispatch has + // settled we know whether it errored, so flush its held `ctx.trace` + // spans (tail bias) or drop them. A no-op for sampled-in traces (their + // spans already streamed live). + this.flushSampledOutTrace(dispatchTrace, dispatchError !== undefined); + this.traceSampling.delete(dispatchTrace.traceId); this.currentRequestTrace = undefined; this.currentRequestBookmark = undefined; this.currentResponseBookmark = undefined; @@ -4810,6 +4846,15 @@ abstract class ShardDO { * `sink.onSpan`. Best-effort throughout, exactly like {@link recordUserLog}: * a span is recorded *after* its body already settled, so letting a telemetry * failure escape here would turn a succeeded operation into a failed request. + * + * The span is ALWAYS buffered locally (the Studio Traces panel is a full + * "recent traces on this instance" readout, unaffected by sampling). Only the + * export to the sink is sampled: when the active dispatch's trace was sampled + * OUT (its inbound `traceparent` flag was `00`), the span is held back rather + * than streamed — the dispatch `finally` re-decides once the trace's error + * status is known ({@link flushSampledOutTrace}), so an errored trace is still + * exported whole (tail bias). Spans from a sampled-in dispatch, or from another + * trace (a subscription re-run mints its own anchor), stream immediately. */ // eslint-disable-next-line @typescript-eslint/member-ordering -- kept in the span-recording cluster (next to recordDispatchRootSpan) for cohesion rather than hoisted above every private member protected recordSpan(span: SpanEvent, sink?: TelemetrySink): void { @@ -4819,14 +4864,77 @@ abstract class ShardDO { // Best-effort — never let span capture fail the handler. } - if (sink?.onSpan) { - try { - // Thread the DO's `waitUntil` so a network sink (otlpSink) can keep - // its export alive past the response, matching the log path. - sink.onSpan(span, { waitUntil: this.state.waitUntil?.bind(this.state) }); - } catch { - // A buggy span sink must not break the handler. - } + if (!sink?.onSpan) { + return; + } + + // Look the verdict up by the SPAN's own `traceId` (not the shared + // `currentRequestTrace`, which a sibling dispatch may have overwritten by now). + const sampling = this.traceSampling.get(span.traceId); + + if (sampling && !sampling.sampled) { + // Sampled out: hold this span back and remember the sink so the + // `finally` can flush the trace's held spans if it turns out to error. + sampling.sink = sink; + + return; + } + + this.emitSpan(span, sink); + } + + /** + * Hand one span to `sink.onSpan`, swallowing sink throws. The DO's `waitUntil` + * is threaded so a network sink (otlpSink) can keep its export alive past the + * response, matching the log path. Extracted so both the live-stream path in + * {@link recordSpan} and the deferred error-keep flush in + * {@link flushSampledOutTrace} share one guarded emit. + */ + private emitSpan(span: SpanEvent, sink: TelemetrySink): void { + if (!sink.onSpan) { + return; + } + + try { + sink.onSpan(span, { waitUntil: this.state.waitUntil?.bind(this.state) }); + } catch { + // A buggy span sink must not break the handler. + } + } + + /** + * Export-boundary decision for a sampled-out trace, run from the dispatch + * `finally` once the error status is known. A sampled-in trace already + * streamed live, so this returns early for it; a sampled-out trace exports its + * held `ctx.trace` spans only when `alwaysSampleErrors` is set AND the trace + * errored (the dispatch threw, or a held span settled `ok: false`) — the tail + * bias — and otherwise drops them. Held spans are read back from the local ring + * (excluding the synthetic dispatch root, which never goes to `onSpan`). + */ + private flushSampledOutTrace(trace: { traceId: string }, dispatchFailed: boolean): void { + // Read THIS trace's own verdict + held sink (keyed by traceId), so an + // interleaved sibling dispatch's state can't drop this trace's error spans. + const sampling = this.traceSampling.get(trace.traceId); + + if (!sampling || sampling.sampled || !sampling.keepErrors) { + return; + } + + const { sink } = sampling; + + if (!sink?.onSpan) { + return; + } + + const held = this.spans.entries().filter((span) => span.traceId === trace.traceId && span.dispatch !== true); + const traceHasError = dispatchFailed || held.some((span) => !span.ok); + + if (!traceHasError) { + return; + } + + for (const span of held) { + this.emitSpan(span, sink); } } diff --git a/packages/runtime/__tests__/sampling.test.ts b/packages/runtime/__tests__/sampling.test.ts new file mode 100644 index 0000000000..6a949f5b8c --- /dev/null +++ b/packages/runtime/__tests__/sampling.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; + +import { isTraceHeadSampled, resolveTraceSampling, shouldExportTrace, traceIdToUnitInterval } from "../../../shared/sampling"; +import type { ObservabilityEvent, ObservabilitySink } from "../src/observability"; +import { emitRpcEvent } from "../src/observability"; + +// A trace id whose first 8 hex chars are `19999999` → ~0.1 on the unit interval, +// so it is kept at headRate 0.2 and dropped at 0.05. The suffix is padding to a +// full 32-hex id and never read by the sampler. +const LOW_TRACE_ID = "19999999000000000000000000000000"; +// First 8 hex `e6666666` → ~0.9, the mirror image: dropped at 0.2, kept at 0.95. +const HIGH_TRACE_ID = "e6666666000000000000000000000000"; + +describe("shared/sampling — deterministic head decision", () => { + it("derives a stable [0,1) value from the first 8 hex chars of the trace id", () => { + expect.assertions(3); + + expect(traceIdToUnitInterval("00000000ffffffffffffffffffffffff")).toBe(0); + // 0x80000000 / 2^32 = exactly one half. + expect(traceIdToUnitInterval("80000000ffffffffffffffffffffffff")).toBe(0.5); + // A malformed / empty id leans keep (0), never silently dropped. + expect(traceIdToUnitInterval("")).toBe(0); + }); + + it("returns the same verdict for the same id across repeated calls", () => { + expect.assertions(2); + + const first = isTraceHeadSampled(LOW_TRACE_ID, 0.5); + const second = isTraceHeadSampled(LOW_TRACE_ID, 0.5); + + expect(first).toBe(second); + // And the mirror id lands on the opposite side of the same rate. + expect(isTraceHeadSampled(HIGH_TRACE_ID, 0.5)).not.toBe(isTraceHeadSampled(LOW_TRACE_ID, 0.5)); + }); + + it("keeps every trace at headRate 1 (the default) and drops every trace at headRate 0", () => { + expect.assertions(4); + + expect(isTraceHeadSampled(LOW_TRACE_ID)).toBe(true); + expect(isTraceHeadSampled(HIGH_TRACE_ID, 1)).toBe(true); + expect(isTraceHeadSampled(LOW_TRACE_ID, 0)).toBe(false); + // The all-`f` id (unit value at the very top) is still kept at rate 1. + expect(isTraceHeadSampled("ffffffffffffffffffffffffffffffff", 1)).toBe(true); + }); + + it("keeps a trace below the rate and drops one above it", () => { + expect.assertions(2); + + expect(isTraceHeadSampled(LOW_TRACE_ID, 0.2)).toBe(true); + expect(isTraceHeadSampled(HIGH_TRACE_ID, 0.2)).toBe(false); + }); +}); + +describe("shared/sampling — export decision", () => { + it("resolves defaults: keep-all head rate and errors force-kept", () => { + expect.assertions(2); + + const decision = resolveTraceSampling(undefined, LOW_TRACE_ID); + + expect(decision.isTraced).toBe(true); + expect(decision.keepErrors).toBe(true); + }); + + it("keeps a sampled-out trace only when it errored and errors are force-kept", () => { + expect.assertions(4); + + const sampledOut = resolveTraceSampling({ headRate: 0 }, LOW_TRACE_ID); + + // No error → dropped; error → kept by the tail bias. + expect(shouldExportTrace(sampledOut, false)).toBe(false); + expect(shouldExportTrace(sampledOut, true)).toBe(true); + + // With errors NOT force-kept, even an error trace is dropped. + const noErrorKeep = resolveTraceSampling({ alwaysSampleErrors: false, headRate: 0 }, LOW_TRACE_ID); + + expect(shouldExportTrace(noErrorKeep, true)).toBe(false); + + // A head-sampled trace is always kept regardless of error status. + expect(shouldExportTrace(resolveTraceSampling({ headRate: 1 }, LOW_TRACE_ID), false)).toBe(true); + }); +}); + +describe("emitRpcEvent — trace sampling of the SERVER span", () => { + const rpcEvent = (overrides: Partial = {}): ObservabilityEvent => { + return { durationMs: 5, functionPath: "a:b", ok: true, traceId: LOW_TRACE_ID, ...overrides }; + }; + + const sinkInto = (seen: ObservabilityEvent[]): ObservabilitySink => { + return { onRpc: (event) => seen.push(event) }; + }; + + it("keeps every RPC span at headRate 1 (and when no sampling is configured)", () => { + expect.assertions(2); + + const seen: ObservabilityEvent[] = []; + + emitRpcEvent(sinkInto(seen), rpcEvent(), undefined, { headRate: 1 }); + emitRpcEvent(sinkInto(seen), rpcEvent()); + + expect(seen).toHaveLength(2); + expect(seen[0]?.functionPath).toBe("a:b"); + }); + + it("drops a non-error RPC span at headRate 0 but keeps the error one (tail bias)", () => { + expect.assertions(2); + + const seen: ObservabilityEvent[] = []; + + emitRpcEvent(sinkInto(seen), rpcEvent({ ok: true }), undefined, { headRate: 0 }); + emitRpcEvent(sinkInto(seen), rpcEvent({ error: { code: "BAD_REQUEST", message: "nope", status: 400 }, ok: false }), undefined, { headRate: 0 }); + + expect(seen).toHaveLength(1); + expect(seen[0]?.ok).toBe(false); + }); + + it("drops even an error RPC span when alwaysSampleErrors is off", () => { + expect.assertions(1); + + const seen: ObservabilityEvent[] = []; + + emitRpcEvent(sinkInto(seen), rpcEvent({ ok: false }), undefined, { alwaysSampleErrors: false, headRate: 0 }); + + expect(seen).toHaveLength(0); + }); + + it("always keeps an event with no trace id (a fan-out aggregation)", () => { + expect.assertions(1); + + const seen: ObservabilityEvent[] = []; + + emitRpcEvent(sinkInto(seen), rpcEvent({ ok: true, traceId: undefined }), undefined, { headRate: 0 }); + + expect(seen).toHaveLength(1); + }); +}); diff --git a/packages/runtime/src/create-worker.ts b/packages/runtime/src/create-worker.ts index 42322d574c..917140dcd9 100644 --- a/packages/runtime/src/create-worker.ts +++ b/packages/runtime/src/create-worker.ts @@ -6,6 +6,8 @@ import type { ExecutionContextLike } from "../../../shared/execution-context"; import { NOOP_EXECUTION_CONTEXT } from "../../../shared/execution-context"; import { buildTraceparent, otlpRandomHex } from "../../../shared/otlp"; import { relayName } from "../../../shared/relay-name"; +import type { TraceSamplingConfig } from "../../../shared/sampling"; +import { resolveTraceSampling } from "../../../shared/sampling"; import { mintWsAdminToken, verifyWsAdminToken } from "../../../shared/ws-admin-token"; import type { AuthAdmin } from "./auth-admin-routes"; import { buildAuthAdminRoutes } from "./auth-admin-routes"; @@ -1011,6 +1013,26 @@ interface WorkerOptions { */ routes?: Record; + /** + * Trace-sampling policy for the observability pipeline, mirroring Cloudflare + * Workers' `head_sampling_rate`. Governs only trace spans (the per-dispatch + * SERVER span and the `ctx.trace` INTERNAL spans beneath it) — never metrics + * or `ctx.log` lines. + * + * The decision is deterministic per trace: a stable value derived from the + * `traceId` is compared to `headRate`, so the same trace is kept or dropped + * as a whole on the worker and on every shard/container it fans out to (no + * half traces). The head decision is propagated to shards via the + * `traceparent` sampled flag, so they drop the matching `ctx.trace` spans + * coherently. + * + * With `alwaysSampleErrors` (default `true`), a trace that produced an error + * span is kept whole regardless of the head decision — the tail bias, so + * failures are never sampled away even at an aggressive `headRate`. Omit the + * option (or leave `headRate` at its default `1`) to keep every trace. + */ + sampling?: TraceSamplingConfig; + /** * Namespace binding for the `SchedulerDO` (typically `env.SCHEDULER`). When * set, the worker exposes the admin-gated `/_lunora/admin/scheduled` @@ -3021,7 +3043,7 @@ const createWorker = (options: WorkerOptions): LunoraWorker => { sinkContext?: ObservabilitySinkContext, ): Promise => { const rpcStartedAt = Date.now(); - const { observability } = options; + const { observability, sampling } = options; // Trace context for this dispatch, generated at entry (before the shard // runs the handler) so it can both ride the dispatch's own span and reach @@ -3030,10 +3052,21 @@ const createWorker = (options: WorkerOptions): LunoraWorker => { const traceId = otlpRandomHex(16); const spanId = otlpRandomHex(8); + // Head-sampling decision for this trace, made once here where the id is + // minted and propagated to the shard so it drops the matching `ctx.trace` + // spans coherently: the `traceparent` sampled flag carries the head + // verdict; `x-lunora-sample-errors` carries the tail-bias toggle so a + // sampled-out trace that errors is still kept whole on both sides. + const decision = resolveTraceSampling(sampling, traceId); + // Re-emit the RPC body to the shard at its `/rpc` route. const forwarded = new Request(`https://shard.internal/rpc`, { body: JSON.stringify({ args, functionPath }), - headers: { ...forwardedHeaders, traceparent: buildTraceparent(traceId, spanId) }, + headers: { + ...forwardedHeaders, + traceparent: buildTraceparent(traceId, spanId, decision.isTraced), + "x-lunora-sample-errors": decision.keepErrors ? "1" : "0", + }, method: "POST", }); @@ -3055,6 +3088,7 @@ const createWorker = (options: WorkerOptions): LunoraWorker => { ...(response.ok ? {} : { error: { code: "SHARD_ERROR", message: `shard returned ${String(response.status)}`, status: response.status } }), }, sinkContext, + sampling, ); // The DO's `x-d1-bookmark` header (which lets the client pin reads @@ -3064,7 +3098,12 @@ const createWorker = (options: WorkerOptions): LunoraWorker => { // (and would drop `statusText`). return response; } catch (error) { - emitRpcEvent(observability, { ...buildErrorEvent(functionPath, Date.now() - rpcStartedAt, error, { shardKey }), spanId, traceId }, sinkContext); + emitRpcEvent( + observability, + { ...buildErrorEvent(functionPath, Date.now() - rpcStartedAt, error, { shardKey }), spanId, traceId }, + sinkContext, + sampling, + ); throw error; } }; diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index fc3c76fdf2..170f7d149c 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -87,6 +87,7 @@ export type { ObservabilitySink, ObservabilitySinkContext, SpanEvent, + TraceSamplingConfig, } from "./observability"; export { emitLogEvent, emitRpcEvent } from "./observability"; export type { diff --git a/packages/runtime/src/observability.ts b/packages/runtime/src/observability.ts index 35afb3f0b7..dc515e69eb 100644 --- a/packages/runtime/src/observability.ts +++ b/packages/runtime/src/observability.ts @@ -15,6 +15,8 @@ /* eslint-disable no-secrets/no-secrets -- the entropy heuristic flags a CamelCase sink-context type name quoted in a doc comment below, not a credential */ import type { ContextLogLevel, LogEvent, LogSinkContext } from "../../../shared/log-event"; import type { MetricEvent } from "../../../shared/metric-event"; +import type { TraceSamplingConfig } from "../../../shared/sampling"; +import { resolveTraceSampling, shouldExportTrace } from "../../../shared/sampling"; import type { SpanEvent } from "../../../shared/span-event"; /** @@ -109,12 +111,27 @@ export interface ObservabilitySink { * throws. Use at the dispatch boundary; the runtime should never see a * sink-originating throw bubble up past this point. `context.waitUntil`, when * supplied, lets a network sink keep its send alive past the response. + * + * `sampling` applies the trace-sampling verdict to this dispatch's SERVER span: + * the event is dropped unless the trace was head-sampled or (with errors + * force-kept) this dispatch errored — the tail bias. A dispatch with no + * `traceId` (a fan-out aggregation, which mints none) is always kept, and an + * absent `sampling` keeps everything, so both are backward-compatible. */ -export const emitRpcEvent = (sink: ObservabilitySink | undefined, event: ObservabilityEvent, context?: ObservabilitySinkContext): void => { +export const emitRpcEvent = ( + sink: ObservabilitySink | undefined, + event: ObservabilityEvent, + context?: ObservabilitySinkContext, + sampling?: TraceSamplingConfig, +): void => { if (!sink?.onRpc) { return; } + if (sampling !== undefined && event.traceId !== undefined && !shouldExportTrace(resolveTraceSampling(sampling, event.traceId), !event.ok)) { + return; + } + try { sink.onRpc(event, context); } catch { @@ -151,4 +168,5 @@ export const emitLogEvent = (sink: ObservabilitySink | undefined, event: LogEven export { type LogEvent } from "../../../shared/log-event"; export { type LogFields } from "../../../shared/log-fields"; export { type MetricEvent, type MetricKind } from "../../../shared/metric-event"; +export { type TraceSamplingConfig } from "../../../shared/sampling"; export { type SpanEvent } from "../../../shared/span-event"; diff --git a/shared/otlp.ts b/shared/otlp.ts index 15b3076448..802e65fbbe 100644 --- a/shared/otlp.ts +++ b/shared/otlp.ts @@ -83,11 +83,15 @@ const HEX_ONLY = /^[0-9a-f]+$/; /** * Build a W3C `traceparent` header from a 32-hex trace id + 16-hex span id: - * `00---01` (version 0, sampled). The ids are the same - * lowercase-hex form {@link otlpRandomHex} produces, so a worker's trace/span id - * composes into a `traceparent` with no reformatting. + * `00---` (version 0). The `sampled` flag (bit 0 of the + * trace-flags octet) is `01` when the trace was sampled in and `00` when it was + * sampled out — how the runtime propagates its head-sampling decision to the + * shard and any container beneath it, so the whole trace is kept or dropped + * coherently. Defaults to sampled for callers that don't sample. The ids are the + * same lowercase-hex form {@link otlpRandomHex} produces, so a worker's + * trace/span id composes into a `traceparent` with no reformatting. */ -const buildTraceparent = (traceId: string, spanId: string): string => `00-${traceId}-${spanId}-01`; +const buildTraceparent = (traceId: string, spanId: string, sampled = true): string => `00-${traceId}-${spanId}-${sampled ? "01" : "00"}`; /** * Parse a W3C `traceparent` into `{ traceId, parentSpanId }`, or `undefined` when @@ -100,8 +104,12 @@ const buildTraceparent = (traceId: string, spanId: string): string => `00-${trac * but a future version may append fields, so a `>= 4`-field header on a higher * version parses off its first four and ignores the rest rather than being * dropped. The reserved version `ff` is rejected. + * + * `sampled` is bit 0 of the trace-flags octet — the head-sampling decision the + * upstream (the Lunora worker) propagated; consumers use it to keep or drop the + * whole trace coherently. */ -const parseTraceparent = (header: null | string | undefined): { parentSpanId: string; traceId: string } | undefined => { +const parseTraceparent = (header: null | string | undefined): { parentSpanId: string; sampled: boolean; traceId: string } | undefined => { if (header === null || header === undefined) { return undefined; } @@ -132,7 +140,8 @@ const parseTraceparent = (header: null | string | undefined): { parentSpanId: st return undefined; } - return { parentSpanId, traceId }; + // Bit 0 of the trace-flags octet is the W3C `sampled` flag. + return { parentSpanId, sampled: (Number.parseInt(flags, 16) & 0x01) === 0x01, traceId }; }; /** Encode one attribute, choosing the OTLP value kind from the JS type. */ diff --git a/shared/sampling.ts b/shared/sampling.ts new file mode 100644 index 0000000000..6cc1a48942 --- /dev/null +++ b/shared/sampling.ts @@ -0,0 +1,113 @@ +/** + * Deterministic trace-sampling primitives, bundler-inlined (like + * {@link file://./otlp.ts}) so the worker runtime (`@lunora/runtime`) and the + * Durable Object (`@lunora/do`) reach the SAME keep/drop verdict for a trace from + * ONE encoder instead of two drifting mirrors. The two consumers sit on opposite + * tiers and must not gain a runtime dependency edge on each other, so this lives + * in `shared/` and is inlined into each `dist` rather than published. + * + * The model mirrors Cloudflare Workers' `head_sampling_rate`: a per-trace head + * decision derived deterministically from the trace id — so the same trace yields + * the same verdict on the worker and on every shard/container beneath it, and a + * trace is kept or dropped *whole* (never half) — biased to always keep traces + * that produced an error (tail bias) so failures are never sampled away. + * + * Pure and dependency-free (only built-ins) so inlining stays sound. Only trace + * spans are governed here; metrics and application logs are never sampled. + */ + +/** Default head-sampling rate: keep every trace. */ +const DEFAULT_TRACE_HEAD_RATE = 1; + +/** + * Trace-sampling configuration — the `sampling` block on the worker's + * observability options. + */ +interface TraceSamplingConfig { + /** + * Always keep a whole trace that produced an error span (root or any child + * `ok: false`), regardless of the head decision — the tail bias that keeps + * failures observable under aggressive head sampling. Default `true`. + */ + alwaysSampleErrors?: boolean; + + /** + * Fraction of traces to keep by the deterministic head decision, in `[0, 1]`. + * `1` keeps every trace (the default), `0` drops every non-error trace, `0.1` + * keeps ~10%. Values outside the range are clamped by the decision helpers. + */ + headRate?: number; +} + +/** + * The resolved per-trace decision. `isTraced` is the cheap head verdict callers + * check to skip work up front; `keepErrors` carries the tail-bias toggle to the + * export boundary, where the trace's final error status is known. + */ +interface TraceSamplingDecision { + /** + * True when the trace passed head sampling. Callers may cheaply skip buffering + * or exporting a trace's spans when this is false — subject to the error-keep + * re-check at the export boundary (see {@link shouldExportTrace}). + */ + isTraced: boolean; + /** Whether error traces are force-kept at export even when `isTraced` is false. */ + keepErrors: boolean; +} + +/** + * Map a trace id to a stable value in `[0, 1)` from its first 8 hex characters + * (32 bits) divided by `2^32`. Same id → same value everywhere, so the head + * decision is identical on the worker, the shard, and any container. A malformed + * or empty id parses to `NaN` and falls back to `0` (the keep-leaning end), so a + * bad id is never silently dropped. + */ +const traceIdToUnitInterval = (traceId: string): number => { + const int = Number.parseInt(traceId.slice(0, 8), 16); + + return Number.isFinite(int) ? int / 0x1_0000_0000 : 0; +}; + +/** + * The deterministic head-sampling verdict for a trace id at a given rate. `>= 1` + * keeps every trace (so the all-`f` id is never dropped by a rounding boundary); + * `<= 0` drops every trace; otherwise keep when the id's stable unit value is + * below the rate. + */ +const isTraceHeadSampled = (traceId: string, headRate: number = DEFAULT_TRACE_HEAD_RATE): boolean => { + if (headRate >= 1) { + return true; + } + + if (headRate <= 0) { + return false; + } + + return traceIdToUnitInterval(traceId) < headRate; +}; + +/** + * Resolve a {@link TraceSamplingConfig} against a trace id into the concrete + * {@link TraceSamplingDecision} for that trace. Applies the defaults (keep-all + * head rate, errors force-kept) so an absent config keeps every trace — fully + * backward-compatible. + */ +const resolveTraceSampling = (config: TraceSamplingConfig | undefined, traceId: string): TraceSamplingDecision => { + return { + isTraced: isTraceHeadSampled(traceId, config?.headRate ?? DEFAULT_TRACE_HEAD_RATE), + keepErrors: config?.alwaysSampleErrors ?? true, + }; +}; + +/** + * The export-boundary verdict: keep the trace when it was head-sampled, or when + * it produced an error and errors are force-kept. This is where the tail bias is + * applied — a trace dropped by the head decision is still exported whole if it + * turned out to contain an error span. + */ +const shouldExportTrace = (decision: TraceSamplingDecision, traceHasError: boolean): boolean => { + return decision.isTraced || (decision.keepErrors && traceHasError); +}; + +export type { TraceSamplingConfig, TraceSamplingDecision }; +export { DEFAULT_TRACE_HEAD_RATE, isTraceHeadSampled, resolveTraceSampling, shouldExportTrace, traceIdToUnitInterval };