From 6dbe534c6345a8b282de644658f0d375b74d2f75 Mon Sep 17 00:00:00 2001 From: Nightingalelyy Date: Sat, 15 Aug 2026 16:18:52 +0800 Subject: [PATCH 1/2] fix(instrumentations): repair LangChain, LlamaIndex, and Mastra spans --- ...60815-otel2-js-langchain-llama-mastra.json | 8 + .../src/_callback.ts | 85 +++++++- .../src/_callback_helpers.ts | 2 +- .../src/index.ts | 4 +- .../tests/langchain_instrumentation.test.mjs | 58 +++++- .../src/_emitter.ts | 96 ++++++++- .../src/_helpers.ts | 15 ++ .../src/index.ts | 194 +++++++++++++++++- .../tests/llama_index_instrumentor.test.mjs | 119 +++++++++++ .../src/index.ts | 80 ++++++-- .../tests/mastra_instrumentor.test.mjs | 66 ++++++ 11 files changed, 687 insertions(+), 40 deletions(-) create mode 100644 .release-intents/20260815-otel2-js-langchain-llama-mastra.json diff --git a/.release-intents/20260815-otel2-js-langchain-llama-mastra.json b/.release-intents/20260815-otel2-js-langchain-llama-mastra.json new file mode 100644 index 00000000..056b12ed --- /dev/null +++ b/.release-intents/20260815-otel2-js-langchain-llama-mastra.json @@ -0,0 +1,8 @@ +{ + "summary": "Repair OpenTelemetry 2.x hierarchy, error, and workflow fields for LangChain, LlamaIndex, and Mastra", + "packages": { + "@respan/instrumentation-langchain": "patch", + "@respan/instrumentation-llama-index": "patch", + "@respan/instrumentation-mastra": "patch" + } +} diff --git a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback.ts b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback.ts index e9885a27..4fd8f79c 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback.ts +++ b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback.ts @@ -76,11 +76,13 @@ export function getCallbackHandler( }); } -export function addRespanCallback( - config: LangChainCallbackConfig = {}, +export function addRespanCallback< + TConfig extends object = LangChainCallbackConfig, +>( + config: TConfig = {} as TConfig, handler: RespanCallbackHandler = getCallbackHandler(), -): LangChainCallbackConfig { - const nextConfig: LangChainCallbackConfig = { ...config }; +): TConfig { + const nextConfig = { ...config } as TConfig & LangChainCallbackConfig; nextConfig.callbacks = withRespanCallback(nextConfig.callbacks, handler); return nextConfig; } @@ -140,7 +142,10 @@ export class RespanCallbackHandler { private readonly _runs = new Map(); private readonly _runTraceIds = new Map(); private readonly _runPaths = new Map(); + private readonly _runWorkflowNames = new Map(); private readonly _langflowTraceId = generateTraceId(); + private _langflowWorkflowName?: string; + private _langflowRootSpanId?: string; constructor(options: RespanCallbackHandlerOptions = {}) { this.includeContent = options.includeContent ?? true; @@ -152,8 +157,10 @@ export class RespanCallbackHandler { private _rememberRun(record: RunRecord): void { this._runTraceIds.set(record.runId, record.traceId); this._runPaths.set(record.runId, record.entityPath); + this._runWorkflowNames.set(record.runId, record.workflowName); trimMap(this._runTraceIds, this.maxCachedRuns); trimMap(this._runPaths, this.maxCachedRuns); + trimMap(this._runWorkflowNames, this.maxCachedRuns); } private _startRun({ @@ -199,24 +206,59 @@ export class RespanCallbackHandler { activeParent?.traceId ?? fallbackTraceId; - const parentSpanId = + let parentSpanId = parentHex !== undefined ? deriveSpanId(parentHex) : activeParent?.spanId; + const isGroupedLangflowRoot = + !parentHex && + !activeParent && + framework === "langflow" && + this.groupLangflowRootRuns; + if (isGroupedLangflowRoot && this._langflowRootSpanId) { + parentSpanId = this._langflowRootSpanId; + } const parentPath = (parentHex && this._runs.get(parentHex)?.entityPath) ?? this._runPaths.get(parentHex ?? ""); const entityPath = parentPath ? `${parentPath}.${name}` : name; - + const parentWorkflowName = + (parentHex && this._runs.get(parentHex)?.workflowName) ?? + this._runWorkflowNames.get(parentHex ?? ""); + if ( + !parentHex && + !activeParent && + framework === "langflow" && + this.groupLangflowRootRuns && + !this._langflowWorkflowName + ) { + const componentName = metadata?.langflow_component; + this._langflowWorkflowName = + typeof componentName === "string" && componentName.trim() + ? componentName.trim() + : name; + } + const workflowName = + parentWorkflowName ?? + (framework === "langflow" && this.groupLangflowRootRuns + ? this._langflowWorkflowName + : undefined) ?? + name; + + const spanId = deriveSpanId(runHex); + if (isGroupedLangflowRoot && !this._langflowRootSpanId) { + this._langflowRootSpanId = spanId; + } this._runs.set(runHex, { runId: runHex, traceId, - spanId: deriveSpanId(runHex), + spanId, parentRunId: parentHex, parentSpanId, name, entityPath, + workflowName, logType, spanKind, startTime: hrTime(), @@ -237,10 +279,17 @@ export class RespanCallbackHandler { [SpanAttributes.TRACELOOP_SPAN_KIND]: record.spanKind, [SpanAttributes.TRACELOOP_ENTITY_NAME]: record.name, [SpanAttributes.TRACELOOP_ENTITY_PATH]: record.entityPath, + [SpanAttributes.TRACELOOP_WORKFLOW_NAME]: record.workflowName, + [RespanSpanAttributes.RESPAN_TRACE_GROUP_ID]: record.workflowName, [LANGCHAIN_RUN_ID_ATTR]: record.runId, [LANGCHAIN_FRAMEWORK_ATTR]: record.framework, }; setIfPresent(attrs, LANGCHAIN_PARENT_RUN_ID_ATTR, record.parentRunId); + setIfPresent( + attrs, + RespanSpanAttributes.RESPAN_SPAN_CUSTOM_ID, + record.metadata?.custom_identifier, + ); if (this.includeMetadata) { setIfPresent(attrs, LANGCHAIN_TAGS_ATTR, safeJsonString(record.tags)); @@ -356,6 +405,10 @@ export class RespanCallbackHandler { (parentHex && this._runs.get(parentHex)?.entityPath) ?? this._runPaths.get(parentHex ?? ""); const entityPath = parentPath ? `${parentPath}.${name}` : name; + const workflowName = + (parentHex && this._runs.get(parentHex)?.workflowName) ?? + this._runWorkflowNames.get(parentHex ?? "") ?? + name; const record: RunRecord = { runId: runIdToHex(`${traceId}:${parentHex ?? ""}:${name}:${Date.now()}:${Math.random()}`), traceId, @@ -364,6 +417,7 @@ export class RespanCallbackHandler { parentSpanId, name, entityPath, + workflowName, logType, spanKind, startTime: hrTime(), @@ -407,16 +461,27 @@ export class RespanCallbackHandler { runName?: unknown, ): void { const isRoot = parentRunId === undefined || parentRunId === null; + const normalizedMetadata = normalizeMetadata(metadata); + const isCreateAgentRoot = + isRoot && normalizedMetadata?.ls_integration === "langchain_create_agent"; this._startRun({ runId, parentRunId, name: extractName(serialized, "chain", runName), - logType: isRoot ? RespanLogType.WORKFLOW : RespanLogType.TASK, - spanKind: isRoot ? TraceloopSpanKindValues.WORKFLOW : TraceloopSpanKindValues.TASK, + logType: isCreateAgentRoot + ? RespanLogType.AGENT + : isRoot + ? RespanLogType.WORKFLOW + : RespanLogType.TASK, + spanKind: isCreateAgentRoot + ? TraceloopSpanKindValues.AGENT + : isRoot + ? TraceloopSpanKindValues.WORKFLOW + : TraceloopSpanKindValues.TASK, inputValue: inputs, serialized, tags: normalizeTags(tags), - metadata: normalizeMetadata(metadata), + metadata: normalizedMetadata, }); } diff --git a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback_helpers.ts b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback_helpers.ts index ec9a5860..64420fbb 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback_helpers.ts +++ b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback_helpers.ts @@ -28,7 +28,6 @@ export type HrTimeTuple = [number, number]; export type FrameworkName = "langchain" | "langgraph" | "langflow"; export interface LangChainCallbackConfig { callbacks?: unknown; - [key: string]: unknown; } export type SpanAttributesRecord = Record; @@ -58,6 +57,7 @@ export interface RunRecord { parentSpanId?: string; name: string; entityPath: string; + workflowName: string; logType: string; spanKind: string; startTime: HrTimeTuple; diff --git a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/index.ts b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/index.ts index e17dd487..4b6042c8 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/index.ts +++ b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/index.ts @@ -53,7 +53,9 @@ export class LangChainInstrumentor { return this._active; } - addCallback(config: LangChainCallbackConfig = {}): LangChainCallbackConfig { + addCallback( + config: TConfig = {} as TConfig, + ): TConfig { return addRespanCallback(config, this.callbackHandler); } } diff --git a/javascript-sdks/instrumentations/respan-instrumentation-langchain/tests/langchain_instrumentation.test.mjs b/javascript-sdks/instrumentations/respan-instrumentation-langchain/tests/langchain_instrumentation.test.mjs index 4f6f173d..d297318b 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-langchain/tests/langchain_instrumentation.test.mjs +++ b/javascript-sdks/instrumentations/respan-instrumentation-langchain/tests/langchain_instrumentation.test.mjs @@ -115,6 +115,10 @@ test("chain root and child emit workflow and task spans with parent linkage", () assert.equal(childSpan.spanContext().traceId, rootSpan.spanContext().traceId); assert.equal(childSpan.parentSpanContext?.spanId, rootSpan.spanContext().spanId); assert.equal(rootSpan.attributes["langchain.framework"], "langgraph"); + assert.equal(rootSpan.attributes["traceloop.workflow.name"], "root_chain"); + assert.equal(childSpan.attributes["traceloop.workflow.name"], "root_chain"); + assert.equal(rootSpan.attributes["respan.trace.trace_group_identifier"], "root_chain"); + assert.equal(childSpan.attributes["respan.trace.trace_group_identifier"], "root_chain"); }); test("explicit Langflow handler groups independent root runs into one trace", () => { @@ -147,7 +151,53 @@ test("explicit Langflow handler groups independent root runs into one trace", () assert.equal(captured.length, 2); assert.equal(captured[0].spanContext().traceId, captured[1].spanContext().traceId); assert.equal(captured[0].parentSpanContext?.spanId, undefined); - assert.equal(captured[1].parentSpanContext?.spanId, undefined); + assert.equal( + captured[1].parentSpanContext?.spanId, + captured[0].spanContext().spanId, + ); + assert.equal(captured[0].attributes["traceloop.workflow.name"], "DemoComponent"); + assert.equal(captured[1].attributes["traceloop.workflow.name"], "DemoComponent"); +}); + +test("createAgent root emits one agent boundary and propagates its workflow name", () => { + resetProvider(); + const handler = new RespanCallbackHandler(); + const agentRunId = runId(41); + const childRunId = runId(42); + + handler.handleChainStart( + { name: "LangGraph" }, + { messages: [{ role: "user", content: "help" }] }, + agentRunId, + undefined, + undefined, + { + ls_integration: "langchain_create_agent", + custom_identifier: "otel2-fix-marker", + }, + undefined, + "support_agent", + ); + handler.handleToolStart( + { name: "lookup" }, + { query: "help" }, + childRunId, + agentRunId, + ); + handler.handleToolEnd({ result: "done" }, childRunId); + handler.handleChainEnd({ messages: [{ role: "assistant", content: "done" }] }, agentRunId); + + const [toolSpan, agentSpan] = captured; + assert.equal(agentSpan.attributes["respan.entity.log_type"], "agent"); + assert.equal(agentSpan.attributes["traceloop.span.kind"], "agent"); + assert.equal(toolSpan.parentSpanContext?.spanId, agentSpan.spanContext().spanId); + assert.equal(toolSpan.attributes["traceloop.workflow.name"], "support_agent"); + assert.equal(agentSpan.attributes["traceloop.workflow.name"], "support_agent"); + assert.equal(agentSpan.attributes["respan.span_params.custom_identifier"], "otel2-fix-marker"); + assert.equal( + captured.filter((span) => span.attributes["respan.entity.log_type"] === "agent").length, + 1, + ); }); test("chat model output maps messages, usage, model, tool calls, and strips JSON fences", () => { @@ -349,6 +399,12 @@ test("agent action, agent end, and custom event emit event spans", () => { assert.equal(eventSpan.name, "custom_step"); assert.equal(eventSpan.attributes["respan.entity.log_type"], "task"); assert.equal(chainSpan.attributes["respan.entity.log_type"], "workflow"); + assert.deepEqual( + [toolSpan, agentSpan, eventSpan, chainSpan].map( + (span) => span.attributes["traceloop.workflow.name"], + ), + ["agent_chain", "agent_chain", "agent_chain", "agent_chain"], + ); }); test("pure helpers normalize messages, usage, serializable values, and tool definitions", () => { diff --git a/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/_emitter.ts b/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/_emitter.ts index 6ec02b77..ea982716 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/_emitter.ts +++ b/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/_emitter.ts @@ -44,6 +44,8 @@ export interface SpanRecord { parentId?: string; startTime: HrTime; input?: unknown; + model?: string; + caller?: object; } export interface EmitterOptions { @@ -126,8 +128,8 @@ export class LlamaIndexSpanEmitter { }): void { const activeContext = activeSpanContext(); const parent = this.currentRecord(); - const traceId = activeContext?.traceId ?? parent?.traceId; - const parentId = activeContext?.spanId ?? parent?.spanId; + const traceId = parent?.traceId ?? activeContext?.traceId; + const parentId = parent?.spanId ?? activeContext?.spanId; const record: SpanRecord = { id: params.id, name: params.name, @@ -162,6 +164,10 @@ export class LlamaIndexSpanEmitter { output: params.output, workflowName: this.options.workflowName, }); + if (params.errorMessage) { + attrs[ATTR_ERROR_MESSAGE] = params.errorMessage; + attrs.status_code = 500; + } emitReadableSpan({ name: record.name, @@ -175,7 +181,13 @@ export class LlamaIndexSpanEmitter { }); } - startLLM(params: { id: string; messages: unknown; startTime: HrTime }): void { + startLLM(params: { + id: string; + messages: unknown; + startTime: HrTime; + model?: string; + caller?: object; + }): void { this.startRecord({ id: params.id, name: "llamaindex.llm", @@ -183,6 +195,11 @@ export class LlamaIndexSpanEmitter { startTime: params.startTime, input: formatMessages(params.messages), }); + const record = this.records.get(params.id); + if (record) { + record.model = params.model; + record.caller = params.caller; + } } endLLM(params: { @@ -202,7 +219,7 @@ export class LlamaIndexSpanEmitter { ? (record.input as Record[]) : []; const completion = extractResponseMessage(params.response); - const model = extractResponseModel(params.response); + const model = extractResponseModel(params.response) ?? record.model; const usage = extractUsage(params.response); const attrs = baseAttrs({ name: record.name, @@ -234,6 +251,10 @@ export class LlamaIndexSpanEmitter { if (usage.totalTokens !== undefined) { attrs[TraceloopSpanAttributes.LLM_USAGE_TOTAL_TOKENS] = usage.totalTokens; } + if (params.errorMessage) { + attrs[ATTR_ERROR_MESSAGE] = params.errorMessage; + attrs.status_code = 500; + } emitReadableSpan({ name: record.name, @@ -346,6 +367,73 @@ export class LlamaIndexSpanEmitter { }); } + failLLMForCaller(params: { + caller?: object; + errorMessage: string; + endTime: HrTime; + }): boolean { + const candidates = [...this.records.values()].reverse(); + const record = + candidates.find( + (candidate) => + candidate.logType === RespanLogType.CHAT && + params.caller !== undefined && + candidate.caller === params.caller, + ) ?? candidates.find((candidate) => candidate.logType === RespanLogType.CHAT); + if (!record) { + return false; + } + + const ancestorIds: string[] = []; + let parentId = record.parentId; + while (parentId) { + const parent = [...this.records.values()].find( + (candidate) => candidate.spanId === parentId, + ); + if (!parent) break; + ancestorIds.push(parent.id); + parentId = parent.parentId; + } + + this.endLLM({ + id: record.id, + response: undefined, + errorMessage: params.errorMessage, + endTime: params.endTime, + }); + for (const id of ancestorIds) { + this.endRecord({ + id, + errorMessage: params.errorMessage, + endTime: params.endTime, + }); + } + return true; + } + + flushPending(params: { errorMessage: string; endTime: HrTime }): void { + const pendingIds = [...this.records.keys()].reverse(); + for (const id of pendingIds) { + const record = this.records.get(id); + if (!record) continue; + if (record.logType === RespanLogType.CHAT) { + this.endLLM({ + id, + response: undefined, + errorMessage: params.errorMessage, + endTime: params.endTime, + }); + } else { + this.endRecord({ + id, + errorMessage: params.errorMessage, + endTime: params.endTime, + }); + } + } + this.pendingToolCalls.clear(); + } + clear(): void { this.records.clear(); this.stack.length = 0; diff --git a/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/_helpers.ts b/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/_helpers.ts index 9f99dffd..2b240fa6 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/_helpers.ts +++ b/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/_helpers.ts @@ -26,6 +26,21 @@ export function safeJson(value: unknown): string { } } +export function sanitizeErrorMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message + .replace( + /\b(authorization\s*:\s*bearer\s+)([^\s,;]+)/gi, + "$1[REDACTED]", + ) + .replace( + /\b((?:api[_ -]?key|token|secret)\s*[=:]\s*)([^\s,;]+)/gi, + "$1[REDACTED]", + ) + .replace(/\bsk-[A-Za-z0-9_-]{8,}\b/g, "[REDACTED]") + .slice(0, 4096); +} + export function normalizeText(value: unknown): string { if (value === undefined || value === null) { return ""; diff --git a/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/index.ts b/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/index.ts index b803e8a1..4de61a0a 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/index.ts +++ b/javascript-sdks/instrumentations/respan-instrumentation-llama-index/src/index.ts @@ -17,6 +17,7 @@ import { formatTaskInput, formatTaskOutput, } from "./_emitter.js"; +import { sanitizeErrorMessage } from "./_helpers.js"; export interface LlamaIndexInstrumentorOptions { workflowName?: string; @@ -33,12 +34,51 @@ type HandlerBinding = { handler: (event: any) => void; }; +type LlamaIndexSettingsLike = { + llm?: unknown; + withLLM?: (llm: unknown, fn: (...args: any[]) => any) => any; +}; + +type PatchedLlm = Record & { + chat: (...args: any[]) => any; +}; + function eventDetail(event: any): Record { return event?.detail && typeof event.detail === "object" ? event.detail : {}; } -function eventId(detail: Record, fallbackPrefix: string): string { - return String(detail.id ?? `${fallbackPrefix}-${Math.random().toString(16).slice(2)}`); +function eventId( + detail: Record, + fallbackPrefix: string, + event?: any, +): string { + return eventCorrelationId(detail, event) ?? + `${fallbackPrefix}-${Math.random().toString(16).slice(2)}`; +} + +function eventCorrelationId( + detail: Record, + event?: any, +): string | undefined { + const value = detail.id ?? event?.reason?.id; + return value === undefined || value === null || value === "" + ? undefined + : String(value); +} + +function eventCaller(event: any): object | undefined { + const callers = event?.reason?.computedCallers; + if (!Array.isArray(callers)) return undefined; + return callers.find( + (caller) => caller && typeof caller === "object" && typeof caller.chat === "function", + ); +} + +function callerModel(caller: object | undefined): string | undefined { + if (!caller) return undefined; + const callerRecord = caller as Record; + const model = callerRecord.model ?? callerRecord.metadata?.model; + return typeof model === "string" && model.trim() ? model : undefined; } export class LlamaIndexInstrumentor { @@ -48,6 +88,12 @@ export class LlamaIndexInstrumentor { private _callbackManager: CallbackManagerLike | null = null; private _bindings: HandlerBinding[] = []; private _isInstrumented = false; + private _settings: LlamaIndexSettingsLike | null = null; + private _llmDescriptor?: PropertyDescriptor; + private _llmDescriptorWasOwn = false; + private _settingsWithLLM?: LlamaIndexSettingsLike["withLLM"]; + private _withLLMWasOwn = false; + private readonly _patchedLlms = new Map(); constructor(options: LlamaIndexInstrumentorOptions = {}) { this._options = options; @@ -85,6 +131,7 @@ export class LlamaIndexInstrumentor { } this._callbackManager = callbackManager; + this._patchSettings(llamaIndex.Settings as LlamaIndexSettingsLike); this._bindings = this._buildHandlers(); for (const binding of this._bindings) { callbackManager.on(binding.event, binding.handler); @@ -105,6 +152,11 @@ export class LlamaIndexInstrumentor { } } + this._emitter.flushPending({ + errorMessage: "LlamaIndex operation ended without a matching completion event", + endTime: hrTime(), + }); + this._restoreSettings(); this._emitter.clear(); this._bindings = []; this._callbackManager = null; @@ -117,10 +169,13 @@ export class LlamaIndexInstrumentor { event: LLAMA_INDEX_EVENTS.LLM_START, handler: (event) => { const detail = eventDetail(event); + const caller = eventCaller(event); this._emitter.startLLM({ id: eventId(detail, "llm"), messages: detail.messages, startTime: hrTime(), + model: callerModel(caller) ?? this._settingsModel(), + caller, }); }, }, @@ -227,13 +282,16 @@ export class LlamaIndexInstrumentor { logType: string, fallbackPrefix: string, ): HandlerBinding[] { + const pendingIds: string[] = []; return [ { event: startEvent, handler: (event) => { const detail = eventDetail(event); + const id = eventId(detail, fallbackPrefix, event); + pendingIds.push(id); this._emitter.startRecord({ - id: eventId(detail, fallbackPrefix), + id, name, logType, startTime: hrTime(), @@ -245,8 +303,15 @@ export class LlamaIndexInstrumentor { event: endEvent, handler: (event) => { const detail = eventDetail(event); + const correlatedId = eventCorrelationId(detail, event); + const correlatedIndex = correlatedId + ? pendingIds.lastIndexOf(correlatedId) + : -1; + const id = correlatedIndex >= 0 + ? pendingIds.splice(correlatedIndex, 1)[0] + : pendingIds.pop() ?? correlatedId ?? eventId(detail, fallbackPrefix, event); this._emitter.endRecord({ - id: eventId(detail, fallbackPrefix), + id, output: formatTaskOutput(detail), endTime: hrTime(), }); @@ -254,6 +319,127 @@ export class LlamaIndexInstrumentor { }, ]; } + + private _patchSettings(settings: LlamaIndexSettingsLike): void { + this._settings = settings; + this._llmDescriptorWasOwn = Object.hasOwn(settings, "llm"); + let descriptorOwner: object | null = settings; + while (descriptorOwner && !Object.getOwnPropertyDescriptor(descriptorOwner, "llm")) { + descriptorOwner = Object.getPrototypeOf(descriptorOwner); + } + this._llmDescriptor = descriptorOwner + ? Object.getOwnPropertyDescriptor(descriptorOwner, "llm") + : undefined; + const originalDescriptor = this._llmDescriptor; + if (originalDescriptor?.get && originalDescriptor.set && originalDescriptor.configurable) { + const instrumentor = this; + Object.defineProperty(settings, "llm", { + ...originalDescriptor, + get: originalDescriptor.get, + set(value: unknown) { + originalDescriptor.set?.call(settings, instrumentor._patchLlm(value)); + }, + }); + try { + this._patchLlm(originalDescriptor.get.call(settings)); + } catch { + // LlamaIndex throws while no default LLM has been configured. + } + } + + if (typeof settings.withLLM === "function") { + this._withLLMWasOwn = Object.hasOwn(settings, "withLLM"); + this._settingsWithLLM = settings.withLLM; + const originalWithLLM = settings.withLLM; + settings.withLLM = (llm, fn) => originalWithLLM.call(settings, this._patchLlm(llm), fn); + } + } + + private _settingsModel(): string | undefined { + try { + const llm = this._settings?.llm; + return llm && typeof llm === "object" ? callerModel(llm) : undefined; + } catch { + return undefined; + } + } + + private _patchLlm(value: unknown): unknown { + if (!value || typeof value !== "object") return value; + const llm = value as PatchedLlm; + if (typeof llm.chat !== "function" || this._patchedLlms.has(llm)) return llm; + + const ownDescriptor = Object.getOwnPropertyDescriptor(llm, "chat"); + const originalChat = llm.chat; + const instrumentor = this; + const wrappedChat = async function (this: object, ...args: any[]): Promise { + const caller = this; + try { + const response = await originalChat.apply(this, args); + if (response && typeof response === "object" && Symbol.asyncIterator in response) { + const originalIterator = response[Symbol.asyncIterator].bind(response); + response[Symbol.asyncIterator] = async function* () { + try { + yield* originalIterator(); + } catch (error) { + instrumentor._recordLlmFailure(caller, error); + throw error; + } + }; + } + return response; + } catch (error) { + instrumentor._recordLlmFailure(this, error); + throw error; + } + }; + Object.defineProperty(llm, "chat", { + configurable: true, + enumerable: ownDescriptor?.enumerable ?? false, + writable: true, + value: wrappedChat, + }); + this._patchedLlms.set(llm, ownDescriptor); + return llm; + } + + private _recordLlmFailure(caller: object, error: unknown): void { + this._emitter.failLLMForCaller({ + caller, + errorMessage: sanitizeErrorMessage(error), + endTime: hrTime(), + }); + } + + private _restoreSettings(): void { + if (this._settings && this._llmDescriptor) { + if (this._llmDescriptorWasOwn) { + Object.defineProperty(this._settings, "llm", this._llmDescriptor); + } else { + delete (this._settings as Record).llm; + } + } + if (this._settings && this._settingsWithLLM) { + if (this._withLLMWasOwn) { + this._settings.withLLM = this._settingsWithLLM; + } else { + delete (this._settings as Record).withLLM; + } + } + for (const [llm, descriptor] of this._patchedLlms) { + if (descriptor) { + Object.defineProperty(llm, "chat", descriptor); + } else { + delete (llm as Record).chat; + } + } + this._patchedLlms.clear(); + this._settings = null; + this._llmDescriptor = undefined; + this._llmDescriptorWasOwn = false; + this._settingsWithLLM = undefined; + this._withLLMWasOwn = false; + } } export { LlamaIndexSpanEmitter } from "./_emitter.js"; diff --git a/javascript-sdks/instrumentations/respan-instrumentation-llama-index/tests/llama_index_instrumentor.test.mjs b/javascript-sdks/instrumentations/respan-instrumentation-llama-index/tests/llama_index_instrumentor.test.mjs index 61c7ff58..48365edc 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-llama-index/tests/llama_index_instrumentor.test.mjs +++ b/javascript-sdks/instrumentations/respan-instrumentation-llama-index/tests/llama_index_instrumentor.test.mjs @@ -212,3 +212,122 @@ test("deactivate removes handlers", async () => { assert.equal(captureState.spans.length, 0); }); + +test("rejected LLM calls emit failed chat and ancestor spans with sanitized errors", async () => { + const instrumentor = new LlamaIndexInstrumentor({ + workflowName: "llama_index_ts_unit_failure", + }); + await instrumentor.activate(); + + const failingLlm = { + model: "gpt-4.1-nano", + async chat() { + Settings.callbackManager.dispatchEvent( + "query-start", + { id: "query-failure", query: "Fail safely" }, + true, + ); + Settings.callbackManager.dispatchEvent( + "llm-start", + { + id: "llm-failure", + messages: [{ role: "user", content: "Fail safely" }], + }, + true, + ); + throw new Error("401 invalid api_key=sk-super-secret-token"); + }, + }; + Settings.llm = failingLlm; + + await assert.rejects(() => Settings.llm.chat({ messages: [] }), /401 invalid/); + + assert.equal(captureState.spans.length, 2); + const chatSpan = captureState.spans.find( + (span) => span.attributes["respan.entity.log_type"] === "chat", + ); + const workflowSpan = captureState.spans.find( + (span) => span.attributes["respan.entity.log_type"] === "workflow", + ); + assert.ok(chatSpan); + assert.ok(workflowSpan); + assert.equal(chatSpan.status.code, 2); + assert.equal(workflowSpan.status.code, 2); + assert.equal(chatSpan.attributes.status_code, 500); + assert.equal(workflowSpan.attributes.status_code, 500); + assert.equal(chatSpan.attributes["gen_ai.request.model"], "gpt-4.1-nano"); + assert.equal(chatSpan.attributes["gen_ai.prompt.0.content"], "Fail safely"); + assert.equal(chatSpan.attributes["error.message"], "401 invalid api_key=[REDACTED]"); + assert.equal(workflowSpan.attributes["error.message"], "401 invalid api_key=[REDACTED]"); + assert.equal(chatSpan.parentSpanContext?.spanId, workflowSpan.spanContext().spanId); + + instrumentor.deactivate(); + assert.equal(Object.hasOwn(failingLlm, "chat"), true); + assert.equal(failingLlm.chat.name, "chat"); +}); + +test("deactivate exports unfinished callbacks as failed spans instead of dropping them", async () => { + const instrumentor = new LlamaIndexInstrumentor({ + workflowName: "llama_index_ts_unit_pending", + }); + await instrumentor.activate(); + + Settings.callbackManager.dispatchEvent( + "llm-start", + { + id: "llm-pending", + messages: [{ role: "user", content: "Pending request" }], + }, + true, + ); + instrumentor.deactivate(); + + assert.equal(captureState.spans.length, 1); + assert.equal(captureState.spans[0].status.code, 2); + assert.equal(captureState.spans[0].attributes.status_code, 500); + assert.match(captureState.spans[0].attributes["error.message"], /without a matching/); +}); + +test("correlates id-less start and end events by LlamaIndex event reason", async () => { + const handlers = new Map(); + const callbackManager = { + on(event, handler) { + handlers.set(event, handler); + }, + off(event) { + handlers.delete(event); + }, + }; + const instrumentor = new LlamaIndexInstrumentor({ + workflowName: "llama_index_ts_unit_idless", + llamaIndexModule: { Settings: { callbackManager } }, + }); + await instrumentor.activate(); + + const reason = { id: "stable-event-reason" }; + handlers.get("chunking-start")({ + detail: { chunks: ["input text"] }, + reason, + }); + handlers.get("chunking-end")({ + detail: { chunks: ["output chunk"] }, + reason, + }); + + handlers.get("node-parsing-start")({ + detail: { documents: [{ text: "input document" }] }, + reason: null, + }); + handlers.get("node-parsing-end")({ + detail: { nodes: [{ text: "output node" }] }, + reason: null, + }); + + assert.equal(captureState.spans.length, 2); + for (const span of captureState.spans) { + assert.equal(span.status.code, 1); + assert.equal(span.attributes["error.message"], undefined); + } + instrumentor.deactivate(); + assert.equal(captureState.spans.length, 2); +}); diff --git a/javascript-sdks/instrumentations/respan-instrumentation-mastra/src/index.ts b/javascript-sdks/instrumentations/respan-instrumentation-mastra/src/index.ts index df7fd16e..cbb5c0ce 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-mastra/src/index.ts +++ b/javascript-sdks/instrumentations/respan-instrumentation-mastra/src/index.ts @@ -1,6 +1,7 @@ import { context, trace } from "@opentelemetry/api"; import type { ReadableSpan } from "@opentelemetry/sdk-trace-base"; import { + ATTR_ERROR_MESSAGE, ATTR_GEN_AI_COMPLETION, ATTR_GEN_AI_PROMPT, ATTR_GEN_AI_REQUEST_MODEL, @@ -76,7 +77,8 @@ export class MastraInstrumentor { private readonly _excludedSpanTypes: Set; private readonly _traceIdMap = new Map(); private readonly _emittedSpanIds = new Set(); - private readonly _pendingToolSpans = new Map(); + private readonly _emittedErrors = new Map(); + private readonly _pendingChildSpans = new Map(); constructor(options: MastraInstrumentorOptions = {}) { this._excludedSpanTypes = new Set([ @@ -91,9 +93,11 @@ export class MastraInstrumentor { deactivate(): void { this._enabled = false; + this._drainAllPendingChildSpans(); this._traceIdMap.clear(); this._emittedSpanIds.clear(); - this._pendingToolSpans.clear(); + this._emittedErrors.clear(); + this._pendingChildSpans.clear(); } onTracingEvent(event: MastraTracingEvent): void { @@ -110,11 +114,11 @@ export class MastraInstrumentor { return; } - if (this._shouldBufferToolSpan(span)) { + if (this._shouldBufferChildSpan(span)) { const pendingKey = resolvePendingToolKey(span); - const pendingSpans = this._pendingToolSpans.get(pendingKey) ?? []; + const pendingSpans = this._pendingChildSpans.get(pendingKey) ?? []; pendingSpans.push(span); - this._pendingToolSpans.set(pendingKey, pendingSpans); + this._pendingChildSpans.set(pendingKey, pendingSpans); return; } @@ -123,7 +127,7 @@ export class MastraInstrumentor { this._rememberEmittedSpan(span, readableSpan); if (span.type === "agent_run") { - this._drainPendingToolSpans(span, readableSpan.spanContext().spanId); + this._drainPendingChildSpans(span, readableSpan.spanContext().spanId); } } @@ -132,10 +136,11 @@ export class MastraInstrumentor { } async shutdown(): Promise { - this._drainAllPendingToolSpans(); + this._drainAllPendingChildSpans(); this._traceIdMap.clear(); this._emittedSpanIds.clear(); - this._pendingToolSpans.clear(); + this._emittedErrors.clear(); + this._pendingChildSpans.clear(); } private _buildReadableSpan( @@ -152,6 +157,17 @@ export class MastraInstrumentor { ? activeSpanContext?.spanId : undefined ); + const rawErrorInfo = + span.errorInfo ?? (parentId ? this._emittedErrors.get(parentId) : undefined); + const effectiveErrorInfo = rawErrorInfo + ? { + ...rawErrorInfo, + message: sanitizeErrorMessage(rawErrorInfo.message), + } + : undefined; + const exportedSpan = effectiveErrorInfo + ? { ...span, errorInfo: effectiveErrorInfo } + : span; const readableSpan = buildReadableSpan({ name: span.name || resolveEntityName(span), @@ -160,9 +176,9 @@ export class MastraInstrumentor { parentId, startTimeIso: toIsoString(span.startTime), endTimeIso: toIsoString(span.endTime ?? new Date()), - attributes: buildMastraAttributes(span), - statusCode: span.errorInfo ? 500 : 200, - errorMessage: span.errorInfo?.message, + attributes: buildMastraAttributes(exportedSpan), + statusCode: effectiveErrorInfo ? 500 : 200, + errorMessage: effectiveErrorInfo?.message, }) as ReadableSpan & { instrumentationScope?: { name: string; version?: string }; }; @@ -171,6 +187,10 @@ export class MastraInstrumentor { name: INSTRUMENTATION_NAME, version: PACKAGE_VERSION, }; + if (effectiveErrorInfo) { + this._emittedErrors.set(span.id, effectiveErrorInfo); + this._emittedErrors.set(readableSpan.spanContext().spanId, effectiveErrorInfo); + } return readableSpan; } @@ -185,8 +205,12 @@ export class MastraInstrumentor { return resolvedTraceId; } - private _shouldBufferToolSpan(span: MastraExportedSpan): boolean { - return isToolSpan(span) && (!(span.parentSpanId ?? (span as any).parentSpanContext?.spanId) || !this._emittedSpanIds.has((span.parentSpanId ?? (span as any).parentSpanContext?.spanId))); + private _shouldBufferChildSpan(span: MastraExportedSpan): boolean { + const parentSpanId = span.parentSpanId ?? (span as any).parentSpanContext?.spanId; + return ( + (isToolSpan(span) || isModelSpan(span.type)) && + (!parentSpanId || !this._emittedSpanIds.has(parentSpanId)) + ); } private _rememberEmittedSpan(span: MastraExportedSpan, readableSpan: ReadableSpan): void { @@ -194,14 +218,14 @@ export class MastraInstrumentor { this._emittedSpanIds.add(readableSpan.spanContext().spanId); } - private _drainPendingToolSpans(agentSpan: MastraExportedSpan, parentId: string): void { + private _drainPendingChildSpans(agentSpan: MastraExportedSpan, parentId: string): void { const pendingKey = resolvePendingToolKey(agentSpan); - const pendingSpans = this._pendingToolSpans.get(pendingKey); + const pendingSpans = this._pendingChildSpans.get(pendingKey); if (!pendingSpans || pendingSpans.length === 0) { return; } - this._pendingToolSpans.delete(pendingKey); + this._pendingChildSpans.delete(pendingKey); for (const pendingSpan of pendingSpans) { const readableSpan = this._buildReadableSpan(pendingSpan, parentId); injectSpan(readableSpan); @@ -209,15 +233,15 @@ export class MastraInstrumentor { } } - private _drainAllPendingToolSpans(): void { - for (const pendingSpans of this._pendingToolSpans.values()) { + private _drainAllPendingChildSpans(): void { + for (const pendingSpans of this._pendingChildSpans.values()) { for (const pendingSpan of pendingSpans) { const readableSpan = this._buildReadableSpan(pendingSpan); injectSpan(readableSpan); this._rememberEmittedSpan(pendingSpan, readableSpan); } } - this._pendingToolSpans.clear(); + this._pendingChildSpans.clear(); } } @@ -236,6 +260,10 @@ function buildMastraAttributes(span: MastraExportedSpan): Record, span: MastraExportedSpan, diff --git a/javascript-sdks/instrumentations/respan-instrumentation-mastra/tests/mastra_instrumentor.test.mjs b/javascript-sdks/instrumentations/respan-instrumentation-mastra/tests/mastra_instrumentor.test.mjs index 8c0f355d..2c53849c 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-mastra/tests/mastra_instrumentor.test.mjs +++ b/javascript-sdks/instrumentations/respan-instrumentation-mastra/tests/mastra_instrumentor.test.mjs @@ -225,3 +225,69 @@ test("drops model chunk spans by default", async () => { assert.equal(captureState.spans.length, 0); }); + +test("propagates a late agent failure to its buffered model span", async () => { + captureState.spans = []; + const instrumentor = new MastraInstrumentor(); + + await instrumentor.exportTracingEvent({ + type: "span_ended", + exportedSpan: { + id: "failed-model", + traceId: "failed-trace", + parentSpanId: "failed-agent", + name: "model generation", + type: "model_generation", + startTime: new Date("2026-05-21T00:00:00.100Z"), + endTime: new Date("2026-05-21T00:00:00.800Z"), + attributes: { + model: "openai/gpt-4.1-nano", + provider: "openai", + }, + input: [{ role: "user", content: "Fail safely" }], + metadata: { runId: "failed-run" }, + }, + }); + assert.equal(captureState.spans.length, 0); + + await instrumentor.exportTracingEvent({ + type: "span_ended", + exportedSpan: { + id: "failed-agent", + traceId: "failed-trace", + name: "Mastra Failure Example.workflow", + type: "agent_run", + isRootSpan: true, + startTime: new Date("2026-05-21T00:00:00.000Z"), + endTime: new Date("2026-05-21T00:00:01.000Z"), + input: "Fail safely", + metadata: { runId: "failed-run" }, + errorInfo: { message: "provider request failed api_key=sk-super-secret-token" }, + }, + }); + + assert.equal(captureState.spans.length, 2); + const agentSpan = captureState.spans.find( + (span) => span.attributes["respan.entity.log_type"] === "agent", + ); + const modelSpan = captureState.spans.find( + (span) => span.attributes["respan.entity.log_type"] === "chat", + ); + assert.ok(agentSpan); + assert.ok(modelSpan); + assert.equal(agentSpan.status.code, 2); + assert.equal(modelSpan.status.code, 2); + assert.equal(agentSpan.attributes.status_code, 500); + assert.equal(modelSpan.attributes.status_code, 500); + assert.equal( + agentSpan.attributes["error.message"], + "provider request failed api_key=[REDACTED]", + ); + assert.equal( + modelSpan.attributes["error.message"], + "provider request failed api_key=[REDACTED]", + ); + assert.equal(modelSpan.attributes["gen_ai.request.model"], "gpt-4.1-nano"); + assert.equal(modelSpan.attributes["gen_ai.prompt.0.content"], "Fail safely"); + assert.equal(modelSpan.parentSpanContext?.spanId, agentSpan.spanContext().spanId); +}); From f37d4a87392821e36d3ae0c7e42117c826e52b6b Mon Sep 17 00:00:00 2001 From: Nightingalelyy Date: Tue, 18 Aug 2026 13:39:13 +0800 Subject: [PATCH 2/2] fix(instrumentation-js): align LangChain spans with contract --- .../src/_callback.ts | 40 ++++++++----------- .../src/_callback_helpers.ts | 8 ---- .../tests/langchain_instrumentation.test.mjs | 40 ++++++++++++------- 3 files changed, 43 insertions(+), 45 deletions(-) diff --git a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback.ts b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback.ts index 4fd8f79c..c0cd73b8 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback.ts +++ b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback.ts @@ -20,13 +20,6 @@ import { TraceloopSpanKindValues, } from "@traceloop/ai-semantic-conventions"; import { - DIRECT_COMPLETION_TOKENS, - DIRECT_INPUT, - DIRECT_MODEL, - DIRECT_OUTPUT, - DIRECT_PROMPT_TOKENS, - DIRECT_TOOLS, - DIRECT_TOTAL_REQUEST_TOKENS, LANGCHAIN_FRAMEWORK_ATTR, LANGCHAIN_METADATA_ATTR, LANGCHAIN_PARENT_RUN_ID_ATTR, @@ -276,7 +269,6 @@ export class RespanCallbackHandler { const attrs: SpanAttributesRecord = { [RespanSpanAttributes.RESPAN_LOG_METHOD]: RESPAN_LOG_METHOD_TS_TRACING, [RespanSpanAttributes.RESPAN_LOG_TYPE]: record.logType, - [SpanAttributes.TRACELOOP_SPAN_KIND]: record.spanKind, [SpanAttributes.TRACELOOP_ENTITY_NAME]: record.name, [SpanAttributes.TRACELOOP_ENTITY_PATH]: record.entityPath, [SpanAttributes.TRACELOOP_WORKFLOW_NAME]: record.workflowName, @@ -301,9 +293,7 @@ export class RespanCallbackHandler { const inputString = safeJsonString(record.inputValue); const outputString = safeJsonString(normalizeOutputForLogging(outputValue)); setIfPresent(attrs, SpanAttributes.TRACELOOP_ENTITY_INPUT, inputString); - setIfPresent(attrs, DIRECT_INPUT, inputString); setIfPresent(attrs, SpanAttributes.TRACELOOP_ENTITY_OUTPUT, outputString); - setIfPresent(attrs, DIRECT_OUTPUT, outputString); } Object.assign(attrs, record.extraAttributes); @@ -511,7 +501,6 @@ export class RespanCallbackHandler { }; const model = extractModel(serialized, undefined, normalizedMetadata); setIfPresent(extraAttributes, ATTR_GEN_AI_REQUEST_MODEL, model); - setIfPresent(extraAttributes, DIRECT_MODEL, model); for (const [index, message] of firstConversation.entries()) { for (const [key, value] of Object.entries(message)) { setIfPresent( @@ -522,8 +511,11 @@ export class RespanCallbackHandler { } } const tools = extractTools({ serialized, extraParams }); - setIfPresent(extraAttributes, RespanSpanAttributes.RESPAN_SPAN_TOOLS, tools); - setIfPresent(extraAttributes, DIRECT_TOOLS, tools); + setIfPresent( + extraAttributes, + SpanAttributes.LLM_REQUEST_FUNCTIONS, + tools ? safeJsonString(tools) : undefined, + ); this._startRun({ runId, @@ -552,14 +544,16 @@ export class RespanCallbackHandler { const normalizedPrompts = Array.isArray(prompts) ? prompts : [prompts]; const normalizedMetadata = normalizeMetadata(metadata); const extraAttributes: SpanAttributesRecord = { - [SpanAttributes.LLM_REQUEST_TYPE]: LLMRequestTypeValues.COMPLETION, + [SpanAttributes.LLM_REQUEST_TYPE]: LLMRequestTypeValues.CHAT, }; const model = extractModel(serialized, undefined, normalizedMetadata); setIfPresent(extraAttributes, ATTR_GEN_AI_REQUEST_MODEL, model); - setIfPresent(extraAttributes, DIRECT_MODEL, model); const tools = extractTools({ serialized, extraParams }); - setIfPresent(extraAttributes, RespanSpanAttributes.RESPAN_SPAN_TOOLS, tools); - setIfPresent(extraAttributes, DIRECT_TOOLS, tools); + setIfPresent( + extraAttributes, + SpanAttributes.LLM_REQUEST_FUNCTIONS, + tools ? safeJsonString(tools) : undefined, + ); for (const [index, prompt] of normalizedPrompts.entries()) { extraAttributes[`${ATTR_GEN_AI_PROMPT}.${index}.role`] = "user"; extraAttributes[`${ATTR_GEN_AI_PROMPT}.${index}.content`] = String(prompt ?? ""); @@ -570,7 +564,7 @@ export class RespanCallbackHandler { parentRunId, name: extractName(serialized, "llm", runName), logType: RespanLogType.TEXT, - spanKind: LLMRequestTypeValues.COMPLETION, + spanKind: LLMRequestTypeValues.CHAT, inputValue: normalizedPrompts, serialized, tags: normalizeTags(tags), @@ -598,7 +592,6 @@ export class RespanCallbackHandler { if (record) { const model = extractModel(record.serialized, output, record.metadata); setIfPresent(extraAttributes, ATTR_GEN_AI_REQUEST_MODEL, model); - setIfPresent(extraAttributes, DIRECT_MODEL, model); } for (const [index, message] of completionMessages.entries()) { @@ -612,17 +605,18 @@ export class RespanCallbackHandler { } const toolCalls = extractToolCallsFromMessages(completionMessages); - setIfPresent(extraAttributes, RespanSpanAttributes.RESPAN_SPAN_TOOL_CALLS, toolCalls); + setIfPresent( + extraAttributes, + `${ATTR_GEN_AI_COMPLETION}.0.tool_calls`, + toolCalls ? safeJsonString(toolCalls) : undefined, + ); const usage = extractUsage(output); setIfPresent(extraAttributes, ATTR_GEN_AI_USAGE_PROMPT_TOKENS, usage.promptTokens); setIfPresent(extraAttributes, ATTR_GEN_AI_USAGE_INPUT_TOKENS, usage.promptTokens); - setIfPresent(extraAttributes, DIRECT_PROMPT_TOKENS, usage.promptTokens); setIfPresent(extraAttributes, ATTR_GEN_AI_USAGE_COMPLETION_TOKENS, usage.completionTokens); setIfPresent(extraAttributes, ATTR_GEN_AI_USAGE_OUTPUT_TOKENS, usage.completionTokens); - setIfPresent(extraAttributes, DIRECT_COMPLETION_TOKENS, usage.completionTokens); setIfPresent(extraAttributes, SpanAttributes.LLM_USAGE_TOTAL_TOKENS, usage.totalTokens); - setIfPresent(extraAttributes, DIRECT_TOTAL_REQUEST_TOKENS, usage.totalTokens); this._endRun({ runId, outputValue: outputPayload, extraAttributes }); } diff --git a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback_helpers.ts b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback_helpers.ts index 64420fbb..89915a76 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback_helpers.ts +++ b/javascript-sdks/instrumentations/respan-instrumentation-langchain/src/_callback_helpers.ts @@ -11,14 +11,6 @@ export const LANGCHAIN_TAGS_ATTR = "langchain.tags"; export const LANGCHAIN_METADATA_ATTR = "langchain.metadata"; export const LANGCHAIN_SERIALIZED_ATTR = "langchain.serialized"; -export const DIRECT_INPUT = "input"; -export const DIRECT_OUTPUT = "output"; -export const DIRECT_MODEL = "model"; -export const DIRECT_PROMPT_TOKENS = "prompt_tokens"; -export const DIRECT_COMPLETION_TOKENS = "completion_tokens"; -export const DIRECT_TOTAL_REQUEST_TOKENS = "total_request_tokens"; -export const DIRECT_TOOLS = "tools"; - const JSON_CODE_FENCE_RE = /^\s*(?`{3,}|~{3,})[ \t]*(?jsonc?)?[ \t]*\r?\n(?.*?)(?:\r?\n)?\k\s*$/is; diff --git a/javascript-sdks/instrumentations/respan-instrumentation-langchain/tests/langchain_instrumentation.test.mjs b/javascript-sdks/instrumentations/respan-instrumentation-langchain/tests/langchain_instrumentation.test.mjs index d297318b..9e918e33 100644 --- a/javascript-sdks/instrumentations/respan-instrumentation-langchain/tests/langchain_instrumentation.test.mjs +++ b/javascript-sdks/instrumentations/respan-instrumentation-langchain/tests/langchain_instrumentation.test.mjs @@ -189,7 +189,7 @@ test("createAgent root emits one agent boundary and propagates its workflow name const [toolSpan, agentSpan] = captured; assert.equal(agentSpan.attributes["respan.entity.log_type"], "agent"); - assert.equal(agentSpan.attributes["traceloop.span.kind"], "agent"); + assert.equal(agentSpan.attributes["traceloop.span.kind"], undefined); assert.equal(toolSpan.parentSpanContext?.spanId, agentSpan.spanContext().spanId); assert.equal(toolSpan.attributes["traceloop.workflow.name"], "support_agent"); assert.equal(agentSpan.attributes["traceloop.workflow.name"], "support_agent"); @@ -232,20 +232,31 @@ test("chat model output maps messages, usage, model, tool calls, and strips JSON assert.equal(span.attributes["respan.entity.log_type"], "chat"); assert.equal(span.attributes["llm.request.type"], "chat"); assert.equal(span.attributes["gen_ai.request.model"], "gpt-4o-mini"); - assert.equal(span.attributes.model, "gpt-4o-mini"); assert.equal(span.attributes["gen_ai.usage.prompt_tokens"], 12); assert.equal(span.attributes["gen_ai.usage.completion_tokens"], 4); - assert.equal(span.attributes.prompt_tokens, 12); - assert.equal(span.attributes.completion_tokens, 4); - assert.equal(span.attributes.total_request_tokens, 16); assert.equal(span.attributes["gen_ai.prompt.0.role"], "user"); assert.equal( span.attributes["gen_ai.completion.0.content"], '{"owner":"Security Operations Team"}', ); - assert.equal(span.attributes.output.includes("```"), false); - assert.ok(Array.isArray(span.attributes["respan.span.tool_calls"])); - assert.equal(span.attributes["respan.span.tool_calls"][0].function.name, "router"); + assert.equal(span.attributes["traceloop.entity.output"].includes("```"), false); + assert.equal( + JSON.parse(span.attributes["gen_ai.completion.0.tool_calls"])[0].function.name, + "router", + ); + for (const alias of [ + "traceloop.span.kind", + "respan.span.tools", + "respan.span.tool_calls", + "model", + "prompt_tokens", + "completion_tokens", + "total_request_tokens", + "tools", + "tool_calls", + ]) { + assert.equal(span.attributes[alias], undefined); + } }); test("chat model start preserves full tool definitions from LangChain extra params", () => { @@ -284,8 +295,9 @@ test("chat model start preserves full tool definitions from LangChain extra para ); const span = captured[0]; - assert.deepEqual(span.attributes["respan.span.tools"], tools); - assert.deepEqual(span.attributes.tools, tools); + assert.deepEqual(JSON.parse(span.attributes["llm.request.functions"]), tools); + assert.equal(span.attributes["respan.span.tools"], undefined); + assert.equal(span.attributes.tools, undefined); }); test("LLM streaming falls back to collected text when final output is empty", () => { @@ -304,8 +316,8 @@ test("LLM streaming falls back to collected text when final output is empty", () const span = captured[0]; assert.equal(span.attributes["respan.entity.log_type"], "text"); - assert.equal(span.attributes["llm.request.type"], "completion"); - assert.equal(span.attributes.output, "old pond"); + assert.equal(span.attributes["llm.request.type"], "chat"); + assert.equal(span.attributes["traceloop.entity.output"], "old pond"); }); test("handleText streaming fallback records chain text output", () => { @@ -318,7 +330,7 @@ test("handleText streaming fallback records chain text output", () => { handler.handleText("world", chainRunId); handler.handleChainEnd(undefined, chainRunId); - assert.equal(captured[0].attributes.output, "hello world"); + assert.equal(captured[0].attributes["traceloop.entity.output"], "hello world"); }); test("tool and retriever callbacks map fields and errors", () => { @@ -341,7 +353,7 @@ test("tool and retriever callbacks map fields and errors", () => { assert.equal(toolSpan.attributes["gen_ai.tool.call.arguments"], '{"expression":"2+2"}'); assert.equal(toolSpan.attributes["gen_ai.tool.call.result"], '{"answer":4}'); assert.equal(retrieverSpan.attributes["respan.entity.log_type"], "task"); - assert.equal(retrieverSpan.attributes.output.includes("doc text"), true); + assert.equal(retrieverSpan.attributes["traceloop.entity.output"].includes("doc text"), true); assert.equal(errorSpan.status.code, 2); assert.equal(errorSpan.attributes["error.message"], "tool failed"); assert.equal(errorSpan.attributes.status_code, 500);