fix(instrumentation-py): repair Claude Agent SDK, Cohere, and CrewAI OTel 2.x spans

.release-intents/20260816-python-claude-agent-cohere-crewai-tracing.json

@@ -0,0 +1,9 @@
+{
+  "summary": "Repair canonical agent, rerank, and tool span content for the Claude Agent SDK, Cohere, and CrewAI instrumentations, including Claude agent/chat export splitting",
+  "packages": {
+    "respan-instrumentation-claude-agent-sdk": "patch",
+    "respan-instrumentation-cohere": "patch",
+    "respan-instrumentation-crewai": "patch",
+    "respan-tracing": "patch"
+  }
+}

python-sdks/instrumentations/respan-instrumentation-claude-agent-sdk/src/respan_instrumentation_claude_agent_sdk/_constants.py

@@ -49,10 +49,6 @@
         CLAUDE_AGENT_SDK_RESPONSE_FINISH_REASONS_ATTR,
         CLAUDE_AGENT_SDK_SYSTEM_INSTRUCTIONS_ATTR,
         CLAUDE_AGENT_SDK_TOOL_DEFINITIONS_ATTR,
-        CLAUDE_AGENT_SDK_USAGE_INPUT_TOKENS_ATTR,
-        CLAUDE_AGENT_SDK_USAGE_OUTPUT_TOKENS_ATTR,
-        SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS,
-        SpanAttributes.LLM_USAGE_CACHE_CREATION_INPUT_TOKENS,
         INPUT_VALUE_ATTR,
         OUTPUT_VALUE_ATTR,
         RESPAN_OVERRIDE_COMPLETION_TOKENS_ATTR,

python-sdks/instrumentations/respan-instrumentation-claude-agent-sdk/src/respan_instrumentation_claude_agent_sdk/_processor.py

@@ -714,7 +714,9 @@ def enrich_claude_agent_sdk_span(span: ReadableSpan) -> None:
             _extract_usage(attrs)
         )
 
-        _set_if_missing(attrs, RESPAN_LOG_TYPE, LOG_TYPE_AGENT)
+        # Upstream marks invoke_agent spans as chat because they also carry GenAI
+        # message attributes. The operation boundary is authoritative here.
+        attrs[RESPAN_LOG_TYPE] = LOG_TYPE_AGENT
         _set_if_missing(attrs, SpanAttributes.TRACELOOP_ENTITY_NAME, agent_name)
         _set_if_missing(attrs, SpanAttributes.TRACELOOP_ENTITY_PATH, agent_name)
         _set_if_missing(attrs, SpanAttributes.TRACELOOP_WORKFLOW_NAME, agent_name)
@@ -762,6 +764,7 @@ def enrich_claude_agent_sdk_span(span: ReadableSpan) -> None:
         key: value
         for key, value in attrs.items()
         if key not in CLAUDE_AGENT_SDK_STRIP_ATTRS
+        and not key.startswith("gen_ai.tool.")
     }
     _set_if_unset_span_status(span, span._attributes)
 
@@ -776,12 +779,18 @@ def __init__(self) -> None:
         ] = {}
         self._pending_tool_calls_lock = threading.Lock()
 
-    def _store_pending_tool_call(self, span: ReadableSpan) -> None:
+    def _store_pending_tool_call(
+        self,
+        span: ReadableSpan,
+        source_attrs: Mapping[str, Any] | None = None,
+    ) -> None:
         parent_span_key = _get_parent_span_key(span)
         if parent_span_key is None:
             return
 
-        attrs = getattr(span, "_attributes", None)
+        attrs = source_attrs
+        if attrs is None:
+            attrs = getattr(span, "_attributes", None)
         if not isinstance(attrs, Mapping):
             return
 
@@ -821,6 +830,9 @@ def on_start(self, span: Any, parent_context: Any = None) -> None:
 
     def on_end(self, span: ReadableSpan) -> None:
         try:
+            original_attrs = getattr(span, "_attributes", None)
+            if isinstance(original_attrs, Mapping):
+                original_attrs = dict(original_attrs)
             enrich_claude_agent_sdk_span(span)
 
             attrs = getattr(span, "_attributes", None)
@@ -829,7 +841,19 @@ def on_end(self, span: ReadableSpan) -> None:
                 return
 
             if attrs.get(RESPAN_LOG_TYPE) == LOG_TYPE_TOOL:
-                self._store_pending_tool_call(span)
+                # Correlate with the upstream call ID before helper attributes
+                # are stripped, while using the normalized canonical name and
+                # arguments from the exported tool span.
+                pending_attrs = dict(attrs)
+                if isinstance(original_attrs, Mapping):
+                    tool_call_id = original_attrs.get(
+                        CLAUDE_AGENT_SDK_TOOL_CALL_ID_ATTR
+                    )
+                    if tool_call_id:
+                        pending_attrs[CLAUDE_AGENT_SDK_TOOL_CALL_ID_ATTR] = (
+                            tool_call_id
+                        )
+                self._store_pending_tool_call(span, pending_attrs)
                 # Only agent spans merge queued tool calls into their final attrs.
                 # Drop any child calls queued against non-agent parents on span end.
                 self._consume_pending_tool_calls(span)

python-sdks/instrumentations/respan-instrumentation-claude-agent-sdk/tests/test_instrumentation.py

@@ -786,11 +786,14 @@ def test_enrich_claude_agent_sdk_span_maps_agent_fields():
     assert span._attributes[SpanAttributes.LLM_USAGE_PROMPT_TOKENS] == 19
     assert span._attributes[SpanAttributes.LLM_USAGE_COMPLETION_TOKENS] == 7
     assert span._attributes[SpanAttributes.LLM_USAGE_TOTAL_TOKENS] == 26
+    assert span._attributes["gen_ai.usage.input_tokens"] == 19
+    assert span._attributes["gen_ai.usage.output_tokens"] == 7
     assert span._attributes[RESPAN_SESSION_ID] == "session-123"
     assert span._status.status_code == StatusCode.OK
     assert json.loads(span._attributes[SpanAttributes.LLM_REQUEST_FUNCTIONS]) == [
         {"type": "function", "function": {"name": "get_weather"}}
     ]
+    assert not any(key.startswith("gen_ai.tool.") for key in span._attributes)
     assert json.loads(span._attributes[_COMPLETION_TOOL_CALLS_ATTR]) == [
         {
             "id": "toolu_123",
@@ -810,6 +813,36 @@ def test_enrich_claude_agent_sdk_span_maps_agent_fields():
     _assert_no_banned_aliases(span._attributes)
 
 
+def test_enrich_agent_overrides_chat_and_keeps_session_cost_and_cache_usage():
+    span = _make_span(
+        name="invoke_agent research_agent",
+        attributes={
+            "gen_ai.operation.name": "invoke_agent",
+            "gen_ai.agent.name": "research_agent",
+            "gen_ai.conversation.id": "session-resume-1",
+            "gen_ai.usage.input_tokens": 20,
+            "gen_ai.usage.output_tokens": 4,
+            SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS: 11,
+            SpanAttributes.LLM_USAGE_CACHE_CREATION_INPUT_TOKENS: 3,
+            "respan.metadata.response_cost": "0.003",
+            RESPAN_LOG_TYPE: LOG_TYPE_CHAT,
+        },
+    )
+
+    _processor.enrich_claude_agent_sdk_span(span)
+
+    assert span._attributes[RESPAN_LOG_TYPE] == LOG_TYPE_AGENT
+    assert span._attributes[RESPAN_SESSION_ID] == "session-resume-1"
+    assert span._attributes["respan.metadata.response_cost"] == "0.003"
+    assert span._attributes[SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS] == 11
+    assert (
+        span._attributes[SpanAttributes.LLM_USAGE_CACHE_CREATION_INPUT_TOKENS] == 3
+    )
+    assert span._attributes["gen_ai.usage.input_tokens"] == 20
+    assert span._attributes["gen_ai.usage.output_tokens"] == 4
+    _assert_no_banned_aliases(span._attributes)
+
+
 def test_enrich_claude_agent_sdk_span_maps_tool_fields():
     span = _make_span(
         name="execute_tool mcp__demo__calculator",
@@ -846,6 +879,7 @@ def test_enrich_claude_agent_sdk_span_maps_tool_fields():
     assert CLAUDE_AGENT_SDK_TOOL_NAME_ATTR not in span._attributes
     assert CLAUDE_AGENT_SDK_TOOL_CALL_ARGUMENTS_ATTR not in span._attributes
     assert CLAUDE_AGENT_SDK_TOOL_CALL_RESULT_ATTR not in span._attributes
+    assert not any(key.startswith("gen_ai.tool.") for key in span._attributes)
 
 
 def test_enrich_claude_agent_sdk_span_overrides_upstream_tool_chat_defaults():
@@ -1020,6 +1054,7 @@ def test_span_processor_on_end_merges_pending_tool_calls_into_parent_agent_span(
     processor.on_end(tool_span)
     processor.on_end(agent_span)
 
+    assert not any(key.startswith("gen_ai.tool.") for key in tool_span._attributes)
     assert json.loads(agent_span._attributes[_COMPLETION_TOOL_CALLS_ATTR]) == [
         {
             "id": "toolu_123",
@@ -1104,15 +1139,15 @@ def test_span_processor_on_end_discards_pending_tool_calls_for_tool_parent_span(
     assert (57, 1) in processor._pending_tool_calls_by_parent
 
 
-def test_span_processor_on_end_leaves_final_chat_child_to_shared_exporter():
+def test_span_processor_on_end_splits_real_scope_agent_and_chat_contracts():
     processor = _processor.ClaudeAgentSDKSpanProcessor()
 
     agent_span = _make_span(
-        name="ClaudeAgentSDK.query",
+        name="invoke_agent weather_agent",
         trace_id=88,
         span_id=7,
         start_time=100,
-        scope_name="openinference.instrumentation.claude_agent_sdk",
+        scope_name="opentelemetry.instrumentation.claude_agent_sdk",
         attributes={
             "gen_ai.operation.name": "invoke_agent",
             "gen_ai.agent.name": "weather_agent",
@@ -1138,6 +1173,10 @@ def test_span_processor_on_end_leaves_final_chat_child_to_shared_exporter():
                 ]
             ),
             "gen_ai.response.model": "claude-sonnet-4-5",
+            "gen_ai.usage.input_tokens": 19,
+            "gen_ai.usage.output_tokens": 7,
+            SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS: 3,
+            SpanAttributes.LLM_REQUEST_TYPE: "chat",
         },
     )
 
@@ -1176,12 +1215,30 @@ def test_span_processor_on_end_leaves_final_chat_child_to_shared_exporter():
 
     prepared_spans = _prepare_spans_for_export(spans=[agent_span])
     assert [span.name for span in prepared_spans] == [
-        "ClaudeAgentSDK.query",
+        "invoke_agent weather_agent",
         "assistant_message",
     ]
-    assert prepared_spans[1].attributes[_COMPLETION_TOOL_CALLS_ATTR] == json.loads(
+    prepared_agent_attrs = prepared_spans[0].attributes
+    prepared_chat_attrs = prepared_spans[1].attributes
+    assert prepared_agent_attrs[RESPAN_LOG_TYPE] == LOG_TYPE_AGENT
+    assert SpanAttributes.LLM_REQUEST_TYPE not in prepared_agent_attrs
+    assert SpanAttributes.LLM_REQUEST_MODEL not in prepared_agent_attrs
+    assert SpanAttributes.LLM_SYSTEM not in prepared_agent_attrs
+    assert not any(key.startswith("gen_ai.usage.") for key in prepared_agent_attrs)
+    assert not any(key.startswith("llm.usage.") for key in prepared_agent_attrs)
+    assert _COMPLETION_TOOL_CALLS_ATTR not in prepared_agent_attrs
+    assert prepared_chat_attrs[RESPAN_LOG_TYPE] == LOG_TYPE_CHAT
+    assert prepared_chat_attrs[SpanAttributes.LLM_REQUEST_TYPE] == "chat"
+    assert prepared_chat_attrs[SpanAttributes.LLM_REQUEST_MODEL] == "claude-sonnet-4-5"
+    assert prepared_chat_attrs["gen_ai.usage.input_tokens"] == 19
+    assert prepared_chat_attrs["gen_ai.usage.output_tokens"] == 7
+    assert prepared_chat_attrs[SpanAttributes.LLM_USAGE_CACHE_READ_INPUT_TOKENS] == 3
+    assert json.loads(prepared_chat_attrs[_COMPLETION_TOOL_CALLS_ATTR]) == json.loads(
         agent_span._attributes[_COMPLETION_TOOL_CALLS_ATTR]
     )
+    assert prepared_chat_attrs[f"{SpanAttributes.LLM_COMPLETIONS}.0.content"] == (
+        "Tokyo is sunny."
+    )
 
 
 def test_span_processor_shutdown_clears_pending_calls_and_force_flush_returns_true():