From 79d5a01c77e8c24a3754ba35dc797248227a807f Mon Sep 17 00:00:00 2001 From: Nightingalelyy Date: Mon, 17 Aug 2026 00:33:52 +0800 Subject: [PATCH] fix(examples): repair Claude Agent SDK, Cohere, and CrewAI tracing --- .../claude-agent-sdk/01_hello_world.py | 9 + .../claude-agent-sdk/02_wrapped_query.py | 12 + .../tracing/claude-agent-sdk/03_multi_turn.py | 13 + .../claude-agent-sdk/04_stream_messages.py | 12 + .../tracing/claude-agent-sdk/05_tool_use.py | 13 + .../tracing/claude-agent-sdk/06_multi_tool.py | 20 ++ python/tracing/claude-agent-sdk/_shared.py | 305 ++++++++++++++++++ .../tracing/claude-agent-sdk/requirements.txt | 4 + .../claude-agent-sdk/run_all_examples.py | 36 +++ python/tracing/cohere/01_chat.py | 17 +- python/tracing/cohere/02_streaming_chat.py | 17 +- python/tracing/cohere/03_embed_rerank.py | 17 +- python/tracing/crewai/01_basic_crew.py | 11 +- python/tracing/crewai/02_tool_use.py | 11 +- python/tracing/crewai/03_attributes.py | 11 +- python/tracing/crewai/_shared.py | 93 ++++++ python/tracing/crewai/requirements.txt | 2 +- 17 files changed, 572 insertions(+), 31 deletions(-) create mode 100644 python/tracing/claude-agent-sdk/01_hello_world.py create mode 100644 python/tracing/claude-agent-sdk/02_wrapped_query.py create mode 100644 python/tracing/claude-agent-sdk/03_multi_turn.py create mode 100644 python/tracing/claude-agent-sdk/04_stream_messages.py create mode 100644 python/tracing/claude-agent-sdk/05_tool_use.py create mode 100644 python/tracing/claude-agent-sdk/06_multi_tool.py create mode 100644 python/tracing/claude-agent-sdk/_shared.py create mode 100644 python/tracing/claude-agent-sdk/requirements.txt create mode 100644 python/tracing/claude-agent-sdk/run_all_examples.py diff --git a/python/tracing/claude-agent-sdk/01_hello_world.py b/python/tracing/claude-agent-sdk/01_hello_world.py new file mode 100644 index 0000000..b0c78f0 --- /dev/null +++ b/python/tracing/claude-agent-sdk/01_hello_world.py @@ -0,0 +1,9 @@ +import asyncio + +from _shared import run_example + + +if __name__ == "__main__": + asyncio.run( + run_example(example_name="01_hello_world", prompts=["What is 2 + 2?"]) + ) diff --git a/python/tracing/claude-agent-sdk/02_wrapped_query.py b/python/tracing/claude-agent-sdk/02_wrapped_query.py new file mode 100644 index 0000000..a48b825 --- /dev/null +++ b/python/tracing/claude-agent-sdk/02_wrapped_query.py @@ -0,0 +1,12 @@ +import asyncio + +from _shared import run_example + + +if __name__ == "__main__": + asyncio.run( + run_example( + example_name="02_wrapped_query", + prompts=["Name three primary colors."], + ) + ) diff --git a/python/tracing/claude-agent-sdk/03_multi_turn.py b/python/tracing/claude-agent-sdk/03_multi_turn.py new file mode 100644 index 0000000..0558670 --- /dev/null +++ b/python/tracing/claude-agent-sdk/03_multi_turn.py @@ -0,0 +1,13 @@ +import asyncio + +from _shared import run_example + + +if __name__ == "__main__": + asyncio.run( + run_example( + example_name="03_multi_turn", + prompts=["My name is Alice.", "What is my name?"], + resume=True, + ) + ) diff --git a/python/tracing/claude-agent-sdk/04_stream_messages.py b/python/tracing/claude-agent-sdk/04_stream_messages.py new file mode 100644 index 0000000..ad7cda4 --- /dev/null +++ b/python/tracing/claude-agent-sdk/04_stream_messages.py @@ -0,0 +1,12 @@ +import asyncio + +from _shared import run_example + + +if __name__ == "__main__": + asyncio.run( + run_example( + example_name="04_stream_messages", + prompts=["Write a haiku about programming."], + ) + ) diff --git a/python/tracing/claude-agent-sdk/05_tool_use.py b/python/tracing/claude-agent-sdk/05_tool_use.py new file mode 100644 index 0000000..081f277 --- /dev/null +++ b/python/tracing/claude-agent-sdk/05_tool_use.py @@ -0,0 +1,13 @@ +import asyncio + +from _shared import ToolSpec, run_example + + +if __name__ == "__main__": + asyncio.run( + run_example( + example_name="05_tool_use", + prompts=["List the Python files in the current directory."], + tools=[ToolSpec("Glob", {"pattern": "*.py"}, ["01_hello_world.py"])], + ) + ) diff --git a/python/tracing/claude-agent-sdk/06_multi_tool.py b/python/tracing/claude-agent-sdk/06_multi_tool.py new file mode 100644 index 0000000..70f9fc5 --- /dev/null +++ b/python/tracing/claude-agent-sdk/06_multi_tool.py @@ -0,0 +1,20 @@ +import asyncio + +from _shared import ToolSpec, run_example + + +if __name__ == "__main__": + asyncio.run( + run_example( + example_name="06_multi_tool", + prompts=["Find all Python files and read the first one."], + tools=[ + ToolSpec("Glob", {"pattern": "*.py"}, ["01_hello_world.py"]), + ToolSpec( + "Read", + {"file_path": "01_hello_world.py"}, + "import asyncio", + ), + ], + ) + ) diff --git a/python/tracing/claude-agent-sdk/_shared.py b/python/tracing/claude-agent-sdk/_shared.py new file mode 100644 index 0000000..39c8859 --- /dev/null +++ b/python/tracing/claude-agent-sdk/_shared.py @@ -0,0 +1,305 @@ +"""Deterministic Claude Agent SDK examples using its real query protocol.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator, Sequence +from dataclasses import dataclass +import json +import os +from pathlib import Path +from typing import Any +import uuid + +from dotenv import load_dotenv + + +ROOT_DIR = Path(__file__).resolve().parents[3] +load_dotenv(ROOT_DIR / ".env", override=True) + +MODEL = os.getenv("CLAUDE_AGENT_SDK_MODEL", "claude-sonnet-4-5-20250514") + + +@dataclass(frozen=True) +class ToolSpec: + name: str + arguments: dict[str, Any] + result: Any + + +class FakeClaudeTransport: + """Fake CLI transport that exercises real SDK hooks and message parsing.""" + + def __init__( + self, + *, + session_id: str, + prompt_text: str, + tools: Sequence[ToolSpec], + ) -> None: + self._session_id = session_id + self._prompt_text = prompt_text + self._tools = list(tools) + self._queue: asyncio.Queue[dict[str, Any] | None] = asyncio.Queue() + self._hook_ids: dict[str, str] = {} + self._tool_index = 0 + self._closed = False + + async def connect(self) -> None: + return None + + async def write(self, data: str) -> None: + message = json.loads(data) + message_type = message.get("type") + if message_type == "control_request": + self._capture_hook_ids(message.get("request", {}).get("hooks")) + await self._queue.put( + { + "type": "control_response", + "response": { + "subtype": "success", + "request_id": message["request_id"], + "response": {"ok": True}, + }, + } + ) + return + + if message_type == "user": + await self._queue.put( + { + "type": "system", + "subtype": "init", + "session_id": self._session_id, + "data": {"session_id": self._session_id}, + } + ) + if self._tools: + await self._enqueue_tool_hook("PreToolUse") + else: + await self._enqueue_final_response() + return + + if message_type != "control_response": + return + + request_id = message.get("response", {}).get("request_id", "") + if request_id.startswith("pre-"): + await self._enqueue_tool_assistant_message() + await self._enqueue_tool_hook("PostToolUse") + elif request_id.startswith("post-"): + self._tool_index += 1 + if self._tool_index < len(self._tools): + await self._enqueue_tool_hook("PreToolUse") + else: + await self._enqueue_final_response() + + def _capture_hook_ids(self, hooks: Any) -> None: + if not isinstance(hooks, dict): + return + for event_name, matchers in hooks.items(): + if not isinstance(matchers, list) or not matchers: + continue + callback_ids = matchers[0].get("hookCallbackIds", []) + if callback_ids: + self._hook_ids[event_name] = callback_ids[0] + + async def _enqueue_tool_hook(self, event_name: str) -> None: + callback_id = self._hook_ids.get(event_name) + if callback_id is None: + raise RuntimeError(f"Instrumentation did not register {event_name}") + tool = self._tools[self._tool_index] + tool_use_id = f"tool-{self._tool_index + 1}" + phase = "pre" if event_name == "PreToolUse" else "post" + hook_input: dict[str, Any] = { + "hook_event_name": event_name, + "session_id": self._session_id, + "transcript_path": "/tmp/respan-claude-agent-sdk.jsonl", + "cwd": str(Path.cwd()), + "permission_mode": "default", + "tool_name": tool.name, + "tool_input": tool.arguments, + "tool_use_id": tool_use_id, + } + if event_name == "PostToolUse": + hook_input["tool_response"] = tool.result + await self._queue.put( + { + "type": "control_request", + "request_id": f"{phase}-{tool_use_id}", + "request": { + "subtype": "hook_callback", + "callback_id": callback_id, + "input": hook_input, + "tool_use_id": tool_use_id, + }, + } + ) + + async def _enqueue_tool_assistant_message(self) -> None: + tool = self._tools[self._tool_index] + tool_use_id = f"tool-{self._tool_index + 1}" + await self._queue.put( + { + "type": "assistant", + "session_id": self._session_id, + "message": { + "id": f"msg-{uuid.uuid4().hex[:8]}", + "model": MODEL, + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": tool_use_id, + "name": tool.name, + "input": tool.arguments, + } + ], + "usage": {"input_tokens": 30, "output_tokens": 5}, + }, + } + ) + + async def _enqueue_final_response(self) -> None: + response_text = f"Completed: {self._prompt_text}" + await self._queue.put( + { + "type": "assistant", + "session_id": self._session_id, + "message": { + "id": f"msg-{uuid.uuid4().hex[:8]}", + "model": MODEL, + "role": "assistant", + "content": [{"type": "text", "text": response_text}], + "usage": {"input_tokens": 4, "output_tokens": 10}, + }, + } + ) + await self._queue.put( + { + "type": "result", + "subtype": "success", + "duration_ms": 800, + "duration_api_ms": 500, + "is_error": False, + "num_turns": 1, + "session_id": self._session_id, + "total_cost_usd": 0.003, + "usage": { + "input_tokens": 4, + "output_tokens": 10, + "cache_read_input_tokens": 5, + "cache_creation_input_tokens": 2, + }, + "result": response_text, + } + ) + await self._queue.put(None) + + async def close(self) -> None: + if not self._closed: + self._closed = True + await self._queue.put(None) + + async def end_input(self) -> None: + return None + + def is_ready(self) -> bool: + return True + + async def read_messages(self) -> AsyncIterator[dict[str, Any]]: + while True: + item = await self._queue.get() + if item is None: + break + yield item + + +async def run_example( + *, + example_name: str, + prompts: Sequence[str], + tools: Sequence[ToolSpec] = (), + resume: bool = False, +) -> None: + import claude_agent_sdk + from claude_agent_sdk import ClaudeAgentOptions, ResultMessage + from respan import Respan + from respan_instrumentation_claude_agent_sdk import ClaudeAgentSDKInstrumentor + + run_id = os.environ["RESPAN_EXAMPLE_RUN_ID"] + session_id = str(uuid.uuid4()) + respan = Respan( + api_key=os.environ["RESPAN_API_KEY"], + base_url=os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api"), + app_name=f"claude-agent-sdk-{example_name}", + instrumentations=[ + ClaudeAgentSDKInstrumentor( + agent_name=example_name, + capture_content=True, + ) + ], + is_batching_enabled=False, + environment=os.getenv("RESPAN_ENVIRONMENT", "examples"), + metadata={ + "example_set": "claude-agent-sdk", + "example_name": example_name, + "example_run_id": run_id, + }, + ) + instrumented_query = claude_agent_sdk.query + current_transport: FakeClaudeTransport | None = None + + async def dispatch_query(*, prompt, options=None, transport=None): + selected_transport = transport or current_transport + if selected_transport is None: + raise RuntimeError("Fake Claude transport was not configured") + async for message in instrumented_query( + prompt=prompt, + options=options, + transport=selected_transport, + ): + yield message + + claude_agent_sdk.query = dispatch_query + try: + with respan.propagate_attributes( + customer_identifier="claude-agent-sdk-example-user", + trace_group_identifier=f"claude-agent-sdk:{example_name}:{run_id}", + custom_identifier=f"{example_name}:{run_id}", + metadata={ + "example_set": "claude-agent-sdk", + "example_name": example_name, + "example_run_id": run_id, + }, + ): + for turn_index, prompt in enumerate(prompts): + current_transport = FakeClaudeTransport( + session_id=session_id, + prompt_text=prompt, + tools=tools if turn_index == 0 else (), + ) + options = ClaudeAgentOptions( + model=MODEL, + tools=[tool.name for tool in tools] or None, + system_prompt="Return concise deterministic example output.", + resume=session_id if resume and turn_index else None, + ) + result = None + async for message in claude_agent_sdk.query( + prompt=prompt, + options=options, + ): + if isinstance(message, ResultMessage): + result = message + if result is None: + raise RuntimeError("Claude Agent SDK example returned no result") + if result.session_id != session_id: + raise RuntimeError("Resumed turn changed the Claude session id") + print( + f"{example_name} turn={turn_index + 1} " + f"session={result.session_id} result={result.subtype}" + ) + finally: + claude_agent_sdk.query = instrumented_query + respan.shutdown() diff --git a/python/tracing/claude-agent-sdk/requirements.txt b/python/tracing/claude-agent-sdk/requirements.txt new file mode 100644 index 0000000..843c03f --- /dev/null +++ b/python/tracing/claude-agent-sdk/requirements.txt @@ -0,0 +1,4 @@ +claude-agent-sdk>=0.1.39 +python-dotenv>=1.0.0 +respan-ai>=2.5.0 +respan-instrumentation-claude-agent-sdk>=0.1.0 diff --git a/python/tracing/claude-agent-sdk/run_all_examples.py b/python/tracing/claude-agent-sdk/run_all_examples.py new file mode 100644 index 0000000..1ec0b90 --- /dev/null +++ b/python/tracing/claude-agent-sdk/run_all_examples.py @@ -0,0 +1,36 @@ +"""Run all Claude Agent SDK tracing examples in isolated processes.""" + +from __future__ import annotations + +import os +from pathlib import Path +import subprocess +import sys + + +EXAMPLE_DIR = Path(__file__).resolve().parent +EXAMPLES = [ + "01_hello_world.py", + "02_wrapped_query.py", + "03_multi_turn.py", + "04_stream_messages.py", + "05_tool_use.py", + "06_multi_tool.py", +] + + +def main() -> None: + run_id = os.environ["RESPAN_EXAMPLE_RUN_ID"] + print(f"Claude Agent SDK example run id: {run_id}", flush=True) + for example in EXAMPLES: + print(f"\n=== {example} ===", flush=True) + subprocess.run( + [sys.executable, str(EXAMPLE_DIR / example)], + cwd=EXAMPLE_DIR, + env=os.environ.copy(), + check=True, + ) + + +if __name__ == "__main__": + main() diff --git a/python/tracing/cohere/01_chat.py b/python/tracing/cohere/01_chat.py index 7b0f74d..9af79ab 100644 --- a/python/tracing/cohere/01_chat.py +++ b/python/tracing/cohere/01_chat.py @@ -32,11 +32,16 @@ def cohere_chat() -> str: def main() -> None: - output = run_with_example_attributes( - respan, - workflow_name=WORKFLOW_NAME, - action=cohere_chat, - ) - print(output) + try: + output = run_with_example_attributes( + respan, + workflow_name=WORKFLOW_NAME, + action=cohere_chat, + ) + print(output) + finally: + respan.shutdown() + + if __name__ == "__main__": main() diff --git a/python/tracing/cohere/02_streaming_chat.py b/python/tracing/cohere/02_streaming_chat.py index 916a34a..21f0f47 100644 --- a/python/tracing/cohere/02_streaming_chat.py +++ b/python/tracing/cohere/02_streaming_chat.py @@ -36,11 +36,16 @@ def cohere_streaming_chat() -> str: def main() -> None: - output = run_with_example_attributes( - respan, - workflow_name=WORKFLOW_NAME, - action=cohere_streaming_chat, - ) - print(output) + try: + output = run_with_example_attributes( + respan, + workflow_name=WORKFLOW_NAME, + action=cohere_streaming_chat, + ) + print(output) + finally: + respan.shutdown() + + if __name__ == "__main__": main() diff --git a/python/tracing/cohere/03_embed_rerank.py b/python/tracing/cohere/03_embed_rerank.py index 367f9b4..0d21945 100644 --- a/python/tracing/cohere/03_embed_rerank.py +++ b/python/tracing/cohere/03_embed_rerank.py @@ -42,11 +42,16 @@ def cohere_embed_rerank() -> dict[str, object]: def main() -> None: - output = run_with_example_attributes( - respan, - workflow_name=WORKFLOW_NAME, - action=cohere_embed_rerank, - ) - print(output) + try: + output = run_with_example_attributes( + respan, + workflow_name=WORKFLOW_NAME, + action=cohere_embed_rerank, + ) + print(output) + finally: + respan.shutdown() + + if __name__ == "__main__": main() diff --git a/python/tracing/crewai/01_basic_crew.py b/python/tracing/crewai/01_basic_crew.py index a81a3b0..6601139 100644 --- a/python/tracing/crewai/01_basic_crew.py +++ b/python/tracing/crewai/01_basic_crew.py @@ -44,10 +44,13 @@ def main() -> None: example_name="01_basic_crew", workflow_name=WORKFLOW_NAME, ) - output = run_with_attributes(context, lambda: run_basic_crew(context)) - print_result("Crew output", output) - print_result("Workflow name", WORKFLOW_NAME) - print_result("Example run id", context.run_id) + try: + output = run_with_attributes(context, lambda: run_basic_crew(context)) + print_result("Crew output", output) + print_result("Workflow name", WORKFLOW_NAME) + print_result("Example run id", context.run_id) + finally: + context.respan.shutdown() if __name__ == "__main__": diff --git a/python/tracing/crewai/02_tool_use.py b/python/tracing/crewai/02_tool_use.py index 45f43c3..1d9d8b3 100644 --- a/python/tracing/crewai/02_tool_use.py +++ b/python/tracing/crewai/02_tool_use.py @@ -72,10 +72,13 @@ def main() -> None: example_name="02_tool_use", workflow_name=WORKFLOW_NAME, ) - output = run_with_attributes(context, lambda: run_tool_crew(context)) - print_result("Crew output", output) - print_result("Workflow name", WORKFLOW_NAME) - print_result("Example run id", context.run_id) + try: + output = run_with_attributes(context, lambda: run_tool_crew(context)) + print_result("Crew output", output) + print_result("Workflow name", WORKFLOW_NAME) + print_result("Example run id", context.run_id) + finally: + context.respan.shutdown() if __name__ == "__main__": diff --git a/python/tracing/crewai/03_attributes.py b/python/tracing/crewai/03_attributes.py index 38e0f35..ddef321 100644 --- a/python/tracing/crewai/03_attributes.py +++ b/python/tracing/crewai/03_attributes.py @@ -48,10 +48,13 @@ def main() -> None: workflow_name=WORKFLOW_NAME, metadata={"scenario": "support_triage"}, ) - output = run_with_attributes(context, lambda: run_attribute_crew(context)) - print_result("Crew output", output) - print_result("Workflow name", WORKFLOW_NAME) - print_result("Example run id", context.run_id) + try: + output = run_with_attributes(context, lambda: run_attribute_crew(context)) + print_result("Crew output", output) + print_result("Workflow name", WORKFLOW_NAME) + print_result("Example run id", context.run_id) + finally: + context.respan.shutdown() if __name__ == "__main__": diff --git a/python/tracing/crewai/_shared.py b/python/tracing/crewai/_shared.py index 5f284fa..ef5ddc3 100644 --- a/python/tracing/crewai/_shared.py +++ b/python/tracing/crewai/_shared.py @@ -5,6 +5,7 @@ from collections.abc import Iterable from dataclasses import dataclass from datetime import datetime, timezone +import json import os from pathlib import Path from typing import Any @@ -132,6 +133,9 @@ def create_respan( def build_llm(settings: GatewaySettings): + if os.getenv("CREWAI_USE_LIVE_LLM", "").lower() not in {"1", "true", "yes"}: + return _build_deterministic_llm(settings) + from crewai import LLM if settings.uses_gateway: @@ -147,6 +151,95 @@ def build_llm(settings: GatewaySettings): ) +def _build_deterministic_llm(settings: GatewaySettings): + """Use real CrewAI lifecycle events without requiring a provider credential.""" + from crewai.events.types.llm_events import LLMCallType + from crewai.llms.base_llm import BaseLLM, llm_call_context + + class DeterministicCrewAILLM(BaseLLM): + tool_round_complete: bool = False + + def supports_function_calling(self) -> bool: + return True + + def call( + self, + messages, + tools=None, + callbacks=None, + available_functions=None, + from_task=None, + from_agent=None, + response_model=None, + ): + _ = callbacks, response_model + with llm_call_context(): + self._emit_call_started_event( + messages=messages, + tools=tools, + available_functions=available_functions, + from_task=from_task, + from_agent=from_agent, + ) + if tools and not self.tool_round_complete: + tool_calls = [] + for index, tool in enumerate(tools): + function = tool.get("function", {}) + name = function.get("name", f"tool_{index + 1}") + arguments = {"city": "Paris"} + call_id = f"crewai-example-call-{index + 1}" + tool_calls.append( + { + "id": call_id, + "type": "function", + "function": { + "name": name, + "arguments": json.dumps(arguments), + }, + } + ) + + self._emit_call_completed_event( + response=tool_calls, + call_type=LLMCallType.TOOL_CALL, + from_task=from_task, + from_agent=from_agent, + messages=messages, + usage={"prompt_tokens": 23, "completion_tokens": 7}, + finish_reason="tool_calls", + response_id="crewai-example-tool-response", + ) + self.tool_round_complete = True + return tool_calls + + if tools: + response = ( + "Paris weather is sunny at 22C and its population is " + "about 2.1 million people." + ) + else: + response = ( + "CrewAI coordinates agents and tasks while Respan records " + "their workflow, model, and output spans." + ) + self._emit_call_completed_event( + response=response, + call_type=LLMCallType.LLM_CALL, + from_task=from_task, + from_agent=from_agent, + messages=messages, + usage={"prompt_tokens": 19, "completion_tokens": 14}, + finish_reason="stop", + response_id="crewai-example-text-response", + ) + return response + + return DeterministicCrewAILLM( + model=settings.model, + provider="openai", + ) + + def run_with_attributes(context: ExampleContext, fn): with context.respan.propagate_attributes( customer_identifier=os.getenv( diff --git a/python/tracing/crewai/requirements.txt b/python/tracing/crewai/requirements.txt index 9cc04cf..d58fece 100644 --- a/python/tracing/crewai/requirements.txt +++ b/python/tracing/crewai/requirements.txt @@ -1,5 +1,5 @@ crewai>=1.10.1 python-dotenv>=1.0.0 respan-ai>=2.5.0 -respan-instrumentation-crewai>=0.2.0 +respan-instrumentation-crewai>=0.1.0 anthropic>=0.73.0