diff --git a/python/tracing/cursor-sdk/03_stop_cleanup.py b/python/tracing/cursor-sdk/03_stop_cleanup.py index f24bb6f..7e77c4c 100644 --- a/python/tracing/cursor-sdk/03_stop_cleanup.py +++ b/python/tracing/cursor-sdk/03_stop_cleanup.py @@ -22,31 +22,40 @@ def main() -> None: respan, instrumentor = make_respan(EXAMPLE_NAME, state_file) print_start(EXAMPLE_NAME, run_id) - with example_attributes(EXAMPLE_NAME, run_id): - for event in [ - make_event( - EXAMPLE_NAME, - run_id, - "beforeSubmitPrompt", - prompt="Start a refactor, then cancel the agent turn.", - ), - make_event( - EXAMPLE_NAME, - run_id, - "afterAgentThought", - text="I found the affected files and am preparing a minimal change.", - duration_ms=300, - ), - make_event(EXAMPLE_NAME, run_id, "stop", status="cancelled", loop_count=1), - ]: - result = instrumentor.process_event(event) - print( - f"event={result.event_name} emitted={result.emitted} span={result.span_name}", - flush=True, - ) - - state = json.loads(state_file.read_text()) if state_file.exists() else {} - print(f"state_after_stop={state}", flush=True) + try: + with example_attributes(EXAMPLE_NAME, run_id): + for event in [ + make_event( + EXAMPLE_NAME, + run_id, + "beforeSubmitPrompt", + prompt="Start a refactor, then cancel the agent turn.", + ), + make_event( + EXAMPLE_NAME, + run_id, + "afterAgentThought", + text="I found the affected files and am preparing a minimal change.", + duration_ms=300, + ), + make_event( + EXAMPLE_NAME, + run_id, + "stop", + status="cancelled", + loop_count=1, + ), + ]: + result = instrumentor.process_event(event) + print( + f"event={result.event_name} emitted={result.emitted} span={result.span_name}", + flush=True, + ) + + state = json.loads(state_file.read_text()) if state_file.exists() else {} + print(f"state_after_stop={state}", flush=True) + finally: + respan.shutdown() if __name__ == "__main__": diff --git a/python/tracing/cursor-sdk/_shared.py b/python/tracing/cursor-sdk/_shared.py index 39b39ab..404e3c3 100644 --- a/python/tracing/cursor-sdk/_shared.py +++ b/python/tracing/cursor-sdk/_shared.py @@ -61,7 +61,9 @@ def state_path(run_id: str) -> Path: def make_custom_identifier(example_name: str) -> str: - return f"cursor-sdk-{example_name}-{uuid4().hex[:8]}" + return os.getenv("RESPAN_EXAMPLE_RUN_ID") or ( + f"cursor-sdk-{example_name}-{uuid4().hex[:8]}" + ) def make_event( @@ -81,7 +83,9 @@ def make_event( return event -def make_respan(example_name: str, state_file: Path) -> tuple[Respan, CursorSDKInstrumentor]: +def make_respan( + example_name: str, state_file: Path +) -> tuple[Respan, CursorSDKInstrumentor]: api_key = require_respan_api_key() instrumentor = CursorSDKInstrumentor(state_path=state_file) respan = Respan( @@ -126,15 +130,17 @@ def replay_events( respan, instrumentor = make_respan(example_name, state_file) print_start(example_name, run_id) results = [] - with example_attributes(example_name, run_id): - for event in events: - result = instrumentor.process_event(event) - results.append(result) - print( - f"event={result.event_name} emitted={result.emitted} span={result.span_name}", - flush=True, - ) - finish_respan(respan) + try: + with example_attributes(example_name, run_id): + for event in events: + result = instrumentor.process_event(event) + results.append(result) + print( + f"event={result.event_name} emitted={result.emitted} span={result.span_name}", + flush=True, + ) + finally: + finish_respan(respan) return results @@ -152,4 +158,4 @@ def print_json(label: str, value: Any) -> None: def finish_respan(respan: Respan) -> None: - pass + respan.shutdown() diff --git a/python/tracing/dify/01_chat_blocking.py b/python/tracing/dify/01_chat_blocking.py index d11d28a..1fd973f 100644 --- a/python/tracing/dify/01_chat_blocking.py +++ b/python/tracing/dify/01_chat_blocking.py @@ -17,7 +17,9 @@ def main() -> None: ) response.raise_for_status() result = response.json() - print_result("chat_blocking", {"workflow": workflow_name, "answer": result.get("answer")}) + summary = {"workflow": workflow_name, "answer": result.get("answer")} + runtime.set_result(summary) + print_result("chat_blocking", summary) if __name__ == "__main__": diff --git a/python/tracing/dify/02_chat_streaming.py b/python/tracing/dify/02_chat_streaming.py index 4595308..cd77f24 100644 --- a/python/tracing/dify/02_chat_streaming.py +++ b/python/tracing/dify/02_chat_streaming.py @@ -16,7 +16,9 @@ def main() -> None: ) response.raise_for_status() answer = collect_stream_answer(response) - print_result("chat_streaming", {"workflow": workflow_name, "answer": answer}) + summary = {"workflow": workflow_name, "answer": answer} + runtime.set_result(summary) + print_result("chat_streaming", summary) if __name__ == "__main__": diff --git a/python/tracing/dify/03_completion.py b/python/tracing/dify/03_completion.py index 41e80c8..bc32c4b 100644 --- a/python/tracing/dify/03_completion.py +++ b/python/tracing/dify/03_completion.py @@ -15,7 +15,9 @@ def main() -> None: ) response.raise_for_status() result = response.json() - print_result("completion", {"workflow": workflow_name, "answer": result.get("answer")}) + summary = {"workflow": workflow_name, "answer": result.get("answer")} + runtime.set_result(summary) + print_result("completion", summary) if __name__ == "__main__": diff --git a/python/tracing/dify/04_workflow_and_api.py b/python/tracing/dify/04_workflow_and_api.py index 53a38ec..5cf79a7 100644 --- a/python/tracing/dify/04_workflow_and_api.py +++ b/python/tracing/dify/04_workflow_and_api.py @@ -44,18 +44,17 @@ def main() -> None: ) rename.raise_for_status() - print_result( - "workflow_and_api", - { - "workflow": workflow_name, - "workflow_run_id": workflow_response.json().get("workflow_run_id"), - "parameters_keys": sorted(parameters.json().keys()), - "conversations": len(conversations.json().get("data", [])), - "messages": len(messages.json().get("data", [])), - "feedback": feedback.json().get("result"), - "rename": rename.json().get("result"), - }, - ) + summary = { + "workflow": workflow_name, + "workflow_run_id": workflow_response.json().get("workflow_run_id"), + "parameters_keys": sorted(parameters.json().keys()), + "conversations": len(conversations.json().get("data", [])), + "messages": len(messages.json().get("data", [])), + "feedback": feedback.json().get("result"), + "rename": rename.json().get("result"), + } + runtime.set_result(summary) + print_result("workflow_and_api", summary) if __name__ == "__main__": diff --git a/python/tracing/dify/05_respan_context_and_files.py b/python/tracing/dify/05_respan_context_and_files.py index 7cd71c9..b610ca9 100644 --- a/python/tracing/dify/05_respan_context_and_files.py +++ b/python/tracing/dify/05_respan_context_and_files.py @@ -45,14 +45,13 @@ def main() -> None: ) response.raise_for_status() - print_result( - "context_and_files", - { - "workflow": workflow_name, - "upload_id": upload_id, - "answer": response.json().get("answer"), - }, - ) + summary = { + "workflow": workflow_name, + "upload_id": upload_id, + "answer": response.json().get("answer"), + } + runtime.set_result(summary) + print_result("context_and_files", summary) if __name__ == "__main__": diff --git a/python/tracing/dify/_shared.py b/python/tracing/dify/_shared.py index d705a84..cd47cbc 100644 --- a/python/tracing/dify/_shared.py +++ b/python/tracing/dify/_shared.py @@ -13,6 +13,7 @@ from typing import Any from dotenv import load_dotenv +from opentelemetry.semconv_ai import SpanAttributes EXAMPLE_DIR = Path(__file__).resolve().parent REPO_ROOT = EXAMPLE_DIR.parents[2] @@ -67,6 +68,7 @@ def _send_sse(self, events: list[dict[str, Any]]) -> None: @staticmethod def _usage(prompt_tokens: int = 9, completion_tokens: int = 5) -> dict[str, Any]: return { + "model": "dify/local-test-model", "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": prompt_tokens + completion_tokens, @@ -267,10 +269,16 @@ def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None: class DifyExampleRuntime(AbstractContextManager): def __init__(self, workflow_name: str) -> None: self.workflow_name = workflow_name + self.run_id = os.getenv("RESPAN_EXAMPLE_RUN_ID") or ( + f"dify-{workflow_name}-{uuid.uuid4().hex[:8]}" + ) self.respan = None self.base_url = "" self._server_context: LocalDifyServer | None = None + self._attributes_context = None self._workflow_context = None + self._workflow_span = None + self._result: Any = None def __enter__(self) -> "DifyExampleRuntime": load_repo_env() @@ -284,25 +292,58 @@ def __enter__(self) -> "DifyExampleRuntime": base_url=os.getenv("RESPAN_BASE_URL", "https://api.respan.ai/api"), app_name=self.workflow_name, instrumentations=[DifyInstrumentor()], + metadata={ + "integration": "dify", + "run_id": self.run_id, + "workflow_name": self.workflow_name, + }, is_batching_enabled=False, log_level=os.getenv("RESPAN_LOG_LEVEL", "WARNING"), ) + self._attributes_context = self.respan.propagate_attributes( + custom_identifier=self.run_id, + trace_group_identifier=self.workflow_name, + metadata={ + "integration": "dify", + "run_id": self.run_id, + "workflow_name": self.workflow_name, + }, + ) + self._attributes_context.__enter__() self._workflow_context = get_client().start_span( self.workflow_name, kind="workflow", ) - self._workflow_context.__enter__() + self._workflow_span = self._workflow_context.__enter__() + if self._workflow_span is not None: + self._workflow_span.set_attribute( + SpanAttributes.TRACELOOP_ENTITY_INPUT, + json.dumps({"scenario": self.workflow_name}, separators=(",", ":")), + ) + print(f"example_run_id={self.run_id}") return self def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> None: - if self._workflow_context is not None: - self._workflow_context.__exit__(exc_type, exc, tb) - if self.respan is not None: - shutdown = getattr(self.respan, "shutdown", None) - if callable(shutdown): - shutdown() - if self._server_context is not None: - self._server_context.__exit__(exc_type, exc, tb) + try: + if self._workflow_span is not None: + output = ( + {"error": str(exc)} + if exc is not None + else self._result or {"status": "completed"} + ) + self._workflow_span.set_attribute( + SpanAttributes.TRACELOOP_ENTITY_OUTPUT, + json.dumps(output, default=str, separators=(",", ":")), + ) + if self._workflow_context is not None: + self._workflow_context.__exit__(exc_type, exc, tb) + finally: + if self._attributes_context is not None: + self._attributes_context.__exit__(exc_type, exc, tb) + if self.respan is not None: + self.respan.shutdown() + if self._server_context is not None: + self._server_context.__exit__(exc_type, exc, tb) def _configure_dify_endpoint(self) -> None: base_url = os.getenv("DIFY_BASE_URL") @@ -342,6 +383,9 @@ def raw_client(self): def user(self, suffix: str) -> str: return f"respan-dify-{suffix}-{uuid.uuid4().hex[:8]}" + def set_result(self, value: Any) -> None: + self._result = value + class sample_file(AbstractContextManager): def __enter__(self) -> Any: diff --git a/python/tracing/dspy/01_predict_signature.py b/python/tracing/dspy/01_predict_signature.py index 7ba688d..0cce841 100644 --- a/python/tracing/dspy/01_predict_signature.py +++ b/python/tracing/dspy/01_predict_signature.py @@ -4,7 +4,7 @@ import dspy -from _shared import create_respan, print_result, traced_example +from _shared import managed_example, print_result, traced_example class BasicQuestion(dspy.Signature): @@ -15,19 +15,19 @@ class BasicQuestion(dspy.Signature): def run_predict_signature_example() -> None: - context = create_respan( + with managed_example( app_name="dspy-01-predict-signature", example_name="01_predict_signature", temperature=0.1, - ) - predict = dspy.Predict(BasicQuestion) - question = "What does DSPy help developers build?" + ) as context: + predict = dspy.Predict(BasicQuestion) + question = "What does DSPy help developers build?" - with traced_example(context, input_data={"question": question}) as span: - prediction = predict(question=question) - span.set_output({"answer": prediction.answer}) + with traced_example(context, input_data={"question": question}) as span: + prediction = predict(question=question) + span.set_output({"answer": prediction.answer}) - print_result("Answer", prediction.answer) + print_result("Answer", prediction.answer) if __name__ == "__main__": diff --git a/python/tracing/dspy/02_chain_of_thought.py b/python/tracing/dspy/02_chain_of_thought.py index dc39acb..0e8416e 100644 --- a/python/tracing/dspy/02_chain_of_thought.py +++ b/python/tracing/dspy/02_chain_of_thought.py @@ -4,7 +4,7 @@ import dspy -from _shared import create_respan, print_result, traced_example +from _shared import managed_example, print_result, traced_example class IncidentSummary(dspy.Signature): @@ -15,27 +15,27 @@ class IncidentSummary(dspy.Signature): def run_chain_of_thought_example() -> None: - context = create_respan( + with managed_example( app_name="dspy-02-chain-of-thought", example_name="02_chain_of_thought", temperature=0.1, - ) - summarize = dspy.ChainOfThought(IncidentSummary) - incident = ( - "A support bot produced slow responses after a prompt update. " - "The trace shows longer retrieval time and two extra LLM calls." - ) - - with traced_example(context, input_data={"incident": incident}) as span: - prediction = summarize(incident=incident) - span.set_output( - { - "reasoning": prediction.reasoning, - "summary": prediction.summary, - } + ) as context: + summarize = dspy.ChainOfThought(IncidentSummary) + incident = ( + "A support bot produced slow responses after a prompt update. " + "The trace shows longer retrieval time and two extra LLM calls." ) - print_result("Summary", prediction.summary) + with traced_example(context, input_data={"incident": incident}) as span: + prediction = summarize(incident=incident) + span.set_output( + { + "reasoning": prediction.reasoning, + "summary": prediction.summary, + } + ) + + print_result("Summary", prediction.summary) if __name__ == "__main__": diff --git a/python/tracing/dspy/03_module_workflow.py b/python/tracing/dspy/03_module_workflow.py index 80e25d2..e387af7 100644 --- a/python/tracing/dspy/03_module_workflow.py +++ b/python/tracing/dspy/03_module_workflow.py @@ -4,7 +4,7 @@ import dspy -from _shared import create_respan, print_result, traced_example +from _shared import managed_example, print_result, traced_example class ContextBuilder(dspy.Signature): @@ -35,25 +35,25 @@ def forward(self, question: str) -> dspy.Prediction: def run_module_workflow_example() -> None: - context = create_respan( + with managed_example( app_name="dspy-03-module-workflow", example_name="03_module_workflow", temperature=0.1, - ) - answerer = SupportAnswerer() - question = "Why is a single trace tree useful for DSPy programs?" - - with traced_example(context, input_data={"question": question}) as span: - prediction = answerer(question=question) - span.set_output( - { - "context": prediction.context, - "answer": prediction.answer, - } - ) - - print_result("Context", prediction.context) - print_result("Answer", prediction.answer) + ) as context: + answerer = SupportAnswerer() + question = "Why is a single trace tree useful for DSPy programs?" + + with traced_example(context, input_data={"question": question}) as span: + prediction = answerer(question=question) + span.set_output( + { + "context": prediction.context, + "answer": prediction.answer, + } + ) + + print_result("Context", prediction.context) + print_result("Answer", prediction.answer) if __name__ == "__main__": diff --git a/python/tracing/dspy/04_tool_call.py b/python/tracing/dspy/04_tool_call.py index bfa98eb..9539e2b 100644 --- a/python/tracing/dspy/04_tool_call.py +++ b/python/tracing/dspy/04_tool_call.py @@ -4,7 +4,7 @@ import dspy -from _shared import create_respan, print_result, traced_example +from _shared import managed_example, print_result, traced_example def lookup_order_status(order_id: str) -> str: @@ -17,22 +17,22 @@ def lookup_order_status(order_id: str) -> str: def run_tool_call_example() -> None: - context = create_respan( + with managed_example( app_name="dspy-04-tool-call", example_name="04_tool_call", - ) - tool = dspy.Tool( - lookup_order_status, - name="lookup_order_status", - desc="Look up the shipping status for an order id.", - ) - order_id = "ord-1001" - - with traced_example(context, input_data={"order_id": order_id}) as span: - status = tool(order_id=order_id) - span.set_output({"status": status}) - - print_result("Status", status) + ) as context: + tool = dspy.Tool( + lookup_order_status, + name="lookup_order_status", + desc="Look up the shipping status for an order id.", + ) + order_id = "ord-1001" + + with traced_example(context, input_data={"order_id": order_id}) as span: + status = tool(order_id=order_id) + span.set_output({"status": status}) + + print_result("Status", status) if __name__ == "__main__": diff --git a/python/tracing/dspy/05_react_agent.py b/python/tracing/dspy/05_react_agent.py index ad7ccfc..3b8b308 100644 --- a/python/tracing/dspy/05_react_agent.py +++ b/python/tracing/dspy/05_react_agent.py @@ -4,7 +4,7 @@ import dspy -from _shared import create_respan, print_result, traced_example +from _shared import managed_example, print_result, traced_example class CityQuestion(dspy.Signature): @@ -24,22 +24,22 @@ def lookup_city_fact(city: str) -> str: def run_react_agent_example() -> None: - context = create_respan( + with managed_example( app_name="dspy-05-react-agent", example_name="05_react_agent", temperature=0.0, - ) - agent = dspy.ReAct(CityQuestion, tools=[lookup_city_fact], max_iters=3) - question = ( - "Use lookup_city_fact for Tokyo, then answer with the fact in " - "one sentence." - ) - - with traced_example(context, input_data={"question": question}) as span: - prediction = agent(question=question) - span.set_output({"answer": prediction.answer}) - - print_result("Answer", prediction.answer) + ) as context: + agent = dspy.ReAct(CityQuestion, tools=[lookup_city_fact], max_iters=3) + question = ( + "Use lookup_city_fact for Tokyo, then answer with the fact in " + "one sentence." + ) + + with traced_example(context, input_data={"question": question}) as span: + prediction = agent(question=question) + span.set_output({"answer": prediction.answer}) + + print_result("Answer", prediction.answer) if __name__ == "__main__": diff --git a/python/tracing/dspy/06_evaluate_program.py b/python/tracing/dspy/06_evaluate_program.py index d1484f1..b2aadda 100644 --- a/python/tracing/dspy/06_evaluate_program.py +++ b/python/tracing/dspy/06_evaluate_program.py @@ -4,7 +4,7 @@ import dspy -from _shared import create_respan, print_result, traced_example +from _shared import managed_example, print_result, traced_example class CapitalQuestion(dspy.Signature): @@ -29,37 +29,37 @@ def contains_expected_answer(example: dspy.Example, prediction: dspy.Prediction) def run_evaluate_program_example() -> None: - context = create_respan( + with managed_example( app_name="dspy-06-evaluate-program", example_name="06_evaluate_program", temperature=0.0, - ) - program = CapitalAnswerer() - devset = [ - dspy.Example( - question="What is the capital of France?", - answer="Paris", - ).with_inputs("question") - ] - evaluator = dspy.Evaluate( - devset=devset, - metric=contains_expected_answer, - num_threads=1, - display_progress=False, - display_table=False, - ) - - with traced_example( - context, - input_data={ - "devset_size": len(devset), - "questions": [example.question for example in devset], - }, - ) as span: - result = evaluator(program) - span.set_output({"score": result.score}) - - print_result("Score", result.score) + ) as context: + program = CapitalAnswerer() + devset = [ + dspy.Example( + question="What is the capital of France?", + answer="Paris", + ).with_inputs("question") + ] + evaluator = dspy.Evaluate( + devset=devset, + metric=contains_expected_answer, + num_threads=1, + display_progress=False, + display_table=False, + ) + + with traced_example( + context, + input_data={ + "devset_size": len(devset), + "questions": [example.question for example in devset], + }, + ) as span: + result = evaluator(program) + span.set_output({"score": result.score}) + + print_result("Score", result.score) if __name__ == "__main__": diff --git a/python/tracing/dspy/README.md b/python/tracing/dspy/README.md index 79f8aa6..aa03d11 100644 --- a/python/tracing/dspy/README.md +++ b/python/tracing/dspy/README.md @@ -36,11 +36,12 @@ pip install -e /path/to/respan/python-sdks/respan-sdk \ | `05_react_agent.py` | `dspy.ReAct` with a Python tool | | `06_evaluate_program.py` | `dspy.Evaluate` on a small devset | -Each script sets distinct `app_name`, `example_name`, `example_run_id`, -`trace_group_identifier`, and `custom_identifier` values so exported results can -be traced back to the script that produced them. The root workflow span also -records a compact example input and output, while child spans contain the DSPy -module, adapter, LM, evaluation, and tool details. +Each script sets distinct `app_name`, `example_name`, and +`trace_group_identifier` values while honoring one exact +`RESPAN_EXAMPLE_RUN_ID` as its `custom_identifier` and metadata `run_id`. The +root workflow span records compact input/output, child spans contain the DSPy +module, adapter, LM, evaluation, and tool details, and shutdown runs explicitly +on success or failure. Run any example: @@ -48,6 +49,12 @@ Run any example: python 01_predict_signature.py ``` +Run the complete set: + +```bash +python run_all.py +``` + ## Environment Variables | Variable | Required | Description | diff --git a/python/tracing/dspy/_shared.py b/python/tracing/dspy/_shared.py index e4ed555..0841d38 100644 --- a/python/tracing/dspy/_shared.py +++ b/python/tracing/dspy/_shared.py @@ -89,6 +89,7 @@ def create_respan( "example_set": "dspy", "example_name": example_name, "example_run_id": run_id, + "run_id": run_id, } respan = Respan( api_key=settings.api_key, @@ -116,6 +117,27 @@ def create_respan( ) +@contextmanager +def managed_example( + *, + app_name: str, + example_name: str, + include_content: bool = True, + temperature: float = 0.2, +): + """Create one example context and always shut down its exporter.""" + context = create_respan( + app_name=app_name, + example_name=example_name, + include_content=include_content, + temperature=temperature, + ) + try: + yield context + finally: + context.respan.shutdown() + + @contextmanager def traced_example( context: ExampleContext, @@ -127,12 +149,13 @@ def traced_example( span_name = root_span_name or f"dspy_example_{context.example_name}" with context.respan.propagate_attributes( trace_group_identifier=f"{context.example_name}-{context.run_id}", - custom_identifier=f"{context.example_name}-{context.run_id}", + custom_identifier=context.run_id, thread_identifier=f"dspy_example_{context.example_name}", metadata={ "example_set": "dspy", "example_name": context.example_name, "example_run_id": context.run_id, + "run_id": context.run_id, }, ): client = context.respan.telemetry.get_client() diff --git a/python/tracing/dspy/run_all.py b/python/tracing/dspy/run_all.py new file mode 100644 index 0000000..9314877 --- /dev/null +++ b/python/tracing/dspy/run_all.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Run the full DSPy tracing example set in separate processes.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +EXAMPLE_DIR = Path(__file__).resolve().parent +SCRIPTS = [ + "01_predict_signature.py", + "02_chain_of_thought.py", + "03_module_workflow.py", + "04_tool_call.py", + "05_react_agent.py", + "06_evaluate_program.py", +] + + +def main() -> None: + for script in SCRIPTS: + print(f"\n### Running {script}", flush=True) + subprocess.run([sys.executable, script], cwd=EXAMPLE_DIR, check=True) + + +if __name__ == "__main__": + main()