fix(examples-py): validate Arize, AutoGen, and AWS Bedrock OTel 2.x spans

python/tracing/arize/02_datasets_projects_spaces.py

@@ -43,6 +43,14 @@ def run_dataset_project_space_operations() -> str:
         print_result("datasets.list", client.datasets.list(space="space-offline"))
         print_result("datasets.create", client.datasets.create(name="offline-dataset", space="space-offline", examples=examples))
         print_result("datasets.get", client.datasets.get(dataset="offline-dataset", space="space-offline"))
+        try:
+            client.datasets.get(
+                dataset="missing-dataset",
+                space="space-offline",
+                _respan_force_error=True,
+            )
+        except RuntimeError as error:
+            print(f"datasets.get expected error: {error}")
         print_result("datasets.update", client.datasets.update(dataset="offline-dataset", name="renamed-dataset", space="space-offline"))
         print_result("datasets.list_examples", client.datasets.list_examples(dataset="offline-dataset", space="space-offline"))
         print_result("datasets.append_examples", client.datasets.append_examples(dataset="offline-dataset", space="space-offline", examples=examples))

python/tracing/arize/03_experiments_prompts_evaluators.py

@@ -48,7 +48,7 @@ def _template_config() -> models.TemplateConfig:
 
 def _code_config() -> models.CustomCodeConfig:
     return models.CustomCodeConfig(
-        type="custom",
+        type="CUSTOM",
         name="offline-code",
         code="def evaluate(row): return 1",
         variables=[],

python/tracing/arize/04_admin_operations.py

@@ -7,6 +7,7 @@
 from __future__ import annotations
 
 from arize._generated.api_client import models
+from arize.api_keys.types import OrgBinding, SpaceBinding
 
 from _shared import (
     create_arize_client,
@@ -25,7 +26,7 @@
 
 def _run_configuration() -> models.TemplateEvaluationRunConfig:
     return models.TemplateEvaluationRunConfig(
-        experiment_type="template_evaluation",
+        experiment_type="TEMPLATE_EVALUATION",
         ai_integration_id="ai-integration-offline",
         model_name="gpt-4o-mini",
         template="score {{output}}",
@@ -50,19 +51,11 @@ def run_admin_operations() -> str:
         example_name=EXAMPLE_NAME,
     ):
         print_result("ai_integrations.list", client.ai_integrations.list(space="space-offline"))
-        print_result("ai_integrations.create", client.ai_integrations.create(name="offline-integration", provider=models.AiIntegrationProvider.OPENAI))
+        print_result("ai_integrations.create", client.ai_integrations.create(name="offline-integration", provider=models.AiIntegrationProvider.OPEN_AI))
         print_result("ai_integrations.get", client.ai_integrations.get(integration="offline-integration", space="space-offline"))
         print_result("ai_integrations.update", client.ai_integrations.update(integration="offline-integration", space="space-offline", name="renamed-integration"))
         print_result("ai_integrations.delete", client.ai_integrations.delete(integration="offline-integration", space="space-offline"))
         print_result("annotation_configs.list", client.annotation_configs.list(space="space-offline"))
-        print_result(
-            "annotation_configs.create",
-            client.annotation_configs.create(
-                name="quality",
-                config_type=models.AnnotationConfigType.FREEFORM,
-                space="space-offline",
-            ),
-        )
         print_result("annotation_configs.get", client.annotation_configs.get(annotation_config="quality", space="space-offline"))
         print_result("annotation_configs.delete", client.annotation_configs.delete(annotation_config="quality", space="space-offline"))
         print_result("annotation_queues.list", client.annotation_queues.list(space="space-offline"))
@@ -176,8 +169,12 @@ def run_admin_operations() -> str:
             "api_keys.create_service_key",
             client.api_keys.create_service_key(
                 name="offline-service-key",
-                space="space-offline",
-                space_role=models.ApiKeySpaceRole.MEMBER,
+                orgs=[
+                    OrgBinding(
+                        org_id="org-offline",
+                        spaces=[SpaceBinding(space="space-offline")],
+                    )
+                ],
             ),
         )
         print_result("api_keys.refresh", client.api_keys.refresh(api_key_id="key-offline"))

python/tracing/arize/_shared.py

@@ -60,6 +60,8 @@ def _offline_result(resource: str, method_name: str, kwargs: dict[str, Any]) ->
 
 def _make_offline_method(resource: str, method_name: str) -> Callable[..., Any]:
     def offline_method(self: Any, *args: Any, **kwargs: Any) -> Any:
+        if kwargs.pop("_respan_force_error", False):
+            raise RuntimeError(f"offline {resource}.{method_name} failure")
         return _offline_result(resource, method_name, kwargs)
 
     offline_method.__name__ = method_name

python/tracing/arize/run_all.py

@@ -14,9 +14,14 @@
 
 def run() -> None:
     here = Path(__file__).resolve().parent
+    failures: list[str] = []
     for example in EXAMPLES:
         print(f"\n### running {example}", flush=True)
-        subprocess.run([sys.executable, str(here / example)], check=True)
+        result = subprocess.run([sys.executable, str(here / example)], check=False)
+        if result.returncode:
+            failures.append(f"{example} (exit {result.returncode})")
+    if failures:
+        raise SystemExit(f"Arize example failures: {', '.join(failures)}")
 
 
 if __name__ == "__main__":

python/tracing/autogen/01_assistant_run.py

@@ -29,7 +29,7 @@
 
 
 @workflow(name=SCRIPT_NAME)
-async def run_assistant_agent() -> None:
+async def run_assistant_agent() -> str:
     model_client = OpenAIChatCompletionClient(
         model=RESPAN_MODEL,
         api_key=RESPAN_API_KEY,
@@ -43,26 +43,39 @@ async def run_assistant_agent() -> None:
     )
 
     try:
-        with propagate_attributes(
-            customer_identifier="autogen-example-user",
-            thread_identifier="autogen-assistant-thread",
-            metadata={"script": SCRIPT_NAME},
-        ):
-            result = await agent.run(
-                task="In one sentence, explain why tracing helps multi-agent apps."
-            )
-        print(result.messages[-1].content)
+        result = await agent.run(
+            task="In one sentence, explain why tracing helps multi-agent apps."
+        )
+        return str(result.messages[-1].content)
     finally:
         await model_client.close()
 
 
 async def main() -> None:
+    run_id = os.getenv("RESPAN_EXAMPLE_RUN_ID", f"autogen-{Path(__file__).stem}")
     respan = Respan(
         api_key=RESPAN_API_KEY,
         base_url=RESPAN_BASE_URL,
         instrumentations=[AutoGenInstrumentor()],
-        metadata={"example": "autogen-assistant-run", "script": SCRIPT_NAME},
+        metadata={
+            "example": "autogen-assistant-run",
+            "script": SCRIPT_NAME,
+            "run_id": run_id,
+        },
     )
-    await run_assistant_agent()
+    try:
+        with propagate_attributes(
+            customer_identifier="autogen-example-user",
+            thread_identifier="autogen-assistant-thread",
+            group_identifier=SCRIPT_NAME,
+            custom_identifier=run_id,
+            metadata={"script": SCRIPT_NAME, "run_id": run_id},
+        ):
+            print(await run_assistant_agent())
+    finally:
+        respan.shutdown()
+    print(f"RESPAN_EXAMPLE_RUN_ID={run_id}")
+
+
 if __name__ == "__main__":
     asyncio.run(main())

python/tracing/autogen/02_tool_use.py

@@ -29,7 +29,7 @@
 
 
 @workflow(name=SCRIPT_NAME)
-async def run_tool_agent() -> None:
+async def run_tool_agent() -> str:
     async def estimate_latency(service: str, requests_per_minute: int) -> str:
         """Estimate API latency for a service under load."""
         baseline_ms = 120
@@ -54,29 +54,42 @@ async def estimate_latency(service: str, requests_per_minute: int) -> str:
     )
 
     try:
-        with propagate_attributes(
-            customer_identifier="autogen-example-user",
-            thread_identifier="autogen-tool-thread",
-            metadata={"script": SCRIPT_NAME},
-        ):
-            result = await agent.run(
-                task=(
-                    "Estimate the p95 latency for the tracing-api service at "
-                    "240 requests per minute."
-                )
+        result = await agent.run(
+            task=(
+                "Estimate the p95 latency for the tracing-api service at "
+                "240 requests per minute."
             )
-        print(result.messages[-1].content)
+        )
+        return str(result.messages[-1].content)
     finally:
         await model_client.close()
 
 
 async def main() -> None:
+    run_id = os.getenv("RESPAN_EXAMPLE_RUN_ID", f"autogen-{Path(__file__).stem}")
     respan = Respan(
         api_key=RESPAN_API_KEY,
         base_url=RESPAN_BASE_URL,
         instrumentations=[AutoGenInstrumentor()],
-        metadata={"example": "autogen-tool-use", "script": SCRIPT_NAME},
+        metadata={
+            "example": "autogen-tool-use",
+            "script": SCRIPT_NAME,
+            "run_id": run_id,
+        },
     )
-    await run_tool_agent()
+    try:
+        with propagate_attributes(
+            customer_identifier="autogen-example-user",
+            thread_identifier="autogen-tool-thread",
+            group_identifier=SCRIPT_NAME,
+            custom_identifier=run_id,
+            metadata={"script": SCRIPT_NAME, "run_id": run_id},
+        ):
+            print(await run_tool_agent())
+    finally:
+        respan.shutdown()
+    print(f"RESPAN_EXAMPLE_RUN_ID={run_id}")
+
+
 if __name__ == "__main__":
     asyncio.run(main())

python/tracing/autogen/03_round_robin_team.py

@@ -31,7 +31,7 @@
 
 
 @workflow(name=SCRIPT_NAME)
-async def run_round_robin_team() -> None:
+async def run_round_robin_team() -> str:
     model_client = OpenAIChatCompletionClient(
         model=RESPAN_MODEL,
         api_key=RESPAN_API_KEY,
@@ -59,29 +59,44 @@ async def run_round_robin_team() -> None:
     )
 
     try:
-        with propagate_attributes(
-            customer_identifier="autogen-example-user",
-            thread_identifier="autogen-team-thread",
-            metadata={"script": SCRIPT_NAME},
-        ):
-            result = await team.run(
-                task="Plan a small release checklist for a new Python tracing plugin."
-            )
+        result = await team.run(
+            task="Plan a small release checklist for a new Python tracing plugin."
+        )
+        lines: list[str] = []
         for message in result.messages:
             content = getattr(message, "content", None)
             if content:
-                print(f"{message.source}: {content}")
+                lines.append(f"{message.source}: {content}")
+        return "\n".join(lines)
     finally:
         await model_client.close()
 
 
 async def main() -> None:
+    run_id = os.getenv("RESPAN_EXAMPLE_RUN_ID", f"autogen-{Path(__file__).stem}")
     respan = Respan(
         api_key=RESPAN_API_KEY,
         base_url=RESPAN_BASE_URL,
         instrumentations=[AutoGenInstrumentor()],
-        metadata={"example": "autogen-round-robin-team", "script": SCRIPT_NAME},
+        metadata={
+            "example": "autogen-round-robin-team",
+            "script": SCRIPT_NAME,
+            "run_id": run_id,
+        },
     )
-    await run_round_robin_team()
+    try:
+        with propagate_attributes(
+            customer_identifier="autogen-example-user",
+            thread_identifier="autogen-team-thread",
+            group_identifier=SCRIPT_NAME,
+            custom_identifier=run_id,
+            metadata={"script": SCRIPT_NAME, "run_id": run_id},
+        ):
+            print(await run_round_robin_team())
+    finally:
+        respan.shutdown()
+    print(f"RESPAN_EXAMPLE_RUN_ID={run_id}")
+
+
 if __name__ == "__main__":
     asyncio.run(main())

python/tracing/autogen/run_all.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+EXAMPLES = [
+    "01_assistant_run.py",
+    "02_tool_use.py",
+    "03_round_robin_team.py",
+]
+
+
+def run() -> None:
+    here = Path(__file__).resolve().parent
+    failures: list[str] = []
+    for example in EXAMPLES:
+        print(f"\n### running {example}", flush=True)
+        result = subprocess.run([sys.executable, str(here / example)], check=False)
+        if result.returncode:
+            failures.append(f"{example} (exit {result.returncode})")
+    if failures:
+        raise SystemExit(f"AutoGen example failures: {', '.join(failures)}")
+
+
+if __name__ == "__main__":
+    run()

python/tracing/aws-bedrock/01_invoke_model.py

@@ -11,10 +11,12 @@
     deactivate_stubber,
     get_model_id,
     maybe_stub_invoke_model,
+    new_run_id,
 )
 
 
 WORKFLOW_NAME = "aws_bedrock_invoke_model"
+EXAMPLE_NAME = "01_invoke_model"
 
 
 @workflow(name=WORKFLOW_NAME)
@@ -24,22 +26,33 @@ def run_invoke_model() -> str:
     client = create_bedrock_client()
     stubber = maybe_stub_invoke_model(client, model_id=model_id, body=body)
     try:
-        with propagate_attributes(
-            trace_group_identifier=WORKFLOW_NAME,
-            metadata={"example": "invoke_model"},
-        ):
-            response = client.invoke_model(
-                modelId=model_id,
-                body=body,
-                contentType="application/json",
-                accept="application/json",
-            )
+        response = client.invoke_model(
+            modelId=model_id,
+            body=body,
+            contentType="application/json",
+            accept="application/json",
+        )
         payload = json.loads(response["body"].read())
         return payload["content"][0]["text"]
     finally:
         deactivate_stubber(stubber)
 
 
+def main() -> str:
+    run_id = new_run_id(EXAMPLE_NAME)
+    respan = create_respan(example_name=EXAMPLE_NAME, run_id=run_id)
+    try:
+        with propagate_attributes(
+            group_identifier=WORKFLOW_NAME,
+            custom_identifier=run_id,
+            metadata={"example": "invoke_model", "run_id": run_id},
+        ):
+            print(run_invoke_model())
+    finally:
+        respan.shutdown()
+    print(f"RESPAN_EXAMPLE_RUN_ID={run_id}")
+    return run_id
+
+
 if __name__ == "__main__":
-    respan = create_respan()
-    print(run_invoke_model())
+    main()