diff --git a/py/noxfile.py b/py/noxfile.py index c92395da..1083ca5b 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -77,6 +77,7 @@ def _pinned_python_version(): "opentelemetry-exporter-otlp-proto-http", "google.genai", "google.adk", + "langsmith", "temporalio", ) @@ -104,6 +105,7 @@ def _pinned_python_version(): DSPY_VERSIONS = (LATEST,) GOOGLE_ADK_VERSIONS = (LATEST, "1.14.1") LANGCHAIN_VERSIONS = (LATEST, "0.3.28") +LANGSMITH_VERSIONS = (LATEST, "0.7.12") OPENROUTER_VERSIONS = (LATEST, "0.6.0") # temporalio 1.19.0+ requires Python >= 3.10; skip Python 3.9 entirely TEMPORAL_VERSIONS = (LATEST, "1.20.0", "1.19.0") @@ -235,6 +237,17 @@ def test_langchain(session, version): _run_core_tests(session) +@nox.session() +@nox.parametrize("version", LANGSMITH_VERSIONS, ids=LANGSMITH_VERSIONS) +def test_langsmith(session, version): + """Test LangSmith integration.""" + _install_test_deps(session) + _install(session, "langsmith", version) + _install(session, "langchain-core") + _install(session, "langchain-openai") + _run_tests(session, f"{INTEGRATION_DIR}/langsmith/test_langsmith.py") + + @nox.session() @nox.parametrize("version", OPENAI_VERSIONS, ids=OPENAI_VERSIONS) def test_openai(session, version): @@ -371,9 +384,8 @@ def pylint(session): session.install("pydantic_ai>=1.10.0") session.install("google-adk") session.install("opentelemetry.instrumentation.openai") - # langsmith is needed for the langsmith_wrapper module but not in VENDOR_PACKAGES # langchain-core, langchain-openai, langchain-anthropic are needed for the langchain integration - session.install("langsmith", "langchain-core", "langchain-openai", "langchain-anthropic") + session.install("langchain-core", "langchain-openai", "langchain-anthropic") result = session.run("git", "ls-files", "**/*.py", silent=True, log=False) files = [path for path in result.strip().splitlines() if path not in GENERATED_LINT_EXCLUDES] diff --git a/py/src/braintrust/auto.py b/py/src/braintrust/auto.py index dc44c7d2..6cf8597c 100644 --- a/py/src/braintrust/auto.py +++ b/py/src/braintrust/auto.py @@ -16,6 +16,7 @@ DSPyIntegration, GoogleGenAIIntegration, LangChainIntegration, + LangSmithIntegration, LiteLLMIntegration, OpenRouterIntegration, PydanticAIIntegration, @@ -52,6 +53,7 @@ def auto_instrument( dspy: bool = True, adk: bool = True, langchain: bool = True, + langsmith: bool = True, ) -> dict[str, bool]: """ Auto-instrument supported AI/ML libraries for Braintrust tracing. @@ -75,6 +77,7 @@ def auto_instrument( dspy: Enable DSPy instrumentation (default: True) adk: Enable Google ADK instrumentation (default: True) langchain: Enable LangChain instrumentation (default: True) + langsmith: Enable LangSmith instrumentation (default: True) Returns: Dict mapping integration name to whether it was successfully instrumented. @@ -146,6 +149,8 @@ def auto_instrument( results["adk"] = _instrument_integration(ADKIntegration) if langchain: results["langchain"] = _instrument_integration(LangChainIntegration) + if langsmith: + results["langsmith"] = _instrument_integration(LangSmithIntegration) return results diff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py index 0062ec77..2dca1110 100644 --- a/py/src/braintrust/integrations/__init__.py +++ b/py/src/braintrust/integrations/__init__.py @@ -6,6 +6,7 @@ from .dspy import DSPyIntegration from .google_genai import GoogleGenAIIntegration from .langchain import LangChainIntegration +from .langsmith import LangSmithIntegration from .litellm import LiteLLMIntegration from .openrouter import OpenRouterIntegration from .pydantic_ai import PydanticAIIntegration @@ -21,6 +22,7 @@ "GoogleGenAIIntegration", "LiteLLMIntegration", "LangChainIntegration", + "LangSmithIntegration", "OpenRouterIntegration", "PydanticAIIntegration", ] diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_langsmith.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_langsmith.py new file mode 100644 index 00000000..710dab80 --- /dev/null +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_langsmith.py @@ -0,0 +1,98 @@ +"""Test auto_instrument for LangSmith.""" + +import os +from pathlib import Path + +import langsmith.client +import langsmith.evaluation._arunner +import langsmith.evaluation._runner +import langsmith.run_helpers +from braintrust.auto import auto_instrument +from braintrust.wrappers.test_utils import autoinstrument_test_context +from langchain_core.prompts import ChatPromptTemplate +from langchain_openai import ChatOpenAI + + +_CASSETTES_DIR = Path(__file__).resolve().parent.parent / "langsmith" / "cassettes" + + +# 1. Verify not patched initially. +assert not getattr(langsmith.run_helpers.traceable, "__braintrust_patched_langsmith_traceable__", False) +assert not getattr(langsmith.evaluation._runner.evaluate, "__braintrust_patched_langsmith_evaluate_sync__", False) +assert not getattr(langsmith.evaluation._arunner.aevaluate, "__braintrust_patched_langsmith_evaluate_async__", False) +assert not getattr(langsmith.client.Client.evaluate, "__braintrust_patched_langsmith_client_evaluate__", False) +assert not getattr(langsmith.client.Client.aevaluate, "__braintrust_patched_langsmith_client_aevaluate__", False) + + +# 2. Instrument with standalone mode so only Braintrust runs. +os.environ["BRAINTRUST_LANGSMITH_STANDALONE"] = "1" +results = auto_instrument( + openai=False, + anthropic=False, + litellm=False, + pydantic_ai=False, + google_genai=False, + openrouter=False, + agno=False, + agentscope=False, + claude_agent_sdk=False, + dspy=False, + adk=False, + langchain=False, + langsmith=True, +) +assert results.get("langsmith") == True + +assert getattr(langsmith.run_helpers.traceable, "__braintrust_patched_langsmith_traceable__", False) +assert getattr(langsmith.evaluation._runner.evaluate, "__braintrust_patched_langsmith_evaluate_sync__", False) +assert getattr(langsmith.evaluation._arunner.aevaluate, "__braintrust_patched_langsmith_evaluate_async__", False) +assert getattr(langsmith.client.Client.evaluate, "__braintrust_patched_langsmith_client_evaluate__", False) +assert getattr(langsmith.client.Client.aevaluate, "__braintrust_patched_langsmith_client_aevaluate__", False) + + +# 3. Idempotent. +results2 = auto_instrument( + openai=False, + anthropic=False, + litellm=False, + pydantic_ai=False, + google_genai=False, + openrouter=False, + agno=False, + agentscope=False, + claude_agent_sdk=False, + dspy=False, + adk=False, + langchain=False, + langsmith=True, +) +assert results2.get("langsmith") == True + + +# 4. Make an API call and verify span. +with autoinstrument_test_context("test_auto_langsmith", cassettes_dir=_CASSETTES_DIR) as memory_logger: + prompt = ChatPromptTemplate.from_template("What is 1 + {number}?") + model = ChatOpenAI( + model="gpt-4o-mini", + temperature=1, + top_p=1, + frequency_penalty=0, + presence_penalty=0, + n=1, + ) + chain = prompt | model + + @langsmith.traceable(name="auto-langsmith") + def run_chain(inputs: dict[str, str]) -> dict[str, str]: + return {"answer": chain.invoke(inputs).content} + + result = run_chain({"number": "2"}) + assert result == {"answer": "1 + 2 equals 3."} + + spans = memory_logger.pop() + assert len(spans) == 1, f"Expected 1 span, got {len(spans)}" + span = spans[0] + assert span["span_attributes"]["name"] == "auto-langsmith" + assert span["output"] == {"answer": "1 + 2 equals 3."} + +print("SUCCESS") diff --git a/py/src/braintrust/integrations/langsmith/__init__.py b/py/src/braintrust/integrations/langsmith/__init__.py new file mode 100644 index 00000000..22500447 --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/__init__.py @@ -0,0 +1,38 @@ +"""Braintrust integration for LangSmith.""" + +import logging +import os + +from braintrust.logger import NOOP_SPAN, current_span, init_logger + +from .integration import LangSmithIntegration + + +logger = logging.getLogger(__name__) + +__all__ = [ + "LangSmithIntegration", + "setup_langsmith", +] + + +def setup_langsmith( + api_key: str | None = None, + project_id: str | None = None, + project_name: str | None = None, + standalone: bool = False, +) -> bool: + """Setup Braintrust integration with LangSmith.""" + resolved_project_name = project_name or os.environ.get("LANGCHAIN_PROJECT") + if current_span() == NOOP_SPAN: + init_logger(project=resolved_project_name, api_key=api_key, project_id=project_id) + + try: + import langsmith # noqa: F401 + except ImportError as exc: + logger.error("Failed to import langsmith: %s", exc) + logger.error("langsmith is not installed. Please install it with: pip install langsmith") + return False + + logger.info("LangSmith integration with Braintrust enabled") + return LangSmithIntegration.setup(standalone=True if standalone else None) diff --git a/py/src/braintrust/integrations/langsmith/cassettes/test_auto_langsmith.yaml b/py/src/braintrust/integrations/langsmith/cassettes/test_auto_langsmith.yaml new file mode 100644 index 00000000..ba9f4fa9 --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/cassettes/test_auto_langsmith.yaml @@ -0,0 +1,225 @@ +interactions: +- request: + body: '{"messages": [{"content": "What is 1 + 2?", "role": "user"}], "model": + "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": + false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ44VUVp2sk1koSWXX64CaLEy1mWy\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659919,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:38:40 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cd8d01c33943a-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '930' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=XPwF0fhMV9JwjYuWwUMNbzPKxvSJ.HOkXEftYzjXRew-1758659920459-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 93acad0503781eb98ab6ea3412173537 + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '1026' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_181413148bbe4814a905514521d6dc34 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jJJBb9swDIXv/hUCr4uL2IlTJ9et2y2HHQZ0Q2EoMm1rk0VNoocNRf77 + IDuN3a0FevGBHx/1Hs3HRAjQNRwEqE6y6p1J33/9XH7adscPRXGXnTb7+49fNse7/b2iYylhFRV0 + +o6Kn1Q3inpnkDXZCSuPkjFOzW53ebHLyiwfQU81mihrHadbSnttdZqv8226vk2z8qLuSCsMcBDf + EiGEeBy/0aet8TccxHr1VOkxBNkiHK5NQoAnEysgQ9CBpWVYzVCRZbSj9Uy8E7nAn4M0QWxull0e + myHI6NQOxiyAtJZYxqSjv4cLOV8dGWqdp1P4RwqNtjp0lUcZyMbXA5ODkZ4TIR7G5MOzMOA89Y4r + ph84PpcV0ziY9z3D8sKYWJq5nG9WLwyramSpTVgsDpRUHdazct6yHGpNC5AsIv/v5aXZU2xt27eM + n4FS6BjrynmstXqed27zGI/xtbbrikfDEND/0gor1ujjb6ixkYOZTgTCn8DYV422LXrn9XQnjauK + 3Vo2OyyKPSTn5C8AAAD//wMAcIbFgjUDAAA= + headers: + CF-RAY: + - 99b0f5db9f1cbffc-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:30:12 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=vfpKl6dvzcujjwigai_kp7UkNhR2ltT1SwFsT05VrS8-1762561812-1.0.1.1-UAyuy134RWxRUzjbClH59IJarw95du8Dl347lkXcDkbXBBx7vCmRuxRccJQB2f1T6oobZSgBj7O8hdaLY4hef6ypZ2uHUshy880EnptiWEY; + path=/; expires=Sat, 08-Nov-25 01:00:12 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=N6FAUGU_qhcPvlVWdt0kvrpbt1SzTvQ0v29fL2QCNbA-1762561812358-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '319' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '489' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3e940a310adf4d9a88c8da6b70645bb7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/langsmith/cassettes/test_langsmith_traceable_coexists_with_langchain.yaml b/py/src/braintrust/integrations/langsmith/cassettes/test_langsmith_traceable_coexists_with_langchain.yaml new file mode 100644 index 00000000..ba9f4fa9 --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/cassettes/test_langsmith_traceable_coexists_with_langchain.yaml @@ -0,0 +1,225 @@ +interactions: +- request: + body: '{"messages": [{"content": "What is 1 + 2?", "role": "user"}], "model": + "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": + false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ44VUVp2sk1koSWXX64CaLEy1mWy\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659919,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:38:40 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cd8d01c33943a-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '930' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=XPwF0fhMV9JwjYuWwUMNbzPKxvSJ.HOkXEftYzjXRew-1758659920459-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 93acad0503781eb98ab6ea3412173537 + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '1026' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_181413148bbe4814a905514521d6dc34 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jJJBb9swDIXv/hUCr4uL2IlTJ9et2y2HHQZ0Q2EoMm1rk0VNoocNRf77 + IDuN3a0FevGBHx/1Hs3HRAjQNRwEqE6y6p1J33/9XH7adscPRXGXnTb7+49fNse7/b2iYylhFRV0 + +o6Kn1Q3inpnkDXZCSuPkjFOzW53ebHLyiwfQU81mihrHadbSnttdZqv8226vk2z8qLuSCsMcBDf + EiGEeBy/0aet8TccxHr1VOkxBNkiHK5NQoAnEysgQ9CBpWVYzVCRZbSj9Uy8E7nAn4M0QWxull0e + myHI6NQOxiyAtJZYxqSjv4cLOV8dGWqdp1P4RwqNtjp0lUcZyMbXA5ODkZ4TIR7G5MOzMOA89Y4r + ph84PpcV0ziY9z3D8sKYWJq5nG9WLwyramSpTVgsDpRUHdazct6yHGpNC5AsIv/v5aXZU2xt27eM + n4FS6BjrynmstXqed27zGI/xtbbrikfDEND/0gor1ujjb6ixkYOZTgTCn8DYV422LXrn9XQnjauK + 3Vo2OyyKPSTn5C8AAAD//wMAcIbFgjUDAAA= + headers: + CF-RAY: + - 99b0f5db9f1cbffc-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:30:12 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=vfpKl6dvzcujjwigai_kp7UkNhR2ltT1SwFsT05VrS8-1762561812-1.0.1.1-UAyuy134RWxRUzjbClH59IJarw95du8Dl347lkXcDkbXBBx7vCmRuxRccJQB2f1T6oobZSgBj7O8hdaLY4hef6ypZ2uHUshy880EnptiWEY; + path=/; expires=Sat, 08-Nov-25 01:00:12 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=N6FAUGU_qhcPvlVWdt0kvrpbt1SzTvQ0v29fL2QCNbA-1762561812358-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '319' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '489' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3e940a310adf4d9a88c8da6b70645bb7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_module_aevaluate_uses_braintrust_eval.yaml b/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_module_aevaluate_uses_braintrust_eval.yaml new file mode 100644 index 00000000..3612f973 --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_module_aevaluate_uses_braintrust_eval.yaml @@ -0,0 +1,3401 @@ +interactions: +- request: + body: '{"messages": [{"content": "What is 1 + 2?", "role": "user"}], "model": + "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": + false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ44VUVp2sk1koSWXX64CaLEy1mWy\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659919,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:38:40 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cd8d01c33943a-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '930' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=XPwF0fhMV9JwjYuWwUMNbzPKxvSJ.HOkXEftYzjXRew-1758659920459-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 93acad0503781eb98ab6ea3412173537 + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '1026' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_181413148bbe4814a905514521d6dc34 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CZR8G4hND55E1b39YFV3NE9YcoN8a\",\n \"object\": + \"chat.completion\",\n \"created\": 1762561812,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + CF-RAY: + - 99b0f5db9f1cbffc-SJC + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:30:12 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=vfpKl6dvzcujjwigai_kp7UkNhR2ltT1SwFsT05VrS8-1762561812-1.0.1.1-UAyuy134RWxRUzjbClH59IJarw95du8Dl347lkXcDkbXBBx7vCmRuxRccJQB2f1T6oobZSgBj7O8hdaLY4hef6ypZ2uHUshy880EnptiWEY; + path=/; expires=Sat, 08-Nov-25 01:00:12 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=N6FAUGU_qhcPvlVWdt0kvrpbt1SzTvQ0v29fL2QCNbA-1762561812358-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '821' + openai-organization: + - braintrust-data + openai-processing-ms: + - '319' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '489' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3e940a310adf4d9a88c8da6b70645bb7 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"f5883013-b5c1-438d-9044-5182b4682337","name":"abhi-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NDFmZDZlNDYtNTUxNS00ZjcxLWFmZjctMzliOTQ5NGY4OGRh'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:17 GMT + Etag: + - '"13nbfem8i3p75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - NDFmZDZlNDYtNTUxNS00ZjcxLWFmZjctMzliOTQ5NGY4OGRh + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::4zjzv-1775254817145-bc7be41f021e + status: + code: 200 + message: OK +- request: + body: '{"project_name": "langsmith-py", "project_id": null, "org_id": "f5883013-b5c1-438d-9044-5182b4682337", + "update": false, "experiment_name": "langsmith-aeval", "repo_info": {"commit": + "5c35051a0102f850f2e09c66f9376a0fc658ed9f", "branch": "main", "tag": null, "dirty": + true, "author_name": "Abhijeet Prasad", "author_email": "abhijeet@braintrustdata.com", + "commit_message": "fix(anthropic): capture server-side tool usage metrics (#192)\n\nFlatten + Anthropic usage.server_tool_use into Braintrust span metrics so\nserver-side + tool invocations are preserved for tracing and cost analysis.\n\nAdd regression + tests using a red/green workflow to cover dict-backed span\nlogging and object-backed + usage extraction.\n\nFixes #171", "commit_time": "2026-04-02T21:10:00-04:00", + "git_diff": "diff --git a/py/noxfile.py b/py/noxfile.py\nindex c92395da..1083ca5b + 100644\n--- a/py/noxfile.py\n+++ b/py/noxfile.py\n@@ -77,6 +77,7 @@ VENDOR_PACKAGES + = (\n \"opentelemetry-exporter-otlp-proto-http\",\n \"google.genai\",\n \"google.adk\",\n+ \"langsmith\",\n \"temporalio\",\n + )\n \n@@ -104,6 +105,7 @@ GENAI_VERSIONS = (LATEST,)\n DSPY_VERSIONS = (LATEST,)\n + GOOGLE_ADK_VERSIONS = (LATEST, \"1.14.1\")\n LANGCHAIN_VERSIONS = (LATEST, \"0.3.28\")\n+LANGSMITH_VERSIONS + = (LATEST, \"0.7.12\")\n OPENROUTER_VERSIONS = (LATEST, \"0.6.0\")\n # temporalio + 1.19.0+ requires Python >= 3.10; skip Python 3.9 entirely\n TEMPORAL_VERSIONS + = (LATEST, \"1.20.0\", \"1.19.0\")\n@@ -235,6 +237,17 @@ def test_langchain(session, + version):\n _run_core_tests(session)\n \n \n+@nox.session()\n+@nox.parametrize(\"version\", + LANGSMITH_VERSIONS, ids=LANGSMITH_VERSIONS)\n+def test_langsmith(session, version):\n+ \"\"\"Test + LangSmith integration.\"\"\"\n+ _install_test_deps(session)\n+ _install(session, + \"langsmith\", version)\n+ _install(session, \"langchain-core\")\n+ _install(session, + \"langchain-openai\")\n+ _run_tests(session, f\"{INTEGRATION_DIR}/langsmith/test_langsmith.py\")\n+\n+\n + @nox.session()\n @nox.parametrize(\"version\", OPENAI_VERSIONS, ids=OPENAI_VERSIONS)\n + def test_openai(session, version):\n@@ -371,9 +384,8 @@ def pylint(session):\n session.install(\"pydantic_ai>=1.10.0\")\n session.install(\"google-adk\")\n session.install(\"opentelemetry.instrumentation.openai\")\n- # + langsmith is needed for the langsmith_wrapper module but not in VENDOR_PACKAGES\n # + langchain-core, langchain-openai, langchain-anthropic are needed for the langchain + integration\n- session.install(\"langsmith\", \"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n+ session.install(\"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n \n result = session.run(\"git\", \"ls-files\", + \"**/*.py\", silent=True, log=False)\n files = [path for path in result.strip().splitlines() + if path not in GENERATED_LINT_EXCLUDES]\ndiff --git a/py/src/braintrust/auto.py + b/py/src/braintrust/auto.py\nindex dc44c7d2..6cf8597c 100644\n--- a/py/src/braintrust/auto.py\n+++ + b/py/src/braintrust/auto.py\n@@ -16,6 +16,7 @@ from braintrust.integrations + import (\n DSPyIntegration,\n GoogleGenAIIntegration,\n LangChainIntegration,\n+ LangSmithIntegration,\n LiteLLMIntegration,\n OpenRouterIntegration,\n PydanticAIIntegration,\n@@ + -52,6 +53,7 @@ def auto_instrument(\n dspy: bool = True,\n adk: bool + = True,\n langchain: bool = True,\n+ langsmith: bool = True,\n ) -> dict[str, + bool]:\n \"\"\"\n Auto-instrument supported AI/ML libraries for Braintrust + tracing.\n@@ -75,6 +77,7 @@ def auto_instrument(\n dspy: Enable DSPy + instrumentation (default: True)\n adk: Enable Google ADK instrumentation + (default: True)\n langchain: Enable LangChain instrumentation (default: + True)\n+ langsmith: Enable LangSmith instrumentation (default: True)\n + \n Returns:\n Dict mapping integration name to whether it was successfully + instrumented.\n@@ -146,6 +149,8 @@ def auto_instrument(\n results[\"adk\"] + = _instrument_integration(ADKIntegration)\n if langchain:\n results[\"langchain\"] + = _instrument_integration(LangChainIntegration)\n+ if langsmith:\n+ results[\"langsmith\"] + = _instrument_integration(LangSmithIntegration)\n \n return results\n \ndiff + --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py\nindex + 0062ec77..2dca1110 100644\n--- a/py/src/braintrust/integrations/__init__.py\n+++ + b/py/src/braintrust/integrations/__init__.py\n@@ -6,6 +6,7 @@ from .claude_agent_sdk + import ClaudeAgentSDKIntegration\n from .dspy import DSPyIntegration\n from + .google_genai import GoogleGenAIIntegration\n from .langchain import LangChainIntegration\n+from + .langsmith import LangSmithIntegration\n from .litellm import LiteLLMIntegration\n + from .openrouter import OpenRouterIntegration\n from .pydantic_ai import PydanticAIIntegration\n@@ + -21,6 +22,7 @@ __all__ = [\n \"GoogleGenAIIntegration\",\n \"LiteLLMIntegration\",\n \"LangChainIntegration\",\n+ \"LangSmithIntegration\",\n \"OpenRouterIntegration\",\n \"PydanticAIIntegration\",\n + ]\ndiff --git a/py/src/braintrust/wrappers/langsmith_wrapper.py b/py/src/braintrust/wrappers/langsmith_wrapper.py\nindex + b22117df..7d4170fd 100644\n--- a/py/src/braintrust/wrappers/langsmith_wrapper.py\n+++ + b/py/src/braintrust/wrappers/langsmith_wrapper.py\n@@ -1,518 +1,4 @@\n-\"\"\"\n-Braintrust + integration for LangSmith - provides a migration path from LangSmith to Braintrust.\n+from + braintrust.integrations.langsmith import setup_langsmith # noqa: F401\n \n-This + module patches LangSmith''s tracing and evaluation APIs to use Braintrust under + the hood,\n-allowing users to migrate with minimal code changes.\n \n-Usage:\n- ```python\n- import + os\n-\n- # Enable LangSmith tracing and set project name (used by both services)\n- os.environ.setdefault(\"LANGCHAIN_TRACING_V2\", + \"true\")\n- os.environ.setdefault(\"LANGCHAIN_PROJECT\", \"my-project\")\n-\n- from + braintrust.wrappers.langsmith_wrapper import setup_langsmith\n-\n- # Call + setup BEFORE importing from langsmith\n- # project_name defaults to LANGCHAIN_PROJECT + env var\n- setup_langsmith()\n-\n- # Continue using langsmith imports + - they now use Braintrust\n- from langsmith import traceable, Client\n-\n- @traceable\n- def + my_function(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- client = Client()\n- results = client.evaluate(\n- my_function,\n- data=[{\"inputs\": + {\"x\": 1}, \"outputs\": {\"result\": 2}}],\n- evaluators=[my_evaluator],\n- )\n- ```\n-\n- Set + BRAINTRUST_STANDALONE=1 to completely replace LangSmith with Braintrust\n- (no + LangSmith code runs). Otherwise, both services run in tandem.\n-\"\"\"\n-\n-import + inspect\n-import logging\n-import os\n-from typing import Any, Callable, Dict, + Iterable, Iterator, List, Optional, ParamSpec, TypeVar\n-\n-from braintrust.framework + import EvalCase\n-from braintrust.logger import NOOP_SPAN, current_span, init_logger, + traced\n-from wrapt import wrap_function_wrapper\n-\n-\n-logger = logging.getLogger(__name__)\n-\n-# + Global list to store Braintrust eval results when running in tandem mode\n-_braintrust_eval_results: + List[Any] = []\n-\n-# TODO: langsmith.test/unit/expect, langsmith.AsyncClient, + trace\n-__all__ = [\n- \"setup_langsmith\",\n- \"wrap_traceable\",\n- \"wrap_client\",\n- \"wrap_evaluate\",\n- \"wrap_aevaluate\",\n- \"get_braintrust_results\",\n- \"clear_braintrust_results\",\n-]\n-\n-F + = TypeVar(\"F\", bound=Callable[..., Any])\n-P = ParamSpec(\"P\")\n-R = TypeVar(\"R\")\n-\n-\n-def + get_braintrust_results() -> List[Any]:\n- \"\"\"Get all Braintrust eval results + collected during tandem mode.\"\"\"\n- return _braintrust_eval_results.copy()\n-\n-\n-def + clear_braintrust_results() -> None:\n- \"\"\"Clear all stored Braintrust + eval results.\"\"\"\n- _braintrust_eval_results.clear()\n-\n-\n-def setup_langsmith(\n- api_key: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- project_name: + Optional[str] = None,\n- standalone: bool = False,\n-) -> bool:\n- \"\"\"\n- Setup + Braintrust integration with LangSmith.\n-\n- This patches LangSmith''s @traceable, + Client.evaluate(), and aevaluate()\n- to use Braintrust under the hood.\n-\n- Args:\n- api_key: + Braintrust API key (optional, can use env var BRAINTRUST_API_KEY)\n- project_id: + Braintrust project ID (optional)\n- project_name: Braintrust project + name (optional, falls back to LANGCHAIN_PROJECT\n- env var, + then BRAINTRUST_PROJECT env var)\n- standalone: If True, completely replace + LangSmith with Braintrust (no LangSmith\n- code runs). If + False (default), run both LangSmith and Braintrust\n- in tandem.\n-\n- Returns:\n- True + if setup was successful, False otherwise\n- \"\"\"\n- # Use LANGCHAIN_PROJECT + as fallback for project_name to keep both services in sync\n- if project_name + is None:\n- project_name = os.environ.get(\"LANGCHAIN_PROJECT\")\n-\n- span + = current_span()\n- if span == NOOP_SPAN:\n- init_logger(project=project_name, + api_key=api_key, project_id=project_id)\n-\n- try:\n- import langsmith\n-\n- langsmith.traceable + = wrap_traceable(langsmith.traceable, standalone=standalone)\n- wrap_client(langsmith.Client, + project_name=project_name, project_id=project_id, standalone=standalone)\n- langsmith.evaluate + = wrap_evaluate(\n- langsmith.evaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n- langsmith.aevaluate + = wrap_aevaluate(\n- langsmith.aevaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n-\n- logger.info(\"LangSmith + integration with Braintrust enabled\")\n- return True\n-\n- except + ImportError as e:\n- logger.error(f\"Failed to import langsmith: {e}\")\n- logger.error(\"langsmith + is not installed. Please install it with: pip install langsmith\")\n- return + False\n-\n-\n-def wrap_traceable(traceable: F, standalone: bool = False) -> + F:\n- \"\"\"\n- Wrap langsmith.traceable to also use Braintrust''s @traced + decorator.\n-\n- Args:\n- traceable: The langsmith.traceable function\n- standalone: + If True, replace LangSmith tracing entirely with Braintrust.\n- If + False, add Braintrust tracing alongside LangSmith tracing.\n-\n- Returns:\n- The + wrapped traceable function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(traceable):\n- return traceable\n-\n- def traceable_wrapper(*args: + Any, **kwargs: Any) -> Any:\n- # Handle both @traceable and @traceable(...) + patterns\n- func = args[0] if args and callable(args[0]) else None\n-\n- def + decorator(fn: Callable[P, R]) -> Callable[P, R]:\n- span_name = kwargs.get(\"name\") + or fn.__name__\n-\n- # Conditionally apply LangSmith decorator first\n- if + not standalone:\n- fn = traceable(fn, **kwargs)\n-\n- # + Always apply Braintrust tracing\n- return traced(name=span_name)(fn) # + type: ignore[return-value]\n-\n- if func is not None:\n- return + decorator(func)\n- return decorator\n-\n- traceable_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return traceable_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_client(\n- Client: Any, project_name: + Optional[str] = None, project_id: Optional[str] = None, standalone: bool = False\n-) + -> Any:\n- \"\"\"\n- Wrap langsmith.Client to redirect evaluate() and + aevaluate() to Braintrust''s Eval.\n-\n- Args:\n- Client: The langsmith.Client + class\n- project_name: Braintrust project name to use for evaluations\n- project_id: + Braintrust project ID to use for evaluations\n- standalone: If True, + only run Braintrust. If False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + Client class (modified in place)\n- \"\"\"\n-\n- if hasattr(Client, \"evaluate\") + and not _is_patched(Client.evaluate):\n- wrap_function_wrapper(\n- Client,\n- \"evaluate\",\n- make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.evaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- if hasattr(Client, \"aevaluate\") + and not _is_patched(Client.aevaluate):\n- wrap_function_wrapper(\n- Client,\n- \"aevaluate\",\n- make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.aevaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- return Client\n-\n-\n-def make_evaluate_wrapper(\n- *, + project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: + bool = False\n-):\n- def evaluate_wrapper(wrapped: Any, instance: Any, args: + Any, kwargs: Any) -> Any:\n- result = None\n- if not standalone:\n- result + = wrapped(*args, **kwargs)\n-\n- try:\n- result = _run_braintrust_eval(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + evaluate failed: {e}\")\n-\n- return result\n-\n- return evaluate_wrapper\n-\n-\n-def + make_aevaluate_wrapper(\n- *, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-):\n- async def aevaluate_wrapper(wrapped: + Any, instance: Any, args: Any, kwargs: Any) -> Any:\n- result = None\n- if + not standalone:\n- result = await wrapped(*args, **kwargs)\n-\n- try:\n- result + = await _run_braintrust_eval_async(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + aevaluate failed: {e}\")\n-\n- return result\n-\n- return aevaluate_wrapper\n-\n-\n-def + wrap_evaluate(\n- evaluate: F, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.evaluate to redirect to Braintrust''s Eval.\n-\n- Args:\n- evaluate: + The langsmith.evaluate function\n- project_name: Braintrust project name + to use for evaluations\n- project_id: Braintrust project ID to use for + evaluations\n- standalone: If True, only run Braintrust. If False, run + both LangSmith and Braintrust.\n-\n- Returns:\n- The wrapped evaluate + function (or the original if already patched)\n- \"\"\"\n- if _is_patched(evaluate):\n- return + evaluate\n-\n- evaluate_wrapper = make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id)\n- evaluate_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return evaluate_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_aevaluate(\n- aevaluate: F,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- standalone: + bool = False,\n-) -> F:\n- \"\"\"\n- Wrap module-level langsmith.aevaluate + to redirect to Braintrust''s EvalAsync.\n-\n- Args:\n- aevaluate: + The langsmith.aevaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to use + for evaluations\n- standalone: If True, only run Braintrust. If False, + run both LangSmith and Braintrust.\n-\n- Returns:\n- The wrapped aevaluate + function (or the original if already patched)\n- \"\"\"\n- if _is_patched(aevaluate):\n- return + aevaluate\n-\n- aevaluate_wrapper = make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id)\n- aevaluate_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return aevaluate_wrapper # type: + ignore[return-value]\n-\n-\n-def _is_patched(obj: Any) -> bool:\n- return + getattr(obj, \"_braintrust_patched\", False)\n-\n-\n-# =============================================================================\n-# + Braintrust evaluation logic\n-# =============================================================================\n-\n-\n-def + _run_braintrust_eval(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust Eval with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import Eval\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + Eval(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-async + def _run_braintrust_eval_async(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust EvalAsync with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import EvalAsync\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + await EvalAsync(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-# + =============================================================================\n-# + Data conversion helpers\n-# =============================================================================\n-\n-\n-def + _wrap_output(output: Any) -> Dict[str, Any]:\n- \"\"\"Wrap non-dict outputs + the same way LangSmith does.\"\"\"\n- if not isinstance(output, dict):\n- return + {\"output\": output}\n- return output\n-\n-\n-def _make_braintrust_scorer(\n- evaluator: + Callable[..., Any],\n-) -> Callable[..., Any]:\n- \"\"\"\n- Create a Braintrust + scorer from a LangSmith evaluator.\n-\n- Always runs the evaluator through + Braintrust for full tracing (span duration, child LLM calls, etc.).\n- \"\"\"\n- evaluator_name + = getattr(evaluator, \"__name__\", \"score\")\n-\n- def braintrust_scorer(input: + Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any:\n- from + braintrust.score import Score\n-\n- # Run the evaluator with LangSmith''s + signature\n- # LangSmith evaluators use: (inputs, outputs, reference_outputs) + -> bool | dict\n- # LangSmith auto-wraps non-dict outputs as {\"output\": + value}\n- outputs = _wrap_output(output)\n-\n- # expected is the + real LangSmith Example object passed through from data loading\n- reference_outputs + = expected.outputs if hasattr(expected, \"outputs\") else expected\n-\n- result + = evaluator(input, outputs, reference_outputs)\n-\n- return Score(\n- name=result.get(\"key\", + evaluator_name),\n- score=result.get(\"score\"),\n- metadata=result.get(\"metadata\", + {}),\n- )\n-\n- braintrust_scorer.__name__ = evaluator_name\n- return + braintrust_scorer\n-\n-\n-def _convert_langsmith_data(data: Any) -> Callable[[], + Iterator[EvalCase[Any, Any]]]:\n- \"\"\"Convert LangSmith data format to + Braintrust data format.\"\"\"\n-\n- def load_data() -> Iterator[EvalCase[Any, + Any]]:\n- # Determine the source iterable without loading everything + into memory\n- source: Iterable[Any]\n- if callable(data):\n- source + = data() # type: ignore\n- elif isinstance(data, str):\n- # + Load examples from LangSmith dataset by name\n- try:\n- from + langsmith import Client # pylint: disable=import-error\n-\n- client + = Client()\n- source = client.list_examples(dataset_name=data)\n- except + Exception as e:\n- logger.warning(f\"Failed to load LangSmith + dataset ''{data}'': {e}\")\n- return\n- elif hasattr(data, + \"__iter__\"):\n- source = data\n- else:\n- source + = [data]\n-\n- # Process items as a generator - yield one at a time\n- for + item in source:\n- # Pass through LangSmith Example objects directly\n- if + hasattr(item, \"inputs\"):\n- yield EvalCase(\n- input=item.inputs,\n- expected=item, # + Pass the whole Example object\n- metadata=getattr(item, \"metadata\", + None),\n- )\n- elif isinstance(item, dict):\n- if + \"inputs\" in item:\n- # LangSmith dict format\n- yield + EvalCase(\n- input=item[\"inputs\"],\n- expected=item, # + Pass the whole dict\n- metadata=item.get(\"metadata\"),\n- )\n- elif + \"input\" in item:\n- # Braintrust format\n- yield + EvalCase(\n- input=item[\"input\"],\n- expected=item.get(\"expected\"),\n- metadata=item.get(\"metadata\"),\n- )\n- else:\n- yield + EvalCase(input=item)\n- else:\n- yield EvalCase(input=item)\n-\n- return + load_data\n-\n-\n-def _make_braintrust_task(target: Callable[..., Any]) -> Callable[..., + Any]:\n- \"\"\"Convert a LangSmith target function to Braintrust task format.\"\"\"\n-\n- def + task_fn(task_input: Any, hooks: Any) -> Any:\n- if isinstance(task_input, + dict):\n- # Try to get the original function''s signature (unwrap + decorators)\n- unwrapped = inspect.unwrap(target)\n-\n- try:\n- sig + = inspect.signature(unwrapped)\n- params = list(sig.parameters.keys())\n- if + len(params) == 1:\n- return target(task_input)\n- if + all(p in task_input for p in params):\n- return target(**task_input)\n- return + target(task_input)\n- except (ValueError, TypeError):\n- # + Fallback: try kwargs first, then single arg\n- try:\n- return + target(**task_input)\n- except TypeError:\n- return + target(task_input)\n- return target(task_input)\n-\n- return task_fn\n+__all__ + = [\"setup_langsmith\"]\ndiff --git a/py/src/braintrust/wrappers/test_langsmith_wrapper.py + b/py/src/braintrust/wrappers/test_langsmith_wrapper.py\ndeleted file mode 100644\nindex + c25256c2..00000000\n--- a/py/src/braintrust/wrappers/test_langsmith_wrapper.py\n+++ + /dev/null\n@@ -1,341 +0,0 @@\n-# pyright: reportPrivateUsage=false\n-# pyright: + reportMissingParameterType=false\n-# pyright: reportUnknownParameterType=false\n-# + pyright: reportUnknownArgumentType=false\n-# pyright: reportUnknownMemberType=false\n-# + pylint: disable=protected-access\n-\n-\"\"\"\n-Tests for the LangSmith wrapper + to ensure compatibility with LangSmith''s API.\n-\"\"\"\n-\n-from braintrust.wrappers.langsmith_wrapper + import (\n- _convert_langsmith_data,\n- _is_patched,\n- _make_braintrust_scorer,\n- _make_braintrust_task,\n- wrap_aevaluate,\n- wrap_client,\n- wrap_traceable,\n-)\n-\n-\n-def + test_is_patched_false():\n- \"\"\"Test that _is_patched returns False for + unpatched objects.\"\"\"\n-\n- def unpatched():\n- pass\n-\n- assert + _is_patched(unpatched) is False\n-\n-\n-def test_is_patched_true():\n- \"\"\"Test + that _is_patched returns True for patched objects.\"\"\"\n-\n- def patched():\n- pass\n-\n- patched._braintrust_patched + = True # type: ignore\n-\n- assert _is_patched(patched) is True\n-\n-\n-def + test_make_braintrust_scorer_dict_result():\n- \"\"\"Test converting a LangSmith + evaluator that returns a dict.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- return {\"key\": \"accuracy\", \"score\": + 0.9, \"metadata\": {\"note\": \"good\"}}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- # + Create a mock Example object\n- class MockExample:\n- outputs = {\"y\": + 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": 2}, expected=MockExample())\n-\n- assert + result.name == \"accuracy\"\n- assert result.score == 0.9\n- assert result.metadata + == {\"note\": \"good\"}\n-\n-\n-def test_make_braintrust_scorer_numeric_result():\n- \"\"\"Test + converting a LangSmith evaluator that returns a numeric score in a dict.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return {\"score\": + 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class + MockExample:\n- outputs = {\"y\": 2}\n-\n- result = converted(input={\"x\": + 1}, output={\"y\": 2}, expected=MockExample())\n-\n- assert result.name == + \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def test_make_braintrust_scorer_with_plain_dict_expected():\n- \"\"\"Test + converting a LangSmith evaluator with plain dict as expected.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return {\"score\": + 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n- result + = converted(input={\"x\": 1}, output={\"y\": 2}, expected={\"y\": 2})\n-\n- assert + result.name == \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def + test_convert_langsmith_data_from_list():\n- \"\"\"Test converting LangSmith + data from a list of dicts.\"\"\"\n- data = [\n- {\"inputs\": {\"x\": + 1}, \"outputs\": {\"y\": 2}},\n- {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- assert result[1].input + == {\"x\": 2}\n- assert result[1].expected == {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}}\n-\n-\n-def test_convert_langsmith_data_from_callable():\n- \"\"\"Test + converting LangSmith data from a callable.\"\"\"\n-\n- def data_generator():\n- yield + {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- yield {\"inputs\": + {\"x\": 2}, \"outputs\": {\"y\": 4}}\n-\n- data_fn = _convert_langsmith_data(data_generator)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n-\n-\n-def test_convert_langsmith_data_with_example_objects():\n- \"\"\"Test + converting LangSmith data with Example-like objects.\"\"\"\n-\n- class MockExample:\n- def + __init__(self, inputs, outputs):\n- self.inputs = inputs\n- self.outputs + = outputs\n-\n- data = [\n- MockExample(inputs={\"x\": 1}, outputs={\"y\": + 2}),\n- MockExample(inputs={\"x\": 2}, outputs={\"y\": 4}),\n- ]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 2\n- assert result[0].input == {\"x\": 1}\n- # The whole + Example object is passed as expected\n- assert result[0].expected.inputs + == {\"x\": 1}\n- assert result[0].expected.outputs == {\"y\": 2}\n-\n-\n-def + test_make_braintrust_task_with_dict_input():\n- \"\"\"Test that task function + handles dict inputs correctly.\"\"\"\n-\n- def target_fn(inputs):\n- return + inputs[\"x\"] * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == 10\n-\n-\n-def test_make_braintrust_task_with_kwargs_expansion():\n- \"\"\"Test + that task function expands dict kwargs when signature matches.\"\"\"\n-\n- def + target_fn(x, y):\n- return x + y\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 2, \"y\": 3}, None)\n-\n- assert result == 5\n-\n-\n-def test_make_braintrust_task_simple_input():\n- \"\"\"Test + that task function handles simple inputs.\"\"\"\n-\n- def target_fn(inp):\n- return + inp * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result = task(5, + None)\n-\n- assert result == 10\n-\n-\n-class TestWrapTraceable:\n- \"\"\"Tests + for wrap_traceable functionality.\"\"\"\n-\n- def test_wrap_traceable_returns_wrapper(self):\n- \"\"\"Test + that wrap_traceable returns a wrapped version.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable, + standalone=False)\n- assert callable(wrapped)\n- assert _is_patched(wrapped)\n-\n- def + test_wrap_traceable_standalone_mode(self):\n- \"\"\"Test that wrap_traceable + works in standalone mode.\"\"\"\n-\n- def mock_traceable(func, **kwargs):\n- return + func\n-\n- wrapped = wrap_traceable(mock_traceable, standalone=True)\n- assert + callable(wrapped)\n- assert _is_patched(wrapped)\n-\n-\n-class TestWrapFunctions:\n- \"\"\"Tests + for the wrap_* functions.\"\"\"\n-\n- def test_wrap_functions_exist(self):\n- \"\"\"Test + that wrap functions are callable.\"\"\"\n- assert callable(wrap_traceable)\n- assert + callable(wrap_client)\n- assert callable(wrap_aevaluate)\n-\n- def + test_wrap_traceable_returns_patched_function(self):\n- \"\"\"Test that + wrap_traceable returns a patched function.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable)\n- assert + _is_patched(wrapped)\n-\n- def test_wrap_traceable_skips_if_already_patched(self):\n- \"\"\"Test + that wrap_traceable skips if already patched.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- mock_traceable._braintrust_patched + = True # type: ignore\n-\n- result = wrap_traceable(mock_traceable)\n- # + Should return the same function\n- assert result is mock_traceable\n-\n- def + test_wrap_client_sets_flag(self):\n- \"\"\"Test that wrap_client sets + the patched flag.\"\"\"\n-\n- class MockClient:\n- def evaluate(self, + *args, **kwargs):\n- return \"original\"\n-\n- wrap_client(MockClient)\n- assert + _is_patched(MockClient.evaluate)\n-\n- def test_wrap_aevaluate_returns_patched_function(self):\n- \"\"\"Test + that wrap_aevaluate returns a patched function.\"\"\"\n-\n- async def + mock_aevaluate(*args, **kwargs):\n- pass\n-\n- wrapped = wrap_aevaluate(mock_aevaluate)\n- assert + _is_patched(wrapped)\n-\n-\n-class TestTandemModeIntegration:\n- \"\"\"Integration + tests for tandem mode (LangSmith + Braintrust together).\"\"\"\n-\n- def + test_make_braintrust_task_with_inputs_parameter(self):\n- \"\"\"Test + that task handles LangSmith''s required ''inputs'' parameter name.\"\"\"\n-\n- def + target_fn(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- task = _make_braintrust_task(target_fn)\n- result = + task({\"x\": 5}, None)\n-\n- assert result == {\"result\": 10}\n-\n- def + test_convert_langsmith_data_handles_different_output_types(self):\n- \"\"\"Test + that data conversion handles various output types.\"\"\"\n- data = [\n- {\"inputs\": + {\"x\": 1}, \"outputs\": 2}, # outputs is int, not dict\n- {\"inputs\": + {\"x\": 2}, \"outputs\": {\"result\": 4}}, # outputs is already dict\n- ]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- # + Both should work - Braintrust''s EvalCase accepts any type for expected\n- assert + len(result) == 2\n- assert result[0].input == {\"x\": 1}\n- assert + result[1].input == {\"x\": 2}\n-\n- def test_make_braintrust_scorer_handles_wrapped_outputs(self):\n- \"\"\"Test + that scorers handle output wrapping correctly.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- # outputs will be wrapped as {\"output\": + value} for non-dict results\n- actual = outputs.get(\"output\", outputs)\n- expected + = (\n- reference_outputs.get(\"output\", reference_outputs)\n- if + isinstance(reference_outputs, dict)\n- else reference_outputs\n- )\n- return + {\"key\": \"match\", \"score\": 1.0 if actual == expected else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class MockExample:\n- outputs + = {\"output\": 42}\n-\n- # Test with wrapped output\n- result + = converted(input={\"x\": 1}, output=42, expected=MockExample())\n- assert + result.name == \"match\"\n- assert result.score == 1.0\n-\n-\n-class + TestDataConversion:\n- \"\"\"Tests for data conversion utilities.\"\"\"\n-\n- def + test_convert_data_with_braintrust_format(self):\n- \"\"\"Test that Braintrust + format is properly handled.\"\"\"\n- data = [\n- {\"input\": + {\"x\": 1}, \"expected\": {\"y\": 2}},\n- {\"input\": {\"x\": 2}, + \"expected\": {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- assert result[0].expected == {\"y\": 2}\n- assert + result[1].input == {\"x\": 2}\n- assert result[1].expected == {\"y\": + 4}\n-\n- def test_convert_data_with_simple_items(self):\n- \"\"\"Test + that simple items (not dicts) are handled.\"\"\"\n- data = [1, 2, 3]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 3\n- assert result[0].input == 1\n- assert result[1].input + == 2\n- assert result[2].input == 3"}, "ancestor_commits": ["5c35051a0102f850f2e09c66f9376a0fc658ed9f", + "0daac2f277a337206cbb03cc46c695f7931d7f3d", "d807b7b4a48b4bd920570cdad5f01c350cdd9f35", + "19ecb8a09557f830675a2dded6bdd04f36ae9f76", "0a708e578ce19f61c17a9f4aa44a96d7ed0277df", + "2a174d884e2034fd9a8c20c20c4c0c384f7aa687", "a656bab37f99445ca442fef41f032907c79e3fc0", + "5fda66e087f78c4edf8c95f24c9454e9c80c6417", "07eae203f645dd661ea0fb83b5256f2144dbf07f", + "2c68b8894223a9b562b7d922a53430efe067ede3", "ba0bc10b5f79e952e69a721be89023773f46fe54", + "e09f625c67fab1f2d2f7b31f0370dab3e0cafd1b", "4e5c83f5456b5c043e1e0830227805d28f922f70", + "161fa36a51946f2212d28354609e442f83889684", "3462f3f3d4ce82b3c59f5dacec9a523b9c99ee2d", + "62a2963911d6c65cdbaef1712fdb7c890d3b5777", "89122d4cca5ea2b766b9416c68b611bee79294dd", + "1c1a07e7dfd96811fd883d81adb3b7f7f4061e75", "f6a2c3b3aa589ab8606b7b525ce79ca019f98717", + "051870af3d7bc8c183edac119bcb7166f1ea9ef0", "4267bde681b41aaa31036e14508708a28bde70ac", + "0b717a9e6240a162a3931116f44bd86f4900852b", "7a4328e7db5636ffe7898d99fd392bc204ec7a08", + "e0780bfc33ab6ced1b982aaf27bd7b5072bbe04e", "183dbb0d034c3588b26a8a94a10f3d2f03237670", + "8b15ad67f697a8218217d9674f0fb919d9c1e8d6", "01d0cea1268051909665e4911af1f1be54b333cd", + "b335e66c1f458de9cd73c12d9d9fc97114f3d505", "0b2f34802606d033bb3471075be95de5d9621b98", + "dd73fb4ac1f38bee8e55133a0348f1bd5c46c30e", "716f9e45b99874f17531f9b5ae4ccfa57cab5ae3", + "72adf8bbf0b48756e18423c5477d58bbf6c401b9", "1163e11507676601e265e637e8916a71a310ce87", + "80308c49a0309d13e02c3203c099b3dc87de1ad9", "e112d705d338ef08d449afab5f2bef3ae0269622", + "10996518f60c0cd2796304c122cd05b16c6ffb1c", "b9a78fa767379bfc14e9ec0f9dc1a016f7ca74a2", + "3fc8912b4063d524f357733549ff1c58bbad00ba", "4252662870b266b32d23a16a11c6df755e5dcdcb", + "92165c3f00478c316e7af29d5c2dba51ffbd7090", "a8144eabce4ef1a46257be77c8055bd3fc64978c", + "363baa083335500aebe129c670eb2c49153ad3a6", "0d487b688de037fa6eebf5410c12b9b226f12cff", + "824861d780be7bcbec92f274a13e221cffef04a4", "84398747d12f52f2b6089d5e832ff6cca82fa3d5", + "46c18b284d0d14855fb66a6de8891ea2c39e08e7", "4f1d9510a188a7a9645fe72966a099aff70478ca", + "d82c4258901176056c1eb30a837896becada5a0f", "265a46272e1f808251c0da03061a9b1843a22779", + "42bed48d1b31a428f189ba174ff4f759ec34e94d", "98aa3e6923d32ed264abd11315cd247c085e8a16", + "ff18889b8f8ef8af644f6eeab1179f6bf9b14de5", "fd34303ea427a9bfc038b78ef1e8e65bb44b6f29", + "08210a4fd97cc17d90556afaaba51dbd67917ddd", "a1fae25c088942b155a31b243f07b9862f79c5ba", + "4fb512eaa52733c1daa4cd6995140f6722f17ba4", "992e036a9199d1b03976ed4d22eebe9921410b40", + "5d41698059f77d9c332cfc4e706ea77669d1cb6a", "2349418ff85b75917d869a26be6df0682686caac", + "fd9f3f932a646f07438db3dbda8cab73cfa70c2f", "fc2d5fe0f6d94754e3c705686c3c69eaa58c258c", + "b3651b577d6ebf617a9991252e19d5fcfe7e693c", "106691a2cbfafc38b3d5d5064c9bb922ae7a3901", + "1b7818ade91e40155c7c126cb92cb21726a25c66", "49ab8392e413992b9728666999593a94ebd00236", + "fb80c5412e011c5b3d343809710c92b75874261e", "ab9fd17f71c277f0d05a809c3c9cfe22379df511", + "7184711ed04957af8f2bbe36590c1cec057ca0a8", "5976a47fe82f90599edc06b7e4f9af64a8ce4ed7", + "962e499f18549cdea2e65946c79bacbb9159f35d", "28875eb8ce2b4114d60d365ad5b61c124b5e3900", + "e68dfce97ee7fa014a30d694d7220a1de47fa73e", "404589ac15776fe04f6dd8bdf36c5d74a3684986", + "5f5b88701fedc4cafed4c037ee2a8cff6b4a40fb", "aeb15581fefc2b0b867ca55c21ea80d067f43361", + "2cd52871898658315cc43e78f25546c3c8df1c8a", "6c4b637d65e047f01052b0749d4c3608f202e073", + "864aafda1789476b01c6991931fb0a6e1968e2dc", "9548ae35e49fc7aac9263811ae3c3a4b26af2e81", + "97da34df8474129e1ad16c8eb67b00a906330970", "fdd020262b6620a4823d336741ebbcf0a02ff9a2", + "0bb83ac92e0ccb38dee1f56c11d15180e8c8c565", "34262bcfb1c5babc03da6d27d1b93ca2c3e969a7", + "7bb761580056ac43c2636964b5f1d2f08b3cc01a", "9424d6a04f73de02038dcbacd518864397ddcafb", + "a51b84c54ba9269d28f17e7355dbc8aaf9ccc0b2", "a5ef7778c00dd0c3782a423e4fb6ee3a0a2a73a3", + "33cfa00beffa04151a7d874d5671b720bb49dbe8", "eb530110ee5746d87c1d2b9972f3a3dc4a2aa582", + "8695a8e3ef85a173cfffd98ae25f6a31a7e23d2e", "7da14b1a295c8f912b6b93ec55ec759242a50108", + "f60a2d39cce59a000017bc193b84ef25ba4491d5", "447d3a96d0679451a2c87a5bf28531ca7744b9ab", + "abc8bed8fbdac570891517c76233e18d9fff4828", "a7053505083fb7b7ae6ed1c10a4141a0d8673cc4", + "8ab9dc06de08f7a925fabe52dbbdb6df3b480e89", "009a839b197b63d6a688d53d91743fb84cbed04f", + "fe75d23b3b61e1693c3b3bfe896380ce16a3ef66", "3a4dde05d882a4eb27d21d1d41784295b3a4168e", + "2b0d9e276db08298b83a10b877f3ccf08bfafde6", "fe1732c7b4df75d49cb42017bb2b2af0136a73c0", + "971559d598a53d361c50da7126e5681124244f32", "6e172e761f95e59dda9fab0409f85491beef2240", + "e3cd507afaaa62bc8ef736b27c1f0e0064c3931f", "3a3ba47a007e0af9af528b63b271391e0c538dc7", + "c3d5ea96b897dc179e4775b14c7bb97b838861db", "1ec7d11f39c97471973de7e5eb0b0784025bc851", + "99a5cb75b08c5e2bff6e96614cfe8c0573f0c19d", "49f1e3846c3c8d264e693bab999270d4392d153e", + "824cf93c402abf51934aebd890c9805457108a7c", "d9e62442aa51d540345c1bfbd1df043b9f00657d", + "c933fe5625cfe8b7e57e4b2953fe612ad37d78b5", "ec0c35e2fadc7bd6a734ff3013226193b1f81d09", + "d462219aa0d5e6525299e3a9cc1dabb19dbdc9c5", "9f8937030053fc43137805b66fef5d1d335fae47", + "1c827b6442cf42fbfa2868abfcb2c640752769bd", "892c41480277cab14f3751eafe678af0c4966bfb", + "6a65a37fb5f7e11e9b8551759d44879e732355d1", "c047819f46a9bf82b7f11124802247055e709c80", + "102257d62684977c39f6e78559d73ac0f873048a", "94f13c6a1c7306e23ba45c2cf0d3600ef8cf8673", + "786d48c8e9cf363508f0c30b872968d9ceec69d7", "7b62001e98bba2fcad8254aff0f0cb11bf35db33", + "2eeee24fc7c3b7c6a0aac79563e1cb0253e0bea1", "7aa19caa8d893488dedf5d6e985b766710b66eb7", + "17b5cae858690e270104ba1c5520508525e6c687", "8b1a5aa0a4b4d547e09c07d69be841c3ebde525f", + "174b4061aab369725a43719d5e966013686b4c38", "7bf81f8ebc7877956e2b8c3c556743426ef05038", + "e151663947390534a2037912cad663daba83de6d", "99bc21b047a09063813a733d659e84ad740424e3", + "1bc2d475bc64f99dd11c3bfd569bd15140a87389", "e3d62fddfab1774e5cae1c1ae2d0d46f164a768f", + "d91f76c24230054edef8774579a0a036b844af7b", "dbea65347acd60addcfbfa7bdb97ddbaeee6ba21", + "a28ef5f8895857ad6d945644f78679271fd10df3", "ec1c107ab6133f5810f9501e3ce57349c0bf4ae9", + "10e2d2237b0791683386e551bd4be79cf86367db", "e3f25b9b6abc2f8e1a376788d11d82a61c41645d", + "a88b44d6852fae824705415d2795f5c5201f2dde", "6210d435deb094c603c9e8debdd0d48255be7dae", + "6b2ab80cc64c1b869fb9ea6002fb1670e9ceb5cb", "1e440103b7521973aa5fb33addc8f9ce1d050232", + "449f6eeddd54f73d88e185fcf499087d3e61fbe6", "52579035063a1b7d1e8b64fbaf1b32bf55a04004", + "4db047355d602c53d85f43c1e7276dd3bd8335df", "4bc366daa82312b79ccb2d6471abef3ad16e4512", + "c710e0c271880b2ea1d506dd3d6f23c282f33fe8", "ce261c13723e8985e61819f6df56298f863155d0", + "f2b6f15f1db2d3e3a930e3274c0c1d1245e851c2", "f11481fe89d024f1b2deb651653ea8ad30da5e8a", + "633cb4c201d614ff5ac5e9775fd68764601d59d8", "0bb08d89c6fe9c043cb75178571715460bc32603", + "a8b9f42b80cfc3a835162fc4e69945edae9a2dd2", "e9f46b8d41234acba81068f7c00e3063204b4231", + "ef1c89e8f840a14f44cd6682bc92f7ce2ea9d07d", "6d086d6cfee47cc1e83c7522a58e1d412e91d197", + "7894ea68db9bcdaa55e66cd4d2a634878f901965", "753298d7ada8377c173de6fad4b490082372b7db", + "25868bc58450dad2058b6499ce3bb9400330fbd1", "95f3c141b7f8ad81bf1d34d857c2fe60bbd15d7f", + "f416cea3139a9a29ed5cae633cb3097e70fc93e2", "5e89c7299d9cc476594fc73933bc7006c842cdf4", + "ddf619048666d883cd4325e21f860e28632a456b", "2385b1aa98998820bcb55cfae76d063a57bba938", + "b798003d9609603b43c98ccc2a5c995d0ac1a488", "4d9333cef5aae8889f9697f38a87f694893f17a8", + "2194150ee020def2d7c4a675f2f001c8c0fba99e", "36788d5a20f45762991b1ba72fb52f8020e6c61b", + "16f72b7d1cb1e0801285abbb7b3138909d6e7f79", "617d9b730b37e96b7d05a099b95f5387944d0951", + "51bb20bec1dc870ec4f521e3c9464054ab9abdec", "15cde9f7795249fbb3c71f23210092265f509e19", + "ea3fd2ff145cbfcd53bb1d79ec6dfb659debdb4f", "eacfc8036be4cee52c619f278b61d32862f3a521", + "252eb0be986ee236c4f095f0078b816c7bd5c894", "6774ffa8b89b2da3644f07e4f411b20f108e5786", + "77fc0472011fb9ea9b3ff7bc5e7dbc5c57f33fed", "3e4002f6b2b3e540db12a2355f447fabd7a83bec", + "3110f96fcbfea2b60d1f8b580e28c00c2fe38690", "1cfad0ba63198f2480a22721501757d9913a1d52", + "2dc8187fd9ace26f7684ff5ba4fa75d7f28d7703", "0687d03758cd295a1976a1b5eb40c5187332be7c", + "8b3614791454b2b4e15a47a09948b3699ad7334a", "54002cedbe48b5036ba94109782d78686c0df7aa", + "9c21b41dd5a19131bd4bf236503ebd0a1464816f", "9a5c1c4a0d50d2d70ab6acb2c38e6abcbfcfc547", + "d0bf7aec410bdd29bf4a8f43af0a8bea633788e6", "7580576751a6890b3314fcd4c313991fb9f311a6", + "d734e8ffc272ee65fe0588df00fd9390614ccd2e", "d9fd9bb0c6bf4c6d5536765e0f8005e9326021a2", + "a751e399b66d396cdf13599c93b1445d81934ddc", "8e50db374a548c26830613a78f70072086ce1f13", + "791a2da1e44f02902e1c31ec28c2c7aba5795f47", "8a63569906d8824faf9c1aaecdd62ebb9e47aeb4", + "e541543ce30054cc10fa3e27e2b0aa5afd297d0f", "b017a02240f978dcfe7feec58aeb85f8cbc91892", + "072d3765416d78e465a728df1b4a812992fba58a", "1671a73c21d09a1b4b73debce7cb2fa1d1b9ee54", + "7b6371f41f6a109ea068fd547b17a131d109ee85", "530a4be0108ff8cc783a49c6390f59b238782e58", + "3ca420e53e77d4665b91ccc7631c95dc97ce566d", "41aceab354ed312d4758b5a5b5c32bbdac40da48", + "6f303beaf59f892b9ed86eb4338900c7f82f7a68", "2d2f7e260b2027576963fad1c432197b15f2811e", + "04acf467f93d2b1064ef648c6583f76d96864578", "3dcfc326ee12b116d919974ef8bbf3cf307ee7fd", + "0740dc169ac9f267e4ec2d435165ca882df0fbc1", "270970d692adaf5d204d336ecdb4ee7f51042187", + "37e49012471ef81326f1313583497f48d612eed1", "8d9100b8341085366c11a36d93faa6009ecd4837", + "f53d7b868f610f1b215c799f25d4af171b59a000", "23b9a8d6b977d6849503433f8b38c49cf90e8203", + "5a31cd02719a5a282825a0fec7cb8d0fc44d0884", "7079cb29799d9fb9fbc919b7ddd1224935a551c7", + "d9c624ea93ca6bf62c2412abce1b3a2ef1a2be67", "126c367b3002ad57bb6aa7e08d04657db6f61980", + "b610cd36638ace60bca2aed261165afbf1c8c081", "52a8fa6d53c58908d95bb865f5907d829987fb97", + "65c037efdffceeec554b2706e16621cfc1704202", "c3bc923fafbc33485c9173c2401fa20897cf4ef1", + "6d6cc2f1e8b5da316c409a754ee8c844b8d2b2ee", "0ecb05e34031941426e3b9ce76760add8af6fdfd", + "8ab13f3f48af6a4d3c0b053e4bbabfd4f24f23ec", "10c14b3688c6969ef55951d5d82e744b87d6c356", + "05f569c80bddb61414e8f9a8adf49f8ca6b821b4", "2311d67956183da8b31bf1fdbe6e09b3ec7e242d", + "0b0bdd77c012e3f51c6402edcad35c8ac7ddf4cd", "ca2eb28a0f22a63b2b748d04e17268d11d35e0b0", + "c20c808993bba8d5604c2ac7848037d7ce430e89", "dcd4f5a4be171b1cac28a5eb3534e4b55420cc06", + "74dd883f7c03ba5591ecee0b2b76e9250781e181", "4f94c9548a493de11cd663fc78db21a81fe6f23c", + "b3c2b6f7120402b3f5f0fce9b7a1e6b7f7d04d47", "476a66a5d09ec811c9856f0f0f289b189859721e", + "2468e8006b8edf9ef8f273e142380faf34d380a6", "b88964e5a1b124816771e2c9c0a628c104ecfddb", + "a05dc4df62a20e9715abe697584641f0032742b5", "ad3ca98e3530c7dbdbb07f63a04fe4912e17f85a", + "dbbc1894ef31143816e5913676301261bc44aa4c", "db388916a9bc42b02e26b519c3c5f116c919cafc", + "78753fa68806ca7478572a8b37c1f8d97f99eee3", "759f04ddc9522471b46d139ab347668c849ef8dc", + "3857983398d6c7278686c9f12fc2f841c628bc06", "11d9aff118de5f99bb0b5d55bca04d0efb36746a", + "38ea029efe156f07385cbea0a8743e07f418dd76", "39cc2896ef3cf7f9c2320fa513c645a972063697", + "e085b589790c05aa27f52ced4fb0480230db38b5", "5a0afe994e81c19a69ac9ce3a60529b5e7e7c529", + "e42f891ae64b8844c2d37bf29c25821382ed654f", "1fa9fe250443884c450e92ca44a73f1de6a4a5d3", + "0241b89c84097de0dcc7cf7abd65098ac3020578", "a735eedc1c421d2ef287fd1759e96699dc7aa75c", + "cef88a007fa60f4cd873f1d891a54ce5e173f3aa", "db634461703056448a77d669d907c3861cee6dac", + "bb13adf5fd90e4ca503b6980e737bb0f3396a63a", "b48a316abdb2fab1e7e3a797af20a7af3395587f", + "8ad895a40e7c6e7940aa464acf094e5c4ce2451a", "685755051290c901f95f5f29d8cb313c43833837", + "45e647b94bf47482695fc747a2b8b5575611aa6f", "b6a2d2d34371c5ddebd8ff6176aed0f5104e2d01", + "8c2b2995c2097f5699c101f4ab5e653ff40014a1", "db9aaff1052e5229ad6cef667543f9f6620b119b", + "be421874326c6922eb48277e347ccbd53099117d", "c6f2bf1b0331ef0f7d295e29a86717fa8b5698a1", + "d27524c41de17bec43692c05870d5dcb0f0458a1", "ed35e0ef407e405818bc218566fa79323d7329f7", + "f0473314bed3f901febe06355abe43cc73ea30d0", "7cffe54f8b960931055ae20f5983b4ea34515eff", + "d46056681f07ab1a435c625c2b46ed561240a573", "b58203e04d2cafa99307443c62f5063ca7d2e228", + "38c38c579147306770452db9f2405c2aaa8bcda2", "38f4356286376bb91a2464352b4cceaba3a3f6a0", + "79e066dbd2e0248345b032648c9310e2c7778d23", "160c352d87c121ef1ea43e49831453b9e8d4d0f8", + "be65342f97e4e7234a94ff6dc2e7c25f21f8660d", "058f4abcc85a2fd62fd3e5ff5b3b673824aa6004", + "5fc2dce168e2f622c8695ac0365b7ecf1309eb15", "6cb3c4075881492fe7189c502659caf20ed37a89", + "9860ef0921e59b2bd11767b5382f3815fff1baa0", "e8b9b463f80953fe29eb73b91468d30f5ef62c99", + "87992ef1fac6c582a28f3327f5d7f4ffc553fa78", "eefed6797eeb529dd376ea970c9c79a6ea017f99", + "b37c32b7e9326f350d3287e5870dbded1989e7d0", "990f2380b5c095d71eba74c595bf0535b500cd9c", + "238ae256a3a1ab93e682d603c4b53652fa0059f1", "9ed13156c47c637c5abc48ee20e6c6182c50f778", + "0f80700a25acea6dad9f5b726f220ee8cc227d7e", "dc628580c9eb037386b4e295db74b7f3daa74302", + "d2f914b6abf45a9bffc2823d7ed53a88ba231907", "133d19e1e5c4b0b87017942d4ca37f731af5a91e", + "1cbc14e38442cfbae772c080de6e17b3df09868f", "3106d7492fd3faba686f789ac5b42f2770a446ab", + "d868af0a96ae2ac8c5488c011397af20146017c0", "7db2d14148d17ed3ad5bdeec728cfe863a20f25a", + "feb813ec958af1823c42f4f5502677e38be72d8d", "0b2ee7336487a12eeda4ba66d8f34c332db30b8e", + "54de3627b1038f15d905c05f4c46a1d781b6b44e", "168702d959029c760f29c2ac9dbfe5ae501d7e8d", + "bd99b577781a7ed50e9049a292ac48d2fe33b983", "fede6ecee14c3842ab748664206bd633ea51c524", + "a1d968a1ff87035e1b4bcd5c4db2dfe15474c298", "93683071196fae3e0f92f10bc4533575ec4d058d", + "afc1500cce291ecd2c1bb748d7b627d11b1f03d1", "7a123d33d1a90683ee3df04e5900277519259c75", + "49a541549f2bf86d9bc6037db5a61f5f708756b8", "4a75b61b5f0cf472c5fb17e2c6a271f9c5ba770a", + "27a10de6ca27ebba2d9c667408538da980a85d5e", "3b7c0a467d849acba884de1a8c9a9334e8b7f9af", + "17001bf4ac840128c88152d0b1e6fd62d6a69f2f", "f9c27614dfa3934bffb1f69cc9623e8dcac9d0ec", + "30c615dafa30ea4ecc030e9d03265908f38eb389", "c6134d552ab4bd41e92fd8e2f4a8faad0c15b296", + "ebcaabe082cbe7c9a546a6f156a2fb67c9b98097", "f2a681ba398572b3a88051c1eef80994ddff009e", + "1c9f4f163e65cb4f0f061307d8b3686a042cc9e4", "3db2fc16ad04fc0bb32a41a9ecd013f9f5fa5c36", + "6d2dc1e53a6d16b466ef449fd8e14ebd3fa20f00", "a4dfb710896797d1ec1cab191e39cc51eba18b4d", + "0f51f6ef36ba16737589cab11ea2dd1742671c41", "fa85adab7bc82c5d14250a988664e0f65d23643a", + "2d8afda70e0f82176d7ee80bcb653ed09d4ada76", "17b618dbe38d1374435496778d3cbd0f3d3720a9", + "6116c86a6188a345cb430e55b48e660009a85799", "479f00719a1c04acda05bc52e8198c76b17b767b", + "41b9555dd4375ea0df23a0a6a2333f7f4d9e626e", "c3a3634129a36fb09c4b79e737582e33bd2a058b", + "7b1f9abab1067273355a38909e102fd2d3a5323c", "2cbf3503536ba11f94a975fc0c547529315caa9d", + "4aa4133193a8e8c3fdc766208be7bffb4983bae5", "436fe5351a1bc91413e903a2d2cbf2a8559a3c5f", + "c033d7aed97d788aa88220df3c8cb72fbe1ab96f", "f11265286454e2059d0de2a82ee4356601552bd2", + "f8c91c4de5c8d19881c4f8b0461bc1f8bfd01d94", "25912cd765f1ca1e19486411fb3de9963fd9f925", + "1026cbd8af888621a402e114a1c58c14da41bb72", "1cf1bf68a47746b2d23e4211c03a4b81e9626d10", + "c5980aae60fc5c3577e241a70c0798d8397e6a07", "9e4f93daa40369d2cdb97bfb01ee8c93bf23ec80", + "2b61d410af69dcfe59d11337df54412a7ed495b7", "d9dc67568e7a7147ea23e42083d2cb8632bdbe6c", + "21bb8edd05cc9d1142fabd4aba808d6a8157f4df", "258e8ec05412a73bb0a9e916db941c7af1b3e959", + "98ca8e2ec4264fb7c6a3a733c455f8d0ea089b07", "d6cc3a68587ba633190b18c083b7d56747b7a19d", + "3525059c8b3d3fadcff6f592c0e2ded7fc06294b", "e01077c19252f97215f6ed141d485f877506282f", + "8a4eb9ac7b33af6ffe3e980c685c290a68c93ca1", "567acfb3e9e9fda2c4051c038c6621bdb47ec823", + "675e293cfa98252cef4d8b7980066b038a901d04", "11e271cd1a3df6c75449e0c51a21a6c3b3d3fe22", + "ce4101fb55325aca6989ccfaebfa9992421c9e2b", "8e0211657949932ce0ed8da68e72b5c5981fc8f5", + "fa016eff3df324a5ff9f3d43d324a01177ec8939", "6f4426eac2c30baf20e09b51c5369156eff7abc1", + "2040109acbd819ebfa776fc97631815b122aa7d4", "30a65a2a2ce03eec847aa822f722a0d2f0f993bd", + "2f320a37a7ef0d32ae4fceb09057b1491eb1fc32", "25e7a079b6717ffd08ffbda3254fddb072160d82", + "146489482d21d37e7297d3b085ba786409e328a3", "555e44e3d20abf44db5780cc7fc8c95ae5593239", + "26b2a34cca8cf71e092eb3e7b02e53b88e6c03a5", "d6278e852f56fdb00ee5b9d97226be1589853166", + "5c61d67a3f8068020979f09c9f7be11f56e95c2a", "e0647fd778205f62b1956e5320f682af61fbf5ca", + "5d55b31e47053325b4a345c445c62413aa69177c", "a71670f60bd64ceec2b68fc07608d4ef05277ad5", + "e6751654101127f666768dfecd42fe9c6e0b47e3", "a45fabb93b9405dd04d2d944fef78dc424361698", + "5f31e95c53879bdcb5bd47276a8aff25673655d8", "fe3f9a83f7db420747f402841d25dd73f8b36d30", + "70dec2f04b7567d45d5a85b77c959623eefcfad2", "042f91e4b2b8067e6b672816182ba9ae2465812a", + "6e7aef5b4b4c6cf84ce51a4d8b947eda306a26b5", "3f01a409a7ecc34e825e8532e56d04613397cd13", + "893a546da4fd221501d3c865c7bf1d16ce9d933e", "f5410073902240fb94b50c30506aa02dbf9a1517", + "722657b05ed625309450f4f33953d7a694a1a414", "691d303ce7356e544cbfae5ad8054454c0f69652", + "a6f5a8bb856a84214811cd2b259edf0a30c6144f", "aba9adfef223a27e9ebee5f1e5896763adc71641", + "500c2fddb600fcbe7a7d8a48d4e1fe7c3bdcfea7", "ce957df3478f5f95b668a0dfc87956da190d74a6", + "3548dc0a4e272b21330ceefe9c8c63517941c034", "b252d68c0dc7bc599474380a7a185b9c5a9a5fb6", + "c2bcdec50527d1153a73a5e191d7bb4ab1f4a5f3", "552e48c600134179ece0d1aabd13d17f124338be", + "1a18ac6ef2f20cb3434d1db59bf4ae4c3a1f715d", "ad99dac6002a2975e445e8dc90f5d93c53991fdb", + "04f8a3f464061a9dba2d1c00dd7940488720974a", "b8c2d1ebc7b75ea604e9490fd265a6f797add415", + "4bf10cf4435ff85c7c3e2f50e7790b2eaa8861a4", "66091f34bf277d5491477085551afc1a8a361a0f", + "f7e1f2ac9f1767984bd125184f8b87563f249bfd", "61a1c51b48f7b278c73c13efeba86aa2484a33b5", + "3f54176089005fb6dc4b252b4dce34370ea8bcc6", "ff6f45ce65c0b32578fb92f842f123581413dfe9", + "6163138faeabb67eb3ef1b97d6a4b08a5c45caa9", "64cf4c6299cadfe4e5a7987c8db3383a69a108b9", + "e98267285f704dd8232dcfffc09616ffeda896fd", "61d4167d00783621e8a830f267b799b8f22969b1", + "12551cea0f20963404d37821d889c7b397ce29ba", "9aa042227f5625364418d20187dc76508c5c32fb", + "8213aad94904430b8165119337da1243fa7a88ab", "e8c47b170edc02abda540037fb4752ef03a21849", + "59a6092a06c163dfb2fa1deea179cd84a5e91083", "db2b8564a0ed5193a70c607f23e575c8f023454a", + "12048f9a795a901cab215f869b185d6e46e2a1e9", "98cd9913a99020f97a98297a8f085881e8ff21b7", + "76c1d436b2fba46a7ac7b7d285ee4b5e3a6660fa", "cb1c08bc56644e993505a2ab863c18ebc8a35cd2", + "f532f53f82e2fbd789b654ddd6eebe181c12d5ab", "1aaa6ce090e25893b7b8b46cc99fe8344eec653d", + "83d4ec39274f91989811f845c6fe805544bfa0ab", "0152b43193700d54d9826709132ac2bcb9ce5975", + "57291a113482eb4277f43c414b09afcb1bf8f0b7", "3d5ec4de453ab8ea873a6c8c0d0ef0b2110bd386", + "4168c8645f251d9584b25c5a57ccb089178a98a1", "0dafcc9a1aba7282e98d653730b496d83ba13c16", + "e45a7da9aba5144429cb9019acf99dc668ddadd5", "0844207c03327225f9913cc4b1a8702330841a96", + "329121933af05b575eee9c77bf13b9a4d3443dde", "4793325d739b2d9d0deb39d1118e33d76202e7a3", + "517ce4da9248bbbf77c7c9f9226054bc5412176b", "b9ec5a7fe95a0c7e32d65901da5d33ab8c30c779", + "cae143cbf533885d8bf978f287e9383bea46c3e6", "e2df632753bc6545d94b6f4c76af47a81ea3542b", + "eb3dcb724dfd1cc85e30594a63839ebd46c8a5a8", "c459f75f02d0c3444baef25d6fd228ce2929f924", + "b98731f2971b9ac5d0acda62cfb2cc32c141dc37", "6b1c69123acb73fd0cc23393bb165e99bab2508c", + "50159c19f26b9c1433e50f388b3fa759c92c0ff0", "08715a5559a18ec189f91421618e534344f342a0", + "2d4792e23d4104e5a89c17bf2195c7a8aded89a2", "27eb88f965e9d3df3a1cf219db7883093f339676", + "ee2acbc913cb75dccb8d1bb124b074bacae27ee1", "d9a2df31b1ff1ea6796a25558771ed1a38e6d1f9", + "b0c3bc7c96cd65435e9ff6b88040799959041972", "c340893e4dfd3cbeb4566cec5e8cd9a24c9982d5", + "cd361d820e14ee59bd8daa6bbe3321cc7d4fa436", "89c6916d89c4fec3cbe1ee6c780358cf51ba87e2", + "262aed06bd219cbfbb4e91c0fbdfb606883f74cd", "e32b82fabd2f29cdc9ac9e8b33966b10f8a0ce18", + "1e3688e139a11ce21f3a9e5c74d176c1ace5cd38", "b460d264e4ecde414ddb70e918a1adb48ae610b7", + "41bee95c0185b6c2521073565e11ede0885a7e8a", "22bfb85e284a90cf37d23354f56786941de27a99", + "59c7f2256d0b342d1cc08eeb3f4dd9c7e61e90f7", "6f791793df1d1124936ad53e9948cd221a184aec", + "e5c3b79336ede6948f3302d60de0fb8a1b7a138c", "666d1b61ab34647db7ab8e617a9497b87166d7e8", + "4d8966bad69bca5970ce0ed64c343fac0e1cf698", "852bc11787bae1ac6f99622dffc2eea32ef121e8", + "2b3067a1dfe1648d1e9be9f54f8e3af5ee9cbc3b", "1a001a96c0fd88b7884b9049a6d47f6ec4d086b3", + "740106ef9e67ae17393d081580f5c1493db372a1", "05e0f5184a2a0c6a003dd1654bdb4d1da4b1a847", + "8eeb1017e6f043e43d6eaaa99d2aec62d2aa2776", "1e34543f322aa94c5f5e679cc8713120002bee0c", + "d253a3c83f3a062f0a18220bdec2188a2445f2ff", "2548241b69cdc856b24ed8fe930ee5a608348b8c", + "47c83b2f5b8cc63496aa507f03babde7e79c730c", "1169d83904da1f140c52a51748d1f9920cadc304", + "2551edd52b168e0691cbc4a16b6f9762c42a2ee8", "b07d82b7fe426edc3ea57eca2e54dd1a7865e61d", + "fe52db5b57808551b30af34184e5f11fb12a1a69", "2b29dbd15c15e138057630527d19a67d9ea9198a", + "5cbc7715b58ac8a9d141cf95813149a09d7f323c", "918ea978ecc68b7f3ed541b7ed1e073848bf06f3", + "af49d808446f40d37d64c66517a45cc19a78de32", "f2fb962dacc2a7f1b1a9a075611e300970463a1e", + "a0ac9637dbc9b7b4af6d37256011e8e777379cb6", "3fefc0572263475219c149f4c54fe1b88f23c7e7", + "f227977b57b78fecd2db13ff2b231d0c22a3b274", "7e4c8a38b0fd82cc80df79b1daec14c1cdb58be8", + "ad3269310aba337feac5712f9621a5b8ffdb29b8", "4ad90d121a48a0390744035a7bc068f885abcf89", + "134fe22aeae849409671acc32e13a17a69681569", "4e0adb75d47a88366da89405264d87d88cf358c1", + "f1252436b3427e8fe87ea9ac07829f977ea12752", "0679d7794f638bc86e47ecf8e49b70482281c20c", + "c06dbe314d1f44c63b8bfe6edadef6ebf5c5ff20", "ad6e8fa9bbbe6f40621a826662b858c96eff4057", + "9756e3372243fd818c895910aa53f4148798b7a8", "dbb3e71028ba0573b07f6a39bee4a4ff7f39c486", + "6aaf30534ead2af6285ed5f3ff12f3680f401066", "5f7f1c1613ed4d17875a6d845e8cffd3fc0b0059", + "7c1432135a24132618b786ace67238029bec8591", "76290595b414a9c7ec3bf3cf5fa6ad32920d32bd", + "97d4988c8ccfb194a7d01d2c427e6796ed36c5e1", "344fbf8c2cab7917197614e4e974bb5478e4a653", + "6a2becd209341a5dd760515889e5aa27a26e1213", "997e1ee3a567f849865457e22be43aad5b84fcad", + "b0a9c6b95b650b90e69e3ea25f3ac2f583a9c622", "c830370439922abc9cef0763e9cae3387e549c42", + "9bb1447ddf4fbcd71f68096307fb5e222d38202d", "b862af307888af607c82d4d2a3a53e12d6265720", + "585e55bba7d4833272526051a7e2a47416f8556c", "99ac5bdc45f0bec78af9807ebcd8fd118155df93", + "c7e14a6aa34a6256944ea4d9be85777a0017415e", "cc2dda3439152bcabbc625f21d9251afbf15a076", + "4e84b51660d909910c0ce4cb014a23e20748ed05", "f43e4d0aca4ed483f9b2568f35e70dfbdfbecd7c", + "10039ee5b86d6ab30389abfafc4158da5e051410", "a212609775beaa7909f92b708b8348b920c34bc3", + "26ef38e22f884aabecbcdc02640beb2e8ec55abb", "4689201232928ffd29fa14a2bac10fa44f45a81a", + "fb2c9cb75fdb96ad4016a076773038a6081a4792", "d80b52a085881ac67c9c8748473991852f8a1113", + "47e4afb01417c95ae136318ce8d066c77402f30c", "2a256e4dec0180e6e27cd8bda8b4ef1bc3b1bdd9", + "fd4b1442c33a018cbfe42a859cb4bbbdb61eb54f", "f36d187c3fec787dd3d4128001c545de5551bc08", + "ef30b00ca626b1bc763427aed853246c0ab72534", "75eed65362b1370b90c3a032dac2f366f1469f1f", + "d0149fcec2d2e19ee8851b4ea2e328b2ee689184", "9e8eca00b251efc8bea606b3571208966fb2a2d7", + "28e3df3754c86e9809909aac9fe02515e231d60f", "5d4af1c155c639c81448f1d51db6cd33431f7fae", + "7e6a0c4ba45a0e692bbcf9456460aa56a974ed01", "63b04d5a98be1d3a23003f6518c4c8ee3b065ce6", + "31991c235b014fcf4fc9a99a7fa4824c705af5cd", "79231d6d3fb4196a482cf902439ff727eecb92ed", + "a8c246b8bf081182b08ad2a4ea478b7ba860d006", "33750764f3c324181451b38a1dc5219ab04a448e", + "370e22839b26494ce0ad1a88286f427434f3dcc5", "200b037c3083bd2e28e4e31f015bc87f9ab7196c", + "6a7573a61b2e15fd261863178f1f54d96ebdf408", "adfe5473bf50ce4f88c88826d4055b144ef42bb0", + "9d66b222b35f994a0d3ab6bbfb1c142d9f226d56", "ec03206161f942a59ea5bc91c08086a9b1fd2ae7", + "ca5bc0d3d6326e4df076624d826be350a89ed54a", "b8b6473dd4a5f93cd6eb1c0e41d659cb7bad0286", + "20162a0585c37d0c14b818eda6f695c40678bf65", "369ce701b448551b3b5b7156667687c68f7a4259", + "587b5c41554e3aed002f92535ef2179231f87e6d", "3b81c061b9efb1074e0dd9cbb5f11f71420e0832", + "6007cac67baac66d72a57c024260e9bda13c456d", "b1678a8e2c14c96405dc17557ca68d059ff68b8a", + "d5879f68ec23e30ca8882d93410073a0cd5db92d", "90a76730c4ccc74d3cce95abfff445d70bb98499", + "bce86716fa530eedc657bfd7cf18f045116a967d", "fc46a08726a7821ebccbd4d1492ee05c6bae89f7", + "d056f38506127e986b226fb5ff5683b6deeefe37", "1119b6ba9baba7c6511bd57f21eb642085932790", + "2d317fabf4f5e8d7a858a3d1dc79b88a2640b241", "498c2b6b4c0964e25a76cd8ef9e1e25dee1cda7d", + "803f7b6304b6fdc60bf08212fa280e31365d16a9", "5532c2bb541a785fd145aeaf624491e201ed7759", + "243023d8a29dffb528b567f37b5ef1d2d78f3fc7", "c0312dad18501b449812f9c9aca3c39fc1647432", + "9aa5151cb1622686f76e3e956dc9a6e48f43d3ef", "c9ac4a1dffd0a1fe7a0a339b156259884c54057c", + "cb53da91b05ac93ef8bb2dffb4f0ce8ddd06255e", "d25e73f15b3a94570b0a85e39df20d4f9d6a3ded", + "d0af00f950ccb83770035cd0db90fd8ebbe00d10", "ae260ec3ddf6bd62cb66ccb1c9ba3a6ca9d19ab8", + "846116d64285c2125f01ac2e06060b84ff9604a1", "2261c2c6e3edb6c2327046fa2d5b69d9a39ec10b", + "0c938db82253db2c09d40079dd15f0a1ad43345d", "4e5825c76b332b11e0e3af2ef74d9b98e3784809", + "83b5cadf47b3eaa15841e3815099ba5338a5f683", "2f11fc2b39f6889ae6720c0592344995b5942c37", + "3e43da7045388e7259414209b3baea7ee339bb9d", "4b71f161759ed105daaf49ad74f3306ef1710855", + "ae7070f6b723cb4c5055252017e5ce712ddbb649", "23cff857c47ae04884a1e522af2d5a836ea30f2d", + "424a8b8cd8ccd2ddbdf3c2d10ced6767b53a4ef2", "6391feb5cffcecf562f79be5c7b5c620cc1ff24d", + "9f1df5788377be4be03c705b21ffe72e3fca68b0", "607db9575d0fdcc02810ceab97c9b3faf9dcb601", + "ee0202713209fc3df5cc7c1c75812c325c94e280", "47905a7a246fc0ab81d6620820a96a283c6fe8d9", + "21ff79105eba8e77a638b42b2e18589a59033609", "28ba2f0f2967b8292a7071c3d99c9438b1db7963", + "e1fcdbf48c205c3a724151cf5008809615687dcb", "fe9755c8149d239696e010a0e92e2fab24da5aa5", + "e6b72cfddcaa930408e0d20800add718d44f30cc", "f341256a2cf06c968460034faa6d581f17af65a6", + "23791cd278e193ccedeb2ba80666c96e9152ae62", "f402a8f708787e8e97a9e096dbb47ccda9009c78", + "8dcac59f949b44e36ace133352ff4dad069e17e8", "f36f4c364f6f702f64cae7210f0d32a535e93bc2", + "8e110a0b274f6a11fc3773a1bd0933dca5cc962f", "745a2fe44d82669a5070791037da95a0d499ee3c", + "d1bda5b3a4ba7b9d7490d8511cce4a11bf76a8d9", "c2fb15525ed50bb529279aef0da1ad889a209760", + "ac5d73eea0dbddd6591c6254078018b894c74d9a", "150d6faa9cbb1ab06fda1d19606adc701633f24b", + "98f046a4f73ab3edd7f33d87279dc17c6009c844", "c0afac03a62235c223ef23a29e6f0f868aa89f4a", + "c19ecf91221af8e0d29e9d6693d8922989c8364d", "6cc04cca454e9610729586ce72ad276e8a8be962", + "5be4d55be3d8a516f8b9bfd8fa4b4b6fa072b3fb", "68e5dc0768d1143138b22a4dec7c0796211c515d", + "40b502f762b12d9008d67359a69427246dbf4590", "c7a0c0a1d86245d58f93578f5e71861850b20327", + "45e88dec3420a3081ed070a89974fade34a8cdd8", "a2cca480abdfd271d83ec53f32b9ca7c0534b1ea", + "eb5b56e00f7e3a3a93a151b04de322a54b478a1d", "ee726a3bd27fcd17373550a73f1265123f038e9b", + "ecbef3833e45ff4b51fcbd07be7ae083d0261106", "6ff3d878e2b0739195fef120f3e8920aae5b6c14", + "c8a254b1ae30d6d0372b9421cf164a2f92770bb3", "b6fe64434d331f92023b8ac4a0ae29a436ece676", + "6408dad003df5f902970c9a9b281bbdcab70513b", "b000f8b57d68abac68a8ba92d2e8af1f4cf04e3a", + "d70f6b719ab62e4d30947d46b48c4f86431e27f4", "d8b1b3fd5b14a37edfca20bafb53bb19b68034b5", + "54e2802c7dda0dc48297627148cb8080900b026c", "2a88231e4c45be17c7b59df395c88ad1de1f55fd", + "bd9541f3b7d18ba20fefb5725bd21f29c01a2fdf", "fb211cb3f8ec40bc8d34ef5da91173b85368ddb8", + "302c5986252fee81e024e43b3950ce0c03c1b19d", "c688626a38c6777646e4c5343903c5d3c39451d0", + "97e8d745c998aa9ca3c0402852cfbdebf38afeff", "512ca7ae1cc0043eee4b9fb24bc680e5e439a22c", + "8a1b18f3f1a52393779d7872132a261596718531", "f1f688ce752c4783753ded0c034de72056e5bf0a", + "cf404c790f1926de4fa92a90bc25bd3b89441ee9", "6f67fb37b601d81e607b4784b59a60f5da04ddc9", + "0358ae29870a0c0555826236454eee2a35460bb2", "267d40944d68c9fe031797824c83b7628ac21054", + "28999639cccbd7471536049556b7bb0445df968c", "96c339642cf640e219e83499b89b3f14c40c1a46", + "22ab4abcaeaab3dffd0a0eef49631c41feb35f01", "f3281e89aa93a9ff4b045c561960128ff02bd67f", + "ba7c3d3d19387b725dd93777e264c2d243c56985", "c06958105add69f9c1215959af1592100820f897", + "0246ace3db73d80516951169f697ab82de31935e", "3ed4aa3045ac0679828da5a36e6a159a76397b85", + "a9b3d7258a65a4ffc0188430f23a2b348bf3f71f", "3f80e737293fc727a2a5f5931afef5da8a108889", + "f5c4af7e6ca21899d603dc1c5d6dec724180a617", "83223c9cdcb837c03a1ee706a68c201c6f46aae1", + "7ebd59911482548ff28bc5257f092339747a9847", "a29e7d193abe5f905841b7dea04968b1c973775e", + "4df92c585a35462fd174a94fa6cc34cb104a3338", "480f1b18698079c345470f6eddf82dfe04f372bf", + "e93635e6ad4990b2037c1c42d96e5f4f5c382211", "f27588ba28c79a1abd36e9ab225006ef711bc2dc", + "0780639bded276d6de3f29a949974f9aad888f1e", "7649e12dadd9626c6aa55ce09bc5a77abbc1866c", + "10684acced018de4c9d795b4e3d17f85b831ed05", "1fd08bb5d65fb7deba4ce611c4e4ae61526f70f5", + "4ad405913be05105cfd8169727862962f73ba90c", "e622f46b8ab1a027717765dd7d392879941691c8", + "e53bc56ec46866426314e2214f39986f794f7ec7", "1ee4211e5bc419b24309bccc29d84c71d8153e33", + "13bf7fbafaffb9eb5305cc786ea5327b1bea8c84", "1b17359571b9a657d8c1c259f392bdca507f8048", + "f14bcf0c9d9d4441837089d9406a275ec7bae489", "650d0d424276a764869dfa6d5769e0d33dee29ab", + "9a51d909f6e0ea003b7bc31368889b34cbaf99db", "d25b13c8e67b29bedd33634693fd47107d79c386", + "d4791ab3fc5623770bd37a62f5847a3469ae62b5", "63cf95a96c6f9550a526c3f7c973b3bf77da1d99", + "0880631d8a31a2d656c214fd5326e86d72d99735", "c028799f0dc5cdd928ba41ed57758827d4c33960", + "09285b7758bced6de4de28adb839ce3707bc0801", "ff3ef6de1556d6a817b228634d9bd36c03f91b4c", + "7599af4e7f3c5038845232dc1c3460e757d29e76", "945656d9dd3e48ded82ae075761849b0b6de7d99", + "14d2b00bcdde27acd1e67707fdec920aaff41440", "bead8489f2f77870afbd190228724d8ba9525e77", + "ef888c5167e12709b33fcb9373476ce222d3a777", "11185717aa656979199228d4f134506f306ebe11", + "763619ad1c6fe756de2d6ea598f4a35b6b5d07f5", "de64195cf8f8e0a8a209993c12c37e6a0e9717fb", + "6fcd348a0817c5ba9f692e9f43b157ba53f61736", "184223aaddee8a795fb21a0298bca7cf9f24b86a", + "025924b72b369155a7dc79296b0fa1542b6cf66e", "00b6d09efea3b56ed30d3f626837c105415a2b9c", + "7fc27ae1ff5720a8632dc7180a62f93f0b240971", "68a1871cc2f4239c1e4c717d6cf5d50121aa24f7", + "d86a119c71e8f3db87ef84a3f2eaee3784e514eb", "bf6c04384c5ebbcbb7a2635e885ad48223000515", + "25df57cb358e1c55102d3eb05c633bbbc8d18ad7", "c2ed09db58daafd8711f0729136bcb4bdab4ee39", + "db9ff970f2394e5ddb70b05835e5cd2ceff1758f", "b5b0cdc6e2d0971f11ffb7b83eae62bd2f2e2a63", + "568a605393fdf0261752b980340b56a695993ca9", "53a7d40199c4cd3e0b61387708a5c4b2daaa6a04", + "d764e0dd4cc13227c90ea0dbdb7a17148a37362c", "2b0970782ff6fbc9b574d9af09c7f4387da71167", + "1a2f72aec014fc3b1c7a58660c15cdf8eb78c30a", "ea70291c158d9ac2e034f91837f3aebb9315e74f", + "ddeee68ef8756513d3c91e77705759282dc88f36", "c4c43e1c71a7693a257680b25c4cce05fa68139a", + "cb91e9590a1524dced13be78e13abde059a5358f", "8df5d8b31502fd506a59b0d15f61631dc5dc0d11", + "ef60833d075ac1aca48423ec296b18756a0907d6", "4bd51174d78af4c6afff4a3580c4380eaf53db03", + "f89f34968e6fb5c4400d4133e5b83eaad9ce1d81", "4926ebd2150fca495a5ebccb69d73a843c2a5c76", + "b7d7768027c43c4bb9178b59096058c827249ebe", "b18e01b7ab7b8fdd5e9093a4d10446ccb4c8b461", + "7c0cb0bf58b260d848be8b2fffc6ea81d45bd9ab", "65560b3113f1d79c241d64615c70a95a27a1295c", + "ef7de7cca843f688185aa44caaa11b8d9c5ba419", "577959df764fbe2295c7355420fb88e7259318f4", + "fb4e5fa6f2ea75bd25bf313675ace5ecfb09ce4b", "58c3bb9c95b2b18b54923c1c66a271c5103b5016", + "d15a9ce8720d18dfb1886de4ce23504c690f9849", "09b1961ea1fb0304ed7ad7c2c404af3113af532c", + "689943c3a6fe2d15706f588d5d192de09025f87e", "76c955116afc032044fa820077f0aa0077a9b221", + "e4e28d5880eb5a19eac4f2dafc94b87121315736", "f66dc5b1d3a36e518a2d7581534cf1eac1c0bb86", + "437d9bab241951c67ba4344ba60e7f8c8fe091fe", "d48f36d517120dec77f9f1383c6ff175e490957d", + "3595ca188c62c02f3a21616a40fce4063e5639e2", "9335fe0ddc8db4b8825b6a8e96f2ae314c3daea5", + "93b3baceba14eb826f0691ec927139eddd131418", "b081a6d644c99936ce2b53227aa949e61c1fc1a2", + "7c301a66f04090a5dfa9b97790c2398bebe8c658", "70e640c8b512f69cbf2d3815ade5e08acfdd6ea6", + "a5ff6189fcfa209df6073595ce8d048bb434142b", "865f86d6f58109166006154d4cb1f18d780afc7b", + "0b32bd194264f3d34f207e36da01747250f370b2", "160db72460b1d2cb57b71c5f5005c43b39344a08", + "ca17e8fc42c63b71a64943b2edc546cb483d4987", "62709ee11b06371980bbc365eade2cc09fb201e1", + "d42b892d83a19496adb8a51865d7786bf11cb51f", "151eda2239a8926cdfe11bddd08d4a68dfc06fce", + "48289268a75e7c551d8407da65a6a8d5e96bfeac", "76475e31fb20faa61548ca6c110bfaf0a8509333", + "ff45b9e590737c453377adfef746c419fd024214", "791336c9f92b920c36d87abfcd23717effeb2979", + "7cbdb028c46f69f69ad5b3870fb9976a339a88ea", "6dce9b85f8a4d116d780895772890e62ec554050", + "609a9ddb002e9e03ec1e01fc0d3273defd125f6a", "eb5cd823d83464a85d52abf6fb618ab6b0ceaf14", + "0b33f97dbe25690d5b395aec431714f326835761", "b3c0967b6cc938da94bbc24bbb4960c499eab996", + "55cc0375389bdbe9081930f6aed59e2d1b996d68", "3ab0dd9b211aa79f607aa18900021d331ffec108", + "8b8fc9dc6eaeab845218b9b42c9f1eadb85d9a76", "d95873f271a98500509514a3bd503e111b2eeff6", + "60c22d4758b2ab938d925ebca4195d388976ef0b", "4a71cb75be4c145468bfa41edeaec4cad48b35bb", + "674df8ed0814f769a01ea52a86b37a37db300fdc", "52cd156a808d638be19a621e77ce9d0b705ff959", + "893fae64443d115a0071088287f1047c9c85bd0c", "fe745f1136b3c2b4d5cc703cee8933de6357c502", + "7a9ed9c04bef62538916e432a74f44c034068c16", "c60540472bbd9473a78ffac3a071904552cfdd6e", + "dce6b37c643555216d24705885d35be6f1658323", "d25cd6e77dd4f8683333b68cd17e4ef161a10f82", + "ef59cd7b3e448c27b726dda2624e3a66bcb7c915", "caeb866326a82dacaf9b9ce21de180c99462b0ab", + "300bbd6765f353ca1f7a42c8db1edbe0fca9146f", "d476be09c2c9a67b065eb972d473c862d05520d7", + "f2692ec53c9eaeb976a81908a1ad9a8a6065eb56", "80a904cfc2637be47cf6db433896285c96642f28", + "b4e7c96d36534976b532f6a73d687ecebd20d3ce", "f442e491213973df215b4e5a13ac42dbf26a418d", + "4a990060a137b7405e90d674667b774c2ab3d9df", "ea028084a5ba60179ffcc5c2e7745a5137810674", + "946cd7b549d16cec7f31b99e31a5cd1f97ecb0a7", "f4d1570ca148a075ffea03034aa359a795a7a95d", + "b24a0df950d615d89a5c490856d9efe0c67e6807", "4a3a4973113c414789c798e3b279d04e68f18193", + "853861b52dff3d4e69cea17db319b578dfe36b57", "75cdf79a50fa9f54ee0abe8a7352983e4e15dc8f", + "d2172012e40b489a1b696aee59ee01beb94d2044", "c463cba55508e3410e6c938ca7f07cc49c69685f", + "298753c52149d58e36b63ae01437d7e9b1626550", "edf83ddead1b5bf94530280409c8f8d7a3566036", + "6ad9ebb63aaa22d5cac2c6ab68a8d044575531b8", "4c4e417f31a770c165848c34e9a1c03a88345923", + "01ac78c98a9e83a657895f330ac60b23df81aa19", "077b29278d6047812f8b65e158402f13dff2e638", + "6a3280cb4ecb45a4a949416106e707b4999e2491", "3de477994ebd5b7edf0a48ba6c5e799fe02a53f7", + "2ebbd703eb8308492ff7d0e3604b786931ad8617", "5fc0b82cdf849a6c424721cc5118a196c07caf43", + "851ed5018abedd4f527d5fc1f184016c2b9dcb20", "8271e8c30ed07e236d230df37635819f45b3adf5", + "794d1fbeb2f5f6e2ad81450602d53d232a2fcd66", "765e4955986ec0b20075669f5a7e2dca79a98ae9", + "f94426a7f4b9bd63aed0e75894ea80278dab4320", "c0006f951059f5aa58b4b898967d1dab70377178", + "414f4c053c7c9d294903806e748a132d54325f73", "a3dc4e32a5bb67c422822731ffbad447f2f89dd0", + "9dab95f8797ac80a22cd854761e2dd141be8f436", "c187e0b59cc8710d812cf24e9ad7191c5d0c6206", + "55cc88749fcd34f08c5d0fee8bee92dcaccfb010", "2c3d218a5428646b970895dd88e2f61739ba0732", + "4ad60738ad972e0cbca3e763b556937501c0973c", "543f72fea7ef066ea3111bdac8f0ac61134e0c24", + "a2bf3ec52c5d004e7e5329efdf6b5cd735618b87", "f483494815f89e80dc399163080052fff49031bb", + "3326ec457cfa6d30663eb0ce49a838d7bbb86415", "f6d5647f6b6923b5052619814a84b2e1bf8d0b85", + "f2dcd0fc70ca95f475b68f7335b2d0855534e5f1", "a50cd55ec2258557a20839a0d6219960c9b1a2b0", + "a34ae4d4326a0913d9af50546c558811565ec66a", "842e922b6252301d8df608a1a1a4a732686e2ff9", + "2b6cb59f7846a482432b4ac57fd960f73d2441ba", "e559e01ccf3cc35aee9732e3db97c23165f8a6d9", + "543085108d5513b9fe63eb96fb386ccc84a811d8", "c7474d0e9cfe5c1d8e56f3978eec6871706a38a8", + "fce05c355558c606a7ca815613c6ad3d8f8a1503", "30a17532024869bc353e28aa4c3da17dbbcab80f", + "13d54732573224d7879166a11857046c50e13626", "42868488cc892e63b13e7afbe157006ddc215258", + "94785fac47a5d2ce41d64269203f14de07995186", "82d752a38f9d5598685d58e5ce6080409b980442", + "96f99cdd47de4d5e4180c99bb7015fa77d329687", "e89a84828a38b552efd3095472eb39dd18be141c", + "5a37e8b4031b627270f06125f5c204eb28437c85", "30da9a8dfb0f65559be3f17a3d2f3be0cb297ba7", + "abba9bb98e64624543ac376fdce700ee5e33b392", "c19f87bb786c68bab429f594374c24d09dc98cd0", + "73754ef7ae4615cdfe881adea981bcc93b65e887", "44c25f0a9f3151efbb98056e1c2bae59f4aca0b3", + "51f465b5aa2a63568d0bd82232898b39fa429a14", "1ece6bad7a8d2492da84f5db5a94a9c81a94a3df", + "603617cb2ad5009c06dfc61432287ea41d4dd70a", "ec346b3231969799e3852cd7f4e951a25f3fbc0f", + "46af277dcaf4eed7cc96e021c96b266c24c1fcd3", "0965cb17b3b353174be301989ee8f5b77ab925da", + "b45319782274fd4841da3c70ad3b4754bf2b1b08", "2a2bcc98ea5b188049bd46b4e315ae55b359b7f2", + "cd09081ec384b747181bc2309a9485ee0888516c", "f636e931892c445a7ffb1aa0eae7499caa19932f", + "34ceb6f49765ac0d173d4a0cecc089f34dce2695", "9b18988eed2891623baa37468ef83dc6a9a93e14", + "087397f895ac78aa63b5b7413dd4c948cae4dc50", "98e1abd91d5502799931c25c1c19fa6605ff6dd1", + "279448c0f978c369797336bd660e457e2abccb3b", "20d76b4b81484d694731d6a2046661a025928bdd", + "7a69b783e3e747a0f7a5eff5cc23b9a00d53aa01", "f645296a6d289f80b9960679b8edfb6b0a4d96c7", + "9352d33e1d396110bedaf1906567a6d8139c98ad", "3a4cb9ed1c4e291a738f6a97ae82b7db616ed4f6", + "e97932a4ff925df81b844058a18f2da4f362edad", "2e9b3d733cb1d6a6fda98cfccfd179c35559671c", + "00e2b96be50f143abbaeaef1b42f502eb1c5eb38", "a3b8a036064eade54ac1a7e49a48b5228863f225", + "b050b6089b63a630ca1e102f27f1e33310bdc0f8", "91200850dde71fd5061890bae22cf6264f3a0ce9", + "c4c90fa787e28f241a6149c5706f67cb8c897c3b", "52cda4d77c07f76113b392291e947e02010a8ef8", + "949b21730e48bc645dca6c6a9dedf758035d0eed", "3e0509507736054281c0feb777b5beb62af98ed5", + "994993e838dda5b000ee4e75786a5a826f368f9f", "24e2e8a67ed9977de3c1373968d76736da3c800e", + "9c8374cdb61697a37bdc89b57ac5574af910138c", "7facfd4ee6d102ce1c2103b2bc0861865fffc07b", + "82f7d052e4d562b268730e8d66d4ac694d963f8d", "a5f79e4ae09eafe2892bff810ed21b1a7cbeff6a", + "7a66189f5b487392ddf3bde3b6b2f837711767c2", "756f813ee996e91c50dee3929035b81d5c2f5b7c", + "c0c98d3382858b9de30907003cb1b3f47bcb7e4c", "7a1f959994f3e8665589050ca2e07c5ed5579ff6", + "06944876fd90b41f439d337cfdbef2582caa4b56", "73e779de8b03c705726abbd686ab6f905fb2fe27", + "dd14f8315d4345a1482b6b5f581fefae03154e6f", "b4f629bddb661422c15c4b0459104b6a6387f3a1", + "a65dec2720ca334bb503079db5a5db4649c27c0a", "23d9c64b7d95c1d1c08ac83058dba6204537709c", + "0070ff46a8b72b79d011f59a770cdd2e3a7e52db", "39538c29808bc3b1c85b2f9555eb56296778e5b5", + "05235f2fda6b96b82d3b798f5c06d1c26fbf5556", "da9ac947e5272631161f7415583487cd5b81bd92", + "0afdc9abb0a921b0eb87256c34c2440d7bcdf748", "708eb61ae54de0fcccffafc686218a959fa0e12e", + "921d2ecf681cb2fa40d8cc9390591bdaeba5bddc", "d54e009ec45cfd47d90fa87926ff55f5668cd745", + "b66c3a482974a6950eb27fa2c83df33c0ddaee0f", "d11ee1996ea846c781624a6be16e2cf05c90a3ad", + "d1fafe49cdba1c135a157ab725caf6a73c44d413", "37add1833a4462b08f3d63ac3cfe2b29e8c19daa", + "54b9e091f1f4a2aecc9c3abef4785cd4116e59da", "2e7c5f1ca4868b1bf1681e852460d1a64f6730f4", + "fee6c7a1a74439ade1a4ba1102431cb16ffa40b4", "0b663b2c2e9813fd968ea4d53c27117a37af990a", + "2bcec290f45a8f707462a56e8cc468f5934fe57e", "2e77e56acb8a073a7164052b3015752f55748f13", + "56cc03e04970154b1ae9a3cb49eeb3d131118feb", "17819764b0cc55daa6f9a70e1b7cfbb082504a26", + "b9992bfe65ca57863517d1a44a5de4abff0d891a", "7cfd6ae2ae46bf951a84ab288fbf6381377e4fb5", + "44f321da361e4de492292e465ac2576625c2f04d", "14d021d38ede8d4fe6c55266ea1bc18471146ca2", + "dd5f008277ecf79acb0c32c88f459a46f556d235", "e712c5cfb5144ba8122f13b400324a3db3eb1131", + "2e3b8bdf70024acbd57dbd574b42d8081b22238f", "a27f805365b6074f96d0d72bd877158a6a2683fc", + "1777f7ceba2141c7725cae03325aad9015f5e0ba", "3ed72d7bfdaea45c7741c4adba22824729fccf93", + "1efd9d0cf3d575dceae0143ad80cdb36ae929f19", "ed1e3113a91cba66cb2bdd2fc62d221f0fe98720", + "56830484e8e3a9bb9c3c0e28d2e06c59c2f85814", "612f82971d281213f121bf3940fb3a22ef339b1a", + "9204cfa997dfafbe1d923cae857fd8772140d2b8", "bd7772cc8ad355a2edb8ca92d712bc6522f679e4", + "50013bba2f0c0daa2ba0601ad9d538b8945f49b3", "903166d464af4cbe28c3dcfe87e3b03d2d4361d4", + "498a0b6129cf4bbdfcd1fbe56f41a09711c81515", "4d82da411ae6922528a7e31ba0d9b7781ca7b140", + "d1a59d5be71487c46f1f13cca1329b890864e3a5", "0130298b9910153b918c24cd0305bf004c77f139", + "b4704585677546e3719b0167a48f7352c3135018", "bd2af11cf4b22ffaec8f0f81e41c7a200707984e", + "90bbdf000b6566e32e973a9ed13673babd3d5971", "53b76d4726ba14be0ff600d76f6e94af63bc76ac", + "98b92425f234019ae1a7d7fa763ee86d24a9d4b3", "402419c820cba29219805adb1b50909143962ab4", + "2507dd9d1d952d45d8e29e3da7d9b67261ddc8d6", "0eef7ee0455462714b4398429bc4e0b1e68aa807", + "c15af97c03e51bb5c7c23118564b21d37e41dc38", "5ff6527e38d98cdf140318d98828d7de099d1552", + "7d8d05ecfc3d87c0a798fbb28a7bd63d79317590", "d728b77adbe3f536b9211d9b26b0668cd796a549", + "ff936585cb1595e9b56e14b77c2a8605a312fc2f", "cee364e17aaa6f0d77dbcfe496b188de38123833", + "28f1af82eea595d5db3da22f4433c144fad620ef", "d5bc60e7980decfb5d5cd10ef9ea1bd0a3ce5fae", + "392ce37ce6411c30d436940506dc2cc8278f66e6", "38d291489369e7f222a2711e5afd7d6ac6afb574", + "479506b2cf7c7392c0a96d6e5d550556dfd20ba0", "c71e5676016de6a9b7dc27a123f403283fadec2b", + "5d03ade937932174a2f2b376efe6f761636a270f", "04302da411d5d79477463fa37de2f639afd5127f", + "d4e6cd7852b5b88d88afb1db0cc9d29d8484f143", "1dbdc1d3d9dba7ce5cac3bf7d2268ab237a4c5cb", + "3f263b23507300ad4dd7903e45876c2877e59ffb", "127d0081100e98843f16d26cbea527856d9a9ddb", + "dd5e22108dcceb35e1944813e2524ebfd0a53df9", "5d52758299b341021d135b5166614a46f3ecf2e3", + "a1b3da2881596c60f669fd80479e45a097b38707", "21a63be654b8e5df434d01d420ea2206c8d9a92b", + "36ea46c3da7dbc2bf4d304d81f86224dfe54444c", "29fcf933c2e58f6eb8fab04bf8c511f56896fe15", + "64737770ddca8995c64e1106bfb08ff0ff1fb144", "d8c191fa2b58539d83c7efc1c3346079ecdc566a", + "77f0910c39aa6108e0c2fdb32fe1832126c0489c", "af64db9a99bfd8ca4affe94dcd5cfea05e94e03c", + "71dd27e77432a2d5b446958707286d08097da244", "5ddedb0a995f06b295696a2a7b4551e404d0dd16", + "01f87385a24ece34ed8ddca801fd84f9c86e74e8", "09249256a87d4a90588b026d789fd3cb1aec856b", + "9240b4a3a946be297b435486d1f2a6d941422b93", "399ca25f86719994d4612fdb618983079d6a59ee", + "4db57eaeeec89f4a96d0f2727c15ff083482fbf2", "4a31d4c94a4c6ae379abe24c46817a1157eab07c", + "182f2002e4307f83c62e06dfa4164543fa7e814e", "97a0a7220c2b8e458641c3e6e56889254d2da049", + "3440a5e4b4e31f45c6adbae99b6214b8fbeebb2e", "6f94a7a5fe870501cdcea762e4a7a20631d48d62", + "9da3107576333b6c7f3e878e508e94a4c764e45e", "20cb94f57147f8372022885ac2356e464fdaf7e3", + "68803fd3ffbf592c0432eadf45fffa22e5afa8dc", "c538f4c3d9ed532f02458a7c863ddd549960902d", + "cd14b737781fb608698437bd2e6d20d691093874", "2db66f7bda4464a6f5342f5e021c29a38d42068d", + "37b12212ecc3a32ede8aa04eb0065fa80e3e821d", "887306035bf8e3f0963a254a6e46b52b55a04fcf", + "8747b40002f4a24a2330ee27f36ecc0880083827", "ee262c59ff4c66f77562ef879569193543838393", + "8282282c974675b044c071faf25996f4b6e4ebc3", "f9ca90f109c32ff5889c0c9d84ff11f9fb5b5924", + "e530847cf2aaa328e800919e6eed9739521a9b50", "ad726687605b280a0c190153654f4751ebba4c7c", + "93f6ba66b73bf972a3df6ba722a1d56782c59d47", "b40be55c3b541c1c3a6b46de02f45e3ab9dfad47", + "95ee9b8acb715971bd1abc137a9d54bc88139b8a", "61dc80eef767360c6d48572ca1bda65c693d5362", + "ba44c4692965b0d694c8fd015f1ec3fa4dfcc696", "418a1501e2a14efb4ab940cad4c898f09f92ee83", + "026029e19d4c6f0bcf239abe607026a5a27f5b86", "8cbd5f3c989185d9474f7b5767d7e6adabb2015c", + "55163a23b71b9a3d63f6e4695b07ebd07064c61f", "d9bde5278517a57ff7714edf98ab2361d1663219", + "9ad5f0871e1df9582ea14bf7ba22350086fcd055", "c57f5ae0aaa74eea99032b45f0dc5c927813d8b5", + "3e3f6f7661780b696501a397d8e95ec446fd1fc3", "03d94679ae1908d9a6f6a38c00ce7988c5afbcb1", + "4c8f526d899bf04b4ce3e088302680da64499e5d", "271ddfcd505226549f7674b7e539efc96c8be5b8", + "25708b2d11167437e6de01fa0406dfd3c26bd8d4", "dcf94a3cc15a85bc1dc2fc7e1751e6a343609049", + "f65dd5dea2e8bf5c90b8b093b4ae2db11fce3b5c", "b3722ffbf1e3287cee8109c82b7ab13d187a3d7b", + "ec3af0efeea2836f6a2fc5a2390232ff10e17cf0", "c7176c1b7df0676b474b4786a7e6ba25650df8aa", + "ad97c1b6ab4dcd3d4917cb59427490508523daf4", "7a8e2f675709ba76674a2903653b07d348de32d6", + "bdd754c4219ceadebaebab82c1dacb0a6d819e24", "31df56efa866faac4fced90b742b1800261ef46b", + "1e77fd0a0bd78293212c518291333d3554136b41", "99f906bf47a41245895867f76ad38fc9ca88921f", + "49dd5ec3406bda057407abe5c57e6a027b8c502d", "f36f65ae5480acecfb85d9b8d7a79603853f2115", + "5a29a9db5779e61efafc338b00be9d22efe2a181", "6b28f73e9097b7a7966aa9add138e3dc14c52e75", + "92571e06644eee1606759deeccb9875fdb454b81"], "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80632' + Content-Type: + - application/json + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"208de547-5ee2-4294-955c-5595cd0f7940","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"langsmith-py","description":null,"created":"2026-04-03T20:35:06.233Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"7ff97ad2-7674-4727-a505-2ca57030c1fd","project_id":"208de547-5ee2-4294-955c-5595cd0f7940","name":"langsmith-aeval-adb3f119","description":null,"created":"2026-04-03T22:20:17.794Z","repo_info":{"commit":"5c35051a0102f850f2e09c66f9376a0fc658ed9f","branch":"main","tag":null,"dirty":true,"author_name":"Abhijeet + Prasad","author_email":"abhijeet@braintrustdata.com","commit_message":"fix(anthropic): + capture server-side tool usage metrics (#192)\n\nFlatten Anthropic usage.server_tool_use + into Braintrust span metrics so\nserver-side tool invocations are preserved + for tracing and cost analysis.\n\nAdd regression tests using a red/green workflow + to cover dict-backed span\nlogging and object-backed usage extraction.\n\nFixes + #171","commit_time":"2026-04-02T21:10:00-04:00","git_diff":"diff --git a/py/noxfile.py + b/py/noxfile.py\nindex c92395da..1083ca5b 100644\n--- a/py/noxfile.py\n+++ + b/py/noxfile.py\n@@ -77,6 +77,7 @@ VENDOR_PACKAGES = (\n \"opentelemetry-exporter-otlp-proto-http\",\n \"google.genai\",\n \"google.adk\",\n+ \"langsmith\",\n \"temporalio\",\n + )\n \n@@ -104,6 +105,7 @@ GENAI_VERSIONS = (LATEST,)\n DSPY_VERSIONS = (LATEST,)\n + GOOGLE_ADK_VERSIONS = (LATEST, \"1.14.1\")\n LANGCHAIN_VERSIONS = (LATEST, + \"0.3.28\")\n+LANGSMITH_VERSIONS = (LATEST, \"0.7.12\")\n OPENROUTER_VERSIONS + = (LATEST, \"0.6.0\")\n # temporalio 1.19.0+ requires Python >= 3.10; skip + Python 3.9 entirely\n TEMPORAL_VERSIONS = (LATEST, \"1.20.0\", \"1.19.0\")\n@@ + -235,6 +237,17 @@ def test_langchain(session, version):\n _run_core_tests(session)\n + \n \n+@nox.session()\n+@nox.parametrize(\"version\", LANGSMITH_VERSIONS, ids=LANGSMITH_VERSIONS)\n+def + test_langsmith(session, version):\n+ \"\"\"Test LangSmith integration.\"\"\"\n+ _install_test_deps(session)\n+ _install(session, + \"langsmith\", version)\n+ _install(session, \"langchain-core\")\n+ _install(session, + \"langchain-openai\")\n+ _run_tests(session, f\"{INTEGRATION_DIR}/langsmith/test_langsmith.py\")\n+\n+\n + @nox.session()\n @nox.parametrize(\"version\", OPENAI_VERSIONS, ids=OPENAI_VERSIONS)\n + def test_openai(session, version):\n@@ -371,9 +384,8 @@ def pylint(session):\n session.install(\"pydantic_ai>=1.10.0\")\n session.install(\"google-adk\")\n session.install(\"opentelemetry.instrumentation.openai\")\n- # + langsmith is needed for the langsmith_wrapper module but not in VENDOR_PACKAGES\n # + langchain-core, langchain-openai, langchain-anthropic are needed for the langchain + integration\n- session.install(\"langsmith\", \"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n+ session.install(\"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n \n result = session.run(\"git\", \"ls-files\", + \"**/*.py\", silent=True, log=False)\n files = [path for path in result.strip().splitlines() + if path not in GENERATED_LINT_EXCLUDES]\ndiff --git a/py/src/braintrust/auto.py + b/py/src/braintrust/auto.py\nindex dc44c7d2..6cf8597c 100644\n--- a/py/src/braintrust/auto.py\n+++ + b/py/src/braintrust/auto.py\n@@ -16,6 +16,7 @@ from braintrust.integrations + import (\n DSPyIntegration,\n GoogleGenAIIntegration,\n LangChainIntegration,\n+ LangSmithIntegration,\n LiteLLMIntegration,\n OpenRouterIntegration,\n PydanticAIIntegration,\n@@ + -52,6 +53,7 @@ def auto_instrument(\n dspy: bool = True,\n adk: bool + = True,\n langchain: bool = True,\n+ langsmith: bool = True,\n ) -> + dict[str, bool]:\n \"\"\"\n Auto-instrument supported AI/ML libraries + for Braintrust tracing.\n@@ -75,6 +77,7 @@ def auto_instrument(\n dspy: + Enable DSPy instrumentation (default: True)\n adk: Enable Google ADK + instrumentation (default: True)\n langchain: Enable LangChain instrumentation + (default: True)\n+ langsmith: Enable LangSmith instrumentation (default: + True)\n \n Returns:\n Dict mapping integration name to whether + it was successfully instrumented.\n@@ -146,6 +149,8 @@ def auto_instrument(\n results[\"adk\"] + = _instrument_integration(ADKIntegration)\n if langchain:\n results[\"langchain\"] + = _instrument_integration(LangChainIntegration)\n+ if langsmith:\n+ results[\"langsmith\"] + = _instrument_integration(LangSmithIntegration)\n \n return results\n + \ndiff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py\nindex + 0062ec77..2dca1110 100644\n--- a/py/src/braintrust/integrations/__init__.py\n+++ + b/py/src/braintrust/integrations/__init__.py\n@@ -6,6 +6,7 @@ from .claude_agent_sdk + import ClaudeAgentSDKIntegration\n from .dspy import DSPyIntegration\n from + .google_genai import GoogleGenAIIntegration\n from .langchain import LangChainIntegration\n+from + .langsmith import LangSmithIntegration\n from .litellm import LiteLLMIntegration\n + from .openrouter import OpenRouterIntegration\n from .pydantic_ai import PydanticAIIntegration\n@@ + -21,6 +22,7 @@ __all__ = [\n \"GoogleGenAIIntegration\",\n \"LiteLLMIntegration\",\n \"LangChainIntegration\",\n+ \"LangSmithIntegration\",\n \"OpenRouterIntegration\",\n \"PydanticAIIntegration\",\n + ]\ndiff --git a/py/src/braintrust/wrappers/langsmith_wrapper.py b/py/src/braintrust/wrappers/langsmith_wrapper.py\nindex + b22117df..7d4170fd 100644\n--- a/py/src/braintrust/wrappers/langsmith_wrapper.py\n+++ + b/py/src/braintrust/wrappers/langsmith_wrapper.py\n@@ -1,518 +1,4 @@\n-\"\"\"\n-Braintrust + integration for LangSmith - provides a migration path from LangSmith to Braintrust.\n+from + braintrust.integrations.langsmith import setup_langsmith # noqa: F401\n \n-This + module patches LangSmith''s tracing and evaluation APIs to use Braintrust + under the hood,\n-allowing users to migrate with minimal code changes.\n \n-Usage:\n- ```python\n- import + os\n-\n- # Enable LangSmith tracing and set project name (used by both + services)\n- os.environ.setdefault(\"LANGCHAIN_TRACING_V2\", \"true\")\n- os.environ.setdefault(\"LANGCHAIN_PROJECT\", + \"my-project\")\n-\n- from braintrust.wrappers.langsmith_wrapper import + setup_langsmith\n-\n- # Call setup BEFORE importing from langsmith\n- # + project_name defaults to LANGCHAIN_PROJECT env var\n- setup_langsmith()\n-\n- # + Continue using langsmith imports - they now use Braintrust\n- from langsmith + import traceable, Client\n-\n- @traceable\n- def my_function(inputs: + dict) -> dict:\n- return {\"result\": inputs[\"x\"] * 2}\n-\n- client + = Client()\n- results = client.evaluate(\n- my_function,\n- data=[{\"inputs\": + {\"x\": 1}, \"outputs\": {\"result\": 2}}],\n- evaluators=[my_evaluator],\n- )\n- ```\n-\n- Set + BRAINTRUST_STANDALONE=1 to completely replace LangSmith with Braintrust\n- (no + LangSmith code runs). Otherwise, both services run in tandem.\n-\"\"\"\n-\n-import + inspect\n-import logging\n-import os\n-from typing import Any, Callable, Dict, + Iterable, Iterator, List, Optional, ParamSpec, TypeVar\n-\n-from braintrust.framework + import EvalCase\n-from braintrust.logger import NOOP_SPAN, current_span, init_logger, + traced\n-from wrapt import wrap_function_wrapper\n-\n-\n-logger = logging.getLogger(__name__)\n-\n-# + Global list to store Braintrust eval results when running in tandem mode\n-_braintrust_eval_results: + List[Any] = []\n-\n-# TODO: langsmith.test/unit/expect, langsmith.AsyncClient, + trace\n-__all__ = [\n- \"setup_langsmith\",\n- \"wrap_traceable\",\n- \"wrap_client\",\n- \"wrap_evaluate\",\n- \"wrap_aevaluate\",\n- \"get_braintrust_results\",\n- \"clear_braintrust_results\",\n-]\n-\n-F + = TypeVar(\"F\", bound=Callable[..., Any])\n-P = ParamSpec(\"P\")\n-R = TypeVar(\"R\")\n-\n-\n-def + get_braintrust_results() -> List[Any]:\n- \"\"\"Get all Braintrust eval + results collected during tandem mode.\"\"\"\n- return _braintrust_eval_results.copy()\n-\n-\n-def + clear_braintrust_results() -> None:\n- \"\"\"Clear all stored Braintrust + eval results.\"\"\"\n- _braintrust_eval_results.clear()\n-\n-\n-def setup_langsmith(\n- api_key: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- project_name: + Optional[str] = None,\n- standalone: bool = False,\n-) -> bool:\n- \"\"\"\n- Setup + Braintrust integration with LangSmith.\n-\n- This patches LangSmith''s + @traceable, Client.evaluate(), and aevaluate()\n- to use Braintrust under + the hood.\n-\n- Args:\n- api_key: Braintrust API key (optional, + can use env var BRAINTRUST_API_KEY)\n- project_id: Braintrust project + ID (optional)\n- project_name: Braintrust project name (optional, falls + back to LANGCHAIN_PROJECT\n- env var, then BRAINTRUST_PROJECT + env var)\n- standalone: If True, completely replace LangSmith with + Braintrust (no LangSmith\n- code runs). If False (default), + run both LangSmith and Braintrust\n- in tandem.\n-\n- Returns:\n- True + if setup was successful, False otherwise\n- \"\"\"\n- # Use LANGCHAIN_PROJECT + as fallback for project_name to keep both services in sync\n- if project_name + is None:\n- project_name = os.environ.get(\"LANGCHAIN_PROJECT\")\n-\n- span + = current_span()\n- if span == NOOP_SPAN:\n- init_logger(project=project_name, + api_key=api_key, project_id=project_id)\n-\n- try:\n- import langsmith\n-\n- langsmith.traceable + = wrap_traceable(langsmith.traceable, standalone=standalone)\n- wrap_client(langsmith.Client, + project_name=project_name, project_id=project_id, standalone=standalone)\n- langsmith.evaluate + = wrap_evaluate(\n- langsmith.evaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n- langsmith.aevaluate + = wrap_aevaluate(\n- langsmith.aevaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n-\n- logger.info(\"LangSmith + integration with Braintrust enabled\")\n- return True\n-\n- except + ImportError as e:\n- logger.error(f\"Failed to import langsmith: {e}\")\n- logger.error(\"langsmith + is not installed. Please install it with: pip install langsmith\")\n- return + False\n-\n-\n-def wrap_traceable(traceable: F, standalone: bool = False) -> + F:\n- \"\"\"\n- Wrap langsmith.traceable to also use Braintrust''s @traced + decorator.\n-\n- Args:\n- traceable: The langsmith.traceable function\n- standalone: + If True, replace LangSmith tracing entirely with Braintrust.\n- If + False, add Braintrust tracing alongside LangSmith tracing.\n-\n- Returns:\n- The + wrapped traceable function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(traceable):\n- return traceable\n-\n- def traceable_wrapper(*args: + Any, **kwargs: Any) -> Any:\n- # Handle both @traceable and @traceable(...) + patterns\n- func = args[0] if args and callable(args[0]) else None\n-\n- def + decorator(fn: Callable[P, R]) -> Callable[P, R]:\n- span_name = + kwargs.get(\"name\") or fn.__name__\n-\n- # Conditionally apply + LangSmith decorator first\n- if not standalone:\n- fn + = traceable(fn, **kwargs)\n-\n- # Always apply Braintrust tracing\n- return + traced(name=span_name)(fn) # type: ignore[return-value]\n-\n- if func + is not None:\n- return decorator(func)\n- return decorator\n-\n- traceable_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return traceable_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_client(\n- Client: Any, project_name: + Optional[str] = None, project_id: Optional[str] = None, standalone: bool = + False\n-) -> Any:\n- \"\"\"\n- Wrap langsmith.Client to redirect evaluate() + and aevaluate() to Braintrust''s Eval.\n-\n- Args:\n- Client: The + langsmith.Client class\n- project_name: Braintrust project name to + use for evaluations\n- project_id: Braintrust project ID to use for + evaluations\n- standalone: If True, only run Braintrust. If False, + run both LangSmith and Braintrust.\n-\n- Returns:\n- The Client + class (modified in place)\n- \"\"\"\n-\n- if hasattr(Client, \"evaluate\") + and not _is_patched(Client.evaluate):\n- wrap_function_wrapper(\n- Client,\n- \"evaluate\",\n- make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.evaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- if hasattr(Client, \"aevaluate\") + and not _is_patched(Client.aevaluate):\n- wrap_function_wrapper(\n- Client,\n- \"aevaluate\",\n- make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.aevaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- return Client\n-\n-\n-def make_evaluate_wrapper(\n- *, + project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: + bool = False\n-):\n- def evaluate_wrapper(wrapped: Any, instance: Any, + args: Any, kwargs: Any) -> Any:\n- result = None\n- if not standalone:\n- result + = wrapped(*args, **kwargs)\n-\n- try:\n- result = _run_braintrust_eval(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + evaluate failed: {e}\")\n-\n- return result\n-\n- return evaluate_wrapper\n-\n-\n-def + make_aevaluate_wrapper(\n- *, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-):\n- async def aevaluate_wrapper(wrapped: + Any, instance: Any, args: Any, kwargs: Any) -> Any:\n- result = None\n- if + not standalone:\n- result = await wrapped(*args, **kwargs)\n-\n- try:\n- result + = await _run_braintrust_eval_async(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + aevaluate failed: {e}\")\n-\n- return result\n-\n- return aevaluate_wrapper\n-\n-\n-def + wrap_evaluate(\n- evaluate: F, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.evaluate to redirect to Braintrust''s Eval.\n-\n- Args:\n- evaluate: + The langsmith.evaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to + use for evaluations\n- standalone: If True, only run Braintrust. If + False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + wrapped evaluate function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(evaluate):\n- return evaluate\n-\n- evaluate_wrapper + = make_evaluate_wrapper(standalone=standalone, project_name=project_name, + project_id=project_id)\n- evaluate_wrapper._braintrust_patched = True # + type: ignore[attr-defined]\n- return evaluate_wrapper # type: ignore[return-value]\n-\n-\n-def + wrap_aevaluate(\n- aevaluate: F,\n- project_name: Optional[str] = None,\n- project_id: + Optional[str] = None,\n- standalone: bool = False,\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.aevaluate to redirect to Braintrust''s EvalAsync.\n-\n- Args:\n- aevaluate: + The langsmith.aevaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to + use for evaluations\n- standalone: If True, only run Braintrust. If + False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + wrapped aevaluate function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(aevaluate):\n- return aevaluate\n-\n- aevaluate_wrapper + = make_aevaluate_wrapper(standalone=standalone, project_name=project_name, + project_id=project_id)\n- aevaluate_wrapper._braintrust_patched = True # + type: ignore[attr-defined]\n- return aevaluate_wrapper # type: ignore[return-value]\n-\n-\n-def + _is_patched(obj: Any) -> bool:\n- return getattr(obj, \"_braintrust_patched\", + False)\n-\n-\n-# =============================================================================\n-# + Braintrust evaluation logic\n-# =============================================================================\n-\n-\n-def + _run_braintrust_eval(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust Eval with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import Eval\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + Eval(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-async + def _run_braintrust_eval_async(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust EvalAsync with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import EvalAsync\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + await EvalAsync(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-# + =============================================================================\n-# + Data conversion helpers\n-# =============================================================================\n-\n-\n-def + _wrap_output(output: Any) -> Dict[str, Any]:\n- \"\"\"Wrap non-dict outputs + the same way LangSmith does.\"\"\"\n- if not isinstance(output, dict):\n- return + {\"output\": output}\n- return output\n-\n-\n-def _make_braintrust_scorer(\n- evaluator: + Callable[..., Any],\n-) -> Callable[..., Any]:\n- \"\"\"\n- Create a + Braintrust scorer from a LangSmith evaluator.\n-\n- Always runs the evaluator + through Braintrust for full tracing (span duration, child LLM calls, etc.).\n- \"\"\"\n- evaluator_name + = getattr(evaluator, \"__name__\", \"score\")\n-\n- def braintrust_scorer(input: + Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any:\n- from + braintrust.score import Score\n-\n- # Run the evaluator with LangSmith''s + signature\n- # LangSmith evaluators use: (inputs, outputs, reference_outputs) + -> bool | dict\n- # LangSmith auto-wraps non-dict outputs as {\"output\": + value}\n- outputs = _wrap_output(output)\n-\n- # expected is + the real LangSmith Example object passed through from data loading\n- reference_outputs + = expected.outputs if hasattr(expected, \"outputs\") else expected\n-\n- result + = evaluator(input, outputs, reference_outputs)\n-\n- return Score(\n- name=result.get(\"key\", + evaluator_name),\n- score=result.get(\"score\"),\n- metadata=result.get(\"metadata\", + {}),\n- )\n-\n- braintrust_scorer.__name__ = evaluator_name\n- return + braintrust_scorer\n-\n-\n-def _convert_langsmith_data(data: Any) -> Callable[[], + Iterator[EvalCase[Any, Any]]]:\n- \"\"\"Convert LangSmith data format to + Braintrust data format.\"\"\"\n-\n- def load_data() -> Iterator[EvalCase[Any, + Any]]:\n- # Determine the source iterable without loading everything + into memory\n- source: Iterable[Any]\n- if callable(data):\n- source + = data() # type: ignore\n- elif isinstance(data, str):\n- # + Load examples from LangSmith dataset by name\n- try:\n- from + langsmith import Client # pylint: disable=import-error\n-\n- client + = Client()\n- source = client.list_examples(dataset_name=data)\n- except + Exception as e:\n- logger.warning(f\"Failed to load LangSmith + dataset ''{data}'': {e}\")\n- return\n- elif hasattr(data, + \"__iter__\"):\n- source = data\n- else:\n- source + = [data]\n-\n- # Process items as a generator - yield one at a time\n- for + item in source:\n- # Pass through LangSmith Example objects directly\n- if + hasattr(item, \"inputs\"):\n- yield EvalCase(\n- input=item.inputs,\n- expected=item, # + Pass the whole Example object\n- metadata=getattr(item, + \"metadata\", None),\n- )\n- elif isinstance(item, + dict):\n- if \"inputs\" in item:\n- # LangSmith + dict format\n- yield EvalCase(\n- input=item[\"inputs\"],\n- expected=item, # + Pass the whole dict\n- metadata=item.get(\"metadata\"),\n- )\n- elif + \"input\" in item:\n- # Braintrust format\n- yield + EvalCase(\n- input=item[\"input\"],\n- expected=item.get(\"expected\"),\n- metadata=item.get(\"metadata\"),\n- )\n- else:\n- yield + EvalCase(input=item)\n- else:\n- yield EvalCase(input=item)\n-\n- return + load_data\n-\n-\n-def _make_braintrust_task(target: Callable[..., Any]) -> + Callable[..., Any]:\n- \"\"\"Convert a LangSmith target function to Braintrust + task format.\"\"\"\n-\n- def task_fn(task_input: Any, hooks: Any) -> Any:\n- if + isinstance(task_input, dict):\n- # Try to get the original function''s + signature (unwrap decorators)\n- unwrapped = inspect.unwrap(target)\n-\n- try:\n- sig + = inspect.signature(unwrapped)\n- params = list(sig.parameters.keys())\n- if + len(params) == 1:\n- return target(task_input)\n- if + all(p in task_input for p in params):\n- return target(**task_input)\n- return + target(task_input)\n- except (ValueError, TypeError):\n- # + Fallback: try kwargs first, then single arg\n- try:\n- return + target(**task_input)\n- except TypeError:\n- return + target(task_input)\n- return target(task_input)\n-\n- return task_fn\n+__all__ + = [\"setup_langsmith\"]\ndiff --git a/py/src/braintrust/wrappers/test_langsmith_wrapper.py + b/py/src/braintrust/wrappers/test_langsmith_wrapper.py\ndeleted file mode + 100644\nindex c25256c2..00000000\n--- a/py/src/braintrust/wrappers/test_langsmith_wrapper.py\n+++ + /dev/null\n@@ -1,341 +0,0 @@\n-# pyright: reportPrivateUsage=false\n-# pyright: + reportMissingParameterType=false\n-# pyright: reportUnknownParameterType=false\n-# + pyright: reportUnknownArgumentType=false\n-# pyright: reportUnknownMemberType=false\n-# + pylint: disable=protected-access\n-\n-\"\"\"\n-Tests for the LangSmith wrapper + to ensure compatibility with LangSmith''s API.\n-\"\"\"\n-\n-from braintrust.wrappers.langsmith_wrapper + import (\n- _convert_langsmith_data,\n- _is_patched,\n- _make_braintrust_scorer,\n- _make_braintrust_task,\n- wrap_aevaluate,\n- wrap_client,\n- wrap_traceable,\n-)\n-\n-\n-def + test_is_patched_false():\n- \"\"\"Test that _is_patched returns False for + unpatched objects.\"\"\"\n-\n- def unpatched():\n- pass\n-\n- assert + _is_patched(unpatched) is False\n-\n-\n-def test_is_patched_true():\n- \"\"\"Test + that _is_patched returns True for patched objects.\"\"\"\n-\n- def patched():\n- pass\n-\n- patched._braintrust_patched + = True # type: ignore\n-\n- assert _is_patched(patched) is True\n-\n-\n-def + test_make_braintrust_scorer_dict_result():\n- \"\"\"Test converting a LangSmith + evaluator that returns a dict.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- return {\"key\": \"accuracy\", \"score\": + 0.9, \"metadata\": {\"note\": \"good\"}}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- # + Create a mock Example object\n- class MockExample:\n- outputs = + {\"y\": 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": 2}, + expected=MockExample())\n-\n- assert result.name == \"accuracy\"\n- assert + result.score == 0.9\n- assert result.metadata == {\"note\": \"good\"}\n-\n-\n-def + test_make_braintrust_scorer_numeric_result():\n- \"\"\"Test converting + a LangSmith evaluator that returns a numeric score in a dict.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return + {\"score\": 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class MockExample:\n- outputs + = {\"y\": 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": + 2}, expected=MockExample())\n-\n- assert result.name == \"langsmith_evaluator\"\n- assert + result.score == 1.0\n-\n-\n-def test_make_braintrust_scorer_with_plain_dict_expected():\n- \"\"\"Test + converting a LangSmith evaluator with plain dict as expected.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return + {\"score\": 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n- result = converted(input={\"x\": + 1}, output={\"y\": 2}, expected={\"y\": 2})\n-\n- assert result.name == + \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def test_convert_langsmith_data_from_list():\n- \"\"\"Test + converting LangSmith data from a list of dicts.\"\"\"\n- data = [\n- {\"inputs\": + {\"x\": 1}, \"outputs\": {\"y\": 2}},\n- {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- assert result[1].input + == {\"x\": 2}\n- assert result[1].expected == {\"inputs\": {\"x\": 2}, + \"outputs\": {\"y\": 4}}\n-\n-\n-def test_convert_langsmith_data_from_callable():\n- \"\"\"Test + converting LangSmith data from a callable.\"\"\"\n-\n- def data_generator():\n- yield + {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- yield {\"inputs\": + {\"x\": 2}, \"outputs\": {\"y\": 4}}\n-\n- data_fn = _convert_langsmith_data(data_generator)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n-\n-\n-def test_convert_langsmith_data_with_example_objects():\n- \"\"\"Test + converting LangSmith data with Example-like objects.\"\"\"\n-\n- class + MockExample:\n- def __init__(self, inputs, outputs):\n- self.inputs + = inputs\n- self.outputs = outputs\n-\n- data = [\n- MockExample(inputs={\"x\": + 1}, outputs={\"y\": 2}),\n- MockExample(inputs={\"x\": 2}, outputs={\"y\": + 4}),\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole Example object is passed as expected\n- assert + result[0].expected.inputs == {\"x\": 1}\n- assert result[0].expected.outputs + == {\"y\": 2}\n-\n-\n-def test_make_braintrust_task_with_dict_input():\n- \"\"\"Test + that task function handles dict inputs correctly.\"\"\"\n-\n- def target_fn(inputs):\n- return + inputs[\"x\"] * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == 10\n-\n-\n-def test_make_braintrust_task_with_kwargs_expansion():\n- \"\"\"Test + that task function expands dict kwargs when signature matches.\"\"\"\n-\n- def + target_fn(x, y):\n- return x + y\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 2, \"y\": 3}, None)\n-\n- assert result == 5\n-\n-\n-def + test_make_braintrust_task_simple_input():\n- \"\"\"Test that task function + handles simple inputs.\"\"\"\n-\n- def target_fn(inp):\n- return + inp * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result = task(5, + None)\n-\n- assert result == 10\n-\n-\n-class TestWrapTraceable:\n- \"\"\"Tests + for wrap_traceable functionality.\"\"\"\n-\n- def test_wrap_traceable_returns_wrapper(self):\n- \"\"\"Test + that wrap_traceable returns a wrapped version.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable, + standalone=False)\n- assert callable(wrapped)\n- assert _is_patched(wrapped)\n-\n- def + test_wrap_traceable_standalone_mode(self):\n- \"\"\"Test that wrap_traceable + works in standalone mode.\"\"\"\n-\n- def mock_traceable(func, **kwargs):\n- return + func\n-\n- wrapped = wrap_traceable(mock_traceable, standalone=True)\n- assert + callable(wrapped)\n- assert _is_patched(wrapped)\n-\n-\n-class TestWrapFunctions:\n- \"\"\"Tests + for the wrap_* functions.\"\"\"\n-\n- def test_wrap_functions_exist(self):\n- \"\"\"Test + that wrap functions are callable.\"\"\"\n- assert callable(wrap_traceable)\n- assert + callable(wrap_client)\n- assert callable(wrap_aevaluate)\n-\n- def + test_wrap_traceable_returns_patched_function(self):\n- \"\"\"Test that + wrap_traceable returns a patched function.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable)\n- assert + _is_patched(wrapped)\n-\n- def test_wrap_traceable_skips_if_already_patched(self):\n- \"\"\"Test + that wrap_traceable skips if already patched.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- mock_traceable._braintrust_patched + = True # type: ignore\n-\n- result = wrap_traceable(mock_traceable)\n- # + Should return the same function\n- assert result is mock_traceable\n-\n- def + test_wrap_client_sets_flag(self):\n- \"\"\"Test that wrap_client sets + the patched flag.\"\"\"\n-\n- class MockClient:\n- def evaluate(self, + *args, **kwargs):\n- return \"original\"\n-\n- wrap_client(MockClient)\n- assert + _is_patched(MockClient.evaluate)\n-\n- def test_wrap_aevaluate_returns_patched_function(self):\n- \"\"\"Test + that wrap_aevaluate returns a patched function.\"\"\"\n-\n- async def + mock_aevaluate(*args, **kwargs):\n- pass\n-\n- wrapped = + wrap_aevaluate(mock_aevaluate)\n- assert _is_patched(wrapped)\n-\n-\n-class + TestTandemModeIntegration:\n- \"\"\"Integration tests for tandem mode (LangSmith + + Braintrust together).\"\"\"\n-\n- def test_make_braintrust_task_with_inputs_parameter(self):\n- \"\"\"Test + that task handles LangSmith''s required ''inputs'' parameter name.\"\"\"\n-\n- def + target_fn(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == {\"result\": 10}\n-\n- def + test_convert_langsmith_data_handles_different_output_types(self):\n- \"\"\"Test + that data conversion handles various output types.\"\"\"\n- data = + [\n- {\"inputs\": {\"x\": 1}, \"outputs\": 2}, # outputs is int, + not dict\n- {\"inputs\": {\"x\": 2}, \"outputs\": {\"result\": + 4}}, # outputs is already dict\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- # Both should work - Braintrust''s EvalCase + accepts any type for expected\n- assert len(result) == 2\n- assert + result[0].input == {\"x\": 1}\n- assert result[1].input == {\"x\": + 2}\n-\n- def test_make_braintrust_scorer_handles_wrapped_outputs(self):\n- \"\"\"Test + that scorers handle output wrapping correctly.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- # outputs will be wrapped as {\"output\": + value} for non-dict results\n- actual = outputs.get(\"output\", + outputs)\n- expected = (\n- reference_outputs.get(\"output\", + reference_outputs)\n- if isinstance(reference_outputs, dict)\n- else + reference_outputs\n- )\n- return {\"key\": \"match\", + \"score\": 1.0 if actual == expected else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class + MockExample:\n- outputs = {\"output\": 42}\n-\n- # Test + with wrapped output\n- result = converted(input={\"x\": 1}, output=42, + expected=MockExample())\n- assert result.name == \"match\"\n- assert + result.score == 1.0\n-\n-\n-class TestDataConversion:\n- \"\"\"Tests for + data conversion utilities.\"\"\"\n-\n- def test_convert_data_with_braintrust_format(self):\n- \"\"\"Test + that Braintrust format is properly handled.\"\"\"\n- data = [\n- {\"input\": + {\"x\": 1}, \"expected\": {\"y\": 2}},\n- {\"input\": {\"x\": 2}, + \"expected\": {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- assert result[0].expected == {\"y\": 2}\n- assert + result[1].input == {\"x\": 2}\n- assert result[1].expected == {\"y\": + 4}\n-\n- def test_convert_data_with_simple_items(self):\n- \"\"\"Test + that simple items (not dicts) are handled.\"\"\"\n- data = [1, 2, 3]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 3\n- assert result[0].input == 1\n- assert result[1].input + == 2\n- assert result[2].input == 3"},"commit":"5c35051a0102f850f2e09c66f9376a0fc658ed9f","base_exp_id":"8018513f-a5e8-4b9f-8d47-8c8ff2115691","deleted_at":null,"dataset_id":null,"dataset_version":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZDgwMWNjZTItYzY4Ny00NWI1LWIyODUtMjUyOWMwNGMxOGIw'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:17 GMT + Etag: + - W/"rrubuculs8soz" + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + Transfer-Encoding: + - chunked + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - ZDgwMWNjZTItYzY4Ny00NWI1LWIyODUtMjUyOWMwNGMxOGIw + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::z6v9p-1775254817595-34070bc34e15 + content-length: + - '37187' + status: + code: 200 + message: OK +- request: + body: '{"id": "7ff97ad2-7674-4727-a505-2ca57030c1fd"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '46' + Content-Type: + - application/json + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/base_experiment/get_id + response: + body: + string: '{"id":"7ff97ad2-7674-4727-a505-2ca57030c1fd","project_id":"208de547-5ee2-4294-955c-5595cd0f7940","name":"langsmith-aeval-adb3f119","base_exp_id":"8018513f-a5e8-4b9f-8d47-8c8ff2115691","base_exp_name":"langsmith-eval-daebc7df"}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '226' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MDgzMDdhYjAtZTVlYS00NjkyLWFmZWQtYzBiNjcyZGU5ZTFm'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:18 GMT + Etag: + - '"nwo42ml9oz6a"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/base_experiment/get_id + X-Nonce: + - MDgzMDdhYjAtZTVlYS00NjkyLWFmZWQtYzBiNjcyZGU5ZTFm + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::vgmfr-1775254817939-62a6c4a050a2 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.33.1 + method: GET + uri: https://api.braintrust.dev/experiment-comparison2?experiment_id=7ff97ad2-7674-4727-a505-2ca57030c1fd&base_experiment_id=8018513f-a5e8-4b9f-8d47-8c8ff2115691 + response: + body: + string: '{"scores":{},"metrics":{}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:18 GMT + Via: + - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 cb0c6226aa19d81a39519501df383968.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - AE34LJxRSzMZ0OW16MimYkbWSpHNRLx9AMLCYpWqB_Uk9-J7OVy00A== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-69d03d22-2b89f1ef04783df33ac101ac;Parent=5425b5a21b5f30e1;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id + content-length: + - '26' + etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - bQ59ZGlLoAMEi6g= + x-amzn-RequestId: + - 3a4b7112-3b19-43ea-b34c-feae40b9d194 + x-bt-internal-trace-id: + - 69d03d220000000041f8cdca869ffe60 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"f5883013-b5c1-438d-9044-5182b4682337","name":"abhi-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-NDIwYzgzM2MtYjBiZS00M2ZmLWI5M2ItMTAyOTlkZGU1MjNh'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:45 GMT + Etag: + - '"13nbfem8i3p75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - NDIwYzgzM2MtYjBiZS00M2ZmLWI5M2ItMTAyOTlkZGU1MjNh + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::dbcm5-1775254845391-c9166477c583 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "langsmith-py", "project_id": null, "org_id": "f5883013-b5c1-438d-9044-5182b4682337", + "update": false, "experiment_name": "langsmith-aeval", "repo_info": {"commit": + "5c35051a0102f850f2e09c66f9376a0fc658ed9f", "branch": "main", "tag": null, "dirty": + true, "author_name": "Abhijeet Prasad", "author_email": "abhijeet@braintrustdata.com", + "commit_message": "fix(anthropic): capture server-side tool usage metrics (#192)\n\nFlatten + Anthropic usage.server_tool_use into Braintrust span metrics so\nserver-side + tool invocations are preserved for tracing and cost analysis.\n\nAdd regression + tests using a red/green workflow to cover dict-backed span\nlogging and object-backed + usage extraction.\n\nFixes #171", "commit_time": "2026-04-02T21:10:00-04:00", + "git_diff": "diff --git a/py/noxfile.py b/py/noxfile.py\nindex c92395da..1083ca5b + 100644\n--- a/py/noxfile.py\n+++ b/py/noxfile.py\n@@ -77,6 +77,7 @@ VENDOR_PACKAGES + = (\n \"opentelemetry-exporter-otlp-proto-http\",\n \"google.genai\",\n \"google.adk\",\n+ \"langsmith\",\n \"temporalio\",\n + )\n \n@@ -104,6 +105,7 @@ GENAI_VERSIONS = (LATEST,)\n DSPY_VERSIONS = (LATEST,)\n + GOOGLE_ADK_VERSIONS = (LATEST, \"1.14.1\")\n LANGCHAIN_VERSIONS = (LATEST, \"0.3.28\")\n+LANGSMITH_VERSIONS + = (LATEST, \"0.7.12\")\n OPENROUTER_VERSIONS = (LATEST, \"0.6.0\")\n # temporalio + 1.19.0+ requires Python >= 3.10; skip Python 3.9 entirely\n TEMPORAL_VERSIONS + = (LATEST, \"1.20.0\", \"1.19.0\")\n@@ -235,6 +237,17 @@ def test_langchain(session, + version):\n _run_core_tests(session)\n \n \n+@nox.session()\n+@nox.parametrize(\"version\", + LANGSMITH_VERSIONS, ids=LANGSMITH_VERSIONS)\n+def test_langsmith(session, version):\n+ \"\"\"Test + LangSmith integration.\"\"\"\n+ _install_test_deps(session)\n+ _install(session, + \"langsmith\", version)\n+ _install(session, \"langchain-core\")\n+ _install(session, + \"langchain-openai\")\n+ _run_tests(session, f\"{INTEGRATION_DIR}/langsmith/test_langsmith.py\")\n+\n+\n + @nox.session()\n @nox.parametrize(\"version\", OPENAI_VERSIONS, ids=OPENAI_VERSIONS)\n + def test_openai(session, version):\n@@ -371,9 +384,8 @@ def pylint(session):\n session.install(\"pydantic_ai>=1.10.0\")\n session.install(\"google-adk\")\n session.install(\"opentelemetry.instrumentation.openai\")\n- # + langsmith is needed for the langsmith_wrapper module but not in VENDOR_PACKAGES\n # + langchain-core, langchain-openai, langchain-anthropic are needed for the langchain + integration\n- session.install(\"langsmith\", \"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n+ session.install(\"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n \n result = session.run(\"git\", \"ls-files\", + \"**/*.py\", silent=True, log=False)\n files = [path for path in result.strip().splitlines() + if path not in GENERATED_LINT_EXCLUDES]\ndiff --git a/py/src/braintrust/auto.py + b/py/src/braintrust/auto.py\nindex dc44c7d2..6cf8597c 100644\n--- a/py/src/braintrust/auto.py\n+++ + b/py/src/braintrust/auto.py\n@@ -16,6 +16,7 @@ from braintrust.integrations + import (\n DSPyIntegration,\n GoogleGenAIIntegration,\n LangChainIntegration,\n+ LangSmithIntegration,\n LiteLLMIntegration,\n OpenRouterIntegration,\n PydanticAIIntegration,\n@@ + -52,6 +53,7 @@ def auto_instrument(\n dspy: bool = True,\n adk: bool + = True,\n langchain: bool = True,\n+ langsmith: bool = True,\n ) -> dict[str, + bool]:\n \"\"\"\n Auto-instrument supported AI/ML libraries for Braintrust + tracing.\n@@ -75,6 +77,7 @@ def auto_instrument(\n dspy: Enable DSPy + instrumentation (default: True)\n adk: Enable Google ADK instrumentation + (default: True)\n langchain: Enable LangChain instrumentation (default: + True)\n+ langsmith: Enable LangSmith instrumentation (default: True)\n + \n Returns:\n Dict mapping integration name to whether it was successfully + instrumented.\n@@ -146,6 +149,8 @@ def auto_instrument(\n results[\"adk\"] + = _instrument_integration(ADKIntegration)\n if langchain:\n results[\"langchain\"] + = _instrument_integration(LangChainIntegration)\n+ if langsmith:\n+ results[\"langsmith\"] + = _instrument_integration(LangSmithIntegration)\n \n return results\n \ndiff + --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py\nindex + 0062ec77..2dca1110 100644\n--- a/py/src/braintrust/integrations/__init__.py\n+++ + b/py/src/braintrust/integrations/__init__.py\n@@ -6,6 +6,7 @@ from .claude_agent_sdk + import ClaudeAgentSDKIntegration\n from .dspy import DSPyIntegration\n from + .google_genai import GoogleGenAIIntegration\n from .langchain import LangChainIntegration\n+from + .langsmith import LangSmithIntegration\n from .litellm import LiteLLMIntegration\n + from .openrouter import OpenRouterIntegration\n from .pydantic_ai import PydanticAIIntegration\n@@ + -21,6 +22,7 @@ __all__ = [\n \"GoogleGenAIIntegration\",\n \"LiteLLMIntegration\",\n \"LangChainIntegration\",\n+ \"LangSmithIntegration\",\n \"OpenRouterIntegration\",\n \"PydanticAIIntegration\",\n + ]\ndiff --git a/py/src/braintrust/wrappers/langsmith_wrapper.py b/py/src/braintrust/wrappers/langsmith_wrapper.py\nindex + b22117df..7d4170fd 100644\n--- a/py/src/braintrust/wrappers/langsmith_wrapper.py\n+++ + b/py/src/braintrust/wrappers/langsmith_wrapper.py\n@@ -1,518 +1,4 @@\n-\"\"\"\n-Braintrust + integration for LangSmith - provides a migration path from LangSmith to Braintrust.\n+from + braintrust.integrations.langsmith import setup_langsmith # noqa: F401\n \n-This + module patches LangSmith''s tracing and evaluation APIs to use Braintrust under + the hood,\n-allowing users to migrate with minimal code changes.\n \n-Usage:\n- ```python\n- import + os\n-\n- # Enable LangSmith tracing and set project name (used by both services)\n- os.environ.setdefault(\"LANGCHAIN_TRACING_V2\", + \"true\")\n- os.environ.setdefault(\"LANGCHAIN_PROJECT\", \"my-project\")\n-\n- from + braintrust.wrappers.langsmith_wrapper import setup_langsmith\n-\n- # Call + setup BEFORE importing from langsmith\n- # project_name defaults to LANGCHAIN_PROJECT + env var\n- setup_langsmith()\n-\n- # Continue using langsmith imports + - they now use Braintrust\n- from langsmith import traceable, Client\n-\n- @traceable\n- def + my_function(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- client = Client()\n- results = client.evaluate(\n- my_function,\n- data=[{\"inputs\": + {\"x\": 1}, \"outputs\": {\"result\": 2}}],\n- evaluators=[my_evaluator],\n- )\n- ```\n-\n- Set + BRAINTRUST_STANDALONE=1 to completely replace LangSmith with Braintrust\n- (no + LangSmith code runs). Otherwise, both services run in tandem.\n-\"\"\"\n-\n-import + inspect\n-import logging\n-import os\n-from typing import Any, Callable, Dict, + Iterable, Iterator, List, Optional, ParamSpec, TypeVar\n-\n-from braintrust.framework + import EvalCase\n-from braintrust.logger import NOOP_SPAN, current_span, init_logger, + traced\n-from wrapt import wrap_function_wrapper\n-\n-\n-logger = logging.getLogger(__name__)\n-\n-# + Global list to store Braintrust eval results when running in tandem mode\n-_braintrust_eval_results: + List[Any] = []\n-\n-# TODO: langsmith.test/unit/expect, langsmith.AsyncClient, + trace\n-__all__ = [\n- \"setup_langsmith\",\n- \"wrap_traceable\",\n- \"wrap_client\",\n- \"wrap_evaluate\",\n- \"wrap_aevaluate\",\n- \"get_braintrust_results\",\n- \"clear_braintrust_results\",\n-]\n-\n-F + = TypeVar(\"F\", bound=Callable[..., Any])\n-P = ParamSpec(\"P\")\n-R = TypeVar(\"R\")\n-\n-\n-def + get_braintrust_results() -> List[Any]:\n- \"\"\"Get all Braintrust eval results + collected during tandem mode.\"\"\"\n- return _braintrust_eval_results.copy()\n-\n-\n-def + clear_braintrust_results() -> None:\n- \"\"\"Clear all stored Braintrust + eval results.\"\"\"\n- _braintrust_eval_results.clear()\n-\n-\n-def setup_langsmith(\n- api_key: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- project_name: + Optional[str] = None,\n- standalone: bool = False,\n-) -> bool:\n- \"\"\"\n- Setup + Braintrust integration with LangSmith.\n-\n- This patches LangSmith''s @traceable, + Client.evaluate(), and aevaluate()\n- to use Braintrust under the hood.\n-\n- Args:\n- api_key: + Braintrust API key (optional, can use env var BRAINTRUST_API_KEY)\n- project_id: + Braintrust project ID (optional)\n- project_name: Braintrust project + name (optional, falls back to LANGCHAIN_PROJECT\n- env var, + then BRAINTRUST_PROJECT env var)\n- standalone: If True, completely replace + LangSmith with Braintrust (no LangSmith\n- code runs). If + False (default), run both LangSmith and Braintrust\n- in tandem.\n-\n- Returns:\n- True + if setup was successful, False otherwise\n- \"\"\"\n- # Use LANGCHAIN_PROJECT + as fallback for project_name to keep both services in sync\n- if project_name + is None:\n- project_name = os.environ.get(\"LANGCHAIN_PROJECT\")\n-\n- span + = current_span()\n- if span == NOOP_SPAN:\n- init_logger(project=project_name, + api_key=api_key, project_id=project_id)\n-\n- try:\n- import langsmith\n-\n- langsmith.traceable + = wrap_traceable(langsmith.traceable, standalone=standalone)\n- wrap_client(langsmith.Client, + project_name=project_name, project_id=project_id, standalone=standalone)\n- langsmith.evaluate + = wrap_evaluate(\n- langsmith.evaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n- langsmith.aevaluate + = wrap_aevaluate(\n- langsmith.aevaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n-\n- logger.info(\"LangSmith + integration with Braintrust enabled\")\n- return True\n-\n- except + ImportError as e:\n- logger.error(f\"Failed to import langsmith: {e}\")\n- logger.error(\"langsmith + is not installed. Please install it with: pip install langsmith\")\n- return + False\n-\n-\n-def wrap_traceable(traceable: F, standalone: bool = False) -> + F:\n- \"\"\"\n- Wrap langsmith.traceable to also use Braintrust''s @traced + decorator.\n-\n- Args:\n- traceable: The langsmith.traceable function\n- standalone: + If True, replace LangSmith tracing entirely with Braintrust.\n- If + False, add Braintrust tracing alongside LangSmith tracing.\n-\n- Returns:\n- The + wrapped traceable function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(traceable):\n- return traceable\n-\n- def traceable_wrapper(*args: + Any, **kwargs: Any) -> Any:\n- # Handle both @traceable and @traceable(...) + patterns\n- func = args[0] if args and callable(args[0]) else None\n-\n- def + decorator(fn: Callable[P, R]) -> Callable[P, R]:\n- span_name = kwargs.get(\"name\") + or fn.__name__\n-\n- # Conditionally apply LangSmith decorator first\n- if + not standalone:\n- fn = traceable(fn, **kwargs)\n-\n- # + Always apply Braintrust tracing\n- return traced(name=span_name)(fn) # + type: ignore[return-value]\n-\n- if func is not None:\n- return + decorator(func)\n- return decorator\n-\n- traceable_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return traceable_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_client(\n- Client: Any, project_name: + Optional[str] = None, project_id: Optional[str] = None, standalone: bool = False\n-) + -> Any:\n- \"\"\"\n- Wrap langsmith.Client to redirect evaluate() and + aevaluate() to Braintrust''s Eval.\n-\n- Args:\n- Client: The langsmith.Client + class\n- project_name: Braintrust project name to use for evaluations\n- project_id: + Braintrust project ID to use for evaluations\n- standalone: If True, + only run Braintrust. If False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + Client class (modified in place)\n- \"\"\"\n-\n- if hasattr(Client, \"evaluate\") + and not _is_patched(Client.evaluate):\n- wrap_function_wrapper(\n- Client,\n- \"evaluate\",\n- make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.evaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- if hasattr(Client, \"aevaluate\") + and not _is_patched(Client.aevaluate):\n- wrap_function_wrapper(\n- Client,\n- \"aevaluate\",\n- make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.aevaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- return Client\n-\n-\n-def make_evaluate_wrapper(\n- *, + project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: + bool = False\n-):\n- def evaluate_wrapper(wrapped: Any, instance: Any, args: + Any, kwargs: Any) -> Any:\n- result = None\n- if not standalone:\n- result + = wrapped(*args, **kwargs)\n-\n- try:\n- result = _run_braintrust_eval(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + evaluate failed: {e}\")\n-\n- return result\n-\n- return evaluate_wrapper\n-\n-\n-def + make_aevaluate_wrapper(\n- *, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-):\n- async def aevaluate_wrapper(wrapped: + Any, instance: Any, args: Any, kwargs: Any) -> Any:\n- result = None\n- if + not standalone:\n- result = await wrapped(*args, **kwargs)\n-\n- try:\n- result + = await _run_braintrust_eval_async(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + aevaluate failed: {e}\")\n-\n- return result\n-\n- return aevaluate_wrapper\n-\n-\n-def + wrap_evaluate(\n- evaluate: F, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.evaluate to redirect to Braintrust''s Eval.\n-\n- Args:\n- evaluate: + The langsmith.evaluate function\n- project_name: Braintrust project name + to use for evaluations\n- project_id: Braintrust project ID to use for + evaluations\n- standalone: If True, only run Braintrust. If False, run + both LangSmith and Braintrust.\n-\n- Returns:\n- The wrapped evaluate + function (or the original if already patched)\n- \"\"\"\n- if _is_patched(evaluate):\n- return + evaluate\n-\n- evaluate_wrapper = make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id)\n- evaluate_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return evaluate_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_aevaluate(\n- aevaluate: F,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- standalone: + bool = False,\n-) -> F:\n- \"\"\"\n- Wrap module-level langsmith.aevaluate + to redirect to Braintrust''s EvalAsync.\n-\n- Args:\n- aevaluate: + The langsmith.aevaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to use + for evaluations\n- standalone: If True, only run Braintrust. If False, + run both LangSmith and Braintrust.\n-\n- Returns:\n- The wrapped aevaluate + function (or the original if already patched)\n- \"\"\"\n- if _is_patched(aevaluate):\n- return + aevaluate\n-\n- aevaluate_wrapper = make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id)\n- aevaluate_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return aevaluate_wrapper # type: + ignore[return-value]\n-\n-\n-def _is_patched(obj: Any) -> bool:\n- return + getattr(obj, \"_braintrust_patched\", False)\n-\n-\n-# =============================================================================\n-# + Braintrust evaluation logic\n-# =============================================================================\n-\n-\n-def + _run_braintrust_eval(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust Eval with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import Eval\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + Eval(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-async + def _run_braintrust_eval_async(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust EvalAsync with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import EvalAsync\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + await EvalAsync(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-# + =============================================================================\n-# + Data conversion helpers\n-# =============================================================================\n-\n-\n-def + _wrap_output(output: Any) -> Dict[str, Any]:\n- \"\"\"Wrap non-dict outputs + the same way LangSmith does.\"\"\"\n- if not isinstance(output, dict):\n- return + {\"output\": output}\n- return output\n-\n-\n-def _make_braintrust_scorer(\n- evaluator: + Callable[..., Any],\n-) -> Callable[..., Any]:\n- \"\"\"\n- Create a Braintrust + scorer from a LangSmith evaluator.\n-\n- Always runs the evaluator through + Braintrust for full tracing (span duration, child LLM calls, etc.).\n- \"\"\"\n- evaluator_name + = getattr(evaluator, \"__name__\", \"score\")\n-\n- def braintrust_scorer(input: + Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any:\n- from + braintrust.score import Score\n-\n- # Run the evaluator with LangSmith''s + signature\n- # LangSmith evaluators use: (inputs, outputs, reference_outputs) + -> bool | dict\n- # LangSmith auto-wraps non-dict outputs as {\"output\": + value}\n- outputs = _wrap_output(output)\n-\n- # expected is the + real LangSmith Example object passed through from data loading\n- reference_outputs + = expected.outputs if hasattr(expected, \"outputs\") else expected\n-\n- result + = evaluator(input, outputs, reference_outputs)\n-\n- return Score(\n- name=result.get(\"key\", + evaluator_name),\n- score=result.get(\"score\"),\n- metadata=result.get(\"metadata\", + {}),\n- )\n-\n- braintrust_scorer.__name__ = evaluator_name\n- return + braintrust_scorer\n-\n-\n-def _convert_langsmith_data(data: Any) -> Callable[[], + Iterator[EvalCase[Any, Any]]]:\n- \"\"\"Convert LangSmith data format to + Braintrust data format.\"\"\"\n-\n- def load_data() -> Iterator[EvalCase[Any, + Any]]:\n- # Determine the source iterable without loading everything + into memory\n- source: Iterable[Any]\n- if callable(data):\n- source + = data() # type: ignore\n- elif isinstance(data, str):\n- # + Load examples from LangSmith dataset by name\n- try:\n- from + langsmith import Client # pylint: disable=import-error\n-\n- client + = Client()\n- source = client.list_examples(dataset_name=data)\n- except + Exception as e:\n- logger.warning(f\"Failed to load LangSmith + dataset ''{data}'': {e}\")\n- return\n- elif hasattr(data, + \"__iter__\"):\n- source = data\n- else:\n- source + = [data]\n-\n- # Process items as a generator - yield one at a time\n- for + item in source:\n- # Pass through LangSmith Example objects directly\n- if + hasattr(item, \"inputs\"):\n- yield EvalCase(\n- input=item.inputs,\n- expected=item, # + Pass the whole Example object\n- metadata=getattr(item, \"metadata\", + None),\n- )\n- elif isinstance(item, dict):\n- if + \"inputs\" in item:\n- # LangSmith dict format\n- yield + EvalCase(\n- input=item[\"inputs\"],\n- expected=item, # + Pass the whole dict\n- metadata=item.get(\"metadata\"),\n- )\n- elif + \"input\" in item:\n- # Braintrust format\n- yield + EvalCase(\n- input=item[\"input\"],\n- expected=item.get(\"expected\"),\n- metadata=item.get(\"metadata\"),\n- )\n- else:\n- yield + EvalCase(input=item)\n- else:\n- yield EvalCase(input=item)\n-\n- return + load_data\n-\n-\n-def _make_braintrust_task(target: Callable[..., Any]) -> Callable[..., + Any]:\n- \"\"\"Convert a LangSmith target function to Braintrust task format.\"\"\"\n-\n- def + task_fn(task_input: Any, hooks: Any) -> Any:\n- if isinstance(task_input, + dict):\n- # Try to get the original function''s signature (unwrap + decorators)\n- unwrapped = inspect.unwrap(target)\n-\n- try:\n- sig + = inspect.signature(unwrapped)\n- params = list(sig.parameters.keys())\n- if + len(params) == 1:\n- return target(task_input)\n- if + all(p in task_input for p in params):\n- return target(**task_input)\n- return + target(task_input)\n- except (ValueError, TypeError):\n- # + Fallback: try kwargs first, then single arg\n- try:\n- return + target(**task_input)\n- except TypeError:\n- return + target(task_input)\n- return target(task_input)\n-\n- return task_fn\n+__all__ + = [\"setup_langsmith\"]\ndiff --git a/py/src/braintrust/wrappers/test_langsmith_wrapper.py + b/py/src/braintrust/wrappers/test_langsmith_wrapper.py\ndeleted file mode 100644\nindex + c25256c2..00000000\n--- a/py/src/braintrust/wrappers/test_langsmith_wrapper.py\n+++ + /dev/null\n@@ -1,341 +0,0 @@\n-# pyright: reportPrivateUsage=false\n-# pyright: + reportMissingParameterType=false\n-# pyright: reportUnknownParameterType=false\n-# + pyright: reportUnknownArgumentType=false\n-# pyright: reportUnknownMemberType=false\n-# + pylint: disable=protected-access\n-\n-\"\"\"\n-Tests for the LangSmith wrapper + to ensure compatibility with LangSmith''s API.\n-\"\"\"\n-\n-from braintrust.wrappers.langsmith_wrapper + import (\n- _convert_langsmith_data,\n- _is_patched,\n- _make_braintrust_scorer,\n- _make_braintrust_task,\n- wrap_aevaluate,\n- wrap_client,\n- wrap_traceable,\n-)\n-\n-\n-def + test_is_patched_false():\n- \"\"\"Test that _is_patched returns False for + unpatched objects.\"\"\"\n-\n- def unpatched():\n- pass\n-\n- assert + _is_patched(unpatched) is False\n-\n-\n-def test_is_patched_true():\n- \"\"\"Test + that _is_patched returns True for patched objects.\"\"\"\n-\n- def patched():\n- pass\n-\n- patched._braintrust_patched + = True # type: ignore\n-\n- assert _is_patched(patched) is True\n-\n-\n-def + test_make_braintrust_scorer_dict_result():\n- \"\"\"Test converting a LangSmith + evaluator that returns a dict.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- return {\"key\": \"accuracy\", \"score\": + 0.9, \"metadata\": {\"note\": \"good\"}}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- # + Create a mock Example object\n- class MockExample:\n- outputs = {\"y\": + 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": 2}, expected=MockExample())\n-\n- assert + result.name == \"accuracy\"\n- assert result.score == 0.9\n- assert result.metadata + == {\"note\": \"good\"}\n-\n-\n-def test_make_braintrust_scorer_numeric_result():\n- \"\"\"Test + converting a LangSmith evaluator that returns a numeric score in a dict.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return {\"score\": + 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class + MockExample:\n- outputs = {\"y\": 2}\n-\n- result = converted(input={\"x\": + 1}, output={\"y\": 2}, expected=MockExample())\n-\n- assert result.name == + \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def test_make_braintrust_scorer_with_plain_dict_expected():\n- \"\"\"Test + converting a LangSmith evaluator with plain dict as expected.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return {\"score\": + 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n- result + = converted(input={\"x\": 1}, output={\"y\": 2}, expected={\"y\": 2})\n-\n- assert + result.name == \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def + test_convert_langsmith_data_from_list():\n- \"\"\"Test converting LangSmith + data from a list of dicts.\"\"\"\n- data = [\n- {\"inputs\": {\"x\": + 1}, \"outputs\": {\"y\": 2}},\n- {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- assert result[1].input + == {\"x\": 2}\n- assert result[1].expected == {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}}\n-\n-\n-def test_convert_langsmith_data_from_callable():\n- \"\"\"Test + converting LangSmith data from a callable.\"\"\"\n-\n- def data_generator():\n- yield + {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- yield {\"inputs\": + {\"x\": 2}, \"outputs\": {\"y\": 4}}\n-\n- data_fn = _convert_langsmith_data(data_generator)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n-\n-\n-def test_convert_langsmith_data_with_example_objects():\n- \"\"\"Test + converting LangSmith data with Example-like objects.\"\"\"\n-\n- class MockExample:\n- def + __init__(self, inputs, outputs):\n- self.inputs = inputs\n- self.outputs + = outputs\n-\n- data = [\n- MockExample(inputs={\"x\": 1}, outputs={\"y\": + 2}),\n- MockExample(inputs={\"x\": 2}, outputs={\"y\": 4}),\n- ]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 2\n- assert result[0].input == {\"x\": 1}\n- # The whole + Example object is passed as expected\n- assert result[0].expected.inputs + == {\"x\": 1}\n- assert result[0].expected.outputs == {\"y\": 2}\n-\n-\n-def + test_make_braintrust_task_with_dict_input():\n- \"\"\"Test that task function + handles dict inputs correctly.\"\"\"\n-\n- def target_fn(inputs):\n- return + inputs[\"x\"] * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == 10\n-\n-\n-def test_make_braintrust_task_with_kwargs_expansion():\n- \"\"\"Test + that task function expands dict kwargs when signature matches.\"\"\"\n-\n- def + target_fn(x, y):\n- return x + y\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 2, \"y\": 3}, None)\n-\n- assert result == 5\n-\n-\n-def test_make_braintrust_task_simple_input():\n- \"\"\"Test + that task function handles simple inputs.\"\"\"\n-\n- def target_fn(inp):\n- return + inp * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result = task(5, + None)\n-\n- assert result == 10\n-\n-\n-class TestWrapTraceable:\n- \"\"\"Tests + for wrap_traceable functionality.\"\"\"\n-\n- def test_wrap_traceable_returns_wrapper(self):\n- \"\"\"Test + that wrap_traceable returns a wrapped version.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable, + standalone=False)\n- assert callable(wrapped)\n- assert _is_patched(wrapped)\n-\n- def + test_wrap_traceable_standalone_mode(self):\n- \"\"\"Test that wrap_traceable + works in standalone mode.\"\"\"\n-\n- def mock_traceable(func, **kwargs):\n- return + func\n-\n- wrapped = wrap_traceable(mock_traceable, standalone=True)\n- assert + callable(wrapped)\n- assert _is_patched(wrapped)\n-\n-\n-class TestWrapFunctions:\n- \"\"\"Tests + for the wrap_* functions.\"\"\"\n-\n- def test_wrap_functions_exist(self):\n- \"\"\"Test + that wrap functions are callable.\"\"\"\n- assert callable(wrap_traceable)\n- assert + callable(wrap_client)\n- assert callable(wrap_aevaluate)\n-\n- def + test_wrap_traceable_returns_patched_function(self):\n- \"\"\"Test that + wrap_traceable returns a patched function.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable)\n- assert + _is_patched(wrapped)\n-\n- def test_wrap_traceable_skips_if_already_patched(self):\n- \"\"\"Test + that wrap_traceable skips if already patched.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- mock_traceable._braintrust_patched + = True # type: ignore\n-\n- result = wrap_traceable(mock_traceable)\n- # + Should return the same function\n- assert result is mock_traceable\n-\n- def + test_wrap_client_sets_flag(self):\n- \"\"\"Test that wrap_client sets + the patched flag.\"\"\"\n-\n- class MockClient:\n- def evaluate(self, + *args, **kwargs):\n- return \"original\"\n-\n- wrap_client(MockClient)\n- assert + _is_patched(MockClient.evaluate)\n-\n- def test_wrap_aevaluate_returns_patched_function(self):\n- \"\"\"Test + that wrap_aevaluate returns a patched function.\"\"\"\n-\n- async def + mock_aevaluate(*args, **kwargs):\n- pass\n-\n- wrapped = wrap_aevaluate(mock_aevaluate)\n- assert + _is_patched(wrapped)\n-\n-\n-class TestTandemModeIntegration:\n- \"\"\"Integration + tests for tandem mode (LangSmith + Braintrust together).\"\"\"\n-\n- def + test_make_braintrust_task_with_inputs_parameter(self):\n- \"\"\"Test + that task handles LangSmith''s required ''inputs'' parameter name.\"\"\"\n-\n- def + target_fn(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- task = _make_braintrust_task(target_fn)\n- result = + task({\"x\": 5}, None)\n-\n- assert result == {\"result\": 10}\n-\n- def + test_convert_langsmith_data_handles_different_output_types(self):\n- \"\"\"Test + that data conversion handles various output types.\"\"\"\n- data = [\n- {\"inputs\": + {\"x\": 1}, \"outputs\": 2}, # outputs is int, not dict\n- {\"inputs\": + {\"x\": 2}, \"outputs\": {\"result\": 4}}, # outputs is already dict\n- ]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- # + Both should work - Braintrust''s EvalCase accepts any type for expected\n- assert + len(result) == 2\n- assert result[0].input == {\"x\": 1}\n- assert + result[1].input == {\"x\": 2}\n-\n- def test_make_braintrust_scorer_handles_wrapped_outputs(self):\n- \"\"\"Test + that scorers handle output wrapping correctly.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- # outputs will be wrapped as {\"output\": + value} for non-dict results\n- actual = outputs.get(\"output\", outputs)\n- expected + = (\n- reference_outputs.get(\"output\", reference_outputs)\n- if + isinstance(reference_outputs, dict)\n- else reference_outputs\n- )\n- return + {\"key\": \"match\", \"score\": 1.0 if actual == expected else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class MockExample:\n- outputs + = {\"output\": 42}\n-\n- # Test with wrapped output\n- result + = converted(input={\"x\": 1}, output=42, expected=MockExample())\n- assert + result.name == \"match\"\n- assert result.score == 1.0\n-\n-\n-class + TestDataConversion:\n- \"\"\"Tests for data conversion utilities.\"\"\"\n-\n- def + test_convert_data_with_braintrust_format(self):\n- \"\"\"Test that Braintrust + format is properly handled.\"\"\"\n- data = [\n- {\"input\": + {\"x\": 1}, \"expected\": {\"y\": 2}},\n- {\"input\": {\"x\": 2}, + \"expected\": {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- assert result[0].expected == {\"y\": 2}\n- assert + result[1].input == {\"x\": 2}\n- assert result[1].expected == {\"y\": + 4}\n-\n- def test_convert_data_with_simple_items(self):\n- \"\"\"Test + that simple items (not dicts) are handled.\"\"\"\n- data = [1, 2, 3]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 3\n- assert result[0].input == 1\n- assert result[1].input + == 2\n- assert result[2].input == 3"}, "ancestor_commits": ["5c35051a0102f850f2e09c66f9376a0fc658ed9f", + "0daac2f277a337206cbb03cc46c695f7931d7f3d", "d807b7b4a48b4bd920570cdad5f01c350cdd9f35", + "19ecb8a09557f830675a2dded6bdd04f36ae9f76", "0a708e578ce19f61c17a9f4aa44a96d7ed0277df", + "2a174d884e2034fd9a8c20c20c4c0c384f7aa687", "a656bab37f99445ca442fef41f032907c79e3fc0", + "5fda66e087f78c4edf8c95f24c9454e9c80c6417", "07eae203f645dd661ea0fb83b5256f2144dbf07f", + "2c68b8894223a9b562b7d922a53430efe067ede3", "ba0bc10b5f79e952e69a721be89023773f46fe54", + "e09f625c67fab1f2d2f7b31f0370dab3e0cafd1b", "4e5c83f5456b5c043e1e0830227805d28f922f70", + "161fa36a51946f2212d28354609e442f83889684", "3462f3f3d4ce82b3c59f5dacec9a523b9c99ee2d", + "62a2963911d6c65cdbaef1712fdb7c890d3b5777", "89122d4cca5ea2b766b9416c68b611bee79294dd", + "1c1a07e7dfd96811fd883d81adb3b7f7f4061e75", "f6a2c3b3aa589ab8606b7b525ce79ca019f98717", + "051870af3d7bc8c183edac119bcb7166f1ea9ef0", "4267bde681b41aaa31036e14508708a28bde70ac", + "0b717a9e6240a162a3931116f44bd86f4900852b", "7a4328e7db5636ffe7898d99fd392bc204ec7a08", + "e0780bfc33ab6ced1b982aaf27bd7b5072bbe04e", "183dbb0d034c3588b26a8a94a10f3d2f03237670", + "8b15ad67f697a8218217d9674f0fb919d9c1e8d6", "01d0cea1268051909665e4911af1f1be54b333cd", + "b335e66c1f458de9cd73c12d9d9fc97114f3d505", "0b2f34802606d033bb3471075be95de5d9621b98", + "dd73fb4ac1f38bee8e55133a0348f1bd5c46c30e", "716f9e45b99874f17531f9b5ae4ccfa57cab5ae3", + "72adf8bbf0b48756e18423c5477d58bbf6c401b9", "1163e11507676601e265e637e8916a71a310ce87", + "80308c49a0309d13e02c3203c099b3dc87de1ad9", "e112d705d338ef08d449afab5f2bef3ae0269622", + "10996518f60c0cd2796304c122cd05b16c6ffb1c", "b9a78fa767379bfc14e9ec0f9dc1a016f7ca74a2", + "3fc8912b4063d524f357733549ff1c58bbad00ba", "4252662870b266b32d23a16a11c6df755e5dcdcb", + "92165c3f00478c316e7af29d5c2dba51ffbd7090", "a8144eabce4ef1a46257be77c8055bd3fc64978c", + "363baa083335500aebe129c670eb2c49153ad3a6", "0d487b688de037fa6eebf5410c12b9b226f12cff", + "824861d780be7bcbec92f274a13e221cffef04a4", "84398747d12f52f2b6089d5e832ff6cca82fa3d5", + "46c18b284d0d14855fb66a6de8891ea2c39e08e7", "4f1d9510a188a7a9645fe72966a099aff70478ca", + "d82c4258901176056c1eb30a837896becada5a0f", "265a46272e1f808251c0da03061a9b1843a22779", + "42bed48d1b31a428f189ba174ff4f759ec34e94d", "98aa3e6923d32ed264abd11315cd247c085e8a16", + "ff18889b8f8ef8af644f6eeab1179f6bf9b14de5", "fd34303ea427a9bfc038b78ef1e8e65bb44b6f29", + "08210a4fd97cc17d90556afaaba51dbd67917ddd", "a1fae25c088942b155a31b243f07b9862f79c5ba", + "4fb512eaa52733c1daa4cd6995140f6722f17ba4", "992e036a9199d1b03976ed4d22eebe9921410b40", + "5d41698059f77d9c332cfc4e706ea77669d1cb6a", "2349418ff85b75917d869a26be6df0682686caac", + "fd9f3f932a646f07438db3dbda8cab73cfa70c2f", "fc2d5fe0f6d94754e3c705686c3c69eaa58c258c", + "b3651b577d6ebf617a9991252e19d5fcfe7e693c", "106691a2cbfafc38b3d5d5064c9bb922ae7a3901", + "1b7818ade91e40155c7c126cb92cb21726a25c66", "49ab8392e413992b9728666999593a94ebd00236", + "fb80c5412e011c5b3d343809710c92b75874261e", "ab9fd17f71c277f0d05a809c3c9cfe22379df511", + "7184711ed04957af8f2bbe36590c1cec057ca0a8", "5976a47fe82f90599edc06b7e4f9af64a8ce4ed7", + "962e499f18549cdea2e65946c79bacbb9159f35d", "28875eb8ce2b4114d60d365ad5b61c124b5e3900", + "e68dfce97ee7fa014a30d694d7220a1de47fa73e", "404589ac15776fe04f6dd8bdf36c5d74a3684986", + "5f5b88701fedc4cafed4c037ee2a8cff6b4a40fb", "aeb15581fefc2b0b867ca55c21ea80d067f43361", + "2cd52871898658315cc43e78f25546c3c8df1c8a", "6c4b637d65e047f01052b0749d4c3608f202e073", + "864aafda1789476b01c6991931fb0a6e1968e2dc", "9548ae35e49fc7aac9263811ae3c3a4b26af2e81", + "97da34df8474129e1ad16c8eb67b00a906330970", "fdd020262b6620a4823d336741ebbcf0a02ff9a2", + "0bb83ac92e0ccb38dee1f56c11d15180e8c8c565", "34262bcfb1c5babc03da6d27d1b93ca2c3e969a7", + "7bb761580056ac43c2636964b5f1d2f08b3cc01a", "9424d6a04f73de02038dcbacd518864397ddcafb", + "a51b84c54ba9269d28f17e7355dbc8aaf9ccc0b2", "a5ef7778c00dd0c3782a423e4fb6ee3a0a2a73a3", + "33cfa00beffa04151a7d874d5671b720bb49dbe8", "eb530110ee5746d87c1d2b9972f3a3dc4a2aa582", + "8695a8e3ef85a173cfffd98ae25f6a31a7e23d2e", "7da14b1a295c8f912b6b93ec55ec759242a50108", + "f60a2d39cce59a000017bc193b84ef25ba4491d5", "447d3a96d0679451a2c87a5bf28531ca7744b9ab", + "abc8bed8fbdac570891517c76233e18d9fff4828", "a7053505083fb7b7ae6ed1c10a4141a0d8673cc4", + "8ab9dc06de08f7a925fabe52dbbdb6df3b480e89", "009a839b197b63d6a688d53d91743fb84cbed04f", + "fe75d23b3b61e1693c3b3bfe896380ce16a3ef66", "3a4dde05d882a4eb27d21d1d41784295b3a4168e", + "2b0d9e276db08298b83a10b877f3ccf08bfafde6", "fe1732c7b4df75d49cb42017bb2b2af0136a73c0", + "971559d598a53d361c50da7126e5681124244f32", "6e172e761f95e59dda9fab0409f85491beef2240", + "e3cd507afaaa62bc8ef736b27c1f0e0064c3931f", "3a3ba47a007e0af9af528b63b271391e0c538dc7", + "c3d5ea96b897dc179e4775b14c7bb97b838861db", "1ec7d11f39c97471973de7e5eb0b0784025bc851", + "99a5cb75b08c5e2bff6e96614cfe8c0573f0c19d", "49f1e3846c3c8d264e693bab999270d4392d153e", + "824cf93c402abf51934aebd890c9805457108a7c", "d9e62442aa51d540345c1bfbd1df043b9f00657d", + "c933fe5625cfe8b7e57e4b2953fe612ad37d78b5", "ec0c35e2fadc7bd6a734ff3013226193b1f81d09", + "d462219aa0d5e6525299e3a9cc1dabb19dbdc9c5", "9f8937030053fc43137805b66fef5d1d335fae47", + "1c827b6442cf42fbfa2868abfcb2c640752769bd", "892c41480277cab14f3751eafe678af0c4966bfb", + "6a65a37fb5f7e11e9b8551759d44879e732355d1", "c047819f46a9bf82b7f11124802247055e709c80", + "102257d62684977c39f6e78559d73ac0f873048a", "94f13c6a1c7306e23ba45c2cf0d3600ef8cf8673", + "786d48c8e9cf363508f0c30b872968d9ceec69d7", "7b62001e98bba2fcad8254aff0f0cb11bf35db33", + "2eeee24fc7c3b7c6a0aac79563e1cb0253e0bea1", "7aa19caa8d893488dedf5d6e985b766710b66eb7", + "17b5cae858690e270104ba1c5520508525e6c687", "8b1a5aa0a4b4d547e09c07d69be841c3ebde525f", + "174b4061aab369725a43719d5e966013686b4c38", "7bf81f8ebc7877956e2b8c3c556743426ef05038", + "e151663947390534a2037912cad663daba83de6d", "99bc21b047a09063813a733d659e84ad740424e3", + "1bc2d475bc64f99dd11c3bfd569bd15140a87389", "e3d62fddfab1774e5cae1c1ae2d0d46f164a768f", + "d91f76c24230054edef8774579a0a036b844af7b", "dbea65347acd60addcfbfa7bdb97ddbaeee6ba21", + "a28ef5f8895857ad6d945644f78679271fd10df3", "ec1c107ab6133f5810f9501e3ce57349c0bf4ae9", + "10e2d2237b0791683386e551bd4be79cf86367db", "e3f25b9b6abc2f8e1a376788d11d82a61c41645d", + "a88b44d6852fae824705415d2795f5c5201f2dde", "6210d435deb094c603c9e8debdd0d48255be7dae", + "6b2ab80cc64c1b869fb9ea6002fb1670e9ceb5cb", "1e440103b7521973aa5fb33addc8f9ce1d050232", + "449f6eeddd54f73d88e185fcf499087d3e61fbe6", "52579035063a1b7d1e8b64fbaf1b32bf55a04004", + "4db047355d602c53d85f43c1e7276dd3bd8335df", "4bc366daa82312b79ccb2d6471abef3ad16e4512", + "c710e0c271880b2ea1d506dd3d6f23c282f33fe8", "ce261c13723e8985e61819f6df56298f863155d0", + "f2b6f15f1db2d3e3a930e3274c0c1d1245e851c2", "f11481fe89d024f1b2deb651653ea8ad30da5e8a", + "633cb4c201d614ff5ac5e9775fd68764601d59d8", "0bb08d89c6fe9c043cb75178571715460bc32603", + "a8b9f42b80cfc3a835162fc4e69945edae9a2dd2", "e9f46b8d41234acba81068f7c00e3063204b4231", + "ef1c89e8f840a14f44cd6682bc92f7ce2ea9d07d", "6d086d6cfee47cc1e83c7522a58e1d412e91d197", + "7894ea68db9bcdaa55e66cd4d2a634878f901965", "753298d7ada8377c173de6fad4b490082372b7db", + "25868bc58450dad2058b6499ce3bb9400330fbd1", "95f3c141b7f8ad81bf1d34d857c2fe60bbd15d7f", + "f416cea3139a9a29ed5cae633cb3097e70fc93e2", "5e89c7299d9cc476594fc73933bc7006c842cdf4", + "ddf619048666d883cd4325e21f860e28632a456b", "2385b1aa98998820bcb55cfae76d063a57bba938", + "b798003d9609603b43c98ccc2a5c995d0ac1a488", "4d9333cef5aae8889f9697f38a87f694893f17a8", + "2194150ee020def2d7c4a675f2f001c8c0fba99e", "36788d5a20f45762991b1ba72fb52f8020e6c61b", + "16f72b7d1cb1e0801285abbb7b3138909d6e7f79", "617d9b730b37e96b7d05a099b95f5387944d0951", + "51bb20bec1dc870ec4f521e3c9464054ab9abdec", "15cde9f7795249fbb3c71f23210092265f509e19", + "ea3fd2ff145cbfcd53bb1d79ec6dfb659debdb4f", "eacfc8036be4cee52c619f278b61d32862f3a521", + "252eb0be986ee236c4f095f0078b816c7bd5c894", "6774ffa8b89b2da3644f07e4f411b20f108e5786", + "77fc0472011fb9ea9b3ff7bc5e7dbc5c57f33fed", "3e4002f6b2b3e540db12a2355f447fabd7a83bec", + "3110f96fcbfea2b60d1f8b580e28c00c2fe38690", "1cfad0ba63198f2480a22721501757d9913a1d52", + "2dc8187fd9ace26f7684ff5ba4fa75d7f28d7703", "0687d03758cd295a1976a1b5eb40c5187332be7c", + "8b3614791454b2b4e15a47a09948b3699ad7334a", "54002cedbe48b5036ba94109782d78686c0df7aa", + "9c21b41dd5a19131bd4bf236503ebd0a1464816f", "9a5c1c4a0d50d2d70ab6acb2c38e6abcbfcfc547", + "d0bf7aec410bdd29bf4a8f43af0a8bea633788e6", "7580576751a6890b3314fcd4c313991fb9f311a6", + "d734e8ffc272ee65fe0588df00fd9390614ccd2e", "d9fd9bb0c6bf4c6d5536765e0f8005e9326021a2", + "a751e399b66d396cdf13599c93b1445d81934ddc", "8e50db374a548c26830613a78f70072086ce1f13", + "791a2da1e44f02902e1c31ec28c2c7aba5795f47", "8a63569906d8824faf9c1aaecdd62ebb9e47aeb4", + "e541543ce30054cc10fa3e27e2b0aa5afd297d0f", "b017a02240f978dcfe7feec58aeb85f8cbc91892", + "072d3765416d78e465a728df1b4a812992fba58a", "1671a73c21d09a1b4b73debce7cb2fa1d1b9ee54", + "7b6371f41f6a109ea068fd547b17a131d109ee85", "530a4be0108ff8cc783a49c6390f59b238782e58", + "3ca420e53e77d4665b91ccc7631c95dc97ce566d", "41aceab354ed312d4758b5a5b5c32bbdac40da48", + "6f303beaf59f892b9ed86eb4338900c7f82f7a68", "2d2f7e260b2027576963fad1c432197b15f2811e", + "04acf467f93d2b1064ef648c6583f76d96864578", "3dcfc326ee12b116d919974ef8bbf3cf307ee7fd", + "0740dc169ac9f267e4ec2d435165ca882df0fbc1", "270970d692adaf5d204d336ecdb4ee7f51042187", + "37e49012471ef81326f1313583497f48d612eed1", "8d9100b8341085366c11a36d93faa6009ecd4837", + "f53d7b868f610f1b215c799f25d4af171b59a000", "23b9a8d6b977d6849503433f8b38c49cf90e8203", + "5a31cd02719a5a282825a0fec7cb8d0fc44d0884", "7079cb29799d9fb9fbc919b7ddd1224935a551c7", + "d9c624ea93ca6bf62c2412abce1b3a2ef1a2be67", "126c367b3002ad57bb6aa7e08d04657db6f61980", + "b610cd36638ace60bca2aed261165afbf1c8c081", "52a8fa6d53c58908d95bb865f5907d829987fb97", + "65c037efdffceeec554b2706e16621cfc1704202", "c3bc923fafbc33485c9173c2401fa20897cf4ef1", + "6d6cc2f1e8b5da316c409a754ee8c844b8d2b2ee", "0ecb05e34031941426e3b9ce76760add8af6fdfd", + "8ab13f3f48af6a4d3c0b053e4bbabfd4f24f23ec", "10c14b3688c6969ef55951d5d82e744b87d6c356", + "05f569c80bddb61414e8f9a8adf49f8ca6b821b4", "2311d67956183da8b31bf1fdbe6e09b3ec7e242d", + "0b0bdd77c012e3f51c6402edcad35c8ac7ddf4cd", "ca2eb28a0f22a63b2b748d04e17268d11d35e0b0", + "c20c808993bba8d5604c2ac7848037d7ce430e89", "dcd4f5a4be171b1cac28a5eb3534e4b55420cc06", + "74dd883f7c03ba5591ecee0b2b76e9250781e181", "4f94c9548a493de11cd663fc78db21a81fe6f23c", + "b3c2b6f7120402b3f5f0fce9b7a1e6b7f7d04d47", "476a66a5d09ec811c9856f0f0f289b189859721e", + "2468e8006b8edf9ef8f273e142380faf34d380a6", "b88964e5a1b124816771e2c9c0a628c104ecfddb", + "a05dc4df62a20e9715abe697584641f0032742b5", "ad3ca98e3530c7dbdbb07f63a04fe4912e17f85a", + "dbbc1894ef31143816e5913676301261bc44aa4c", "db388916a9bc42b02e26b519c3c5f116c919cafc", + "78753fa68806ca7478572a8b37c1f8d97f99eee3", "759f04ddc9522471b46d139ab347668c849ef8dc", + "3857983398d6c7278686c9f12fc2f841c628bc06", "11d9aff118de5f99bb0b5d55bca04d0efb36746a", + "38ea029efe156f07385cbea0a8743e07f418dd76", "39cc2896ef3cf7f9c2320fa513c645a972063697", + "e085b589790c05aa27f52ced4fb0480230db38b5", "5a0afe994e81c19a69ac9ce3a60529b5e7e7c529", + "e42f891ae64b8844c2d37bf29c25821382ed654f", "1fa9fe250443884c450e92ca44a73f1de6a4a5d3", + "0241b89c84097de0dcc7cf7abd65098ac3020578", "a735eedc1c421d2ef287fd1759e96699dc7aa75c", + "cef88a007fa60f4cd873f1d891a54ce5e173f3aa", "db634461703056448a77d669d907c3861cee6dac", + "bb13adf5fd90e4ca503b6980e737bb0f3396a63a", "b48a316abdb2fab1e7e3a797af20a7af3395587f", + "8ad895a40e7c6e7940aa464acf094e5c4ce2451a", "685755051290c901f95f5f29d8cb313c43833837", + "45e647b94bf47482695fc747a2b8b5575611aa6f", "b6a2d2d34371c5ddebd8ff6176aed0f5104e2d01", + "8c2b2995c2097f5699c101f4ab5e653ff40014a1", "db9aaff1052e5229ad6cef667543f9f6620b119b", + "be421874326c6922eb48277e347ccbd53099117d", "c6f2bf1b0331ef0f7d295e29a86717fa8b5698a1", + "d27524c41de17bec43692c05870d5dcb0f0458a1", "ed35e0ef407e405818bc218566fa79323d7329f7", + "f0473314bed3f901febe06355abe43cc73ea30d0", "7cffe54f8b960931055ae20f5983b4ea34515eff", + "d46056681f07ab1a435c625c2b46ed561240a573", "b58203e04d2cafa99307443c62f5063ca7d2e228", + "38c38c579147306770452db9f2405c2aaa8bcda2", "38f4356286376bb91a2464352b4cceaba3a3f6a0", + "79e066dbd2e0248345b032648c9310e2c7778d23", "160c352d87c121ef1ea43e49831453b9e8d4d0f8", + "be65342f97e4e7234a94ff6dc2e7c25f21f8660d", "058f4abcc85a2fd62fd3e5ff5b3b673824aa6004", + "5fc2dce168e2f622c8695ac0365b7ecf1309eb15", "6cb3c4075881492fe7189c502659caf20ed37a89", + "9860ef0921e59b2bd11767b5382f3815fff1baa0", "e8b9b463f80953fe29eb73b91468d30f5ef62c99", + "87992ef1fac6c582a28f3327f5d7f4ffc553fa78", "eefed6797eeb529dd376ea970c9c79a6ea017f99", + "b37c32b7e9326f350d3287e5870dbded1989e7d0", "990f2380b5c095d71eba74c595bf0535b500cd9c", + "238ae256a3a1ab93e682d603c4b53652fa0059f1", "9ed13156c47c637c5abc48ee20e6c6182c50f778", + "0f80700a25acea6dad9f5b726f220ee8cc227d7e", "dc628580c9eb037386b4e295db74b7f3daa74302", + "d2f914b6abf45a9bffc2823d7ed53a88ba231907", "133d19e1e5c4b0b87017942d4ca37f731af5a91e", + "1cbc14e38442cfbae772c080de6e17b3df09868f", "3106d7492fd3faba686f789ac5b42f2770a446ab", + "d868af0a96ae2ac8c5488c011397af20146017c0", "7db2d14148d17ed3ad5bdeec728cfe863a20f25a", + "feb813ec958af1823c42f4f5502677e38be72d8d", "0b2ee7336487a12eeda4ba66d8f34c332db30b8e", + "54de3627b1038f15d905c05f4c46a1d781b6b44e", "168702d959029c760f29c2ac9dbfe5ae501d7e8d", + "bd99b577781a7ed50e9049a292ac48d2fe33b983", "fede6ecee14c3842ab748664206bd633ea51c524", + "a1d968a1ff87035e1b4bcd5c4db2dfe15474c298", "93683071196fae3e0f92f10bc4533575ec4d058d", + "afc1500cce291ecd2c1bb748d7b627d11b1f03d1", "7a123d33d1a90683ee3df04e5900277519259c75", + "49a541549f2bf86d9bc6037db5a61f5f708756b8", "4a75b61b5f0cf472c5fb17e2c6a271f9c5ba770a", + "27a10de6ca27ebba2d9c667408538da980a85d5e", "3b7c0a467d849acba884de1a8c9a9334e8b7f9af", + "17001bf4ac840128c88152d0b1e6fd62d6a69f2f", "f9c27614dfa3934bffb1f69cc9623e8dcac9d0ec", + "30c615dafa30ea4ecc030e9d03265908f38eb389", "c6134d552ab4bd41e92fd8e2f4a8faad0c15b296", + "ebcaabe082cbe7c9a546a6f156a2fb67c9b98097", "f2a681ba398572b3a88051c1eef80994ddff009e", + "1c9f4f163e65cb4f0f061307d8b3686a042cc9e4", "3db2fc16ad04fc0bb32a41a9ecd013f9f5fa5c36", + "6d2dc1e53a6d16b466ef449fd8e14ebd3fa20f00", "a4dfb710896797d1ec1cab191e39cc51eba18b4d", + "0f51f6ef36ba16737589cab11ea2dd1742671c41", "fa85adab7bc82c5d14250a988664e0f65d23643a", + "2d8afda70e0f82176d7ee80bcb653ed09d4ada76", "17b618dbe38d1374435496778d3cbd0f3d3720a9", + "6116c86a6188a345cb430e55b48e660009a85799", "479f00719a1c04acda05bc52e8198c76b17b767b", + "41b9555dd4375ea0df23a0a6a2333f7f4d9e626e", "c3a3634129a36fb09c4b79e737582e33bd2a058b", + "7b1f9abab1067273355a38909e102fd2d3a5323c", "2cbf3503536ba11f94a975fc0c547529315caa9d", + "4aa4133193a8e8c3fdc766208be7bffb4983bae5", "436fe5351a1bc91413e903a2d2cbf2a8559a3c5f", + "c033d7aed97d788aa88220df3c8cb72fbe1ab96f", "f11265286454e2059d0de2a82ee4356601552bd2", + "f8c91c4de5c8d19881c4f8b0461bc1f8bfd01d94", "25912cd765f1ca1e19486411fb3de9963fd9f925", + "1026cbd8af888621a402e114a1c58c14da41bb72", "1cf1bf68a47746b2d23e4211c03a4b81e9626d10", + "c5980aae60fc5c3577e241a70c0798d8397e6a07", "9e4f93daa40369d2cdb97bfb01ee8c93bf23ec80", + "2b61d410af69dcfe59d11337df54412a7ed495b7", "d9dc67568e7a7147ea23e42083d2cb8632bdbe6c", + "21bb8edd05cc9d1142fabd4aba808d6a8157f4df", "258e8ec05412a73bb0a9e916db941c7af1b3e959", + "98ca8e2ec4264fb7c6a3a733c455f8d0ea089b07", "d6cc3a68587ba633190b18c083b7d56747b7a19d", + "3525059c8b3d3fadcff6f592c0e2ded7fc06294b", "e01077c19252f97215f6ed141d485f877506282f", + "8a4eb9ac7b33af6ffe3e980c685c290a68c93ca1", "567acfb3e9e9fda2c4051c038c6621bdb47ec823", + "675e293cfa98252cef4d8b7980066b038a901d04", "11e271cd1a3df6c75449e0c51a21a6c3b3d3fe22", + "ce4101fb55325aca6989ccfaebfa9992421c9e2b", "8e0211657949932ce0ed8da68e72b5c5981fc8f5", + "fa016eff3df324a5ff9f3d43d324a01177ec8939", "6f4426eac2c30baf20e09b51c5369156eff7abc1", + "2040109acbd819ebfa776fc97631815b122aa7d4", "30a65a2a2ce03eec847aa822f722a0d2f0f993bd", + "2f320a37a7ef0d32ae4fceb09057b1491eb1fc32", "25e7a079b6717ffd08ffbda3254fddb072160d82", + "146489482d21d37e7297d3b085ba786409e328a3", "555e44e3d20abf44db5780cc7fc8c95ae5593239", + "26b2a34cca8cf71e092eb3e7b02e53b88e6c03a5", "d6278e852f56fdb00ee5b9d97226be1589853166", + "5c61d67a3f8068020979f09c9f7be11f56e95c2a", "e0647fd778205f62b1956e5320f682af61fbf5ca", + "5d55b31e47053325b4a345c445c62413aa69177c", "a71670f60bd64ceec2b68fc07608d4ef05277ad5", + "e6751654101127f666768dfecd42fe9c6e0b47e3", "a45fabb93b9405dd04d2d944fef78dc424361698", + "5f31e95c53879bdcb5bd47276a8aff25673655d8", "fe3f9a83f7db420747f402841d25dd73f8b36d30", + "70dec2f04b7567d45d5a85b77c959623eefcfad2", "042f91e4b2b8067e6b672816182ba9ae2465812a", + "6e7aef5b4b4c6cf84ce51a4d8b947eda306a26b5", "3f01a409a7ecc34e825e8532e56d04613397cd13", + "893a546da4fd221501d3c865c7bf1d16ce9d933e", "f5410073902240fb94b50c30506aa02dbf9a1517", + "722657b05ed625309450f4f33953d7a694a1a414", "691d303ce7356e544cbfae5ad8054454c0f69652", + "a6f5a8bb856a84214811cd2b259edf0a30c6144f", "aba9adfef223a27e9ebee5f1e5896763adc71641", + "500c2fddb600fcbe7a7d8a48d4e1fe7c3bdcfea7", "ce957df3478f5f95b668a0dfc87956da190d74a6", + "3548dc0a4e272b21330ceefe9c8c63517941c034", "b252d68c0dc7bc599474380a7a185b9c5a9a5fb6", + "c2bcdec50527d1153a73a5e191d7bb4ab1f4a5f3", "552e48c600134179ece0d1aabd13d17f124338be", + "1a18ac6ef2f20cb3434d1db59bf4ae4c3a1f715d", "ad99dac6002a2975e445e8dc90f5d93c53991fdb", + "04f8a3f464061a9dba2d1c00dd7940488720974a", "b8c2d1ebc7b75ea604e9490fd265a6f797add415", + "4bf10cf4435ff85c7c3e2f50e7790b2eaa8861a4", "66091f34bf277d5491477085551afc1a8a361a0f", + "f7e1f2ac9f1767984bd125184f8b87563f249bfd", "61a1c51b48f7b278c73c13efeba86aa2484a33b5", + "3f54176089005fb6dc4b252b4dce34370ea8bcc6", "ff6f45ce65c0b32578fb92f842f123581413dfe9", + "6163138faeabb67eb3ef1b97d6a4b08a5c45caa9", "64cf4c6299cadfe4e5a7987c8db3383a69a108b9", + "e98267285f704dd8232dcfffc09616ffeda896fd", "61d4167d00783621e8a830f267b799b8f22969b1", + "12551cea0f20963404d37821d889c7b397ce29ba", "9aa042227f5625364418d20187dc76508c5c32fb", + "8213aad94904430b8165119337da1243fa7a88ab", "e8c47b170edc02abda540037fb4752ef03a21849", + "59a6092a06c163dfb2fa1deea179cd84a5e91083", "db2b8564a0ed5193a70c607f23e575c8f023454a", + "12048f9a795a901cab215f869b185d6e46e2a1e9", "98cd9913a99020f97a98297a8f085881e8ff21b7", + "76c1d436b2fba46a7ac7b7d285ee4b5e3a6660fa", "cb1c08bc56644e993505a2ab863c18ebc8a35cd2", + "f532f53f82e2fbd789b654ddd6eebe181c12d5ab", "1aaa6ce090e25893b7b8b46cc99fe8344eec653d", + "83d4ec39274f91989811f845c6fe805544bfa0ab", "0152b43193700d54d9826709132ac2bcb9ce5975", + "57291a113482eb4277f43c414b09afcb1bf8f0b7", "3d5ec4de453ab8ea873a6c8c0d0ef0b2110bd386", + "4168c8645f251d9584b25c5a57ccb089178a98a1", "0dafcc9a1aba7282e98d653730b496d83ba13c16", + "e45a7da9aba5144429cb9019acf99dc668ddadd5", "0844207c03327225f9913cc4b1a8702330841a96", + "329121933af05b575eee9c77bf13b9a4d3443dde", "4793325d739b2d9d0deb39d1118e33d76202e7a3", + "517ce4da9248bbbf77c7c9f9226054bc5412176b", "b9ec5a7fe95a0c7e32d65901da5d33ab8c30c779", + "cae143cbf533885d8bf978f287e9383bea46c3e6", "e2df632753bc6545d94b6f4c76af47a81ea3542b", + "eb3dcb724dfd1cc85e30594a63839ebd46c8a5a8", "c459f75f02d0c3444baef25d6fd228ce2929f924", + "b98731f2971b9ac5d0acda62cfb2cc32c141dc37", "6b1c69123acb73fd0cc23393bb165e99bab2508c", + "50159c19f26b9c1433e50f388b3fa759c92c0ff0", "08715a5559a18ec189f91421618e534344f342a0", + "2d4792e23d4104e5a89c17bf2195c7a8aded89a2", "27eb88f965e9d3df3a1cf219db7883093f339676", + "ee2acbc913cb75dccb8d1bb124b074bacae27ee1", "d9a2df31b1ff1ea6796a25558771ed1a38e6d1f9", + "b0c3bc7c96cd65435e9ff6b88040799959041972", "c340893e4dfd3cbeb4566cec5e8cd9a24c9982d5", + "cd361d820e14ee59bd8daa6bbe3321cc7d4fa436", "89c6916d89c4fec3cbe1ee6c780358cf51ba87e2", + "262aed06bd219cbfbb4e91c0fbdfb606883f74cd", "e32b82fabd2f29cdc9ac9e8b33966b10f8a0ce18", + "1e3688e139a11ce21f3a9e5c74d176c1ace5cd38", "b460d264e4ecde414ddb70e918a1adb48ae610b7", + "41bee95c0185b6c2521073565e11ede0885a7e8a", "22bfb85e284a90cf37d23354f56786941de27a99", + "59c7f2256d0b342d1cc08eeb3f4dd9c7e61e90f7", "6f791793df1d1124936ad53e9948cd221a184aec", + "e5c3b79336ede6948f3302d60de0fb8a1b7a138c", "666d1b61ab34647db7ab8e617a9497b87166d7e8", + "4d8966bad69bca5970ce0ed64c343fac0e1cf698", "852bc11787bae1ac6f99622dffc2eea32ef121e8", + "2b3067a1dfe1648d1e9be9f54f8e3af5ee9cbc3b", "1a001a96c0fd88b7884b9049a6d47f6ec4d086b3", + "740106ef9e67ae17393d081580f5c1493db372a1", "05e0f5184a2a0c6a003dd1654bdb4d1da4b1a847", + "8eeb1017e6f043e43d6eaaa99d2aec62d2aa2776", "1e34543f322aa94c5f5e679cc8713120002bee0c", + "d253a3c83f3a062f0a18220bdec2188a2445f2ff", "2548241b69cdc856b24ed8fe930ee5a608348b8c", + "47c83b2f5b8cc63496aa507f03babde7e79c730c", "1169d83904da1f140c52a51748d1f9920cadc304", + "2551edd52b168e0691cbc4a16b6f9762c42a2ee8", "b07d82b7fe426edc3ea57eca2e54dd1a7865e61d", + "fe52db5b57808551b30af34184e5f11fb12a1a69", "2b29dbd15c15e138057630527d19a67d9ea9198a", + "5cbc7715b58ac8a9d141cf95813149a09d7f323c", "918ea978ecc68b7f3ed541b7ed1e073848bf06f3", + "af49d808446f40d37d64c66517a45cc19a78de32", "f2fb962dacc2a7f1b1a9a075611e300970463a1e", + "a0ac9637dbc9b7b4af6d37256011e8e777379cb6", "3fefc0572263475219c149f4c54fe1b88f23c7e7", + "f227977b57b78fecd2db13ff2b231d0c22a3b274", "7e4c8a38b0fd82cc80df79b1daec14c1cdb58be8", + "ad3269310aba337feac5712f9621a5b8ffdb29b8", "4ad90d121a48a0390744035a7bc068f885abcf89", + "134fe22aeae849409671acc32e13a17a69681569", "4e0adb75d47a88366da89405264d87d88cf358c1", + "f1252436b3427e8fe87ea9ac07829f977ea12752", "0679d7794f638bc86e47ecf8e49b70482281c20c", + "c06dbe314d1f44c63b8bfe6edadef6ebf5c5ff20", "ad6e8fa9bbbe6f40621a826662b858c96eff4057", + "9756e3372243fd818c895910aa53f4148798b7a8", "dbb3e71028ba0573b07f6a39bee4a4ff7f39c486", + "6aaf30534ead2af6285ed5f3ff12f3680f401066", "5f7f1c1613ed4d17875a6d845e8cffd3fc0b0059", + "7c1432135a24132618b786ace67238029bec8591", "76290595b414a9c7ec3bf3cf5fa6ad32920d32bd", + "97d4988c8ccfb194a7d01d2c427e6796ed36c5e1", "344fbf8c2cab7917197614e4e974bb5478e4a653", + "6a2becd209341a5dd760515889e5aa27a26e1213", "997e1ee3a567f849865457e22be43aad5b84fcad", + "b0a9c6b95b650b90e69e3ea25f3ac2f583a9c622", "c830370439922abc9cef0763e9cae3387e549c42", + "9bb1447ddf4fbcd71f68096307fb5e222d38202d", "b862af307888af607c82d4d2a3a53e12d6265720", + "585e55bba7d4833272526051a7e2a47416f8556c", "99ac5bdc45f0bec78af9807ebcd8fd118155df93", + "c7e14a6aa34a6256944ea4d9be85777a0017415e", "cc2dda3439152bcabbc625f21d9251afbf15a076", + "4e84b51660d909910c0ce4cb014a23e20748ed05", "f43e4d0aca4ed483f9b2568f35e70dfbdfbecd7c", + "10039ee5b86d6ab30389abfafc4158da5e051410", "a212609775beaa7909f92b708b8348b920c34bc3", + "26ef38e22f884aabecbcdc02640beb2e8ec55abb", "4689201232928ffd29fa14a2bac10fa44f45a81a", + "fb2c9cb75fdb96ad4016a076773038a6081a4792", "d80b52a085881ac67c9c8748473991852f8a1113", + "47e4afb01417c95ae136318ce8d066c77402f30c", "2a256e4dec0180e6e27cd8bda8b4ef1bc3b1bdd9", + "fd4b1442c33a018cbfe42a859cb4bbbdb61eb54f", "f36d187c3fec787dd3d4128001c545de5551bc08", + "ef30b00ca626b1bc763427aed853246c0ab72534", "75eed65362b1370b90c3a032dac2f366f1469f1f", + "d0149fcec2d2e19ee8851b4ea2e328b2ee689184", "9e8eca00b251efc8bea606b3571208966fb2a2d7", + "28e3df3754c86e9809909aac9fe02515e231d60f", "5d4af1c155c639c81448f1d51db6cd33431f7fae", + "7e6a0c4ba45a0e692bbcf9456460aa56a974ed01", "63b04d5a98be1d3a23003f6518c4c8ee3b065ce6", + "31991c235b014fcf4fc9a99a7fa4824c705af5cd", "79231d6d3fb4196a482cf902439ff727eecb92ed", + "a8c246b8bf081182b08ad2a4ea478b7ba860d006", "33750764f3c324181451b38a1dc5219ab04a448e", + "370e22839b26494ce0ad1a88286f427434f3dcc5", "200b037c3083bd2e28e4e31f015bc87f9ab7196c", + "6a7573a61b2e15fd261863178f1f54d96ebdf408", "adfe5473bf50ce4f88c88826d4055b144ef42bb0", + "9d66b222b35f994a0d3ab6bbfb1c142d9f226d56", "ec03206161f942a59ea5bc91c08086a9b1fd2ae7", + "ca5bc0d3d6326e4df076624d826be350a89ed54a", "b8b6473dd4a5f93cd6eb1c0e41d659cb7bad0286", + "20162a0585c37d0c14b818eda6f695c40678bf65", "369ce701b448551b3b5b7156667687c68f7a4259", + "587b5c41554e3aed002f92535ef2179231f87e6d", "3b81c061b9efb1074e0dd9cbb5f11f71420e0832", + "6007cac67baac66d72a57c024260e9bda13c456d", "b1678a8e2c14c96405dc17557ca68d059ff68b8a", + "d5879f68ec23e30ca8882d93410073a0cd5db92d", "90a76730c4ccc74d3cce95abfff445d70bb98499", + "bce86716fa530eedc657bfd7cf18f045116a967d", "fc46a08726a7821ebccbd4d1492ee05c6bae89f7", + "d056f38506127e986b226fb5ff5683b6deeefe37", "1119b6ba9baba7c6511bd57f21eb642085932790", + "2d317fabf4f5e8d7a858a3d1dc79b88a2640b241", "498c2b6b4c0964e25a76cd8ef9e1e25dee1cda7d", + "803f7b6304b6fdc60bf08212fa280e31365d16a9", "5532c2bb541a785fd145aeaf624491e201ed7759", + "243023d8a29dffb528b567f37b5ef1d2d78f3fc7", "c0312dad18501b449812f9c9aca3c39fc1647432", + "9aa5151cb1622686f76e3e956dc9a6e48f43d3ef", "c9ac4a1dffd0a1fe7a0a339b156259884c54057c", + "cb53da91b05ac93ef8bb2dffb4f0ce8ddd06255e", "d25e73f15b3a94570b0a85e39df20d4f9d6a3ded", + "d0af00f950ccb83770035cd0db90fd8ebbe00d10", "ae260ec3ddf6bd62cb66ccb1c9ba3a6ca9d19ab8", + "846116d64285c2125f01ac2e06060b84ff9604a1", "2261c2c6e3edb6c2327046fa2d5b69d9a39ec10b", + "0c938db82253db2c09d40079dd15f0a1ad43345d", "4e5825c76b332b11e0e3af2ef74d9b98e3784809", + "83b5cadf47b3eaa15841e3815099ba5338a5f683", "2f11fc2b39f6889ae6720c0592344995b5942c37", + "3e43da7045388e7259414209b3baea7ee339bb9d", "4b71f161759ed105daaf49ad74f3306ef1710855", + "ae7070f6b723cb4c5055252017e5ce712ddbb649", "23cff857c47ae04884a1e522af2d5a836ea30f2d", + "424a8b8cd8ccd2ddbdf3c2d10ced6767b53a4ef2", "6391feb5cffcecf562f79be5c7b5c620cc1ff24d", + "9f1df5788377be4be03c705b21ffe72e3fca68b0", "607db9575d0fdcc02810ceab97c9b3faf9dcb601", + "ee0202713209fc3df5cc7c1c75812c325c94e280", "47905a7a246fc0ab81d6620820a96a283c6fe8d9", + "21ff79105eba8e77a638b42b2e18589a59033609", "28ba2f0f2967b8292a7071c3d99c9438b1db7963", + "e1fcdbf48c205c3a724151cf5008809615687dcb", "fe9755c8149d239696e010a0e92e2fab24da5aa5", + "e6b72cfddcaa930408e0d20800add718d44f30cc", "f341256a2cf06c968460034faa6d581f17af65a6", + "23791cd278e193ccedeb2ba80666c96e9152ae62", "f402a8f708787e8e97a9e096dbb47ccda9009c78", + "8dcac59f949b44e36ace133352ff4dad069e17e8", "f36f4c364f6f702f64cae7210f0d32a535e93bc2", + "8e110a0b274f6a11fc3773a1bd0933dca5cc962f", "745a2fe44d82669a5070791037da95a0d499ee3c", + "d1bda5b3a4ba7b9d7490d8511cce4a11bf76a8d9", "c2fb15525ed50bb529279aef0da1ad889a209760", + "ac5d73eea0dbddd6591c6254078018b894c74d9a", "150d6faa9cbb1ab06fda1d19606adc701633f24b", + "98f046a4f73ab3edd7f33d87279dc17c6009c844", "c0afac03a62235c223ef23a29e6f0f868aa89f4a", + "c19ecf91221af8e0d29e9d6693d8922989c8364d", "6cc04cca454e9610729586ce72ad276e8a8be962", + "5be4d55be3d8a516f8b9bfd8fa4b4b6fa072b3fb", "68e5dc0768d1143138b22a4dec7c0796211c515d", + "40b502f762b12d9008d67359a69427246dbf4590", "c7a0c0a1d86245d58f93578f5e71861850b20327", + "45e88dec3420a3081ed070a89974fade34a8cdd8", "a2cca480abdfd271d83ec53f32b9ca7c0534b1ea", + "eb5b56e00f7e3a3a93a151b04de322a54b478a1d", "ee726a3bd27fcd17373550a73f1265123f038e9b", + "ecbef3833e45ff4b51fcbd07be7ae083d0261106", "6ff3d878e2b0739195fef120f3e8920aae5b6c14", + "c8a254b1ae30d6d0372b9421cf164a2f92770bb3", "b6fe64434d331f92023b8ac4a0ae29a436ece676", + "6408dad003df5f902970c9a9b281bbdcab70513b", "b000f8b57d68abac68a8ba92d2e8af1f4cf04e3a", + "d70f6b719ab62e4d30947d46b48c4f86431e27f4", "d8b1b3fd5b14a37edfca20bafb53bb19b68034b5", + "54e2802c7dda0dc48297627148cb8080900b026c", "2a88231e4c45be17c7b59df395c88ad1de1f55fd", + "bd9541f3b7d18ba20fefb5725bd21f29c01a2fdf", "fb211cb3f8ec40bc8d34ef5da91173b85368ddb8", + "302c5986252fee81e024e43b3950ce0c03c1b19d", "c688626a38c6777646e4c5343903c5d3c39451d0", + "97e8d745c998aa9ca3c0402852cfbdebf38afeff", "512ca7ae1cc0043eee4b9fb24bc680e5e439a22c", + "8a1b18f3f1a52393779d7872132a261596718531", "f1f688ce752c4783753ded0c034de72056e5bf0a", + "cf404c790f1926de4fa92a90bc25bd3b89441ee9", "6f67fb37b601d81e607b4784b59a60f5da04ddc9", + "0358ae29870a0c0555826236454eee2a35460bb2", "267d40944d68c9fe031797824c83b7628ac21054", + "28999639cccbd7471536049556b7bb0445df968c", "96c339642cf640e219e83499b89b3f14c40c1a46", + "22ab4abcaeaab3dffd0a0eef49631c41feb35f01", "f3281e89aa93a9ff4b045c561960128ff02bd67f", + "ba7c3d3d19387b725dd93777e264c2d243c56985", "c06958105add69f9c1215959af1592100820f897", + "0246ace3db73d80516951169f697ab82de31935e", "3ed4aa3045ac0679828da5a36e6a159a76397b85", + "a9b3d7258a65a4ffc0188430f23a2b348bf3f71f", "3f80e737293fc727a2a5f5931afef5da8a108889", + "f5c4af7e6ca21899d603dc1c5d6dec724180a617", "83223c9cdcb837c03a1ee706a68c201c6f46aae1", + "7ebd59911482548ff28bc5257f092339747a9847", "a29e7d193abe5f905841b7dea04968b1c973775e", + "4df92c585a35462fd174a94fa6cc34cb104a3338", "480f1b18698079c345470f6eddf82dfe04f372bf", + "e93635e6ad4990b2037c1c42d96e5f4f5c382211", "f27588ba28c79a1abd36e9ab225006ef711bc2dc", + "0780639bded276d6de3f29a949974f9aad888f1e", "7649e12dadd9626c6aa55ce09bc5a77abbc1866c", + "10684acced018de4c9d795b4e3d17f85b831ed05", "1fd08bb5d65fb7deba4ce611c4e4ae61526f70f5", + "4ad405913be05105cfd8169727862962f73ba90c", "e622f46b8ab1a027717765dd7d392879941691c8", + "e53bc56ec46866426314e2214f39986f794f7ec7", "1ee4211e5bc419b24309bccc29d84c71d8153e33", + "13bf7fbafaffb9eb5305cc786ea5327b1bea8c84", "1b17359571b9a657d8c1c259f392bdca507f8048", + "f14bcf0c9d9d4441837089d9406a275ec7bae489", "650d0d424276a764869dfa6d5769e0d33dee29ab", + "9a51d909f6e0ea003b7bc31368889b34cbaf99db", "d25b13c8e67b29bedd33634693fd47107d79c386", + "d4791ab3fc5623770bd37a62f5847a3469ae62b5", "63cf95a96c6f9550a526c3f7c973b3bf77da1d99", + "0880631d8a31a2d656c214fd5326e86d72d99735", "c028799f0dc5cdd928ba41ed57758827d4c33960", + "09285b7758bced6de4de28adb839ce3707bc0801", "ff3ef6de1556d6a817b228634d9bd36c03f91b4c", + "7599af4e7f3c5038845232dc1c3460e757d29e76", "945656d9dd3e48ded82ae075761849b0b6de7d99", + "14d2b00bcdde27acd1e67707fdec920aaff41440", "bead8489f2f77870afbd190228724d8ba9525e77", + "ef888c5167e12709b33fcb9373476ce222d3a777", "11185717aa656979199228d4f134506f306ebe11", + "763619ad1c6fe756de2d6ea598f4a35b6b5d07f5", "de64195cf8f8e0a8a209993c12c37e6a0e9717fb", + "6fcd348a0817c5ba9f692e9f43b157ba53f61736", "184223aaddee8a795fb21a0298bca7cf9f24b86a", + "025924b72b369155a7dc79296b0fa1542b6cf66e", "00b6d09efea3b56ed30d3f626837c105415a2b9c", + "7fc27ae1ff5720a8632dc7180a62f93f0b240971", "68a1871cc2f4239c1e4c717d6cf5d50121aa24f7", + "d86a119c71e8f3db87ef84a3f2eaee3784e514eb", "bf6c04384c5ebbcbb7a2635e885ad48223000515", + "25df57cb358e1c55102d3eb05c633bbbc8d18ad7", "c2ed09db58daafd8711f0729136bcb4bdab4ee39", + "db9ff970f2394e5ddb70b05835e5cd2ceff1758f", "b5b0cdc6e2d0971f11ffb7b83eae62bd2f2e2a63", + "568a605393fdf0261752b980340b56a695993ca9", "53a7d40199c4cd3e0b61387708a5c4b2daaa6a04", + "d764e0dd4cc13227c90ea0dbdb7a17148a37362c", "2b0970782ff6fbc9b574d9af09c7f4387da71167", + "1a2f72aec014fc3b1c7a58660c15cdf8eb78c30a", "ea70291c158d9ac2e034f91837f3aebb9315e74f", + "ddeee68ef8756513d3c91e77705759282dc88f36", "c4c43e1c71a7693a257680b25c4cce05fa68139a", + "cb91e9590a1524dced13be78e13abde059a5358f", "8df5d8b31502fd506a59b0d15f61631dc5dc0d11", + "ef60833d075ac1aca48423ec296b18756a0907d6", "4bd51174d78af4c6afff4a3580c4380eaf53db03", + "f89f34968e6fb5c4400d4133e5b83eaad9ce1d81", "4926ebd2150fca495a5ebccb69d73a843c2a5c76", + "b7d7768027c43c4bb9178b59096058c827249ebe", "b18e01b7ab7b8fdd5e9093a4d10446ccb4c8b461", + "7c0cb0bf58b260d848be8b2fffc6ea81d45bd9ab", "65560b3113f1d79c241d64615c70a95a27a1295c", + "ef7de7cca843f688185aa44caaa11b8d9c5ba419", "577959df764fbe2295c7355420fb88e7259318f4", + "fb4e5fa6f2ea75bd25bf313675ace5ecfb09ce4b", "58c3bb9c95b2b18b54923c1c66a271c5103b5016", + "d15a9ce8720d18dfb1886de4ce23504c690f9849", "09b1961ea1fb0304ed7ad7c2c404af3113af532c", + "689943c3a6fe2d15706f588d5d192de09025f87e", "76c955116afc032044fa820077f0aa0077a9b221", + "e4e28d5880eb5a19eac4f2dafc94b87121315736", "f66dc5b1d3a36e518a2d7581534cf1eac1c0bb86", + "437d9bab241951c67ba4344ba60e7f8c8fe091fe", "d48f36d517120dec77f9f1383c6ff175e490957d", + "3595ca188c62c02f3a21616a40fce4063e5639e2", "9335fe0ddc8db4b8825b6a8e96f2ae314c3daea5", + "93b3baceba14eb826f0691ec927139eddd131418", "b081a6d644c99936ce2b53227aa949e61c1fc1a2", + "7c301a66f04090a5dfa9b97790c2398bebe8c658", "70e640c8b512f69cbf2d3815ade5e08acfdd6ea6", + "a5ff6189fcfa209df6073595ce8d048bb434142b", "865f86d6f58109166006154d4cb1f18d780afc7b", + "0b32bd194264f3d34f207e36da01747250f370b2", "160db72460b1d2cb57b71c5f5005c43b39344a08", + "ca17e8fc42c63b71a64943b2edc546cb483d4987", "62709ee11b06371980bbc365eade2cc09fb201e1", + "d42b892d83a19496adb8a51865d7786bf11cb51f", "151eda2239a8926cdfe11bddd08d4a68dfc06fce", + "48289268a75e7c551d8407da65a6a8d5e96bfeac", "76475e31fb20faa61548ca6c110bfaf0a8509333", + "ff45b9e590737c453377adfef746c419fd024214", "791336c9f92b920c36d87abfcd23717effeb2979", + "7cbdb028c46f69f69ad5b3870fb9976a339a88ea", "6dce9b85f8a4d116d780895772890e62ec554050", + "609a9ddb002e9e03ec1e01fc0d3273defd125f6a", "eb5cd823d83464a85d52abf6fb618ab6b0ceaf14", + "0b33f97dbe25690d5b395aec431714f326835761", "b3c0967b6cc938da94bbc24bbb4960c499eab996", + "55cc0375389bdbe9081930f6aed59e2d1b996d68", "3ab0dd9b211aa79f607aa18900021d331ffec108", + "8b8fc9dc6eaeab845218b9b42c9f1eadb85d9a76", "d95873f271a98500509514a3bd503e111b2eeff6", + "60c22d4758b2ab938d925ebca4195d388976ef0b", "4a71cb75be4c145468bfa41edeaec4cad48b35bb", + "674df8ed0814f769a01ea52a86b37a37db300fdc", "52cd156a808d638be19a621e77ce9d0b705ff959", + "893fae64443d115a0071088287f1047c9c85bd0c", "fe745f1136b3c2b4d5cc703cee8933de6357c502", + "7a9ed9c04bef62538916e432a74f44c034068c16", "c60540472bbd9473a78ffac3a071904552cfdd6e", + "dce6b37c643555216d24705885d35be6f1658323", "d25cd6e77dd4f8683333b68cd17e4ef161a10f82", + "ef59cd7b3e448c27b726dda2624e3a66bcb7c915", "caeb866326a82dacaf9b9ce21de180c99462b0ab", + "300bbd6765f353ca1f7a42c8db1edbe0fca9146f", "d476be09c2c9a67b065eb972d473c862d05520d7", + "f2692ec53c9eaeb976a81908a1ad9a8a6065eb56", "80a904cfc2637be47cf6db433896285c96642f28", + "b4e7c96d36534976b532f6a73d687ecebd20d3ce", "f442e491213973df215b4e5a13ac42dbf26a418d", + "4a990060a137b7405e90d674667b774c2ab3d9df", "ea028084a5ba60179ffcc5c2e7745a5137810674", + "946cd7b549d16cec7f31b99e31a5cd1f97ecb0a7", "f4d1570ca148a075ffea03034aa359a795a7a95d", + "b24a0df950d615d89a5c490856d9efe0c67e6807", "4a3a4973113c414789c798e3b279d04e68f18193", + "853861b52dff3d4e69cea17db319b578dfe36b57", "75cdf79a50fa9f54ee0abe8a7352983e4e15dc8f", + "d2172012e40b489a1b696aee59ee01beb94d2044", "c463cba55508e3410e6c938ca7f07cc49c69685f", + "298753c52149d58e36b63ae01437d7e9b1626550", "edf83ddead1b5bf94530280409c8f8d7a3566036", + "6ad9ebb63aaa22d5cac2c6ab68a8d044575531b8", "4c4e417f31a770c165848c34e9a1c03a88345923", + "01ac78c98a9e83a657895f330ac60b23df81aa19", "077b29278d6047812f8b65e158402f13dff2e638", + "6a3280cb4ecb45a4a949416106e707b4999e2491", "3de477994ebd5b7edf0a48ba6c5e799fe02a53f7", + "2ebbd703eb8308492ff7d0e3604b786931ad8617", "5fc0b82cdf849a6c424721cc5118a196c07caf43", + "851ed5018abedd4f527d5fc1f184016c2b9dcb20", "8271e8c30ed07e236d230df37635819f45b3adf5", + "794d1fbeb2f5f6e2ad81450602d53d232a2fcd66", "765e4955986ec0b20075669f5a7e2dca79a98ae9", + "f94426a7f4b9bd63aed0e75894ea80278dab4320", "c0006f951059f5aa58b4b898967d1dab70377178", + "414f4c053c7c9d294903806e748a132d54325f73", "a3dc4e32a5bb67c422822731ffbad447f2f89dd0", + "9dab95f8797ac80a22cd854761e2dd141be8f436", "c187e0b59cc8710d812cf24e9ad7191c5d0c6206", + "55cc88749fcd34f08c5d0fee8bee92dcaccfb010", "2c3d218a5428646b970895dd88e2f61739ba0732", + "4ad60738ad972e0cbca3e763b556937501c0973c", "543f72fea7ef066ea3111bdac8f0ac61134e0c24", + "a2bf3ec52c5d004e7e5329efdf6b5cd735618b87", "f483494815f89e80dc399163080052fff49031bb", + "3326ec457cfa6d30663eb0ce49a838d7bbb86415", "f6d5647f6b6923b5052619814a84b2e1bf8d0b85", + "f2dcd0fc70ca95f475b68f7335b2d0855534e5f1", "a50cd55ec2258557a20839a0d6219960c9b1a2b0", + "a34ae4d4326a0913d9af50546c558811565ec66a", "842e922b6252301d8df608a1a1a4a732686e2ff9", + "2b6cb59f7846a482432b4ac57fd960f73d2441ba", "e559e01ccf3cc35aee9732e3db97c23165f8a6d9", + "543085108d5513b9fe63eb96fb386ccc84a811d8", "c7474d0e9cfe5c1d8e56f3978eec6871706a38a8", + "fce05c355558c606a7ca815613c6ad3d8f8a1503", "30a17532024869bc353e28aa4c3da17dbbcab80f", + "13d54732573224d7879166a11857046c50e13626", "42868488cc892e63b13e7afbe157006ddc215258", + "94785fac47a5d2ce41d64269203f14de07995186", "82d752a38f9d5598685d58e5ce6080409b980442", + "96f99cdd47de4d5e4180c99bb7015fa77d329687", "e89a84828a38b552efd3095472eb39dd18be141c", + "5a37e8b4031b627270f06125f5c204eb28437c85", "30da9a8dfb0f65559be3f17a3d2f3be0cb297ba7", + "abba9bb98e64624543ac376fdce700ee5e33b392", "c19f87bb786c68bab429f594374c24d09dc98cd0", + "73754ef7ae4615cdfe881adea981bcc93b65e887", "44c25f0a9f3151efbb98056e1c2bae59f4aca0b3", + "51f465b5aa2a63568d0bd82232898b39fa429a14", "1ece6bad7a8d2492da84f5db5a94a9c81a94a3df", + "603617cb2ad5009c06dfc61432287ea41d4dd70a", "ec346b3231969799e3852cd7f4e951a25f3fbc0f", + "46af277dcaf4eed7cc96e021c96b266c24c1fcd3", "0965cb17b3b353174be301989ee8f5b77ab925da", + "b45319782274fd4841da3c70ad3b4754bf2b1b08", "2a2bcc98ea5b188049bd46b4e315ae55b359b7f2", + "cd09081ec384b747181bc2309a9485ee0888516c", "f636e931892c445a7ffb1aa0eae7499caa19932f", + "34ceb6f49765ac0d173d4a0cecc089f34dce2695", "9b18988eed2891623baa37468ef83dc6a9a93e14", + "087397f895ac78aa63b5b7413dd4c948cae4dc50", "98e1abd91d5502799931c25c1c19fa6605ff6dd1", + "279448c0f978c369797336bd660e457e2abccb3b", "20d76b4b81484d694731d6a2046661a025928bdd", + "7a69b783e3e747a0f7a5eff5cc23b9a00d53aa01", "f645296a6d289f80b9960679b8edfb6b0a4d96c7", + "9352d33e1d396110bedaf1906567a6d8139c98ad", "3a4cb9ed1c4e291a738f6a97ae82b7db616ed4f6", + "e97932a4ff925df81b844058a18f2da4f362edad", "2e9b3d733cb1d6a6fda98cfccfd179c35559671c", + "00e2b96be50f143abbaeaef1b42f502eb1c5eb38", "a3b8a036064eade54ac1a7e49a48b5228863f225", + "b050b6089b63a630ca1e102f27f1e33310bdc0f8", "91200850dde71fd5061890bae22cf6264f3a0ce9", + "c4c90fa787e28f241a6149c5706f67cb8c897c3b", "52cda4d77c07f76113b392291e947e02010a8ef8", + "949b21730e48bc645dca6c6a9dedf758035d0eed", "3e0509507736054281c0feb777b5beb62af98ed5", + "994993e838dda5b000ee4e75786a5a826f368f9f", "24e2e8a67ed9977de3c1373968d76736da3c800e", + "9c8374cdb61697a37bdc89b57ac5574af910138c", "7facfd4ee6d102ce1c2103b2bc0861865fffc07b", + "82f7d052e4d562b268730e8d66d4ac694d963f8d", "a5f79e4ae09eafe2892bff810ed21b1a7cbeff6a", + "7a66189f5b487392ddf3bde3b6b2f837711767c2", "756f813ee996e91c50dee3929035b81d5c2f5b7c", + "c0c98d3382858b9de30907003cb1b3f47bcb7e4c", "7a1f959994f3e8665589050ca2e07c5ed5579ff6", + "06944876fd90b41f439d337cfdbef2582caa4b56", "73e779de8b03c705726abbd686ab6f905fb2fe27", + "dd14f8315d4345a1482b6b5f581fefae03154e6f", "b4f629bddb661422c15c4b0459104b6a6387f3a1", + "a65dec2720ca334bb503079db5a5db4649c27c0a", "23d9c64b7d95c1d1c08ac83058dba6204537709c", + "0070ff46a8b72b79d011f59a770cdd2e3a7e52db", "39538c29808bc3b1c85b2f9555eb56296778e5b5", + "05235f2fda6b96b82d3b798f5c06d1c26fbf5556", "da9ac947e5272631161f7415583487cd5b81bd92", + "0afdc9abb0a921b0eb87256c34c2440d7bcdf748", "708eb61ae54de0fcccffafc686218a959fa0e12e", + "921d2ecf681cb2fa40d8cc9390591bdaeba5bddc", "d54e009ec45cfd47d90fa87926ff55f5668cd745", + "b66c3a482974a6950eb27fa2c83df33c0ddaee0f", "d11ee1996ea846c781624a6be16e2cf05c90a3ad", + "d1fafe49cdba1c135a157ab725caf6a73c44d413", "37add1833a4462b08f3d63ac3cfe2b29e8c19daa", + "54b9e091f1f4a2aecc9c3abef4785cd4116e59da", "2e7c5f1ca4868b1bf1681e852460d1a64f6730f4", + "fee6c7a1a74439ade1a4ba1102431cb16ffa40b4", "0b663b2c2e9813fd968ea4d53c27117a37af990a", + "2bcec290f45a8f707462a56e8cc468f5934fe57e", "2e77e56acb8a073a7164052b3015752f55748f13", + "56cc03e04970154b1ae9a3cb49eeb3d131118feb", "17819764b0cc55daa6f9a70e1b7cfbb082504a26", + "b9992bfe65ca57863517d1a44a5de4abff0d891a", "7cfd6ae2ae46bf951a84ab288fbf6381377e4fb5", + "44f321da361e4de492292e465ac2576625c2f04d", "14d021d38ede8d4fe6c55266ea1bc18471146ca2", + "dd5f008277ecf79acb0c32c88f459a46f556d235", "e712c5cfb5144ba8122f13b400324a3db3eb1131", + "2e3b8bdf70024acbd57dbd574b42d8081b22238f", "a27f805365b6074f96d0d72bd877158a6a2683fc", + "1777f7ceba2141c7725cae03325aad9015f5e0ba", "3ed72d7bfdaea45c7741c4adba22824729fccf93", + "1efd9d0cf3d575dceae0143ad80cdb36ae929f19", "ed1e3113a91cba66cb2bdd2fc62d221f0fe98720", + "56830484e8e3a9bb9c3c0e28d2e06c59c2f85814", "612f82971d281213f121bf3940fb3a22ef339b1a", + "9204cfa997dfafbe1d923cae857fd8772140d2b8", "bd7772cc8ad355a2edb8ca92d712bc6522f679e4", + "50013bba2f0c0daa2ba0601ad9d538b8945f49b3", "903166d464af4cbe28c3dcfe87e3b03d2d4361d4", + "498a0b6129cf4bbdfcd1fbe56f41a09711c81515", "4d82da411ae6922528a7e31ba0d9b7781ca7b140", + "d1a59d5be71487c46f1f13cca1329b890864e3a5", "0130298b9910153b918c24cd0305bf004c77f139", + "b4704585677546e3719b0167a48f7352c3135018", "bd2af11cf4b22ffaec8f0f81e41c7a200707984e", + "90bbdf000b6566e32e973a9ed13673babd3d5971", "53b76d4726ba14be0ff600d76f6e94af63bc76ac", + "98b92425f234019ae1a7d7fa763ee86d24a9d4b3", "402419c820cba29219805adb1b50909143962ab4", + "2507dd9d1d952d45d8e29e3da7d9b67261ddc8d6", "0eef7ee0455462714b4398429bc4e0b1e68aa807", + "c15af97c03e51bb5c7c23118564b21d37e41dc38", "5ff6527e38d98cdf140318d98828d7de099d1552", + "7d8d05ecfc3d87c0a798fbb28a7bd63d79317590", "d728b77adbe3f536b9211d9b26b0668cd796a549", + "ff936585cb1595e9b56e14b77c2a8605a312fc2f", "cee364e17aaa6f0d77dbcfe496b188de38123833", + "28f1af82eea595d5db3da22f4433c144fad620ef", "d5bc60e7980decfb5d5cd10ef9ea1bd0a3ce5fae", + "392ce37ce6411c30d436940506dc2cc8278f66e6", "38d291489369e7f222a2711e5afd7d6ac6afb574", + "479506b2cf7c7392c0a96d6e5d550556dfd20ba0", "c71e5676016de6a9b7dc27a123f403283fadec2b", + "5d03ade937932174a2f2b376efe6f761636a270f", "04302da411d5d79477463fa37de2f639afd5127f", + "d4e6cd7852b5b88d88afb1db0cc9d29d8484f143", "1dbdc1d3d9dba7ce5cac3bf7d2268ab237a4c5cb", + "3f263b23507300ad4dd7903e45876c2877e59ffb", "127d0081100e98843f16d26cbea527856d9a9ddb", + "dd5e22108dcceb35e1944813e2524ebfd0a53df9", "5d52758299b341021d135b5166614a46f3ecf2e3", + "a1b3da2881596c60f669fd80479e45a097b38707", "21a63be654b8e5df434d01d420ea2206c8d9a92b", + "36ea46c3da7dbc2bf4d304d81f86224dfe54444c", "29fcf933c2e58f6eb8fab04bf8c511f56896fe15", + "64737770ddca8995c64e1106bfb08ff0ff1fb144", "d8c191fa2b58539d83c7efc1c3346079ecdc566a", + "77f0910c39aa6108e0c2fdb32fe1832126c0489c", "af64db9a99bfd8ca4affe94dcd5cfea05e94e03c", + "71dd27e77432a2d5b446958707286d08097da244", "5ddedb0a995f06b295696a2a7b4551e404d0dd16", + "01f87385a24ece34ed8ddca801fd84f9c86e74e8", "09249256a87d4a90588b026d789fd3cb1aec856b", + "9240b4a3a946be297b435486d1f2a6d941422b93", "399ca25f86719994d4612fdb618983079d6a59ee", + "4db57eaeeec89f4a96d0f2727c15ff083482fbf2", "4a31d4c94a4c6ae379abe24c46817a1157eab07c", + "182f2002e4307f83c62e06dfa4164543fa7e814e", "97a0a7220c2b8e458641c3e6e56889254d2da049", + "3440a5e4b4e31f45c6adbae99b6214b8fbeebb2e", "6f94a7a5fe870501cdcea762e4a7a20631d48d62", + "9da3107576333b6c7f3e878e508e94a4c764e45e", "20cb94f57147f8372022885ac2356e464fdaf7e3", + "68803fd3ffbf592c0432eadf45fffa22e5afa8dc", "c538f4c3d9ed532f02458a7c863ddd549960902d", + "cd14b737781fb608698437bd2e6d20d691093874", "2db66f7bda4464a6f5342f5e021c29a38d42068d", + "37b12212ecc3a32ede8aa04eb0065fa80e3e821d", "887306035bf8e3f0963a254a6e46b52b55a04fcf", + "8747b40002f4a24a2330ee27f36ecc0880083827", "ee262c59ff4c66f77562ef879569193543838393", + "8282282c974675b044c071faf25996f4b6e4ebc3", "f9ca90f109c32ff5889c0c9d84ff11f9fb5b5924", + "e530847cf2aaa328e800919e6eed9739521a9b50", "ad726687605b280a0c190153654f4751ebba4c7c", + "93f6ba66b73bf972a3df6ba722a1d56782c59d47", "b40be55c3b541c1c3a6b46de02f45e3ab9dfad47", + "95ee9b8acb715971bd1abc137a9d54bc88139b8a", "61dc80eef767360c6d48572ca1bda65c693d5362", + "ba44c4692965b0d694c8fd015f1ec3fa4dfcc696", "418a1501e2a14efb4ab940cad4c898f09f92ee83", + "026029e19d4c6f0bcf239abe607026a5a27f5b86", "8cbd5f3c989185d9474f7b5767d7e6adabb2015c", + "55163a23b71b9a3d63f6e4695b07ebd07064c61f", "d9bde5278517a57ff7714edf98ab2361d1663219", + "9ad5f0871e1df9582ea14bf7ba22350086fcd055", "c57f5ae0aaa74eea99032b45f0dc5c927813d8b5", + "3e3f6f7661780b696501a397d8e95ec446fd1fc3", "03d94679ae1908d9a6f6a38c00ce7988c5afbcb1", + "4c8f526d899bf04b4ce3e088302680da64499e5d", "271ddfcd505226549f7674b7e539efc96c8be5b8", + "25708b2d11167437e6de01fa0406dfd3c26bd8d4", "dcf94a3cc15a85bc1dc2fc7e1751e6a343609049", + "f65dd5dea2e8bf5c90b8b093b4ae2db11fce3b5c", "b3722ffbf1e3287cee8109c82b7ab13d187a3d7b", + "ec3af0efeea2836f6a2fc5a2390232ff10e17cf0", "c7176c1b7df0676b474b4786a7e6ba25650df8aa", + "ad97c1b6ab4dcd3d4917cb59427490508523daf4", "7a8e2f675709ba76674a2903653b07d348de32d6", + "bdd754c4219ceadebaebab82c1dacb0a6d819e24", "31df56efa866faac4fced90b742b1800261ef46b", + "1e77fd0a0bd78293212c518291333d3554136b41", "99f906bf47a41245895867f76ad38fc9ca88921f", + "49dd5ec3406bda057407abe5c57e6a027b8c502d", "f36f65ae5480acecfb85d9b8d7a79603853f2115", + "5a29a9db5779e61efafc338b00be9d22efe2a181", "6b28f73e9097b7a7966aa9add138e3dc14c52e75", + "92571e06644eee1606759deeccb9875fdb454b81"], "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80632' + Content-Type: + - application/json + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"208de547-5ee2-4294-955c-5595cd0f7940","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"langsmith-py","description":null,"created":"2026-04-03T20:35:06.233Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"52d142c7-3e4f-47b6-9d00-688befc3eb02","project_id":"208de547-5ee2-4294-955c-5595cd0f7940","name":"langsmith-aeval-21b39aac","description":null,"created":"2026-04-03T22:20:45.963Z","repo_info":{"commit":"5c35051a0102f850f2e09c66f9376a0fc658ed9f","branch":"main","tag":null,"dirty":true,"author_name":"Abhijeet + Prasad","author_email":"abhijeet@braintrustdata.com","commit_message":"fix(anthropic): + capture server-side tool usage metrics (#192)\n\nFlatten Anthropic usage.server_tool_use + into Braintrust span metrics so\nserver-side tool invocations are preserved + for tracing and cost analysis.\n\nAdd regression tests using a red/green workflow + to cover dict-backed span\nlogging and object-backed usage extraction.\n\nFixes + #171","commit_time":"2026-04-02T21:10:00-04:00","git_diff":"diff --git a/py/noxfile.py + b/py/noxfile.py\nindex c92395da..1083ca5b 100644\n--- a/py/noxfile.py\n+++ + b/py/noxfile.py\n@@ -77,6 +77,7 @@ VENDOR_PACKAGES = (\n \"opentelemetry-exporter-otlp-proto-http\",\n \"google.genai\",\n \"google.adk\",\n+ \"langsmith\",\n \"temporalio\",\n + )\n \n@@ -104,6 +105,7 @@ GENAI_VERSIONS = (LATEST,)\n DSPY_VERSIONS = (LATEST,)\n + GOOGLE_ADK_VERSIONS = (LATEST, \"1.14.1\")\n LANGCHAIN_VERSIONS = (LATEST, + \"0.3.28\")\n+LANGSMITH_VERSIONS = (LATEST, \"0.7.12\")\n OPENROUTER_VERSIONS + = (LATEST, \"0.6.0\")\n # temporalio 1.19.0+ requires Python >= 3.10; skip + Python 3.9 entirely\n TEMPORAL_VERSIONS = (LATEST, \"1.20.0\", \"1.19.0\")\n@@ + -235,6 +237,17 @@ def test_langchain(session, version):\n _run_core_tests(session)\n + \n \n+@nox.session()\n+@nox.parametrize(\"version\", LANGSMITH_VERSIONS, ids=LANGSMITH_VERSIONS)\n+def + test_langsmith(session, version):\n+ \"\"\"Test LangSmith integration.\"\"\"\n+ _install_test_deps(session)\n+ _install(session, + \"langsmith\", version)\n+ _install(session, \"langchain-core\")\n+ _install(session, + \"langchain-openai\")\n+ _run_tests(session, f\"{INTEGRATION_DIR}/langsmith/test_langsmith.py\")\n+\n+\n + @nox.session()\n @nox.parametrize(\"version\", OPENAI_VERSIONS, ids=OPENAI_VERSIONS)\n + def test_openai(session, version):\n@@ -371,9 +384,8 @@ def pylint(session):\n session.install(\"pydantic_ai>=1.10.0\")\n session.install(\"google-adk\")\n session.install(\"opentelemetry.instrumentation.openai\")\n- # + langsmith is needed for the langsmith_wrapper module but not in VENDOR_PACKAGES\n # + langchain-core, langchain-openai, langchain-anthropic are needed for the langchain + integration\n- session.install(\"langsmith\", \"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n+ session.install(\"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n \n result = session.run(\"git\", \"ls-files\", + \"**/*.py\", silent=True, log=False)\n files = [path for path in result.strip().splitlines() + if path not in GENERATED_LINT_EXCLUDES]\ndiff --git a/py/src/braintrust/auto.py + b/py/src/braintrust/auto.py\nindex dc44c7d2..6cf8597c 100644\n--- a/py/src/braintrust/auto.py\n+++ + b/py/src/braintrust/auto.py\n@@ -16,6 +16,7 @@ from braintrust.integrations + import (\n DSPyIntegration,\n GoogleGenAIIntegration,\n LangChainIntegration,\n+ LangSmithIntegration,\n LiteLLMIntegration,\n OpenRouterIntegration,\n PydanticAIIntegration,\n@@ + -52,6 +53,7 @@ def auto_instrument(\n dspy: bool = True,\n adk: bool + = True,\n langchain: bool = True,\n+ langsmith: bool = True,\n ) -> + dict[str, bool]:\n \"\"\"\n Auto-instrument supported AI/ML libraries + for Braintrust tracing.\n@@ -75,6 +77,7 @@ def auto_instrument(\n dspy: + Enable DSPy instrumentation (default: True)\n adk: Enable Google ADK + instrumentation (default: True)\n langchain: Enable LangChain instrumentation + (default: True)\n+ langsmith: Enable LangSmith instrumentation (default: + True)\n \n Returns:\n Dict mapping integration name to whether + it was successfully instrumented.\n@@ -146,6 +149,8 @@ def auto_instrument(\n results[\"adk\"] + = _instrument_integration(ADKIntegration)\n if langchain:\n results[\"langchain\"] + = _instrument_integration(LangChainIntegration)\n+ if langsmith:\n+ results[\"langsmith\"] + = _instrument_integration(LangSmithIntegration)\n \n return results\n + \ndiff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py\nindex + 0062ec77..2dca1110 100644\n--- a/py/src/braintrust/integrations/__init__.py\n+++ + b/py/src/braintrust/integrations/__init__.py\n@@ -6,6 +6,7 @@ from .claude_agent_sdk + import ClaudeAgentSDKIntegration\n from .dspy import DSPyIntegration\n from + .google_genai import GoogleGenAIIntegration\n from .langchain import LangChainIntegration\n+from + .langsmith import LangSmithIntegration\n from .litellm import LiteLLMIntegration\n + from .openrouter import OpenRouterIntegration\n from .pydantic_ai import PydanticAIIntegration\n@@ + -21,6 +22,7 @@ __all__ = [\n \"GoogleGenAIIntegration\",\n \"LiteLLMIntegration\",\n \"LangChainIntegration\",\n+ \"LangSmithIntegration\",\n \"OpenRouterIntegration\",\n \"PydanticAIIntegration\",\n + ]\ndiff --git a/py/src/braintrust/wrappers/langsmith_wrapper.py b/py/src/braintrust/wrappers/langsmith_wrapper.py\nindex + b22117df..7d4170fd 100644\n--- a/py/src/braintrust/wrappers/langsmith_wrapper.py\n+++ + b/py/src/braintrust/wrappers/langsmith_wrapper.py\n@@ -1,518 +1,4 @@\n-\"\"\"\n-Braintrust + integration for LangSmith - provides a migration path from LangSmith to Braintrust.\n+from + braintrust.integrations.langsmith import setup_langsmith # noqa: F401\n \n-This + module patches LangSmith''s tracing and evaluation APIs to use Braintrust + under the hood,\n-allowing users to migrate with minimal code changes.\n \n-Usage:\n- ```python\n- import + os\n-\n- # Enable LangSmith tracing and set project name (used by both + services)\n- os.environ.setdefault(\"LANGCHAIN_TRACING_V2\", \"true\")\n- os.environ.setdefault(\"LANGCHAIN_PROJECT\", + \"my-project\")\n-\n- from braintrust.wrappers.langsmith_wrapper import + setup_langsmith\n-\n- # Call setup BEFORE importing from langsmith\n- # + project_name defaults to LANGCHAIN_PROJECT env var\n- setup_langsmith()\n-\n- # + Continue using langsmith imports - they now use Braintrust\n- from langsmith + import traceable, Client\n-\n- @traceable\n- def my_function(inputs: + dict) -> dict:\n- return {\"result\": inputs[\"x\"] * 2}\n-\n- client + = Client()\n- results = client.evaluate(\n- my_function,\n- data=[{\"inputs\": + {\"x\": 1}, \"outputs\": {\"result\": 2}}],\n- evaluators=[my_evaluator],\n- )\n- ```\n-\n- Set + BRAINTRUST_STANDALONE=1 to completely replace LangSmith with Braintrust\n- (no + LangSmith code runs). Otherwise, both services run in tandem.\n-\"\"\"\n-\n-import + inspect\n-import logging\n-import os\n-from typing import Any, Callable, Dict, + Iterable, Iterator, List, Optional, ParamSpec, TypeVar\n-\n-from braintrust.framework + import EvalCase\n-from braintrust.logger import NOOP_SPAN, current_span, init_logger, + traced\n-from wrapt import wrap_function_wrapper\n-\n-\n-logger = logging.getLogger(__name__)\n-\n-# + Global list to store Braintrust eval results when running in tandem mode\n-_braintrust_eval_results: + List[Any] = []\n-\n-# TODO: langsmith.test/unit/expect, langsmith.AsyncClient, + trace\n-__all__ = [\n- \"setup_langsmith\",\n- \"wrap_traceable\",\n- \"wrap_client\",\n- \"wrap_evaluate\",\n- \"wrap_aevaluate\",\n- \"get_braintrust_results\",\n- \"clear_braintrust_results\",\n-]\n-\n-F + = TypeVar(\"F\", bound=Callable[..., Any])\n-P = ParamSpec(\"P\")\n-R = TypeVar(\"R\")\n-\n-\n-def + get_braintrust_results() -> List[Any]:\n- \"\"\"Get all Braintrust eval + results collected during tandem mode.\"\"\"\n- return _braintrust_eval_results.copy()\n-\n-\n-def + clear_braintrust_results() -> None:\n- \"\"\"Clear all stored Braintrust + eval results.\"\"\"\n- _braintrust_eval_results.clear()\n-\n-\n-def setup_langsmith(\n- api_key: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- project_name: + Optional[str] = None,\n- standalone: bool = False,\n-) -> bool:\n- \"\"\"\n- Setup + Braintrust integration with LangSmith.\n-\n- This patches LangSmith''s + @traceable, Client.evaluate(), and aevaluate()\n- to use Braintrust under + the hood.\n-\n- Args:\n- api_key: Braintrust API key (optional, + can use env var BRAINTRUST_API_KEY)\n- project_id: Braintrust project + ID (optional)\n- project_name: Braintrust project name (optional, falls + back to LANGCHAIN_PROJECT\n- env var, then BRAINTRUST_PROJECT + env var)\n- standalone: If True, completely replace LangSmith with + Braintrust (no LangSmith\n- code runs). If False (default), + run both LangSmith and Braintrust\n- in tandem.\n-\n- Returns:\n- True + if setup was successful, False otherwise\n- \"\"\"\n- # Use LANGCHAIN_PROJECT + as fallback for project_name to keep both services in sync\n- if project_name + is None:\n- project_name = os.environ.get(\"LANGCHAIN_PROJECT\")\n-\n- span + = current_span()\n- if span == NOOP_SPAN:\n- init_logger(project=project_name, + api_key=api_key, project_id=project_id)\n-\n- try:\n- import langsmith\n-\n- langsmith.traceable + = wrap_traceable(langsmith.traceable, standalone=standalone)\n- wrap_client(langsmith.Client, + project_name=project_name, project_id=project_id, standalone=standalone)\n- langsmith.evaluate + = wrap_evaluate(\n- langsmith.evaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n- langsmith.aevaluate + = wrap_aevaluate(\n- langsmith.aevaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n-\n- logger.info(\"LangSmith + integration with Braintrust enabled\")\n- return True\n-\n- except + ImportError as e:\n- logger.error(f\"Failed to import langsmith: {e}\")\n- logger.error(\"langsmith + is not installed. Please install it with: pip install langsmith\")\n- return + False\n-\n-\n-def wrap_traceable(traceable: F, standalone: bool = False) -> + F:\n- \"\"\"\n- Wrap langsmith.traceable to also use Braintrust''s @traced + decorator.\n-\n- Args:\n- traceable: The langsmith.traceable function\n- standalone: + If True, replace LangSmith tracing entirely with Braintrust.\n- If + False, add Braintrust tracing alongside LangSmith tracing.\n-\n- Returns:\n- The + wrapped traceable function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(traceable):\n- return traceable\n-\n- def traceable_wrapper(*args: + Any, **kwargs: Any) -> Any:\n- # Handle both @traceable and @traceable(...) + patterns\n- func = args[0] if args and callable(args[0]) else None\n-\n- def + decorator(fn: Callable[P, R]) -> Callable[P, R]:\n- span_name = + kwargs.get(\"name\") or fn.__name__\n-\n- # Conditionally apply + LangSmith decorator first\n- if not standalone:\n- fn + = traceable(fn, **kwargs)\n-\n- # Always apply Braintrust tracing\n- return + traced(name=span_name)(fn) # type: ignore[return-value]\n-\n- if func + is not None:\n- return decorator(func)\n- return decorator\n-\n- traceable_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return traceable_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_client(\n- Client: Any, project_name: + Optional[str] = None, project_id: Optional[str] = None, standalone: bool = + False\n-) -> Any:\n- \"\"\"\n- Wrap langsmith.Client to redirect evaluate() + and aevaluate() to Braintrust''s Eval.\n-\n- Args:\n- Client: The + langsmith.Client class\n- project_name: Braintrust project name to + use for evaluations\n- project_id: Braintrust project ID to use for + evaluations\n- standalone: If True, only run Braintrust. If False, + run both LangSmith and Braintrust.\n-\n- Returns:\n- The Client + class (modified in place)\n- \"\"\"\n-\n- if hasattr(Client, \"evaluate\") + and not _is_patched(Client.evaluate):\n- wrap_function_wrapper(\n- Client,\n- \"evaluate\",\n- make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.evaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- if hasattr(Client, \"aevaluate\") + and not _is_patched(Client.aevaluate):\n- wrap_function_wrapper(\n- Client,\n- \"aevaluate\",\n- make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.aevaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- return Client\n-\n-\n-def make_evaluate_wrapper(\n- *, + project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: + bool = False\n-):\n- def evaluate_wrapper(wrapped: Any, instance: Any, + args: Any, kwargs: Any) -> Any:\n- result = None\n- if not standalone:\n- result + = wrapped(*args, **kwargs)\n-\n- try:\n- result = _run_braintrust_eval(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + evaluate failed: {e}\")\n-\n- return result\n-\n- return evaluate_wrapper\n-\n-\n-def + make_aevaluate_wrapper(\n- *, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-):\n- async def aevaluate_wrapper(wrapped: + Any, instance: Any, args: Any, kwargs: Any) -> Any:\n- result = None\n- if + not standalone:\n- result = await wrapped(*args, **kwargs)\n-\n- try:\n- result + = await _run_braintrust_eval_async(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + aevaluate failed: {e}\")\n-\n- return result\n-\n- return aevaluate_wrapper\n-\n-\n-def + wrap_evaluate(\n- evaluate: F, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.evaluate to redirect to Braintrust''s Eval.\n-\n- Args:\n- evaluate: + The langsmith.evaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to + use for evaluations\n- standalone: If True, only run Braintrust. If + False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + wrapped evaluate function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(evaluate):\n- return evaluate\n-\n- evaluate_wrapper + = make_evaluate_wrapper(standalone=standalone, project_name=project_name, + project_id=project_id)\n- evaluate_wrapper._braintrust_patched = True # + type: ignore[attr-defined]\n- return evaluate_wrapper # type: ignore[return-value]\n-\n-\n-def + wrap_aevaluate(\n- aevaluate: F,\n- project_name: Optional[str] = None,\n- project_id: + Optional[str] = None,\n- standalone: bool = False,\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.aevaluate to redirect to Braintrust''s EvalAsync.\n-\n- Args:\n- aevaluate: + The langsmith.aevaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to + use for evaluations\n- standalone: If True, only run Braintrust. If + False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + wrapped aevaluate function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(aevaluate):\n- return aevaluate\n-\n- aevaluate_wrapper + = make_aevaluate_wrapper(standalone=standalone, project_name=project_name, + project_id=project_id)\n- aevaluate_wrapper._braintrust_patched = True # + type: ignore[attr-defined]\n- return aevaluate_wrapper # type: ignore[return-value]\n-\n-\n-def + _is_patched(obj: Any) -> bool:\n- return getattr(obj, \"_braintrust_patched\", + False)\n-\n-\n-# =============================================================================\n-# + Braintrust evaluation logic\n-# =============================================================================\n-\n-\n-def + _run_braintrust_eval(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust Eval with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import Eval\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + Eval(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-async + def _run_braintrust_eval_async(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust EvalAsync with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import EvalAsync\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + await EvalAsync(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-# + =============================================================================\n-# + Data conversion helpers\n-# =============================================================================\n-\n-\n-def + _wrap_output(output: Any) -> Dict[str, Any]:\n- \"\"\"Wrap non-dict outputs + the same way LangSmith does.\"\"\"\n- if not isinstance(output, dict):\n- return + {\"output\": output}\n- return output\n-\n-\n-def _make_braintrust_scorer(\n- evaluator: + Callable[..., Any],\n-) -> Callable[..., Any]:\n- \"\"\"\n- Create a + Braintrust scorer from a LangSmith evaluator.\n-\n- Always runs the evaluator + through Braintrust for full tracing (span duration, child LLM calls, etc.).\n- \"\"\"\n- evaluator_name + = getattr(evaluator, \"__name__\", \"score\")\n-\n- def braintrust_scorer(input: + Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any:\n- from + braintrust.score import Score\n-\n- # Run the evaluator with LangSmith''s + signature\n- # LangSmith evaluators use: (inputs, outputs, reference_outputs) + -> bool | dict\n- # LangSmith auto-wraps non-dict outputs as {\"output\": + value}\n- outputs = _wrap_output(output)\n-\n- # expected is + the real LangSmith Example object passed through from data loading\n- reference_outputs + = expected.outputs if hasattr(expected, \"outputs\") else expected\n-\n- result + = evaluator(input, outputs, reference_outputs)\n-\n- return Score(\n- name=result.get(\"key\", + evaluator_name),\n- score=result.get(\"score\"),\n- metadata=result.get(\"metadata\", + {}),\n- )\n-\n- braintrust_scorer.__name__ = evaluator_name\n- return + braintrust_scorer\n-\n-\n-def _convert_langsmith_data(data: Any) -> Callable[[], + Iterator[EvalCase[Any, Any]]]:\n- \"\"\"Convert LangSmith data format to + Braintrust data format.\"\"\"\n-\n- def load_data() -> Iterator[EvalCase[Any, + Any]]:\n- # Determine the source iterable without loading everything + into memory\n- source: Iterable[Any]\n- if callable(data):\n- source + = data() # type: ignore\n- elif isinstance(data, str):\n- # + Load examples from LangSmith dataset by name\n- try:\n- from + langsmith import Client # pylint: disable=import-error\n-\n- client + = Client()\n- source = client.list_examples(dataset_name=data)\n- except + Exception as e:\n- logger.warning(f\"Failed to load LangSmith + dataset ''{data}'': {e}\")\n- return\n- elif hasattr(data, + \"__iter__\"):\n- source = data\n- else:\n- source + = [data]\n-\n- # Process items as a generator - yield one at a time\n- for + item in source:\n- # Pass through LangSmith Example objects directly\n- if + hasattr(item, \"inputs\"):\n- yield EvalCase(\n- input=item.inputs,\n- expected=item, # + Pass the whole Example object\n- metadata=getattr(item, + \"metadata\", None),\n- )\n- elif isinstance(item, + dict):\n- if \"inputs\" in item:\n- # LangSmith + dict format\n- yield EvalCase(\n- input=item[\"inputs\"],\n- expected=item, # + Pass the whole dict\n- metadata=item.get(\"metadata\"),\n- )\n- elif + \"input\" in item:\n- # Braintrust format\n- yield + EvalCase(\n- input=item[\"input\"],\n- expected=item.get(\"expected\"),\n- metadata=item.get(\"metadata\"),\n- )\n- else:\n- yield + EvalCase(input=item)\n- else:\n- yield EvalCase(input=item)\n-\n- return + load_data\n-\n-\n-def _make_braintrust_task(target: Callable[..., Any]) -> + Callable[..., Any]:\n- \"\"\"Convert a LangSmith target function to Braintrust + task format.\"\"\"\n-\n- def task_fn(task_input: Any, hooks: Any) -> Any:\n- if + isinstance(task_input, dict):\n- # Try to get the original function''s + signature (unwrap decorators)\n- unwrapped = inspect.unwrap(target)\n-\n- try:\n- sig + = inspect.signature(unwrapped)\n- params = list(sig.parameters.keys())\n- if + len(params) == 1:\n- return target(task_input)\n- if + all(p in task_input for p in params):\n- return target(**task_input)\n- return + target(task_input)\n- except (ValueError, TypeError):\n- # + Fallback: try kwargs first, then single arg\n- try:\n- return + target(**task_input)\n- except TypeError:\n- return + target(task_input)\n- return target(task_input)\n-\n- return task_fn\n+__all__ + = [\"setup_langsmith\"]\ndiff --git a/py/src/braintrust/wrappers/test_langsmith_wrapper.py + b/py/src/braintrust/wrappers/test_langsmith_wrapper.py\ndeleted file mode + 100644\nindex c25256c2..00000000\n--- a/py/src/braintrust/wrappers/test_langsmith_wrapper.py\n+++ + /dev/null\n@@ -1,341 +0,0 @@\n-# pyright: reportPrivateUsage=false\n-# pyright: + reportMissingParameterType=false\n-# pyright: reportUnknownParameterType=false\n-# + pyright: reportUnknownArgumentType=false\n-# pyright: reportUnknownMemberType=false\n-# + pylint: disable=protected-access\n-\n-\"\"\"\n-Tests for the LangSmith wrapper + to ensure compatibility with LangSmith''s API.\n-\"\"\"\n-\n-from braintrust.wrappers.langsmith_wrapper + import (\n- _convert_langsmith_data,\n- _is_patched,\n- _make_braintrust_scorer,\n- _make_braintrust_task,\n- wrap_aevaluate,\n- wrap_client,\n- wrap_traceable,\n-)\n-\n-\n-def + test_is_patched_false():\n- \"\"\"Test that _is_patched returns False for + unpatched objects.\"\"\"\n-\n- def unpatched():\n- pass\n-\n- assert + _is_patched(unpatched) is False\n-\n-\n-def test_is_patched_true():\n- \"\"\"Test + that _is_patched returns True for patched objects.\"\"\"\n-\n- def patched():\n- pass\n-\n- patched._braintrust_patched + = True # type: ignore\n-\n- assert _is_patched(patched) is True\n-\n-\n-def + test_make_braintrust_scorer_dict_result():\n- \"\"\"Test converting a LangSmith + evaluator that returns a dict.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- return {\"key\": \"accuracy\", \"score\": + 0.9, \"metadata\": {\"note\": \"good\"}}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- # + Create a mock Example object\n- class MockExample:\n- outputs = + {\"y\": 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": 2}, + expected=MockExample())\n-\n- assert result.name == \"accuracy\"\n- assert + result.score == 0.9\n- assert result.metadata == {\"note\": \"good\"}\n-\n-\n-def + test_make_braintrust_scorer_numeric_result():\n- \"\"\"Test converting + a LangSmith evaluator that returns a numeric score in a dict.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return + {\"score\": 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class MockExample:\n- outputs + = {\"y\": 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": + 2}, expected=MockExample())\n-\n- assert result.name == \"langsmith_evaluator\"\n- assert + result.score == 1.0\n-\n-\n-def test_make_braintrust_scorer_with_plain_dict_expected():\n- \"\"\"Test + converting a LangSmith evaluator with plain dict as expected.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return + {\"score\": 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n- result = converted(input={\"x\": + 1}, output={\"y\": 2}, expected={\"y\": 2})\n-\n- assert result.name == + \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def test_convert_langsmith_data_from_list():\n- \"\"\"Test + converting LangSmith data from a list of dicts.\"\"\"\n- data = [\n- {\"inputs\": + {\"x\": 1}, \"outputs\": {\"y\": 2}},\n- {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- assert result[1].input + == {\"x\": 2}\n- assert result[1].expected == {\"inputs\": {\"x\": 2}, + \"outputs\": {\"y\": 4}}\n-\n-\n-def test_convert_langsmith_data_from_callable():\n- \"\"\"Test + converting LangSmith data from a callable.\"\"\"\n-\n- def data_generator():\n- yield + {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- yield {\"inputs\": + {\"x\": 2}, \"outputs\": {\"y\": 4}}\n-\n- data_fn = _convert_langsmith_data(data_generator)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n-\n-\n-def test_convert_langsmith_data_with_example_objects():\n- \"\"\"Test + converting LangSmith data with Example-like objects.\"\"\"\n-\n- class + MockExample:\n- def __init__(self, inputs, outputs):\n- self.inputs + = inputs\n- self.outputs = outputs\n-\n- data = [\n- MockExample(inputs={\"x\": + 1}, outputs={\"y\": 2}),\n- MockExample(inputs={\"x\": 2}, outputs={\"y\": + 4}),\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole Example object is passed as expected\n- assert + result[0].expected.inputs == {\"x\": 1}\n- assert result[0].expected.outputs + == {\"y\": 2}\n-\n-\n-def test_make_braintrust_task_with_dict_input():\n- \"\"\"Test + that task function handles dict inputs correctly.\"\"\"\n-\n- def target_fn(inputs):\n- return + inputs[\"x\"] * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == 10\n-\n-\n-def test_make_braintrust_task_with_kwargs_expansion():\n- \"\"\"Test + that task function expands dict kwargs when signature matches.\"\"\"\n-\n- def + target_fn(x, y):\n- return x + y\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 2, \"y\": 3}, None)\n-\n- assert result == 5\n-\n-\n-def + test_make_braintrust_task_simple_input():\n- \"\"\"Test that task function + handles simple inputs.\"\"\"\n-\n- def target_fn(inp):\n- return + inp * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result = task(5, + None)\n-\n- assert result == 10\n-\n-\n-class TestWrapTraceable:\n- \"\"\"Tests + for wrap_traceable functionality.\"\"\"\n-\n- def test_wrap_traceable_returns_wrapper(self):\n- \"\"\"Test + that wrap_traceable returns a wrapped version.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable, + standalone=False)\n- assert callable(wrapped)\n- assert _is_patched(wrapped)\n-\n- def + test_wrap_traceable_standalone_mode(self):\n- \"\"\"Test that wrap_traceable + works in standalone mode.\"\"\"\n-\n- def mock_traceable(func, **kwargs):\n- return + func\n-\n- wrapped = wrap_traceable(mock_traceable, standalone=True)\n- assert + callable(wrapped)\n- assert _is_patched(wrapped)\n-\n-\n-class TestWrapFunctions:\n- \"\"\"Tests + for the wrap_* functions.\"\"\"\n-\n- def test_wrap_functions_exist(self):\n- \"\"\"Test + that wrap functions are callable.\"\"\"\n- assert callable(wrap_traceable)\n- assert + callable(wrap_client)\n- assert callable(wrap_aevaluate)\n-\n- def + test_wrap_traceable_returns_patched_function(self):\n- \"\"\"Test that + wrap_traceable returns a patched function.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable)\n- assert + _is_patched(wrapped)\n-\n- def test_wrap_traceable_skips_if_already_patched(self):\n- \"\"\"Test + that wrap_traceable skips if already patched.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- mock_traceable._braintrust_patched + = True # type: ignore\n-\n- result = wrap_traceable(mock_traceable)\n- # + Should return the same function\n- assert result is mock_traceable\n-\n- def + test_wrap_client_sets_flag(self):\n- \"\"\"Test that wrap_client sets + the patched flag.\"\"\"\n-\n- class MockClient:\n- def evaluate(self, + *args, **kwargs):\n- return \"original\"\n-\n- wrap_client(MockClient)\n- assert + _is_patched(MockClient.evaluate)\n-\n- def test_wrap_aevaluate_returns_patched_function(self):\n- \"\"\"Test + that wrap_aevaluate returns a patched function.\"\"\"\n-\n- async def + mock_aevaluate(*args, **kwargs):\n- pass\n-\n- wrapped = + wrap_aevaluate(mock_aevaluate)\n- assert _is_patched(wrapped)\n-\n-\n-class + TestTandemModeIntegration:\n- \"\"\"Integration tests for tandem mode (LangSmith + + Braintrust together).\"\"\"\n-\n- def test_make_braintrust_task_with_inputs_parameter(self):\n- \"\"\"Test + that task handles LangSmith''s required ''inputs'' parameter name.\"\"\"\n-\n- def + target_fn(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == {\"result\": 10}\n-\n- def + test_convert_langsmith_data_handles_different_output_types(self):\n- \"\"\"Test + that data conversion handles various output types.\"\"\"\n- data = + [\n- {\"inputs\": {\"x\": 1}, \"outputs\": 2}, # outputs is int, + not dict\n- {\"inputs\": {\"x\": 2}, \"outputs\": {\"result\": + 4}}, # outputs is already dict\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- # Both should work - Braintrust''s EvalCase + accepts any type for expected\n- assert len(result) == 2\n- assert + result[0].input == {\"x\": 1}\n- assert result[1].input == {\"x\": + 2}\n-\n- def test_make_braintrust_scorer_handles_wrapped_outputs(self):\n- \"\"\"Test + that scorers handle output wrapping correctly.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- # outputs will be wrapped as {\"output\": + value} for non-dict results\n- actual = outputs.get(\"output\", + outputs)\n- expected = (\n- reference_outputs.get(\"output\", + reference_outputs)\n- if isinstance(reference_outputs, dict)\n- else + reference_outputs\n- )\n- return {\"key\": \"match\", + \"score\": 1.0 if actual == expected else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class + MockExample:\n- outputs = {\"output\": 42}\n-\n- # Test + with wrapped output\n- result = converted(input={\"x\": 1}, output=42, + expected=MockExample())\n- assert result.name == \"match\"\n- assert + result.score == 1.0\n-\n-\n-class TestDataConversion:\n- \"\"\"Tests for + data conversion utilities.\"\"\"\n-\n- def test_convert_data_with_braintrust_format(self):\n- \"\"\"Test + that Braintrust format is properly handled.\"\"\"\n- data = [\n- {\"input\": + {\"x\": 1}, \"expected\": {\"y\": 2}},\n- {\"input\": {\"x\": 2}, + \"expected\": {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- assert result[0].expected == {\"y\": 2}\n- assert + result[1].input == {\"x\": 2}\n- assert result[1].expected == {\"y\": + 4}\n-\n- def test_convert_data_with_simple_items(self):\n- \"\"\"Test + that simple items (not dicts) are handled.\"\"\"\n- data = [1, 2, 3]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 3\n- assert result[0].input == 1\n- assert result[1].input + == 2\n- assert result[2].input == 3"},"commit":"5c35051a0102f850f2e09c66f9376a0fc658ed9f","base_exp_id":"407a9c6f-27ba-4c89-9bf2-b19985d7b695","deleted_at":null,"dataset_id":null,"dataset_version":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZGU4ODI4MGUtMDExMy00OTJiLWE1NDgtZTI4ZjNkN2Q3M2Y0'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:45 GMT + Etag: + - W/"yip0l2ind5soz" + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + Transfer-Encoding: + - chunked + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - ZGU4ODI4MGUtMDExMy00OTJiLWE1NDgtZTI4ZjNkN2Q3M2Y0 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::t2fxs-1775254845684-965634f05762 + content-length: + - '37187' + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '177' + Content-Type: + - application/json + Cookie: + - __cf_bm=nDUgjs5SwRKkemklPjvjDMMtHAWOcPfIHKgtAzyMdD4-1775254843.2839096-1.0.1.1-jqUdQAsdETEsMo9sqxNwGnd9gL0HZtBZFNPxr9dlJQcp4qDxu6bUFoXVGjuENB0RU7pcHYkEBByXGBg3HcZvtqF5APy11vLfxasT.KBBzRNPTl6_j8TJmYLAKXGfnz7V + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.30.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.30.0 + X-Stainless-Raw-Response: + - 'true' + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-DQhAcRyjbFLZ1Dv8vzUB0GDIPX4xg\",\n \"object\": + \"chat.completion\",\n \"created\": 1775254846,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_43047b3f6b\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e6b76640ae7f288-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 03 Apr 2026 22:20:46 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '821' + openai-organization: + - braintrust-data + openai-processing-ms: + - '594' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_9db119b32e604c7892ff775bf8e478d4 + status: + code: 200 + message: OK +- request: + body: '{"id": "52d142c7-3e4f-47b6-9d00-688befc3eb02"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '46' + Content-Type: + - application/json + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/base_experiment/get_id + response: + body: + string: '{"id":"52d142c7-3e4f-47b6-9d00-688befc3eb02","project_id":"208de547-5ee2-4294-955c-5595cd0f7940","name":"langsmith-aeval-21b39aac","base_exp_id":"407a9c6f-27ba-4c89-9bf2-b19985d7b695","base_exp_name":"langsmith-eval-f5e274f6"}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '226' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-YTc2YTI4MzMtMmM3NS00MGJkLWEwZWQtNWZkZTRhOWExZDM2'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:47 GMT + Etag: + - '"11mb6xv65x66a"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/base_experiment/get_id + X-Nonce: + - YTc2YTI4MzMtMmM3NS00MGJkLWEwZWQtNWZkZTRhOWExZDM2 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::twgzn-1775254847025-125adbc654a9 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.33.1 + method: GET + uri: https://api.braintrust.dev/experiment-comparison2?experiment_id=52d142c7-3e4f-47b6-9d00-688befc3eb02&base_experiment_id=407a9c6f-27ba-4c89-9bf2-b19985d7b695 + response: + body: + string: '{"scores":{},"metrics":{}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:47 GMT + Via: + - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 39d0b6c3836d173e719889fc86d67ce8.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - TJjULdjY7bGQdt7f5bpxltxdQk3X0PzR6ao75Dx_RMeXZIQ6KXx9ng== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-69d03d3f-1d1de0531d9718b52321f85b;Parent=6586a494c8f4a457;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id + content-length: + - '26' + etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - bQ6B8F3JIAMEUkg= + x-amzn-RequestId: + - b8934d78-fb5d-4776-8251-ca3b2e44c531 + x-bt-internal-trace-id: + - 69d03d3f000000005cac486d78ceb84f + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_module_evaluate_uses_braintrust_eval.yaml b/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_module_evaluate_uses_braintrust_eval.yaml new file mode 100644 index 00000000..673f35e3 --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_module_evaluate_uses_braintrust_eval.yaml @@ -0,0 +1,3509 @@ +interactions: +- request: + body: '{"messages": [{"content": "What is 1 + 2?", "role": "user"}], "model": + "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": + false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ44VUVp2sk1koSWXX64CaLEy1mWy\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659919,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:38:40 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cd8d01c33943a-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '930' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=XPwF0fhMV9JwjYuWwUMNbzPKxvSJ.HOkXEftYzjXRew-1758659920459-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 93acad0503781eb98ab6ea3412173537 + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '1026' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_181413148bbe4814a905514521d6dc34 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CZR8G4hND55E1b39YFV3NE9YcoN8a\",\n \"object\": + \"chat.completion\",\n \"created\": 1762561812,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + CF-RAY: + - 99b0f5db9f1cbffc-SJC + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:30:12 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=vfpKl6dvzcujjwigai_kp7UkNhR2ltT1SwFsT05VrS8-1762561812-1.0.1.1-UAyuy134RWxRUzjbClH59IJarw95du8Dl347lkXcDkbXBBx7vCmRuxRccJQB2f1T6oobZSgBj7O8hdaLY4hef6ypZ2uHUshy880EnptiWEY; + path=/; expires=Sat, 08-Nov-25 01:00:12 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=N6FAUGU_qhcPvlVWdt0kvrpbt1SzTvQ0v29fL2QCNbA-1762561812358-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '821' + openai-organization: + - braintrust-data + openai-processing-ms: + - '319' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '489' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3e940a310adf4d9a88c8da6b70645bb7 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"f5883013-b5c1-438d-9044-5182b4682337","name":"abhi-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZGY1M2Y4NDAtOTcwZi00ZjEzLTk5OTgtNjQ2MzEyNTU1YjQ3'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:13 GMT + Etag: + - '"13nbfem8i3p75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - ZGY1M2Y4NDAtOTcwZi00ZjEzLTk5OTgtNjQ2MzEyNTU1YjQ3 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::fd7hr-1775254813580-16321e2be5ba + status: + code: 200 + message: OK +- request: + body: '{"project_name": "langsmith-py", "project_id": null, "org_id": "f5883013-b5c1-438d-9044-5182b4682337", + "update": false, "experiment_name": "langsmith-eval", "repo_info": {"commit": + "5c35051a0102f850f2e09c66f9376a0fc658ed9f", "branch": "main", "tag": null, "dirty": + true, "author_name": "Abhijeet Prasad", "author_email": "abhijeet@braintrustdata.com", + "commit_message": "fix(anthropic): capture server-side tool usage metrics (#192)\n\nFlatten + Anthropic usage.server_tool_use into Braintrust span metrics so\nserver-side + tool invocations are preserved for tracing and cost analysis.\n\nAdd regression + tests using a red/green workflow to cover dict-backed span\nlogging and object-backed + usage extraction.\n\nFixes #171", "commit_time": "2026-04-02T21:10:00-04:00", + "git_diff": "diff --git a/py/noxfile.py b/py/noxfile.py\nindex c92395da..1083ca5b + 100644\n--- a/py/noxfile.py\n+++ b/py/noxfile.py\n@@ -77,6 +77,7 @@ VENDOR_PACKAGES + = (\n \"opentelemetry-exporter-otlp-proto-http\",\n \"google.genai\",\n \"google.adk\",\n+ \"langsmith\",\n \"temporalio\",\n + )\n \n@@ -104,6 +105,7 @@ GENAI_VERSIONS = (LATEST,)\n DSPY_VERSIONS = (LATEST,)\n + GOOGLE_ADK_VERSIONS = (LATEST, \"1.14.1\")\n LANGCHAIN_VERSIONS = (LATEST, \"0.3.28\")\n+LANGSMITH_VERSIONS + = (LATEST, \"0.7.12\")\n OPENROUTER_VERSIONS = (LATEST, \"0.6.0\")\n # temporalio + 1.19.0+ requires Python >= 3.10; skip Python 3.9 entirely\n TEMPORAL_VERSIONS + = (LATEST, \"1.20.0\", \"1.19.0\")\n@@ -235,6 +237,17 @@ def test_langchain(session, + version):\n _run_core_tests(session)\n \n \n+@nox.session()\n+@nox.parametrize(\"version\", + LANGSMITH_VERSIONS, ids=LANGSMITH_VERSIONS)\n+def test_langsmith(session, version):\n+ \"\"\"Test + LangSmith integration.\"\"\"\n+ _install_test_deps(session)\n+ _install(session, + \"langsmith\", version)\n+ _install(session, \"langchain-core\")\n+ _install(session, + \"langchain-openai\")\n+ _run_tests(session, f\"{INTEGRATION_DIR}/langsmith/test_langsmith.py\")\n+\n+\n + @nox.session()\n @nox.parametrize(\"version\", OPENAI_VERSIONS, ids=OPENAI_VERSIONS)\n + def test_openai(session, version):\n@@ -371,9 +384,8 @@ def pylint(session):\n session.install(\"pydantic_ai>=1.10.0\")\n session.install(\"google-adk\")\n session.install(\"opentelemetry.instrumentation.openai\")\n- # + langsmith is needed for the langsmith_wrapper module but not in VENDOR_PACKAGES\n # + langchain-core, langchain-openai, langchain-anthropic are needed for the langchain + integration\n- session.install(\"langsmith\", \"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n+ session.install(\"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n \n result = session.run(\"git\", \"ls-files\", + \"**/*.py\", silent=True, log=False)\n files = [path for path in result.strip().splitlines() + if path not in GENERATED_LINT_EXCLUDES]\ndiff --git a/py/src/braintrust/auto.py + b/py/src/braintrust/auto.py\nindex dc44c7d2..6cf8597c 100644\n--- a/py/src/braintrust/auto.py\n+++ + b/py/src/braintrust/auto.py\n@@ -16,6 +16,7 @@ from braintrust.integrations + import (\n DSPyIntegration,\n GoogleGenAIIntegration,\n LangChainIntegration,\n+ LangSmithIntegration,\n LiteLLMIntegration,\n OpenRouterIntegration,\n PydanticAIIntegration,\n@@ + -52,6 +53,7 @@ def auto_instrument(\n dspy: bool = True,\n adk: bool + = True,\n langchain: bool = True,\n+ langsmith: bool = True,\n ) -> dict[str, + bool]:\n \"\"\"\n Auto-instrument supported AI/ML libraries for Braintrust + tracing.\n@@ -75,6 +77,7 @@ def auto_instrument(\n dspy: Enable DSPy + instrumentation (default: True)\n adk: Enable Google ADK instrumentation + (default: True)\n langchain: Enable LangChain instrumentation (default: + True)\n+ langsmith: Enable LangSmith instrumentation (default: True)\n + \n Returns:\n Dict mapping integration name to whether it was successfully + instrumented.\n@@ -146,6 +149,8 @@ def auto_instrument(\n results[\"adk\"] + = _instrument_integration(ADKIntegration)\n if langchain:\n results[\"langchain\"] + = _instrument_integration(LangChainIntegration)\n+ if langsmith:\n+ results[\"langsmith\"] + = _instrument_integration(LangSmithIntegration)\n \n return results\n \ndiff + --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py\nindex + 0062ec77..2dca1110 100644\n--- a/py/src/braintrust/integrations/__init__.py\n+++ + b/py/src/braintrust/integrations/__init__.py\n@@ -6,6 +6,7 @@ from .claude_agent_sdk + import ClaudeAgentSDKIntegration\n from .dspy import DSPyIntegration\n from + .google_genai import GoogleGenAIIntegration\n from .langchain import LangChainIntegration\n+from + .langsmith import LangSmithIntegration\n from .litellm import LiteLLMIntegration\n + from .openrouter import OpenRouterIntegration\n from .pydantic_ai import PydanticAIIntegration\n@@ + -21,6 +22,7 @@ __all__ = [\n \"GoogleGenAIIntegration\",\n \"LiteLLMIntegration\",\n \"LangChainIntegration\",\n+ \"LangSmithIntegration\",\n \"OpenRouterIntegration\",\n \"PydanticAIIntegration\",\n + ]\ndiff --git a/py/src/braintrust/wrappers/langsmith_wrapper.py b/py/src/braintrust/wrappers/langsmith_wrapper.py\nindex + b22117df..7d4170fd 100644\n--- a/py/src/braintrust/wrappers/langsmith_wrapper.py\n+++ + b/py/src/braintrust/wrappers/langsmith_wrapper.py\n@@ -1,518 +1,4 @@\n-\"\"\"\n-Braintrust + integration for LangSmith - provides a migration path from LangSmith to Braintrust.\n+from + braintrust.integrations.langsmith import setup_langsmith # noqa: F401\n \n-This + module patches LangSmith''s tracing and evaluation APIs to use Braintrust under + the hood,\n-allowing users to migrate with minimal code changes.\n \n-Usage:\n- ```python\n- import + os\n-\n- # Enable LangSmith tracing and set project name (used by both services)\n- os.environ.setdefault(\"LANGCHAIN_TRACING_V2\", + \"true\")\n- os.environ.setdefault(\"LANGCHAIN_PROJECT\", \"my-project\")\n-\n- from + braintrust.wrappers.langsmith_wrapper import setup_langsmith\n-\n- # Call + setup BEFORE importing from langsmith\n- # project_name defaults to LANGCHAIN_PROJECT + env var\n- setup_langsmith()\n-\n- # Continue using langsmith imports + - they now use Braintrust\n- from langsmith import traceable, Client\n-\n- @traceable\n- def + my_function(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- client = Client()\n- results = client.evaluate(\n- my_function,\n- data=[{\"inputs\": + {\"x\": 1}, \"outputs\": {\"result\": 2}}],\n- evaluators=[my_evaluator],\n- )\n- ```\n-\n- Set + BRAINTRUST_STANDALONE=1 to completely replace LangSmith with Braintrust\n- (no + LangSmith code runs). Otherwise, both services run in tandem.\n-\"\"\"\n-\n-import + inspect\n-import logging\n-import os\n-from typing import Any, Callable, Dict, + Iterable, Iterator, List, Optional, ParamSpec, TypeVar\n-\n-from braintrust.framework + import EvalCase\n-from braintrust.logger import NOOP_SPAN, current_span, init_logger, + traced\n-from wrapt import wrap_function_wrapper\n-\n-\n-logger = logging.getLogger(__name__)\n-\n-# + Global list to store Braintrust eval results when running in tandem mode\n-_braintrust_eval_results: + List[Any] = []\n-\n-# TODO: langsmith.test/unit/expect, langsmith.AsyncClient, + trace\n-__all__ = [\n- \"setup_langsmith\",\n- \"wrap_traceable\",\n- \"wrap_client\",\n- \"wrap_evaluate\",\n- \"wrap_aevaluate\",\n- \"get_braintrust_results\",\n- \"clear_braintrust_results\",\n-]\n-\n-F + = TypeVar(\"F\", bound=Callable[..., Any])\n-P = ParamSpec(\"P\")\n-R = TypeVar(\"R\")\n-\n-\n-def + get_braintrust_results() -> List[Any]:\n- \"\"\"Get all Braintrust eval results + collected during tandem mode.\"\"\"\n- return _braintrust_eval_results.copy()\n-\n-\n-def + clear_braintrust_results() -> None:\n- \"\"\"Clear all stored Braintrust + eval results.\"\"\"\n- _braintrust_eval_results.clear()\n-\n-\n-def setup_langsmith(\n- api_key: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- project_name: + Optional[str] = None,\n- standalone: bool = False,\n-) -> bool:\n- \"\"\"\n- Setup + Braintrust integration with LangSmith.\n-\n- This patches LangSmith''s @traceable, + Client.evaluate(), and aevaluate()\n- to use Braintrust under the hood.\n-\n- Args:\n- api_key: + Braintrust API key (optional, can use env var BRAINTRUST_API_KEY)\n- project_id: + Braintrust project ID (optional)\n- project_name: Braintrust project + name (optional, falls back to LANGCHAIN_PROJECT\n- env var, + then BRAINTRUST_PROJECT env var)\n- standalone: If True, completely replace + LangSmith with Braintrust (no LangSmith\n- code runs). If + False (default), run both LangSmith and Braintrust\n- in tandem.\n-\n- Returns:\n- True + if setup was successful, False otherwise\n- \"\"\"\n- # Use LANGCHAIN_PROJECT + as fallback for project_name to keep both services in sync\n- if project_name + is None:\n- project_name = os.environ.get(\"LANGCHAIN_PROJECT\")\n-\n- span + = current_span()\n- if span == NOOP_SPAN:\n- init_logger(project=project_name, + api_key=api_key, project_id=project_id)\n-\n- try:\n- import langsmith\n-\n- langsmith.traceable + = wrap_traceable(langsmith.traceable, standalone=standalone)\n- wrap_client(langsmith.Client, + project_name=project_name, project_id=project_id, standalone=standalone)\n- langsmith.evaluate + = wrap_evaluate(\n- langsmith.evaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n- langsmith.aevaluate + = wrap_aevaluate(\n- langsmith.aevaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n-\n- logger.info(\"LangSmith + integration with Braintrust enabled\")\n- return True\n-\n- except + ImportError as e:\n- logger.error(f\"Failed to import langsmith: {e}\")\n- logger.error(\"langsmith + is not installed. Please install it with: pip install langsmith\")\n- return + False\n-\n-\n-def wrap_traceable(traceable: F, standalone: bool = False) -> + F:\n- \"\"\"\n- Wrap langsmith.traceable to also use Braintrust''s @traced + decorator.\n-\n- Args:\n- traceable: The langsmith.traceable function\n- standalone: + If True, replace LangSmith tracing entirely with Braintrust.\n- If + False, add Braintrust tracing alongside LangSmith tracing.\n-\n- Returns:\n- The + wrapped traceable function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(traceable):\n- return traceable\n-\n- def traceable_wrapper(*args: + Any, **kwargs: Any) -> Any:\n- # Handle both @traceable and @traceable(...) + patterns\n- func = args[0] if args and callable(args[0]) else None\n-\n- def + decorator(fn: Callable[P, R]) -> Callable[P, R]:\n- span_name = kwargs.get(\"name\") + or fn.__name__\n-\n- # Conditionally apply LangSmith decorator first\n- if + not standalone:\n- fn = traceable(fn, **kwargs)\n-\n- # + Always apply Braintrust tracing\n- return traced(name=span_name)(fn) # + type: ignore[return-value]\n-\n- if func is not None:\n- return + decorator(func)\n- return decorator\n-\n- traceable_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return traceable_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_client(\n- Client: Any, project_name: + Optional[str] = None, project_id: Optional[str] = None, standalone: bool = False\n-) + -> Any:\n- \"\"\"\n- Wrap langsmith.Client to redirect evaluate() and + aevaluate() to Braintrust''s Eval.\n-\n- Args:\n- Client: The langsmith.Client + class\n- project_name: Braintrust project name to use for evaluations\n- project_id: + Braintrust project ID to use for evaluations\n- standalone: If True, + only run Braintrust. If False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + Client class (modified in place)\n- \"\"\"\n-\n- if hasattr(Client, \"evaluate\") + and not _is_patched(Client.evaluate):\n- wrap_function_wrapper(\n- Client,\n- \"evaluate\",\n- make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.evaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- if hasattr(Client, \"aevaluate\") + and not _is_patched(Client.aevaluate):\n- wrap_function_wrapper(\n- Client,\n- \"aevaluate\",\n- make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.aevaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- return Client\n-\n-\n-def make_evaluate_wrapper(\n- *, + project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: + bool = False\n-):\n- def evaluate_wrapper(wrapped: Any, instance: Any, args: + Any, kwargs: Any) -> Any:\n- result = None\n- if not standalone:\n- result + = wrapped(*args, **kwargs)\n-\n- try:\n- result = _run_braintrust_eval(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + evaluate failed: {e}\")\n-\n- return result\n-\n- return evaluate_wrapper\n-\n-\n-def + make_aevaluate_wrapper(\n- *, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-):\n- async def aevaluate_wrapper(wrapped: + Any, instance: Any, args: Any, kwargs: Any) -> Any:\n- result = None\n- if + not standalone:\n- result = await wrapped(*args, **kwargs)\n-\n- try:\n- result + = await _run_braintrust_eval_async(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + aevaluate failed: {e}\")\n-\n- return result\n-\n- return aevaluate_wrapper\n-\n-\n-def + wrap_evaluate(\n- evaluate: F, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.evaluate to redirect to Braintrust''s Eval.\n-\n- Args:\n- evaluate: + The langsmith.evaluate function\n- project_name: Braintrust project name + to use for evaluations\n- project_id: Braintrust project ID to use for + evaluations\n- standalone: If True, only run Braintrust. If False, run + both LangSmith and Braintrust.\n-\n- Returns:\n- The wrapped evaluate + function (or the original if already patched)\n- \"\"\"\n- if _is_patched(evaluate):\n- return + evaluate\n-\n- evaluate_wrapper = make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id)\n- evaluate_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return evaluate_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_aevaluate(\n- aevaluate: F,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- standalone: + bool = False,\n-) -> F:\n- \"\"\"\n- Wrap module-level langsmith.aevaluate + to redirect to Braintrust''s EvalAsync.\n-\n- Args:\n- aevaluate: + The langsmith.aevaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to use + for evaluations\n- standalone: If True, only run Braintrust. If False, + run both LangSmith and Braintrust.\n-\n- Returns:\n- The wrapped aevaluate + function (or the original if already patched)\n- \"\"\"\n- if _is_patched(aevaluate):\n- return + aevaluate\n-\n- aevaluate_wrapper = make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id)\n- aevaluate_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return aevaluate_wrapper # type: + ignore[return-value]\n-\n-\n-def _is_patched(obj: Any) -> bool:\n- return + getattr(obj, \"_braintrust_patched\", False)\n-\n-\n-# =============================================================================\n-# + Braintrust evaluation logic\n-# =============================================================================\n-\n-\n-def + _run_braintrust_eval(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust Eval with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import Eval\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + Eval(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-async + def _run_braintrust_eval_async(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust EvalAsync with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import EvalAsync\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + await EvalAsync(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-# + =============================================================================\n-# + Data conversion helpers\n-# =============================================================================\n-\n-\n-def + _wrap_output(output: Any) -> Dict[str, Any]:\n- \"\"\"Wrap non-dict outputs + the same way LangSmith does.\"\"\"\n- if not isinstance(output, dict):\n- return + {\"output\": output}\n- return output\n-\n-\n-def _make_braintrust_scorer(\n- evaluator: + Callable[..., Any],\n-) -> Callable[..., Any]:\n- \"\"\"\n- Create a Braintrust + scorer from a LangSmith evaluator.\n-\n- Always runs the evaluator through + Braintrust for full tracing (span duration, child LLM calls, etc.).\n- \"\"\"\n- evaluator_name + = getattr(evaluator, \"__name__\", \"score\")\n-\n- def braintrust_scorer(input: + Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any:\n- from + braintrust.score import Score\n-\n- # Run the evaluator with LangSmith''s + signature\n- # LangSmith evaluators use: (inputs, outputs, reference_outputs) + -> bool | dict\n- # LangSmith auto-wraps non-dict outputs as {\"output\": + value}\n- outputs = _wrap_output(output)\n-\n- # expected is the + real LangSmith Example object passed through from data loading\n- reference_outputs + = expected.outputs if hasattr(expected, \"outputs\") else expected\n-\n- result + = evaluator(input, outputs, reference_outputs)\n-\n- return Score(\n- name=result.get(\"key\", + evaluator_name),\n- score=result.get(\"score\"),\n- metadata=result.get(\"metadata\", + {}),\n- )\n-\n- braintrust_scorer.__name__ = evaluator_name\n- return + braintrust_scorer\n-\n-\n-def _convert_langsmith_data(data: Any) -> Callable[[], + Iterator[EvalCase[Any, Any]]]:\n- \"\"\"Convert LangSmith data format to + Braintrust data format.\"\"\"\n-\n- def load_data() -> Iterator[EvalCase[Any, + Any]]:\n- # Determine the source iterable without loading everything + into memory\n- source: Iterable[Any]\n- if callable(data):\n- source + = data() # type: ignore\n- elif isinstance(data, str):\n- # + Load examples from LangSmith dataset by name\n- try:\n- from + langsmith import Client # pylint: disable=import-error\n-\n- client + = Client()\n- source = client.list_examples(dataset_name=data)\n- except + Exception as e:\n- logger.warning(f\"Failed to load LangSmith + dataset ''{data}'': {e}\")\n- return\n- elif hasattr(data, + \"__iter__\"):\n- source = data\n- else:\n- source + = [data]\n-\n- # Process items as a generator - yield one at a time\n- for + item in source:\n- # Pass through LangSmith Example objects directly\n- if + hasattr(item, \"inputs\"):\n- yield EvalCase(\n- input=item.inputs,\n- expected=item, # + Pass the whole Example object\n- metadata=getattr(item, \"metadata\", + None),\n- )\n- elif isinstance(item, dict):\n- if + \"inputs\" in item:\n- # LangSmith dict format\n- yield + EvalCase(\n- input=item[\"inputs\"],\n- expected=item, # + Pass the whole dict\n- metadata=item.get(\"metadata\"),\n- )\n- elif + \"input\" in item:\n- # Braintrust format\n- yield + EvalCase(\n- input=item[\"input\"],\n- expected=item.get(\"expected\"),\n- metadata=item.get(\"metadata\"),\n- )\n- else:\n- yield + EvalCase(input=item)\n- else:\n- yield EvalCase(input=item)\n-\n- return + load_data\n-\n-\n-def _make_braintrust_task(target: Callable[..., Any]) -> Callable[..., + Any]:\n- \"\"\"Convert a LangSmith target function to Braintrust task format.\"\"\"\n-\n- def + task_fn(task_input: Any, hooks: Any) -> Any:\n- if isinstance(task_input, + dict):\n- # Try to get the original function''s signature (unwrap + decorators)\n- unwrapped = inspect.unwrap(target)\n-\n- try:\n- sig + = inspect.signature(unwrapped)\n- params = list(sig.parameters.keys())\n- if + len(params) == 1:\n- return target(task_input)\n- if + all(p in task_input for p in params):\n- return target(**task_input)\n- return + target(task_input)\n- except (ValueError, TypeError):\n- # + Fallback: try kwargs first, then single arg\n- try:\n- return + target(**task_input)\n- except TypeError:\n- return + target(task_input)\n- return target(task_input)\n-\n- return task_fn\n+__all__ + = [\"setup_langsmith\"]\ndiff --git a/py/src/braintrust/wrappers/test_langsmith_wrapper.py + b/py/src/braintrust/wrappers/test_langsmith_wrapper.py\ndeleted file mode 100644\nindex + c25256c2..00000000\n--- a/py/src/braintrust/wrappers/test_langsmith_wrapper.py\n+++ + /dev/null\n@@ -1,341 +0,0 @@\n-# pyright: reportPrivateUsage=false\n-# pyright: + reportMissingParameterType=false\n-# pyright: reportUnknownParameterType=false\n-# + pyright: reportUnknownArgumentType=false\n-# pyright: reportUnknownMemberType=false\n-# + pylint: disable=protected-access\n-\n-\"\"\"\n-Tests for the LangSmith wrapper + to ensure compatibility with LangSmith''s API.\n-\"\"\"\n-\n-from braintrust.wrappers.langsmith_wrapper + import (\n- _convert_langsmith_data,\n- _is_patched,\n- _make_braintrust_scorer,\n- _make_braintrust_task,\n- wrap_aevaluate,\n- wrap_client,\n- wrap_traceable,\n-)\n-\n-\n-def + test_is_patched_false():\n- \"\"\"Test that _is_patched returns False for + unpatched objects.\"\"\"\n-\n- def unpatched():\n- pass\n-\n- assert + _is_patched(unpatched) is False\n-\n-\n-def test_is_patched_true():\n- \"\"\"Test + that _is_patched returns True for patched objects.\"\"\"\n-\n- def patched():\n- pass\n-\n- patched._braintrust_patched + = True # type: ignore\n-\n- assert _is_patched(patched) is True\n-\n-\n-def + test_make_braintrust_scorer_dict_result():\n- \"\"\"Test converting a LangSmith + evaluator that returns a dict.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- return {\"key\": \"accuracy\", \"score\": + 0.9, \"metadata\": {\"note\": \"good\"}}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- # + Create a mock Example object\n- class MockExample:\n- outputs = {\"y\": + 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": 2}, expected=MockExample())\n-\n- assert + result.name == \"accuracy\"\n- assert result.score == 0.9\n- assert result.metadata + == {\"note\": \"good\"}\n-\n-\n-def test_make_braintrust_scorer_numeric_result():\n- \"\"\"Test + converting a LangSmith evaluator that returns a numeric score in a dict.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return {\"score\": + 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class + MockExample:\n- outputs = {\"y\": 2}\n-\n- result = converted(input={\"x\": + 1}, output={\"y\": 2}, expected=MockExample())\n-\n- assert result.name == + \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def test_make_braintrust_scorer_with_plain_dict_expected():\n- \"\"\"Test + converting a LangSmith evaluator with plain dict as expected.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return {\"score\": + 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n- result + = converted(input={\"x\": 1}, output={\"y\": 2}, expected={\"y\": 2})\n-\n- assert + result.name == \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def + test_convert_langsmith_data_from_list():\n- \"\"\"Test converting LangSmith + data from a list of dicts.\"\"\"\n- data = [\n- {\"inputs\": {\"x\": + 1}, \"outputs\": {\"y\": 2}},\n- {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- assert result[1].input + == {\"x\": 2}\n- assert result[1].expected == {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}}\n-\n-\n-def test_convert_langsmith_data_from_callable():\n- \"\"\"Test + converting LangSmith data from a callable.\"\"\"\n-\n- def data_generator():\n- yield + {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- yield {\"inputs\": + {\"x\": 2}, \"outputs\": {\"y\": 4}}\n-\n- data_fn = _convert_langsmith_data(data_generator)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n-\n-\n-def test_convert_langsmith_data_with_example_objects():\n- \"\"\"Test + converting LangSmith data with Example-like objects.\"\"\"\n-\n- class MockExample:\n- def + __init__(self, inputs, outputs):\n- self.inputs = inputs\n- self.outputs + = outputs\n-\n- data = [\n- MockExample(inputs={\"x\": 1}, outputs={\"y\": + 2}),\n- MockExample(inputs={\"x\": 2}, outputs={\"y\": 4}),\n- ]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 2\n- assert result[0].input == {\"x\": 1}\n- # The whole + Example object is passed as expected\n- assert result[0].expected.inputs + == {\"x\": 1}\n- assert result[0].expected.outputs == {\"y\": 2}\n-\n-\n-def + test_make_braintrust_task_with_dict_input():\n- \"\"\"Test that task function + handles dict inputs correctly.\"\"\"\n-\n- def target_fn(inputs):\n- return + inputs[\"x\"] * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == 10\n-\n-\n-def test_make_braintrust_task_with_kwargs_expansion():\n- \"\"\"Test + that task function expands dict kwargs when signature matches.\"\"\"\n-\n- def + target_fn(x, y):\n- return x + y\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 2, \"y\": 3}, None)\n-\n- assert result == 5\n-\n-\n-def test_make_braintrust_task_simple_input():\n- \"\"\"Test + that task function handles simple inputs.\"\"\"\n-\n- def target_fn(inp):\n- return + inp * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result = task(5, + None)\n-\n- assert result == 10\n-\n-\n-class TestWrapTraceable:\n- \"\"\"Tests + for wrap_traceable functionality.\"\"\"\n-\n- def test_wrap_traceable_returns_wrapper(self):\n- \"\"\"Test + that wrap_traceable returns a wrapped version.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable, + standalone=False)\n- assert callable(wrapped)\n- assert _is_patched(wrapped)\n-\n- def + test_wrap_traceable_standalone_mode(self):\n- \"\"\"Test that wrap_traceable + works in standalone mode.\"\"\"\n-\n- def mock_traceable(func, **kwargs):\n- return + func\n-\n- wrapped = wrap_traceable(mock_traceable, standalone=True)\n- assert + callable(wrapped)\n- assert _is_patched(wrapped)\n-\n-\n-class TestWrapFunctions:\n- \"\"\"Tests + for the wrap_* functions.\"\"\"\n-\n- def test_wrap_functions_exist(self):\n- \"\"\"Test + that wrap functions are callable.\"\"\"\n- assert callable(wrap_traceable)\n- assert + callable(wrap_client)\n- assert callable(wrap_aevaluate)\n-\n- def + test_wrap_traceable_returns_patched_function(self):\n- \"\"\"Test that + wrap_traceable returns a patched function.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable)\n- assert + _is_patched(wrapped)\n-\n- def test_wrap_traceable_skips_if_already_patched(self):\n- \"\"\"Test + that wrap_traceable skips if already patched.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- mock_traceable._braintrust_patched + = True # type: ignore\n-\n- result = wrap_traceable(mock_traceable)\n- # + Should return the same function\n- assert result is mock_traceable\n-\n- def + test_wrap_client_sets_flag(self):\n- \"\"\"Test that wrap_client sets + the patched flag.\"\"\"\n-\n- class MockClient:\n- def evaluate(self, + *args, **kwargs):\n- return \"original\"\n-\n- wrap_client(MockClient)\n- assert + _is_patched(MockClient.evaluate)\n-\n- def test_wrap_aevaluate_returns_patched_function(self):\n- \"\"\"Test + that wrap_aevaluate returns a patched function.\"\"\"\n-\n- async def + mock_aevaluate(*args, **kwargs):\n- pass\n-\n- wrapped = wrap_aevaluate(mock_aevaluate)\n- assert + _is_patched(wrapped)\n-\n-\n-class TestTandemModeIntegration:\n- \"\"\"Integration + tests for tandem mode (LangSmith + Braintrust together).\"\"\"\n-\n- def + test_make_braintrust_task_with_inputs_parameter(self):\n- \"\"\"Test + that task handles LangSmith''s required ''inputs'' parameter name.\"\"\"\n-\n- def + target_fn(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- task = _make_braintrust_task(target_fn)\n- result = + task({\"x\": 5}, None)\n-\n- assert result == {\"result\": 10}\n-\n- def + test_convert_langsmith_data_handles_different_output_types(self):\n- \"\"\"Test + that data conversion handles various output types.\"\"\"\n- data = [\n- {\"inputs\": + {\"x\": 1}, \"outputs\": 2}, # outputs is int, not dict\n- {\"inputs\": + {\"x\": 2}, \"outputs\": {\"result\": 4}}, # outputs is already dict\n- ]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- # + Both should work - Braintrust''s EvalCase accepts any type for expected\n- assert + len(result) == 2\n- assert result[0].input == {\"x\": 1}\n- assert + result[1].input == {\"x\": 2}\n-\n- def test_make_braintrust_scorer_handles_wrapped_outputs(self):\n- \"\"\"Test + that scorers handle output wrapping correctly.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- # outputs will be wrapped as {\"output\": + value} for non-dict results\n- actual = outputs.get(\"output\", outputs)\n- expected + = (\n- reference_outputs.get(\"output\", reference_outputs)\n- if + isinstance(reference_outputs, dict)\n- else reference_outputs\n- )\n- return + {\"key\": \"match\", \"score\": 1.0 if actual == expected else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class MockExample:\n- outputs + = {\"output\": 42}\n-\n- # Test with wrapped output\n- result + = converted(input={\"x\": 1}, output=42, expected=MockExample())\n- assert + result.name == \"match\"\n- assert result.score == 1.0\n-\n-\n-class + TestDataConversion:\n- \"\"\"Tests for data conversion utilities.\"\"\"\n-\n- def + test_convert_data_with_braintrust_format(self):\n- \"\"\"Test that Braintrust + format is properly handled.\"\"\"\n- data = [\n- {\"input\": + {\"x\": 1}, \"expected\": {\"y\": 2}},\n- {\"input\": {\"x\": 2}, + \"expected\": {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- assert result[0].expected == {\"y\": 2}\n- assert + result[1].input == {\"x\": 2}\n- assert result[1].expected == {\"y\": + 4}\n-\n- def test_convert_data_with_simple_items(self):\n- \"\"\"Test + that simple items (not dicts) are handled.\"\"\"\n- data = [1, 2, 3]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 3\n- assert result[0].input == 1\n- assert result[1].input + == 2\n- assert result[2].input == 3"}, "ancestor_commits": ["5c35051a0102f850f2e09c66f9376a0fc658ed9f", + "0daac2f277a337206cbb03cc46c695f7931d7f3d", "d807b7b4a48b4bd920570cdad5f01c350cdd9f35", + "19ecb8a09557f830675a2dded6bdd04f36ae9f76", "0a708e578ce19f61c17a9f4aa44a96d7ed0277df", + "2a174d884e2034fd9a8c20c20c4c0c384f7aa687", "a656bab37f99445ca442fef41f032907c79e3fc0", + "5fda66e087f78c4edf8c95f24c9454e9c80c6417", "07eae203f645dd661ea0fb83b5256f2144dbf07f", + "2c68b8894223a9b562b7d922a53430efe067ede3", "ba0bc10b5f79e952e69a721be89023773f46fe54", + "e09f625c67fab1f2d2f7b31f0370dab3e0cafd1b", "4e5c83f5456b5c043e1e0830227805d28f922f70", + "161fa36a51946f2212d28354609e442f83889684", "3462f3f3d4ce82b3c59f5dacec9a523b9c99ee2d", + "62a2963911d6c65cdbaef1712fdb7c890d3b5777", "89122d4cca5ea2b766b9416c68b611bee79294dd", + "1c1a07e7dfd96811fd883d81adb3b7f7f4061e75", "f6a2c3b3aa589ab8606b7b525ce79ca019f98717", + "051870af3d7bc8c183edac119bcb7166f1ea9ef0", "4267bde681b41aaa31036e14508708a28bde70ac", + "0b717a9e6240a162a3931116f44bd86f4900852b", "7a4328e7db5636ffe7898d99fd392bc204ec7a08", + "e0780bfc33ab6ced1b982aaf27bd7b5072bbe04e", "183dbb0d034c3588b26a8a94a10f3d2f03237670", + "8b15ad67f697a8218217d9674f0fb919d9c1e8d6", "01d0cea1268051909665e4911af1f1be54b333cd", + "b335e66c1f458de9cd73c12d9d9fc97114f3d505", "0b2f34802606d033bb3471075be95de5d9621b98", + "dd73fb4ac1f38bee8e55133a0348f1bd5c46c30e", "716f9e45b99874f17531f9b5ae4ccfa57cab5ae3", + "72adf8bbf0b48756e18423c5477d58bbf6c401b9", "1163e11507676601e265e637e8916a71a310ce87", + "80308c49a0309d13e02c3203c099b3dc87de1ad9", "e112d705d338ef08d449afab5f2bef3ae0269622", + "10996518f60c0cd2796304c122cd05b16c6ffb1c", "b9a78fa767379bfc14e9ec0f9dc1a016f7ca74a2", + "3fc8912b4063d524f357733549ff1c58bbad00ba", "4252662870b266b32d23a16a11c6df755e5dcdcb", + "92165c3f00478c316e7af29d5c2dba51ffbd7090", "a8144eabce4ef1a46257be77c8055bd3fc64978c", + "363baa083335500aebe129c670eb2c49153ad3a6", "0d487b688de037fa6eebf5410c12b9b226f12cff", + "824861d780be7bcbec92f274a13e221cffef04a4", "84398747d12f52f2b6089d5e832ff6cca82fa3d5", + "46c18b284d0d14855fb66a6de8891ea2c39e08e7", "4f1d9510a188a7a9645fe72966a099aff70478ca", + "d82c4258901176056c1eb30a837896becada5a0f", "265a46272e1f808251c0da03061a9b1843a22779", + "42bed48d1b31a428f189ba174ff4f759ec34e94d", "98aa3e6923d32ed264abd11315cd247c085e8a16", + "ff18889b8f8ef8af644f6eeab1179f6bf9b14de5", "fd34303ea427a9bfc038b78ef1e8e65bb44b6f29", + "08210a4fd97cc17d90556afaaba51dbd67917ddd", "a1fae25c088942b155a31b243f07b9862f79c5ba", + "4fb512eaa52733c1daa4cd6995140f6722f17ba4", "992e036a9199d1b03976ed4d22eebe9921410b40", + "5d41698059f77d9c332cfc4e706ea77669d1cb6a", "2349418ff85b75917d869a26be6df0682686caac", + "fd9f3f932a646f07438db3dbda8cab73cfa70c2f", "fc2d5fe0f6d94754e3c705686c3c69eaa58c258c", + "b3651b577d6ebf617a9991252e19d5fcfe7e693c", "106691a2cbfafc38b3d5d5064c9bb922ae7a3901", + "1b7818ade91e40155c7c126cb92cb21726a25c66", "49ab8392e413992b9728666999593a94ebd00236", + "fb80c5412e011c5b3d343809710c92b75874261e", "ab9fd17f71c277f0d05a809c3c9cfe22379df511", + "7184711ed04957af8f2bbe36590c1cec057ca0a8", "5976a47fe82f90599edc06b7e4f9af64a8ce4ed7", + "962e499f18549cdea2e65946c79bacbb9159f35d", "28875eb8ce2b4114d60d365ad5b61c124b5e3900", + "e68dfce97ee7fa014a30d694d7220a1de47fa73e", "404589ac15776fe04f6dd8bdf36c5d74a3684986", + "5f5b88701fedc4cafed4c037ee2a8cff6b4a40fb", "aeb15581fefc2b0b867ca55c21ea80d067f43361", + "2cd52871898658315cc43e78f25546c3c8df1c8a", "6c4b637d65e047f01052b0749d4c3608f202e073", + "864aafda1789476b01c6991931fb0a6e1968e2dc", "9548ae35e49fc7aac9263811ae3c3a4b26af2e81", + "97da34df8474129e1ad16c8eb67b00a906330970", "fdd020262b6620a4823d336741ebbcf0a02ff9a2", + "0bb83ac92e0ccb38dee1f56c11d15180e8c8c565", "34262bcfb1c5babc03da6d27d1b93ca2c3e969a7", + "7bb761580056ac43c2636964b5f1d2f08b3cc01a", "9424d6a04f73de02038dcbacd518864397ddcafb", + "a51b84c54ba9269d28f17e7355dbc8aaf9ccc0b2", "a5ef7778c00dd0c3782a423e4fb6ee3a0a2a73a3", + "33cfa00beffa04151a7d874d5671b720bb49dbe8", "eb530110ee5746d87c1d2b9972f3a3dc4a2aa582", + "8695a8e3ef85a173cfffd98ae25f6a31a7e23d2e", "7da14b1a295c8f912b6b93ec55ec759242a50108", + "f60a2d39cce59a000017bc193b84ef25ba4491d5", "447d3a96d0679451a2c87a5bf28531ca7744b9ab", + "abc8bed8fbdac570891517c76233e18d9fff4828", "a7053505083fb7b7ae6ed1c10a4141a0d8673cc4", + "8ab9dc06de08f7a925fabe52dbbdb6df3b480e89", "009a839b197b63d6a688d53d91743fb84cbed04f", + "fe75d23b3b61e1693c3b3bfe896380ce16a3ef66", "3a4dde05d882a4eb27d21d1d41784295b3a4168e", + "2b0d9e276db08298b83a10b877f3ccf08bfafde6", "fe1732c7b4df75d49cb42017bb2b2af0136a73c0", + "971559d598a53d361c50da7126e5681124244f32", "6e172e761f95e59dda9fab0409f85491beef2240", + "e3cd507afaaa62bc8ef736b27c1f0e0064c3931f", "3a3ba47a007e0af9af528b63b271391e0c538dc7", + "c3d5ea96b897dc179e4775b14c7bb97b838861db", "1ec7d11f39c97471973de7e5eb0b0784025bc851", + "99a5cb75b08c5e2bff6e96614cfe8c0573f0c19d", "49f1e3846c3c8d264e693bab999270d4392d153e", + "824cf93c402abf51934aebd890c9805457108a7c", "d9e62442aa51d540345c1bfbd1df043b9f00657d", + "c933fe5625cfe8b7e57e4b2953fe612ad37d78b5", "ec0c35e2fadc7bd6a734ff3013226193b1f81d09", + "d462219aa0d5e6525299e3a9cc1dabb19dbdc9c5", "9f8937030053fc43137805b66fef5d1d335fae47", + "1c827b6442cf42fbfa2868abfcb2c640752769bd", "892c41480277cab14f3751eafe678af0c4966bfb", + "6a65a37fb5f7e11e9b8551759d44879e732355d1", "c047819f46a9bf82b7f11124802247055e709c80", + "102257d62684977c39f6e78559d73ac0f873048a", "94f13c6a1c7306e23ba45c2cf0d3600ef8cf8673", + "786d48c8e9cf363508f0c30b872968d9ceec69d7", "7b62001e98bba2fcad8254aff0f0cb11bf35db33", + "2eeee24fc7c3b7c6a0aac79563e1cb0253e0bea1", "7aa19caa8d893488dedf5d6e985b766710b66eb7", + "17b5cae858690e270104ba1c5520508525e6c687", "8b1a5aa0a4b4d547e09c07d69be841c3ebde525f", + "174b4061aab369725a43719d5e966013686b4c38", "7bf81f8ebc7877956e2b8c3c556743426ef05038", + "e151663947390534a2037912cad663daba83de6d", "99bc21b047a09063813a733d659e84ad740424e3", + "1bc2d475bc64f99dd11c3bfd569bd15140a87389", "e3d62fddfab1774e5cae1c1ae2d0d46f164a768f", + "d91f76c24230054edef8774579a0a036b844af7b", "dbea65347acd60addcfbfa7bdb97ddbaeee6ba21", + "a28ef5f8895857ad6d945644f78679271fd10df3", "ec1c107ab6133f5810f9501e3ce57349c0bf4ae9", + "10e2d2237b0791683386e551bd4be79cf86367db", "e3f25b9b6abc2f8e1a376788d11d82a61c41645d", + "a88b44d6852fae824705415d2795f5c5201f2dde", "6210d435deb094c603c9e8debdd0d48255be7dae", + "6b2ab80cc64c1b869fb9ea6002fb1670e9ceb5cb", "1e440103b7521973aa5fb33addc8f9ce1d050232", + "449f6eeddd54f73d88e185fcf499087d3e61fbe6", "52579035063a1b7d1e8b64fbaf1b32bf55a04004", + "4db047355d602c53d85f43c1e7276dd3bd8335df", "4bc366daa82312b79ccb2d6471abef3ad16e4512", + "c710e0c271880b2ea1d506dd3d6f23c282f33fe8", "ce261c13723e8985e61819f6df56298f863155d0", + "f2b6f15f1db2d3e3a930e3274c0c1d1245e851c2", "f11481fe89d024f1b2deb651653ea8ad30da5e8a", + "633cb4c201d614ff5ac5e9775fd68764601d59d8", "0bb08d89c6fe9c043cb75178571715460bc32603", + "a8b9f42b80cfc3a835162fc4e69945edae9a2dd2", "e9f46b8d41234acba81068f7c00e3063204b4231", + "ef1c89e8f840a14f44cd6682bc92f7ce2ea9d07d", "6d086d6cfee47cc1e83c7522a58e1d412e91d197", + "7894ea68db9bcdaa55e66cd4d2a634878f901965", "753298d7ada8377c173de6fad4b490082372b7db", + "25868bc58450dad2058b6499ce3bb9400330fbd1", "95f3c141b7f8ad81bf1d34d857c2fe60bbd15d7f", + "f416cea3139a9a29ed5cae633cb3097e70fc93e2", "5e89c7299d9cc476594fc73933bc7006c842cdf4", + "ddf619048666d883cd4325e21f860e28632a456b", "2385b1aa98998820bcb55cfae76d063a57bba938", + "b798003d9609603b43c98ccc2a5c995d0ac1a488", "4d9333cef5aae8889f9697f38a87f694893f17a8", + "2194150ee020def2d7c4a675f2f001c8c0fba99e", "36788d5a20f45762991b1ba72fb52f8020e6c61b", + "16f72b7d1cb1e0801285abbb7b3138909d6e7f79", "617d9b730b37e96b7d05a099b95f5387944d0951", + "51bb20bec1dc870ec4f521e3c9464054ab9abdec", "15cde9f7795249fbb3c71f23210092265f509e19", + "ea3fd2ff145cbfcd53bb1d79ec6dfb659debdb4f", "eacfc8036be4cee52c619f278b61d32862f3a521", + "252eb0be986ee236c4f095f0078b816c7bd5c894", "6774ffa8b89b2da3644f07e4f411b20f108e5786", + "77fc0472011fb9ea9b3ff7bc5e7dbc5c57f33fed", "3e4002f6b2b3e540db12a2355f447fabd7a83bec", + "3110f96fcbfea2b60d1f8b580e28c00c2fe38690", "1cfad0ba63198f2480a22721501757d9913a1d52", + "2dc8187fd9ace26f7684ff5ba4fa75d7f28d7703", "0687d03758cd295a1976a1b5eb40c5187332be7c", + "8b3614791454b2b4e15a47a09948b3699ad7334a", "54002cedbe48b5036ba94109782d78686c0df7aa", + "9c21b41dd5a19131bd4bf236503ebd0a1464816f", "9a5c1c4a0d50d2d70ab6acb2c38e6abcbfcfc547", + "d0bf7aec410bdd29bf4a8f43af0a8bea633788e6", "7580576751a6890b3314fcd4c313991fb9f311a6", + "d734e8ffc272ee65fe0588df00fd9390614ccd2e", "d9fd9bb0c6bf4c6d5536765e0f8005e9326021a2", + "a751e399b66d396cdf13599c93b1445d81934ddc", "8e50db374a548c26830613a78f70072086ce1f13", + "791a2da1e44f02902e1c31ec28c2c7aba5795f47", "8a63569906d8824faf9c1aaecdd62ebb9e47aeb4", + "e541543ce30054cc10fa3e27e2b0aa5afd297d0f", "b017a02240f978dcfe7feec58aeb85f8cbc91892", + "072d3765416d78e465a728df1b4a812992fba58a", "1671a73c21d09a1b4b73debce7cb2fa1d1b9ee54", + "7b6371f41f6a109ea068fd547b17a131d109ee85", "530a4be0108ff8cc783a49c6390f59b238782e58", + "3ca420e53e77d4665b91ccc7631c95dc97ce566d", "41aceab354ed312d4758b5a5b5c32bbdac40da48", + "6f303beaf59f892b9ed86eb4338900c7f82f7a68", "2d2f7e260b2027576963fad1c432197b15f2811e", + "04acf467f93d2b1064ef648c6583f76d96864578", "3dcfc326ee12b116d919974ef8bbf3cf307ee7fd", + "0740dc169ac9f267e4ec2d435165ca882df0fbc1", "270970d692adaf5d204d336ecdb4ee7f51042187", + "37e49012471ef81326f1313583497f48d612eed1", "8d9100b8341085366c11a36d93faa6009ecd4837", + "f53d7b868f610f1b215c799f25d4af171b59a000", "23b9a8d6b977d6849503433f8b38c49cf90e8203", + "5a31cd02719a5a282825a0fec7cb8d0fc44d0884", "7079cb29799d9fb9fbc919b7ddd1224935a551c7", + "d9c624ea93ca6bf62c2412abce1b3a2ef1a2be67", "126c367b3002ad57bb6aa7e08d04657db6f61980", + "b610cd36638ace60bca2aed261165afbf1c8c081", "52a8fa6d53c58908d95bb865f5907d829987fb97", + "65c037efdffceeec554b2706e16621cfc1704202", "c3bc923fafbc33485c9173c2401fa20897cf4ef1", + "6d6cc2f1e8b5da316c409a754ee8c844b8d2b2ee", "0ecb05e34031941426e3b9ce76760add8af6fdfd", + "8ab13f3f48af6a4d3c0b053e4bbabfd4f24f23ec", "10c14b3688c6969ef55951d5d82e744b87d6c356", + "05f569c80bddb61414e8f9a8adf49f8ca6b821b4", "2311d67956183da8b31bf1fdbe6e09b3ec7e242d", + "0b0bdd77c012e3f51c6402edcad35c8ac7ddf4cd", "ca2eb28a0f22a63b2b748d04e17268d11d35e0b0", + "c20c808993bba8d5604c2ac7848037d7ce430e89", "dcd4f5a4be171b1cac28a5eb3534e4b55420cc06", + "74dd883f7c03ba5591ecee0b2b76e9250781e181", "4f94c9548a493de11cd663fc78db21a81fe6f23c", + "b3c2b6f7120402b3f5f0fce9b7a1e6b7f7d04d47", "476a66a5d09ec811c9856f0f0f289b189859721e", + "2468e8006b8edf9ef8f273e142380faf34d380a6", "b88964e5a1b124816771e2c9c0a628c104ecfddb", + "a05dc4df62a20e9715abe697584641f0032742b5", "ad3ca98e3530c7dbdbb07f63a04fe4912e17f85a", + "dbbc1894ef31143816e5913676301261bc44aa4c", "db388916a9bc42b02e26b519c3c5f116c919cafc", + "78753fa68806ca7478572a8b37c1f8d97f99eee3", "759f04ddc9522471b46d139ab347668c849ef8dc", + "3857983398d6c7278686c9f12fc2f841c628bc06", "11d9aff118de5f99bb0b5d55bca04d0efb36746a", + "38ea029efe156f07385cbea0a8743e07f418dd76", "39cc2896ef3cf7f9c2320fa513c645a972063697", + "e085b589790c05aa27f52ced4fb0480230db38b5", "5a0afe994e81c19a69ac9ce3a60529b5e7e7c529", + "e42f891ae64b8844c2d37bf29c25821382ed654f", "1fa9fe250443884c450e92ca44a73f1de6a4a5d3", + "0241b89c84097de0dcc7cf7abd65098ac3020578", "a735eedc1c421d2ef287fd1759e96699dc7aa75c", + "cef88a007fa60f4cd873f1d891a54ce5e173f3aa", "db634461703056448a77d669d907c3861cee6dac", + "bb13adf5fd90e4ca503b6980e737bb0f3396a63a", "b48a316abdb2fab1e7e3a797af20a7af3395587f", + "8ad895a40e7c6e7940aa464acf094e5c4ce2451a", "685755051290c901f95f5f29d8cb313c43833837", + "45e647b94bf47482695fc747a2b8b5575611aa6f", "b6a2d2d34371c5ddebd8ff6176aed0f5104e2d01", + "8c2b2995c2097f5699c101f4ab5e653ff40014a1", "db9aaff1052e5229ad6cef667543f9f6620b119b", + "be421874326c6922eb48277e347ccbd53099117d", "c6f2bf1b0331ef0f7d295e29a86717fa8b5698a1", + "d27524c41de17bec43692c05870d5dcb0f0458a1", "ed35e0ef407e405818bc218566fa79323d7329f7", + "f0473314bed3f901febe06355abe43cc73ea30d0", "7cffe54f8b960931055ae20f5983b4ea34515eff", + "d46056681f07ab1a435c625c2b46ed561240a573", "b58203e04d2cafa99307443c62f5063ca7d2e228", + "38c38c579147306770452db9f2405c2aaa8bcda2", "38f4356286376bb91a2464352b4cceaba3a3f6a0", + "79e066dbd2e0248345b032648c9310e2c7778d23", "160c352d87c121ef1ea43e49831453b9e8d4d0f8", + "be65342f97e4e7234a94ff6dc2e7c25f21f8660d", "058f4abcc85a2fd62fd3e5ff5b3b673824aa6004", + "5fc2dce168e2f622c8695ac0365b7ecf1309eb15", "6cb3c4075881492fe7189c502659caf20ed37a89", + "9860ef0921e59b2bd11767b5382f3815fff1baa0", "e8b9b463f80953fe29eb73b91468d30f5ef62c99", + "87992ef1fac6c582a28f3327f5d7f4ffc553fa78", "eefed6797eeb529dd376ea970c9c79a6ea017f99", + "b37c32b7e9326f350d3287e5870dbded1989e7d0", "990f2380b5c095d71eba74c595bf0535b500cd9c", + "238ae256a3a1ab93e682d603c4b53652fa0059f1", "9ed13156c47c637c5abc48ee20e6c6182c50f778", + "0f80700a25acea6dad9f5b726f220ee8cc227d7e", "dc628580c9eb037386b4e295db74b7f3daa74302", + "d2f914b6abf45a9bffc2823d7ed53a88ba231907", "133d19e1e5c4b0b87017942d4ca37f731af5a91e", + "1cbc14e38442cfbae772c080de6e17b3df09868f", "3106d7492fd3faba686f789ac5b42f2770a446ab", + "d868af0a96ae2ac8c5488c011397af20146017c0", "7db2d14148d17ed3ad5bdeec728cfe863a20f25a", + "feb813ec958af1823c42f4f5502677e38be72d8d", "0b2ee7336487a12eeda4ba66d8f34c332db30b8e", + "54de3627b1038f15d905c05f4c46a1d781b6b44e", "168702d959029c760f29c2ac9dbfe5ae501d7e8d", + "bd99b577781a7ed50e9049a292ac48d2fe33b983", "fede6ecee14c3842ab748664206bd633ea51c524", + "a1d968a1ff87035e1b4bcd5c4db2dfe15474c298", "93683071196fae3e0f92f10bc4533575ec4d058d", + "afc1500cce291ecd2c1bb748d7b627d11b1f03d1", "7a123d33d1a90683ee3df04e5900277519259c75", + "49a541549f2bf86d9bc6037db5a61f5f708756b8", "4a75b61b5f0cf472c5fb17e2c6a271f9c5ba770a", + "27a10de6ca27ebba2d9c667408538da980a85d5e", "3b7c0a467d849acba884de1a8c9a9334e8b7f9af", + "17001bf4ac840128c88152d0b1e6fd62d6a69f2f", "f9c27614dfa3934bffb1f69cc9623e8dcac9d0ec", + "30c615dafa30ea4ecc030e9d03265908f38eb389", "c6134d552ab4bd41e92fd8e2f4a8faad0c15b296", + "ebcaabe082cbe7c9a546a6f156a2fb67c9b98097", "f2a681ba398572b3a88051c1eef80994ddff009e", + "1c9f4f163e65cb4f0f061307d8b3686a042cc9e4", "3db2fc16ad04fc0bb32a41a9ecd013f9f5fa5c36", + "6d2dc1e53a6d16b466ef449fd8e14ebd3fa20f00", "a4dfb710896797d1ec1cab191e39cc51eba18b4d", + "0f51f6ef36ba16737589cab11ea2dd1742671c41", "fa85adab7bc82c5d14250a988664e0f65d23643a", + "2d8afda70e0f82176d7ee80bcb653ed09d4ada76", "17b618dbe38d1374435496778d3cbd0f3d3720a9", + "6116c86a6188a345cb430e55b48e660009a85799", "479f00719a1c04acda05bc52e8198c76b17b767b", + "41b9555dd4375ea0df23a0a6a2333f7f4d9e626e", "c3a3634129a36fb09c4b79e737582e33bd2a058b", + "7b1f9abab1067273355a38909e102fd2d3a5323c", "2cbf3503536ba11f94a975fc0c547529315caa9d", + "4aa4133193a8e8c3fdc766208be7bffb4983bae5", "436fe5351a1bc91413e903a2d2cbf2a8559a3c5f", + "c033d7aed97d788aa88220df3c8cb72fbe1ab96f", "f11265286454e2059d0de2a82ee4356601552bd2", + "f8c91c4de5c8d19881c4f8b0461bc1f8bfd01d94", "25912cd765f1ca1e19486411fb3de9963fd9f925", + "1026cbd8af888621a402e114a1c58c14da41bb72", "1cf1bf68a47746b2d23e4211c03a4b81e9626d10", + "c5980aae60fc5c3577e241a70c0798d8397e6a07", "9e4f93daa40369d2cdb97bfb01ee8c93bf23ec80", + "2b61d410af69dcfe59d11337df54412a7ed495b7", "d9dc67568e7a7147ea23e42083d2cb8632bdbe6c", + "21bb8edd05cc9d1142fabd4aba808d6a8157f4df", "258e8ec05412a73bb0a9e916db941c7af1b3e959", + "98ca8e2ec4264fb7c6a3a733c455f8d0ea089b07", "d6cc3a68587ba633190b18c083b7d56747b7a19d", + "3525059c8b3d3fadcff6f592c0e2ded7fc06294b", "e01077c19252f97215f6ed141d485f877506282f", + "8a4eb9ac7b33af6ffe3e980c685c290a68c93ca1", "567acfb3e9e9fda2c4051c038c6621bdb47ec823", + "675e293cfa98252cef4d8b7980066b038a901d04", "11e271cd1a3df6c75449e0c51a21a6c3b3d3fe22", + "ce4101fb55325aca6989ccfaebfa9992421c9e2b", "8e0211657949932ce0ed8da68e72b5c5981fc8f5", + "fa016eff3df324a5ff9f3d43d324a01177ec8939", "6f4426eac2c30baf20e09b51c5369156eff7abc1", + "2040109acbd819ebfa776fc97631815b122aa7d4", "30a65a2a2ce03eec847aa822f722a0d2f0f993bd", + "2f320a37a7ef0d32ae4fceb09057b1491eb1fc32", "25e7a079b6717ffd08ffbda3254fddb072160d82", + "146489482d21d37e7297d3b085ba786409e328a3", "555e44e3d20abf44db5780cc7fc8c95ae5593239", + "26b2a34cca8cf71e092eb3e7b02e53b88e6c03a5", "d6278e852f56fdb00ee5b9d97226be1589853166", + "5c61d67a3f8068020979f09c9f7be11f56e95c2a", "e0647fd778205f62b1956e5320f682af61fbf5ca", + "5d55b31e47053325b4a345c445c62413aa69177c", "a71670f60bd64ceec2b68fc07608d4ef05277ad5", + "e6751654101127f666768dfecd42fe9c6e0b47e3", "a45fabb93b9405dd04d2d944fef78dc424361698", + "5f31e95c53879bdcb5bd47276a8aff25673655d8", "fe3f9a83f7db420747f402841d25dd73f8b36d30", + "70dec2f04b7567d45d5a85b77c959623eefcfad2", "042f91e4b2b8067e6b672816182ba9ae2465812a", + "6e7aef5b4b4c6cf84ce51a4d8b947eda306a26b5", "3f01a409a7ecc34e825e8532e56d04613397cd13", + "893a546da4fd221501d3c865c7bf1d16ce9d933e", "f5410073902240fb94b50c30506aa02dbf9a1517", + "722657b05ed625309450f4f33953d7a694a1a414", "691d303ce7356e544cbfae5ad8054454c0f69652", + "a6f5a8bb856a84214811cd2b259edf0a30c6144f", "aba9adfef223a27e9ebee5f1e5896763adc71641", + "500c2fddb600fcbe7a7d8a48d4e1fe7c3bdcfea7", "ce957df3478f5f95b668a0dfc87956da190d74a6", + "3548dc0a4e272b21330ceefe9c8c63517941c034", "b252d68c0dc7bc599474380a7a185b9c5a9a5fb6", + "c2bcdec50527d1153a73a5e191d7bb4ab1f4a5f3", "552e48c600134179ece0d1aabd13d17f124338be", + "1a18ac6ef2f20cb3434d1db59bf4ae4c3a1f715d", "ad99dac6002a2975e445e8dc90f5d93c53991fdb", + "04f8a3f464061a9dba2d1c00dd7940488720974a", "b8c2d1ebc7b75ea604e9490fd265a6f797add415", + "4bf10cf4435ff85c7c3e2f50e7790b2eaa8861a4", "66091f34bf277d5491477085551afc1a8a361a0f", + "f7e1f2ac9f1767984bd125184f8b87563f249bfd", "61a1c51b48f7b278c73c13efeba86aa2484a33b5", + "3f54176089005fb6dc4b252b4dce34370ea8bcc6", "ff6f45ce65c0b32578fb92f842f123581413dfe9", + "6163138faeabb67eb3ef1b97d6a4b08a5c45caa9", "64cf4c6299cadfe4e5a7987c8db3383a69a108b9", + "e98267285f704dd8232dcfffc09616ffeda896fd", "61d4167d00783621e8a830f267b799b8f22969b1", + "12551cea0f20963404d37821d889c7b397ce29ba", "9aa042227f5625364418d20187dc76508c5c32fb", + "8213aad94904430b8165119337da1243fa7a88ab", "e8c47b170edc02abda540037fb4752ef03a21849", + "59a6092a06c163dfb2fa1deea179cd84a5e91083", "db2b8564a0ed5193a70c607f23e575c8f023454a", + "12048f9a795a901cab215f869b185d6e46e2a1e9", "98cd9913a99020f97a98297a8f085881e8ff21b7", + "76c1d436b2fba46a7ac7b7d285ee4b5e3a6660fa", "cb1c08bc56644e993505a2ab863c18ebc8a35cd2", + "f532f53f82e2fbd789b654ddd6eebe181c12d5ab", "1aaa6ce090e25893b7b8b46cc99fe8344eec653d", + "83d4ec39274f91989811f845c6fe805544bfa0ab", "0152b43193700d54d9826709132ac2bcb9ce5975", + "57291a113482eb4277f43c414b09afcb1bf8f0b7", "3d5ec4de453ab8ea873a6c8c0d0ef0b2110bd386", + "4168c8645f251d9584b25c5a57ccb089178a98a1", "0dafcc9a1aba7282e98d653730b496d83ba13c16", + "e45a7da9aba5144429cb9019acf99dc668ddadd5", "0844207c03327225f9913cc4b1a8702330841a96", + "329121933af05b575eee9c77bf13b9a4d3443dde", "4793325d739b2d9d0deb39d1118e33d76202e7a3", + "517ce4da9248bbbf77c7c9f9226054bc5412176b", "b9ec5a7fe95a0c7e32d65901da5d33ab8c30c779", + "cae143cbf533885d8bf978f287e9383bea46c3e6", "e2df632753bc6545d94b6f4c76af47a81ea3542b", + "eb3dcb724dfd1cc85e30594a63839ebd46c8a5a8", "c459f75f02d0c3444baef25d6fd228ce2929f924", + "b98731f2971b9ac5d0acda62cfb2cc32c141dc37", "6b1c69123acb73fd0cc23393bb165e99bab2508c", + "50159c19f26b9c1433e50f388b3fa759c92c0ff0", "08715a5559a18ec189f91421618e534344f342a0", + "2d4792e23d4104e5a89c17bf2195c7a8aded89a2", "27eb88f965e9d3df3a1cf219db7883093f339676", + "ee2acbc913cb75dccb8d1bb124b074bacae27ee1", "d9a2df31b1ff1ea6796a25558771ed1a38e6d1f9", + "b0c3bc7c96cd65435e9ff6b88040799959041972", "c340893e4dfd3cbeb4566cec5e8cd9a24c9982d5", + "cd361d820e14ee59bd8daa6bbe3321cc7d4fa436", "89c6916d89c4fec3cbe1ee6c780358cf51ba87e2", + "262aed06bd219cbfbb4e91c0fbdfb606883f74cd", "e32b82fabd2f29cdc9ac9e8b33966b10f8a0ce18", + "1e3688e139a11ce21f3a9e5c74d176c1ace5cd38", "b460d264e4ecde414ddb70e918a1adb48ae610b7", + "41bee95c0185b6c2521073565e11ede0885a7e8a", "22bfb85e284a90cf37d23354f56786941de27a99", + "59c7f2256d0b342d1cc08eeb3f4dd9c7e61e90f7", "6f791793df1d1124936ad53e9948cd221a184aec", + "e5c3b79336ede6948f3302d60de0fb8a1b7a138c", "666d1b61ab34647db7ab8e617a9497b87166d7e8", + "4d8966bad69bca5970ce0ed64c343fac0e1cf698", "852bc11787bae1ac6f99622dffc2eea32ef121e8", + "2b3067a1dfe1648d1e9be9f54f8e3af5ee9cbc3b", "1a001a96c0fd88b7884b9049a6d47f6ec4d086b3", + "740106ef9e67ae17393d081580f5c1493db372a1", "05e0f5184a2a0c6a003dd1654bdb4d1da4b1a847", + "8eeb1017e6f043e43d6eaaa99d2aec62d2aa2776", "1e34543f322aa94c5f5e679cc8713120002bee0c", + "d253a3c83f3a062f0a18220bdec2188a2445f2ff", "2548241b69cdc856b24ed8fe930ee5a608348b8c", + "47c83b2f5b8cc63496aa507f03babde7e79c730c", "1169d83904da1f140c52a51748d1f9920cadc304", + "2551edd52b168e0691cbc4a16b6f9762c42a2ee8", "b07d82b7fe426edc3ea57eca2e54dd1a7865e61d", + "fe52db5b57808551b30af34184e5f11fb12a1a69", "2b29dbd15c15e138057630527d19a67d9ea9198a", + "5cbc7715b58ac8a9d141cf95813149a09d7f323c", "918ea978ecc68b7f3ed541b7ed1e073848bf06f3", + "af49d808446f40d37d64c66517a45cc19a78de32", "f2fb962dacc2a7f1b1a9a075611e300970463a1e", + "a0ac9637dbc9b7b4af6d37256011e8e777379cb6", "3fefc0572263475219c149f4c54fe1b88f23c7e7", + "f227977b57b78fecd2db13ff2b231d0c22a3b274", "7e4c8a38b0fd82cc80df79b1daec14c1cdb58be8", + "ad3269310aba337feac5712f9621a5b8ffdb29b8", "4ad90d121a48a0390744035a7bc068f885abcf89", + "134fe22aeae849409671acc32e13a17a69681569", "4e0adb75d47a88366da89405264d87d88cf358c1", + "f1252436b3427e8fe87ea9ac07829f977ea12752", "0679d7794f638bc86e47ecf8e49b70482281c20c", + "c06dbe314d1f44c63b8bfe6edadef6ebf5c5ff20", "ad6e8fa9bbbe6f40621a826662b858c96eff4057", + "9756e3372243fd818c895910aa53f4148798b7a8", "dbb3e71028ba0573b07f6a39bee4a4ff7f39c486", + "6aaf30534ead2af6285ed5f3ff12f3680f401066", "5f7f1c1613ed4d17875a6d845e8cffd3fc0b0059", + "7c1432135a24132618b786ace67238029bec8591", "76290595b414a9c7ec3bf3cf5fa6ad32920d32bd", + "97d4988c8ccfb194a7d01d2c427e6796ed36c5e1", "344fbf8c2cab7917197614e4e974bb5478e4a653", + "6a2becd209341a5dd760515889e5aa27a26e1213", "997e1ee3a567f849865457e22be43aad5b84fcad", + "b0a9c6b95b650b90e69e3ea25f3ac2f583a9c622", "c830370439922abc9cef0763e9cae3387e549c42", + "9bb1447ddf4fbcd71f68096307fb5e222d38202d", "b862af307888af607c82d4d2a3a53e12d6265720", + "585e55bba7d4833272526051a7e2a47416f8556c", "99ac5bdc45f0bec78af9807ebcd8fd118155df93", + "c7e14a6aa34a6256944ea4d9be85777a0017415e", "cc2dda3439152bcabbc625f21d9251afbf15a076", + "4e84b51660d909910c0ce4cb014a23e20748ed05", "f43e4d0aca4ed483f9b2568f35e70dfbdfbecd7c", + "10039ee5b86d6ab30389abfafc4158da5e051410", "a212609775beaa7909f92b708b8348b920c34bc3", + "26ef38e22f884aabecbcdc02640beb2e8ec55abb", "4689201232928ffd29fa14a2bac10fa44f45a81a", + "fb2c9cb75fdb96ad4016a076773038a6081a4792", "d80b52a085881ac67c9c8748473991852f8a1113", + "47e4afb01417c95ae136318ce8d066c77402f30c", "2a256e4dec0180e6e27cd8bda8b4ef1bc3b1bdd9", + "fd4b1442c33a018cbfe42a859cb4bbbdb61eb54f", "f36d187c3fec787dd3d4128001c545de5551bc08", + "ef30b00ca626b1bc763427aed853246c0ab72534", "75eed65362b1370b90c3a032dac2f366f1469f1f", + "d0149fcec2d2e19ee8851b4ea2e328b2ee689184", "9e8eca00b251efc8bea606b3571208966fb2a2d7", + "28e3df3754c86e9809909aac9fe02515e231d60f", "5d4af1c155c639c81448f1d51db6cd33431f7fae", + "7e6a0c4ba45a0e692bbcf9456460aa56a974ed01", "63b04d5a98be1d3a23003f6518c4c8ee3b065ce6", + "31991c235b014fcf4fc9a99a7fa4824c705af5cd", "79231d6d3fb4196a482cf902439ff727eecb92ed", + "a8c246b8bf081182b08ad2a4ea478b7ba860d006", "33750764f3c324181451b38a1dc5219ab04a448e", + "370e22839b26494ce0ad1a88286f427434f3dcc5", "200b037c3083bd2e28e4e31f015bc87f9ab7196c", + "6a7573a61b2e15fd261863178f1f54d96ebdf408", "adfe5473bf50ce4f88c88826d4055b144ef42bb0", + "9d66b222b35f994a0d3ab6bbfb1c142d9f226d56", "ec03206161f942a59ea5bc91c08086a9b1fd2ae7", + "ca5bc0d3d6326e4df076624d826be350a89ed54a", "b8b6473dd4a5f93cd6eb1c0e41d659cb7bad0286", + "20162a0585c37d0c14b818eda6f695c40678bf65", "369ce701b448551b3b5b7156667687c68f7a4259", + "587b5c41554e3aed002f92535ef2179231f87e6d", "3b81c061b9efb1074e0dd9cbb5f11f71420e0832", + "6007cac67baac66d72a57c024260e9bda13c456d", "b1678a8e2c14c96405dc17557ca68d059ff68b8a", + "d5879f68ec23e30ca8882d93410073a0cd5db92d", "90a76730c4ccc74d3cce95abfff445d70bb98499", + "bce86716fa530eedc657bfd7cf18f045116a967d", "fc46a08726a7821ebccbd4d1492ee05c6bae89f7", + "d056f38506127e986b226fb5ff5683b6deeefe37", "1119b6ba9baba7c6511bd57f21eb642085932790", + "2d317fabf4f5e8d7a858a3d1dc79b88a2640b241", "498c2b6b4c0964e25a76cd8ef9e1e25dee1cda7d", + "803f7b6304b6fdc60bf08212fa280e31365d16a9", "5532c2bb541a785fd145aeaf624491e201ed7759", + "243023d8a29dffb528b567f37b5ef1d2d78f3fc7", "c0312dad18501b449812f9c9aca3c39fc1647432", + "9aa5151cb1622686f76e3e956dc9a6e48f43d3ef", "c9ac4a1dffd0a1fe7a0a339b156259884c54057c", + "cb53da91b05ac93ef8bb2dffb4f0ce8ddd06255e", "d25e73f15b3a94570b0a85e39df20d4f9d6a3ded", + "d0af00f950ccb83770035cd0db90fd8ebbe00d10", "ae260ec3ddf6bd62cb66ccb1c9ba3a6ca9d19ab8", + "846116d64285c2125f01ac2e06060b84ff9604a1", "2261c2c6e3edb6c2327046fa2d5b69d9a39ec10b", + "0c938db82253db2c09d40079dd15f0a1ad43345d", "4e5825c76b332b11e0e3af2ef74d9b98e3784809", + "83b5cadf47b3eaa15841e3815099ba5338a5f683", "2f11fc2b39f6889ae6720c0592344995b5942c37", + "3e43da7045388e7259414209b3baea7ee339bb9d", "4b71f161759ed105daaf49ad74f3306ef1710855", + "ae7070f6b723cb4c5055252017e5ce712ddbb649", "23cff857c47ae04884a1e522af2d5a836ea30f2d", + "424a8b8cd8ccd2ddbdf3c2d10ced6767b53a4ef2", "6391feb5cffcecf562f79be5c7b5c620cc1ff24d", + "9f1df5788377be4be03c705b21ffe72e3fca68b0", "607db9575d0fdcc02810ceab97c9b3faf9dcb601", + "ee0202713209fc3df5cc7c1c75812c325c94e280", "47905a7a246fc0ab81d6620820a96a283c6fe8d9", + "21ff79105eba8e77a638b42b2e18589a59033609", "28ba2f0f2967b8292a7071c3d99c9438b1db7963", + "e1fcdbf48c205c3a724151cf5008809615687dcb", "fe9755c8149d239696e010a0e92e2fab24da5aa5", + "e6b72cfddcaa930408e0d20800add718d44f30cc", "f341256a2cf06c968460034faa6d581f17af65a6", + "23791cd278e193ccedeb2ba80666c96e9152ae62", "f402a8f708787e8e97a9e096dbb47ccda9009c78", + "8dcac59f949b44e36ace133352ff4dad069e17e8", "f36f4c364f6f702f64cae7210f0d32a535e93bc2", + "8e110a0b274f6a11fc3773a1bd0933dca5cc962f", "745a2fe44d82669a5070791037da95a0d499ee3c", + "d1bda5b3a4ba7b9d7490d8511cce4a11bf76a8d9", "c2fb15525ed50bb529279aef0da1ad889a209760", + "ac5d73eea0dbddd6591c6254078018b894c74d9a", "150d6faa9cbb1ab06fda1d19606adc701633f24b", + "98f046a4f73ab3edd7f33d87279dc17c6009c844", "c0afac03a62235c223ef23a29e6f0f868aa89f4a", + "c19ecf91221af8e0d29e9d6693d8922989c8364d", "6cc04cca454e9610729586ce72ad276e8a8be962", + "5be4d55be3d8a516f8b9bfd8fa4b4b6fa072b3fb", "68e5dc0768d1143138b22a4dec7c0796211c515d", + "40b502f762b12d9008d67359a69427246dbf4590", "c7a0c0a1d86245d58f93578f5e71861850b20327", + "45e88dec3420a3081ed070a89974fade34a8cdd8", "a2cca480abdfd271d83ec53f32b9ca7c0534b1ea", + "eb5b56e00f7e3a3a93a151b04de322a54b478a1d", "ee726a3bd27fcd17373550a73f1265123f038e9b", + "ecbef3833e45ff4b51fcbd07be7ae083d0261106", "6ff3d878e2b0739195fef120f3e8920aae5b6c14", + "c8a254b1ae30d6d0372b9421cf164a2f92770bb3", "b6fe64434d331f92023b8ac4a0ae29a436ece676", + "6408dad003df5f902970c9a9b281bbdcab70513b", "b000f8b57d68abac68a8ba92d2e8af1f4cf04e3a", + "d70f6b719ab62e4d30947d46b48c4f86431e27f4", "d8b1b3fd5b14a37edfca20bafb53bb19b68034b5", + "54e2802c7dda0dc48297627148cb8080900b026c", "2a88231e4c45be17c7b59df395c88ad1de1f55fd", + "bd9541f3b7d18ba20fefb5725bd21f29c01a2fdf", "fb211cb3f8ec40bc8d34ef5da91173b85368ddb8", + "302c5986252fee81e024e43b3950ce0c03c1b19d", "c688626a38c6777646e4c5343903c5d3c39451d0", + "97e8d745c998aa9ca3c0402852cfbdebf38afeff", "512ca7ae1cc0043eee4b9fb24bc680e5e439a22c", + "8a1b18f3f1a52393779d7872132a261596718531", "f1f688ce752c4783753ded0c034de72056e5bf0a", + "cf404c790f1926de4fa92a90bc25bd3b89441ee9", "6f67fb37b601d81e607b4784b59a60f5da04ddc9", + "0358ae29870a0c0555826236454eee2a35460bb2", "267d40944d68c9fe031797824c83b7628ac21054", + "28999639cccbd7471536049556b7bb0445df968c", "96c339642cf640e219e83499b89b3f14c40c1a46", + "22ab4abcaeaab3dffd0a0eef49631c41feb35f01", "f3281e89aa93a9ff4b045c561960128ff02bd67f", + "ba7c3d3d19387b725dd93777e264c2d243c56985", "c06958105add69f9c1215959af1592100820f897", + "0246ace3db73d80516951169f697ab82de31935e", "3ed4aa3045ac0679828da5a36e6a159a76397b85", + "a9b3d7258a65a4ffc0188430f23a2b348bf3f71f", "3f80e737293fc727a2a5f5931afef5da8a108889", + "f5c4af7e6ca21899d603dc1c5d6dec724180a617", "83223c9cdcb837c03a1ee706a68c201c6f46aae1", + "7ebd59911482548ff28bc5257f092339747a9847", "a29e7d193abe5f905841b7dea04968b1c973775e", + "4df92c585a35462fd174a94fa6cc34cb104a3338", "480f1b18698079c345470f6eddf82dfe04f372bf", + "e93635e6ad4990b2037c1c42d96e5f4f5c382211", "f27588ba28c79a1abd36e9ab225006ef711bc2dc", + "0780639bded276d6de3f29a949974f9aad888f1e", "7649e12dadd9626c6aa55ce09bc5a77abbc1866c", + "10684acced018de4c9d795b4e3d17f85b831ed05", "1fd08bb5d65fb7deba4ce611c4e4ae61526f70f5", + "4ad405913be05105cfd8169727862962f73ba90c", "e622f46b8ab1a027717765dd7d392879941691c8", + "e53bc56ec46866426314e2214f39986f794f7ec7", "1ee4211e5bc419b24309bccc29d84c71d8153e33", + "13bf7fbafaffb9eb5305cc786ea5327b1bea8c84", "1b17359571b9a657d8c1c259f392bdca507f8048", + "f14bcf0c9d9d4441837089d9406a275ec7bae489", "650d0d424276a764869dfa6d5769e0d33dee29ab", + "9a51d909f6e0ea003b7bc31368889b34cbaf99db", "d25b13c8e67b29bedd33634693fd47107d79c386", + "d4791ab3fc5623770bd37a62f5847a3469ae62b5", "63cf95a96c6f9550a526c3f7c973b3bf77da1d99", + "0880631d8a31a2d656c214fd5326e86d72d99735", "c028799f0dc5cdd928ba41ed57758827d4c33960", + "09285b7758bced6de4de28adb839ce3707bc0801", "ff3ef6de1556d6a817b228634d9bd36c03f91b4c", + "7599af4e7f3c5038845232dc1c3460e757d29e76", "945656d9dd3e48ded82ae075761849b0b6de7d99", + "14d2b00bcdde27acd1e67707fdec920aaff41440", "bead8489f2f77870afbd190228724d8ba9525e77", + "ef888c5167e12709b33fcb9373476ce222d3a777", "11185717aa656979199228d4f134506f306ebe11", + "763619ad1c6fe756de2d6ea598f4a35b6b5d07f5", "de64195cf8f8e0a8a209993c12c37e6a0e9717fb", + "6fcd348a0817c5ba9f692e9f43b157ba53f61736", "184223aaddee8a795fb21a0298bca7cf9f24b86a", + "025924b72b369155a7dc79296b0fa1542b6cf66e", "00b6d09efea3b56ed30d3f626837c105415a2b9c", + "7fc27ae1ff5720a8632dc7180a62f93f0b240971", "68a1871cc2f4239c1e4c717d6cf5d50121aa24f7", + "d86a119c71e8f3db87ef84a3f2eaee3784e514eb", "bf6c04384c5ebbcbb7a2635e885ad48223000515", + "25df57cb358e1c55102d3eb05c633bbbc8d18ad7", "c2ed09db58daafd8711f0729136bcb4bdab4ee39", + "db9ff970f2394e5ddb70b05835e5cd2ceff1758f", "b5b0cdc6e2d0971f11ffb7b83eae62bd2f2e2a63", + "568a605393fdf0261752b980340b56a695993ca9", "53a7d40199c4cd3e0b61387708a5c4b2daaa6a04", + "d764e0dd4cc13227c90ea0dbdb7a17148a37362c", "2b0970782ff6fbc9b574d9af09c7f4387da71167", + "1a2f72aec014fc3b1c7a58660c15cdf8eb78c30a", "ea70291c158d9ac2e034f91837f3aebb9315e74f", + "ddeee68ef8756513d3c91e77705759282dc88f36", "c4c43e1c71a7693a257680b25c4cce05fa68139a", + "cb91e9590a1524dced13be78e13abde059a5358f", "8df5d8b31502fd506a59b0d15f61631dc5dc0d11", + "ef60833d075ac1aca48423ec296b18756a0907d6", "4bd51174d78af4c6afff4a3580c4380eaf53db03", + "f89f34968e6fb5c4400d4133e5b83eaad9ce1d81", "4926ebd2150fca495a5ebccb69d73a843c2a5c76", + "b7d7768027c43c4bb9178b59096058c827249ebe", "b18e01b7ab7b8fdd5e9093a4d10446ccb4c8b461", + "7c0cb0bf58b260d848be8b2fffc6ea81d45bd9ab", "65560b3113f1d79c241d64615c70a95a27a1295c", + "ef7de7cca843f688185aa44caaa11b8d9c5ba419", "577959df764fbe2295c7355420fb88e7259318f4", + "fb4e5fa6f2ea75bd25bf313675ace5ecfb09ce4b", "58c3bb9c95b2b18b54923c1c66a271c5103b5016", + "d15a9ce8720d18dfb1886de4ce23504c690f9849", "09b1961ea1fb0304ed7ad7c2c404af3113af532c", + "689943c3a6fe2d15706f588d5d192de09025f87e", "76c955116afc032044fa820077f0aa0077a9b221", + "e4e28d5880eb5a19eac4f2dafc94b87121315736", "f66dc5b1d3a36e518a2d7581534cf1eac1c0bb86", + "437d9bab241951c67ba4344ba60e7f8c8fe091fe", "d48f36d517120dec77f9f1383c6ff175e490957d", + "3595ca188c62c02f3a21616a40fce4063e5639e2", "9335fe0ddc8db4b8825b6a8e96f2ae314c3daea5", + "93b3baceba14eb826f0691ec927139eddd131418", "b081a6d644c99936ce2b53227aa949e61c1fc1a2", + "7c301a66f04090a5dfa9b97790c2398bebe8c658", "70e640c8b512f69cbf2d3815ade5e08acfdd6ea6", + "a5ff6189fcfa209df6073595ce8d048bb434142b", "865f86d6f58109166006154d4cb1f18d780afc7b", + "0b32bd194264f3d34f207e36da01747250f370b2", "160db72460b1d2cb57b71c5f5005c43b39344a08", + "ca17e8fc42c63b71a64943b2edc546cb483d4987", "62709ee11b06371980bbc365eade2cc09fb201e1", + "d42b892d83a19496adb8a51865d7786bf11cb51f", "151eda2239a8926cdfe11bddd08d4a68dfc06fce", + "48289268a75e7c551d8407da65a6a8d5e96bfeac", "76475e31fb20faa61548ca6c110bfaf0a8509333", + "ff45b9e590737c453377adfef746c419fd024214", "791336c9f92b920c36d87abfcd23717effeb2979", + "7cbdb028c46f69f69ad5b3870fb9976a339a88ea", "6dce9b85f8a4d116d780895772890e62ec554050", + "609a9ddb002e9e03ec1e01fc0d3273defd125f6a", "eb5cd823d83464a85d52abf6fb618ab6b0ceaf14", + "0b33f97dbe25690d5b395aec431714f326835761", "b3c0967b6cc938da94bbc24bbb4960c499eab996", + "55cc0375389bdbe9081930f6aed59e2d1b996d68", "3ab0dd9b211aa79f607aa18900021d331ffec108", + "8b8fc9dc6eaeab845218b9b42c9f1eadb85d9a76", "d95873f271a98500509514a3bd503e111b2eeff6", + "60c22d4758b2ab938d925ebca4195d388976ef0b", "4a71cb75be4c145468bfa41edeaec4cad48b35bb", + "674df8ed0814f769a01ea52a86b37a37db300fdc", "52cd156a808d638be19a621e77ce9d0b705ff959", + "893fae64443d115a0071088287f1047c9c85bd0c", "fe745f1136b3c2b4d5cc703cee8933de6357c502", + "7a9ed9c04bef62538916e432a74f44c034068c16", "c60540472bbd9473a78ffac3a071904552cfdd6e", + "dce6b37c643555216d24705885d35be6f1658323", "d25cd6e77dd4f8683333b68cd17e4ef161a10f82", + "ef59cd7b3e448c27b726dda2624e3a66bcb7c915", "caeb866326a82dacaf9b9ce21de180c99462b0ab", + "300bbd6765f353ca1f7a42c8db1edbe0fca9146f", "d476be09c2c9a67b065eb972d473c862d05520d7", + "f2692ec53c9eaeb976a81908a1ad9a8a6065eb56", "80a904cfc2637be47cf6db433896285c96642f28", + "b4e7c96d36534976b532f6a73d687ecebd20d3ce", "f442e491213973df215b4e5a13ac42dbf26a418d", + "4a990060a137b7405e90d674667b774c2ab3d9df", "ea028084a5ba60179ffcc5c2e7745a5137810674", + "946cd7b549d16cec7f31b99e31a5cd1f97ecb0a7", "f4d1570ca148a075ffea03034aa359a795a7a95d", + "b24a0df950d615d89a5c490856d9efe0c67e6807", "4a3a4973113c414789c798e3b279d04e68f18193", + "853861b52dff3d4e69cea17db319b578dfe36b57", "75cdf79a50fa9f54ee0abe8a7352983e4e15dc8f", + "d2172012e40b489a1b696aee59ee01beb94d2044", "c463cba55508e3410e6c938ca7f07cc49c69685f", + "298753c52149d58e36b63ae01437d7e9b1626550", "edf83ddead1b5bf94530280409c8f8d7a3566036", + "6ad9ebb63aaa22d5cac2c6ab68a8d044575531b8", "4c4e417f31a770c165848c34e9a1c03a88345923", + "01ac78c98a9e83a657895f330ac60b23df81aa19", "077b29278d6047812f8b65e158402f13dff2e638", + "6a3280cb4ecb45a4a949416106e707b4999e2491", "3de477994ebd5b7edf0a48ba6c5e799fe02a53f7", + "2ebbd703eb8308492ff7d0e3604b786931ad8617", "5fc0b82cdf849a6c424721cc5118a196c07caf43", + "851ed5018abedd4f527d5fc1f184016c2b9dcb20", "8271e8c30ed07e236d230df37635819f45b3adf5", + "794d1fbeb2f5f6e2ad81450602d53d232a2fcd66", "765e4955986ec0b20075669f5a7e2dca79a98ae9", + "f94426a7f4b9bd63aed0e75894ea80278dab4320", "c0006f951059f5aa58b4b898967d1dab70377178", + "414f4c053c7c9d294903806e748a132d54325f73", "a3dc4e32a5bb67c422822731ffbad447f2f89dd0", + "9dab95f8797ac80a22cd854761e2dd141be8f436", "c187e0b59cc8710d812cf24e9ad7191c5d0c6206", + "55cc88749fcd34f08c5d0fee8bee92dcaccfb010", "2c3d218a5428646b970895dd88e2f61739ba0732", + "4ad60738ad972e0cbca3e763b556937501c0973c", "543f72fea7ef066ea3111bdac8f0ac61134e0c24", + "a2bf3ec52c5d004e7e5329efdf6b5cd735618b87", "f483494815f89e80dc399163080052fff49031bb", + "3326ec457cfa6d30663eb0ce49a838d7bbb86415", "f6d5647f6b6923b5052619814a84b2e1bf8d0b85", + "f2dcd0fc70ca95f475b68f7335b2d0855534e5f1", "a50cd55ec2258557a20839a0d6219960c9b1a2b0", + "a34ae4d4326a0913d9af50546c558811565ec66a", "842e922b6252301d8df608a1a1a4a732686e2ff9", + "2b6cb59f7846a482432b4ac57fd960f73d2441ba", "e559e01ccf3cc35aee9732e3db97c23165f8a6d9", + "543085108d5513b9fe63eb96fb386ccc84a811d8", "c7474d0e9cfe5c1d8e56f3978eec6871706a38a8", + "fce05c355558c606a7ca815613c6ad3d8f8a1503", "30a17532024869bc353e28aa4c3da17dbbcab80f", + "13d54732573224d7879166a11857046c50e13626", "42868488cc892e63b13e7afbe157006ddc215258", + "94785fac47a5d2ce41d64269203f14de07995186", "82d752a38f9d5598685d58e5ce6080409b980442", + "96f99cdd47de4d5e4180c99bb7015fa77d329687", "e89a84828a38b552efd3095472eb39dd18be141c", + "5a37e8b4031b627270f06125f5c204eb28437c85", "30da9a8dfb0f65559be3f17a3d2f3be0cb297ba7", + "abba9bb98e64624543ac376fdce700ee5e33b392", "c19f87bb786c68bab429f594374c24d09dc98cd0", + "73754ef7ae4615cdfe881adea981bcc93b65e887", "44c25f0a9f3151efbb98056e1c2bae59f4aca0b3", + "51f465b5aa2a63568d0bd82232898b39fa429a14", "1ece6bad7a8d2492da84f5db5a94a9c81a94a3df", + "603617cb2ad5009c06dfc61432287ea41d4dd70a", "ec346b3231969799e3852cd7f4e951a25f3fbc0f", + "46af277dcaf4eed7cc96e021c96b266c24c1fcd3", "0965cb17b3b353174be301989ee8f5b77ab925da", + "b45319782274fd4841da3c70ad3b4754bf2b1b08", "2a2bcc98ea5b188049bd46b4e315ae55b359b7f2", + "cd09081ec384b747181bc2309a9485ee0888516c", "f636e931892c445a7ffb1aa0eae7499caa19932f", + "34ceb6f49765ac0d173d4a0cecc089f34dce2695", "9b18988eed2891623baa37468ef83dc6a9a93e14", + "087397f895ac78aa63b5b7413dd4c948cae4dc50", "98e1abd91d5502799931c25c1c19fa6605ff6dd1", + "279448c0f978c369797336bd660e457e2abccb3b", "20d76b4b81484d694731d6a2046661a025928bdd", + "7a69b783e3e747a0f7a5eff5cc23b9a00d53aa01", "f645296a6d289f80b9960679b8edfb6b0a4d96c7", + "9352d33e1d396110bedaf1906567a6d8139c98ad", "3a4cb9ed1c4e291a738f6a97ae82b7db616ed4f6", + "e97932a4ff925df81b844058a18f2da4f362edad", "2e9b3d733cb1d6a6fda98cfccfd179c35559671c", + "00e2b96be50f143abbaeaef1b42f502eb1c5eb38", "a3b8a036064eade54ac1a7e49a48b5228863f225", + "b050b6089b63a630ca1e102f27f1e33310bdc0f8", "91200850dde71fd5061890bae22cf6264f3a0ce9", + "c4c90fa787e28f241a6149c5706f67cb8c897c3b", "52cda4d77c07f76113b392291e947e02010a8ef8", + "949b21730e48bc645dca6c6a9dedf758035d0eed", "3e0509507736054281c0feb777b5beb62af98ed5", + "994993e838dda5b000ee4e75786a5a826f368f9f", "24e2e8a67ed9977de3c1373968d76736da3c800e", + "9c8374cdb61697a37bdc89b57ac5574af910138c", "7facfd4ee6d102ce1c2103b2bc0861865fffc07b", + "82f7d052e4d562b268730e8d66d4ac694d963f8d", "a5f79e4ae09eafe2892bff810ed21b1a7cbeff6a", + "7a66189f5b487392ddf3bde3b6b2f837711767c2", "756f813ee996e91c50dee3929035b81d5c2f5b7c", + "c0c98d3382858b9de30907003cb1b3f47bcb7e4c", "7a1f959994f3e8665589050ca2e07c5ed5579ff6", + "06944876fd90b41f439d337cfdbef2582caa4b56", "73e779de8b03c705726abbd686ab6f905fb2fe27", + "dd14f8315d4345a1482b6b5f581fefae03154e6f", "b4f629bddb661422c15c4b0459104b6a6387f3a1", + "a65dec2720ca334bb503079db5a5db4649c27c0a", "23d9c64b7d95c1d1c08ac83058dba6204537709c", + "0070ff46a8b72b79d011f59a770cdd2e3a7e52db", "39538c29808bc3b1c85b2f9555eb56296778e5b5", + "05235f2fda6b96b82d3b798f5c06d1c26fbf5556", "da9ac947e5272631161f7415583487cd5b81bd92", + "0afdc9abb0a921b0eb87256c34c2440d7bcdf748", "708eb61ae54de0fcccffafc686218a959fa0e12e", + "921d2ecf681cb2fa40d8cc9390591bdaeba5bddc", "d54e009ec45cfd47d90fa87926ff55f5668cd745", + "b66c3a482974a6950eb27fa2c83df33c0ddaee0f", "d11ee1996ea846c781624a6be16e2cf05c90a3ad", + "d1fafe49cdba1c135a157ab725caf6a73c44d413", "37add1833a4462b08f3d63ac3cfe2b29e8c19daa", + "54b9e091f1f4a2aecc9c3abef4785cd4116e59da", "2e7c5f1ca4868b1bf1681e852460d1a64f6730f4", + "fee6c7a1a74439ade1a4ba1102431cb16ffa40b4", "0b663b2c2e9813fd968ea4d53c27117a37af990a", + "2bcec290f45a8f707462a56e8cc468f5934fe57e", "2e77e56acb8a073a7164052b3015752f55748f13", + "56cc03e04970154b1ae9a3cb49eeb3d131118feb", "17819764b0cc55daa6f9a70e1b7cfbb082504a26", + "b9992bfe65ca57863517d1a44a5de4abff0d891a", "7cfd6ae2ae46bf951a84ab288fbf6381377e4fb5", + "44f321da361e4de492292e465ac2576625c2f04d", "14d021d38ede8d4fe6c55266ea1bc18471146ca2", + "dd5f008277ecf79acb0c32c88f459a46f556d235", "e712c5cfb5144ba8122f13b400324a3db3eb1131", + "2e3b8bdf70024acbd57dbd574b42d8081b22238f", "a27f805365b6074f96d0d72bd877158a6a2683fc", + "1777f7ceba2141c7725cae03325aad9015f5e0ba", "3ed72d7bfdaea45c7741c4adba22824729fccf93", + "1efd9d0cf3d575dceae0143ad80cdb36ae929f19", "ed1e3113a91cba66cb2bdd2fc62d221f0fe98720", + "56830484e8e3a9bb9c3c0e28d2e06c59c2f85814", "612f82971d281213f121bf3940fb3a22ef339b1a", + "9204cfa997dfafbe1d923cae857fd8772140d2b8", "bd7772cc8ad355a2edb8ca92d712bc6522f679e4", + "50013bba2f0c0daa2ba0601ad9d538b8945f49b3", "903166d464af4cbe28c3dcfe87e3b03d2d4361d4", + "498a0b6129cf4bbdfcd1fbe56f41a09711c81515", "4d82da411ae6922528a7e31ba0d9b7781ca7b140", + "d1a59d5be71487c46f1f13cca1329b890864e3a5", "0130298b9910153b918c24cd0305bf004c77f139", + "b4704585677546e3719b0167a48f7352c3135018", "bd2af11cf4b22ffaec8f0f81e41c7a200707984e", + "90bbdf000b6566e32e973a9ed13673babd3d5971", "53b76d4726ba14be0ff600d76f6e94af63bc76ac", + "98b92425f234019ae1a7d7fa763ee86d24a9d4b3", "402419c820cba29219805adb1b50909143962ab4", + "2507dd9d1d952d45d8e29e3da7d9b67261ddc8d6", "0eef7ee0455462714b4398429bc4e0b1e68aa807", + "c15af97c03e51bb5c7c23118564b21d37e41dc38", "5ff6527e38d98cdf140318d98828d7de099d1552", + "7d8d05ecfc3d87c0a798fbb28a7bd63d79317590", "d728b77adbe3f536b9211d9b26b0668cd796a549", + "ff936585cb1595e9b56e14b77c2a8605a312fc2f", "cee364e17aaa6f0d77dbcfe496b188de38123833", + "28f1af82eea595d5db3da22f4433c144fad620ef", "d5bc60e7980decfb5d5cd10ef9ea1bd0a3ce5fae", + "392ce37ce6411c30d436940506dc2cc8278f66e6", "38d291489369e7f222a2711e5afd7d6ac6afb574", + "479506b2cf7c7392c0a96d6e5d550556dfd20ba0", "c71e5676016de6a9b7dc27a123f403283fadec2b", + "5d03ade937932174a2f2b376efe6f761636a270f", "04302da411d5d79477463fa37de2f639afd5127f", + "d4e6cd7852b5b88d88afb1db0cc9d29d8484f143", "1dbdc1d3d9dba7ce5cac3bf7d2268ab237a4c5cb", + "3f263b23507300ad4dd7903e45876c2877e59ffb", "127d0081100e98843f16d26cbea527856d9a9ddb", + "dd5e22108dcceb35e1944813e2524ebfd0a53df9", "5d52758299b341021d135b5166614a46f3ecf2e3", + "a1b3da2881596c60f669fd80479e45a097b38707", "21a63be654b8e5df434d01d420ea2206c8d9a92b", + "36ea46c3da7dbc2bf4d304d81f86224dfe54444c", "29fcf933c2e58f6eb8fab04bf8c511f56896fe15", + "64737770ddca8995c64e1106bfb08ff0ff1fb144", "d8c191fa2b58539d83c7efc1c3346079ecdc566a", + "77f0910c39aa6108e0c2fdb32fe1832126c0489c", "af64db9a99bfd8ca4affe94dcd5cfea05e94e03c", + "71dd27e77432a2d5b446958707286d08097da244", "5ddedb0a995f06b295696a2a7b4551e404d0dd16", + "01f87385a24ece34ed8ddca801fd84f9c86e74e8", "09249256a87d4a90588b026d789fd3cb1aec856b", + "9240b4a3a946be297b435486d1f2a6d941422b93", "399ca25f86719994d4612fdb618983079d6a59ee", + "4db57eaeeec89f4a96d0f2727c15ff083482fbf2", "4a31d4c94a4c6ae379abe24c46817a1157eab07c", + "182f2002e4307f83c62e06dfa4164543fa7e814e", "97a0a7220c2b8e458641c3e6e56889254d2da049", + "3440a5e4b4e31f45c6adbae99b6214b8fbeebb2e", "6f94a7a5fe870501cdcea762e4a7a20631d48d62", + "9da3107576333b6c7f3e878e508e94a4c764e45e", "20cb94f57147f8372022885ac2356e464fdaf7e3", + "68803fd3ffbf592c0432eadf45fffa22e5afa8dc", "c538f4c3d9ed532f02458a7c863ddd549960902d", + "cd14b737781fb608698437bd2e6d20d691093874", "2db66f7bda4464a6f5342f5e021c29a38d42068d", + "37b12212ecc3a32ede8aa04eb0065fa80e3e821d", "887306035bf8e3f0963a254a6e46b52b55a04fcf", + "8747b40002f4a24a2330ee27f36ecc0880083827", "ee262c59ff4c66f77562ef879569193543838393", + "8282282c974675b044c071faf25996f4b6e4ebc3", "f9ca90f109c32ff5889c0c9d84ff11f9fb5b5924", + "e530847cf2aaa328e800919e6eed9739521a9b50", "ad726687605b280a0c190153654f4751ebba4c7c", + "93f6ba66b73bf972a3df6ba722a1d56782c59d47", "b40be55c3b541c1c3a6b46de02f45e3ab9dfad47", + "95ee9b8acb715971bd1abc137a9d54bc88139b8a", "61dc80eef767360c6d48572ca1bda65c693d5362", + "ba44c4692965b0d694c8fd015f1ec3fa4dfcc696", "418a1501e2a14efb4ab940cad4c898f09f92ee83", + "026029e19d4c6f0bcf239abe607026a5a27f5b86", "8cbd5f3c989185d9474f7b5767d7e6adabb2015c", + "55163a23b71b9a3d63f6e4695b07ebd07064c61f", "d9bde5278517a57ff7714edf98ab2361d1663219", + "9ad5f0871e1df9582ea14bf7ba22350086fcd055", "c57f5ae0aaa74eea99032b45f0dc5c927813d8b5", + "3e3f6f7661780b696501a397d8e95ec446fd1fc3", "03d94679ae1908d9a6f6a38c00ce7988c5afbcb1", + "4c8f526d899bf04b4ce3e088302680da64499e5d", "271ddfcd505226549f7674b7e539efc96c8be5b8", + "25708b2d11167437e6de01fa0406dfd3c26bd8d4", "dcf94a3cc15a85bc1dc2fc7e1751e6a343609049", + "f65dd5dea2e8bf5c90b8b093b4ae2db11fce3b5c", "b3722ffbf1e3287cee8109c82b7ab13d187a3d7b", + "ec3af0efeea2836f6a2fc5a2390232ff10e17cf0", "c7176c1b7df0676b474b4786a7e6ba25650df8aa", + "ad97c1b6ab4dcd3d4917cb59427490508523daf4", "7a8e2f675709ba76674a2903653b07d348de32d6", + "bdd754c4219ceadebaebab82c1dacb0a6d819e24", "31df56efa866faac4fced90b742b1800261ef46b", + "1e77fd0a0bd78293212c518291333d3554136b41", "99f906bf47a41245895867f76ad38fc9ca88921f", + "49dd5ec3406bda057407abe5c57e6a027b8c502d", "f36f65ae5480acecfb85d9b8d7a79603853f2115", + "5a29a9db5779e61efafc338b00be9d22efe2a181", "6b28f73e9097b7a7966aa9add138e3dc14c52e75", + "92571e06644eee1606759deeccb9875fdb454b81"], "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80631' + Content-Type: + - application/json + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"208de547-5ee2-4294-955c-5595cd0f7940","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"langsmith-py","description":null,"created":"2026-04-03T20:35:06.233Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"8018513f-a5e8-4b9f-8d47-8c8ff2115691","project_id":"208de547-5ee2-4294-955c-5595cd0f7940","name":"langsmith-eval-daebc7df","description":null,"created":"2026-04-03T22:20:14.116Z","repo_info":{"commit":"5c35051a0102f850f2e09c66f9376a0fc658ed9f","branch":"main","tag":null,"dirty":true,"author_name":"Abhijeet + Prasad","author_email":"abhijeet@braintrustdata.com","commit_message":"fix(anthropic): + capture server-side tool usage metrics (#192)\n\nFlatten Anthropic usage.server_tool_use + into Braintrust span metrics so\nserver-side tool invocations are preserved + for tracing and cost analysis.\n\nAdd regression tests using a red/green workflow + to cover dict-backed span\nlogging and object-backed usage extraction.\n\nFixes + #171","commit_time":"2026-04-02T21:10:00-04:00","git_diff":"diff --git a/py/noxfile.py + b/py/noxfile.py\nindex c92395da..1083ca5b 100644\n--- a/py/noxfile.py\n+++ + b/py/noxfile.py\n@@ -77,6 +77,7 @@ VENDOR_PACKAGES = (\n \"opentelemetry-exporter-otlp-proto-http\",\n \"google.genai\",\n \"google.adk\",\n+ \"langsmith\",\n \"temporalio\",\n + )\n \n@@ -104,6 +105,7 @@ GENAI_VERSIONS = (LATEST,)\n DSPY_VERSIONS = (LATEST,)\n + GOOGLE_ADK_VERSIONS = (LATEST, \"1.14.1\")\n LANGCHAIN_VERSIONS = (LATEST, + \"0.3.28\")\n+LANGSMITH_VERSIONS = (LATEST, \"0.7.12\")\n OPENROUTER_VERSIONS + = (LATEST, \"0.6.0\")\n # temporalio 1.19.0+ requires Python >= 3.10; skip + Python 3.9 entirely\n TEMPORAL_VERSIONS = (LATEST, \"1.20.0\", \"1.19.0\")\n@@ + -235,6 +237,17 @@ def test_langchain(session, version):\n _run_core_tests(session)\n + \n \n+@nox.session()\n+@nox.parametrize(\"version\", LANGSMITH_VERSIONS, ids=LANGSMITH_VERSIONS)\n+def + test_langsmith(session, version):\n+ \"\"\"Test LangSmith integration.\"\"\"\n+ _install_test_deps(session)\n+ _install(session, + \"langsmith\", version)\n+ _install(session, \"langchain-core\")\n+ _install(session, + \"langchain-openai\")\n+ _run_tests(session, f\"{INTEGRATION_DIR}/langsmith/test_langsmith.py\")\n+\n+\n + @nox.session()\n @nox.parametrize(\"version\", OPENAI_VERSIONS, ids=OPENAI_VERSIONS)\n + def test_openai(session, version):\n@@ -371,9 +384,8 @@ def pylint(session):\n session.install(\"pydantic_ai>=1.10.0\")\n session.install(\"google-adk\")\n session.install(\"opentelemetry.instrumentation.openai\")\n- # + langsmith is needed for the langsmith_wrapper module but not in VENDOR_PACKAGES\n # + langchain-core, langchain-openai, langchain-anthropic are needed for the langchain + integration\n- session.install(\"langsmith\", \"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n+ session.install(\"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n \n result = session.run(\"git\", \"ls-files\", + \"**/*.py\", silent=True, log=False)\n files = [path for path in result.strip().splitlines() + if path not in GENERATED_LINT_EXCLUDES]\ndiff --git a/py/src/braintrust/auto.py + b/py/src/braintrust/auto.py\nindex dc44c7d2..6cf8597c 100644\n--- a/py/src/braintrust/auto.py\n+++ + b/py/src/braintrust/auto.py\n@@ -16,6 +16,7 @@ from braintrust.integrations + import (\n DSPyIntegration,\n GoogleGenAIIntegration,\n LangChainIntegration,\n+ LangSmithIntegration,\n LiteLLMIntegration,\n OpenRouterIntegration,\n PydanticAIIntegration,\n@@ + -52,6 +53,7 @@ def auto_instrument(\n dspy: bool = True,\n adk: bool + = True,\n langchain: bool = True,\n+ langsmith: bool = True,\n ) -> + dict[str, bool]:\n \"\"\"\n Auto-instrument supported AI/ML libraries + for Braintrust tracing.\n@@ -75,6 +77,7 @@ def auto_instrument(\n dspy: + Enable DSPy instrumentation (default: True)\n adk: Enable Google ADK + instrumentation (default: True)\n langchain: Enable LangChain instrumentation + (default: True)\n+ langsmith: Enable LangSmith instrumentation (default: + True)\n \n Returns:\n Dict mapping integration name to whether + it was successfully instrumented.\n@@ -146,6 +149,8 @@ def auto_instrument(\n results[\"adk\"] + = _instrument_integration(ADKIntegration)\n if langchain:\n results[\"langchain\"] + = _instrument_integration(LangChainIntegration)\n+ if langsmith:\n+ results[\"langsmith\"] + = _instrument_integration(LangSmithIntegration)\n \n return results\n + \ndiff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py\nindex + 0062ec77..2dca1110 100644\n--- a/py/src/braintrust/integrations/__init__.py\n+++ + b/py/src/braintrust/integrations/__init__.py\n@@ -6,6 +6,7 @@ from .claude_agent_sdk + import ClaudeAgentSDKIntegration\n from .dspy import DSPyIntegration\n from + .google_genai import GoogleGenAIIntegration\n from .langchain import LangChainIntegration\n+from + .langsmith import LangSmithIntegration\n from .litellm import LiteLLMIntegration\n + from .openrouter import OpenRouterIntegration\n from .pydantic_ai import PydanticAIIntegration\n@@ + -21,6 +22,7 @@ __all__ = [\n \"GoogleGenAIIntegration\",\n \"LiteLLMIntegration\",\n \"LangChainIntegration\",\n+ \"LangSmithIntegration\",\n \"OpenRouterIntegration\",\n \"PydanticAIIntegration\",\n + ]\ndiff --git a/py/src/braintrust/wrappers/langsmith_wrapper.py b/py/src/braintrust/wrappers/langsmith_wrapper.py\nindex + b22117df..7d4170fd 100644\n--- a/py/src/braintrust/wrappers/langsmith_wrapper.py\n+++ + b/py/src/braintrust/wrappers/langsmith_wrapper.py\n@@ -1,518 +1,4 @@\n-\"\"\"\n-Braintrust + integration for LangSmith - provides a migration path from LangSmith to Braintrust.\n+from + braintrust.integrations.langsmith import setup_langsmith # noqa: F401\n \n-This + module patches LangSmith''s tracing and evaluation APIs to use Braintrust + under the hood,\n-allowing users to migrate with minimal code changes.\n \n-Usage:\n- ```python\n- import + os\n-\n- # Enable LangSmith tracing and set project name (used by both + services)\n- os.environ.setdefault(\"LANGCHAIN_TRACING_V2\", \"true\")\n- os.environ.setdefault(\"LANGCHAIN_PROJECT\", + \"my-project\")\n-\n- from braintrust.wrappers.langsmith_wrapper import + setup_langsmith\n-\n- # Call setup BEFORE importing from langsmith\n- # + project_name defaults to LANGCHAIN_PROJECT env var\n- setup_langsmith()\n-\n- # + Continue using langsmith imports - they now use Braintrust\n- from langsmith + import traceable, Client\n-\n- @traceable\n- def my_function(inputs: + dict) -> dict:\n- return {\"result\": inputs[\"x\"] * 2}\n-\n- client + = Client()\n- results = client.evaluate(\n- my_function,\n- data=[{\"inputs\": + {\"x\": 1}, \"outputs\": {\"result\": 2}}],\n- evaluators=[my_evaluator],\n- )\n- ```\n-\n- Set + BRAINTRUST_STANDALONE=1 to completely replace LangSmith with Braintrust\n- (no + LangSmith code runs). Otherwise, both services run in tandem.\n-\"\"\"\n-\n-import + inspect\n-import logging\n-import os\n-from typing import Any, Callable, Dict, + Iterable, Iterator, List, Optional, ParamSpec, TypeVar\n-\n-from braintrust.framework + import EvalCase\n-from braintrust.logger import NOOP_SPAN, current_span, init_logger, + traced\n-from wrapt import wrap_function_wrapper\n-\n-\n-logger = logging.getLogger(__name__)\n-\n-# + Global list to store Braintrust eval results when running in tandem mode\n-_braintrust_eval_results: + List[Any] = []\n-\n-# TODO: langsmith.test/unit/expect, langsmith.AsyncClient, + trace\n-__all__ = [\n- \"setup_langsmith\",\n- \"wrap_traceable\",\n- \"wrap_client\",\n- \"wrap_evaluate\",\n- \"wrap_aevaluate\",\n- \"get_braintrust_results\",\n- \"clear_braintrust_results\",\n-]\n-\n-F + = TypeVar(\"F\", bound=Callable[..., Any])\n-P = ParamSpec(\"P\")\n-R = TypeVar(\"R\")\n-\n-\n-def + get_braintrust_results() -> List[Any]:\n- \"\"\"Get all Braintrust eval + results collected during tandem mode.\"\"\"\n- return _braintrust_eval_results.copy()\n-\n-\n-def + clear_braintrust_results() -> None:\n- \"\"\"Clear all stored Braintrust + eval results.\"\"\"\n- _braintrust_eval_results.clear()\n-\n-\n-def setup_langsmith(\n- api_key: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- project_name: + Optional[str] = None,\n- standalone: bool = False,\n-) -> bool:\n- \"\"\"\n- Setup + Braintrust integration with LangSmith.\n-\n- This patches LangSmith''s + @traceable, Client.evaluate(), and aevaluate()\n- to use Braintrust under + the hood.\n-\n- Args:\n- api_key: Braintrust API key (optional, + can use env var BRAINTRUST_API_KEY)\n- project_id: Braintrust project + ID (optional)\n- project_name: Braintrust project name (optional, falls + back to LANGCHAIN_PROJECT\n- env var, then BRAINTRUST_PROJECT + env var)\n- standalone: If True, completely replace LangSmith with + Braintrust (no LangSmith\n- code runs). If False (default), + run both LangSmith and Braintrust\n- in tandem.\n-\n- Returns:\n- True + if setup was successful, False otherwise\n- \"\"\"\n- # Use LANGCHAIN_PROJECT + as fallback for project_name to keep both services in sync\n- if project_name + is None:\n- project_name = os.environ.get(\"LANGCHAIN_PROJECT\")\n-\n- span + = current_span()\n- if span == NOOP_SPAN:\n- init_logger(project=project_name, + api_key=api_key, project_id=project_id)\n-\n- try:\n- import langsmith\n-\n- langsmith.traceable + = wrap_traceable(langsmith.traceable, standalone=standalone)\n- wrap_client(langsmith.Client, + project_name=project_name, project_id=project_id, standalone=standalone)\n- langsmith.evaluate + = wrap_evaluate(\n- langsmith.evaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n- langsmith.aevaluate + = wrap_aevaluate(\n- langsmith.aevaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n-\n- logger.info(\"LangSmith + integration with Braintrust enabled\")\n- return True\n-\n- except + ImportError as e:\n- logger.error(f\"Failed to import langsmith: {e}\")\n- logger.error(\"langsmith + is not installed. Please install it with: pip install langsmith\")\n- return + False\n-\n-\n-def wrap_traceable(traceable: F, standalone: bool = False) -> + F:\n- \"\"\"\n- Wrap langsmith.traceable to also use Braintrust''s @traced + decorator.\n-\n- Args:\n- traceable: The langsmith.traceable function\n- standalone: + If True, replace LangSmith tracing entirely with Braintrust.\n- If + False, add Braintrust tracing alongside LangSmith tracing.\n-\n- Returns:\n- The + wrapped traceable function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(traceable):\n- return traceable\n-\n- def traceable_wrapper(*args: + Any, **kwargs: Any) -> Any:\n- # Handle both @traceable and @traceable(...) + patterns\n- func = args[0] if args and callable(args[0]) else None\n-\n- def + decorator(fn: Callable[P, R]) -> Callable[P, R]:\n- span_name = + kwargs.get(\"name\") or fn.__name__\n-\n- # Conditionally apply + LangSmith decorator first\n- if not standalone:\n- fn + = traceable(fn, **kwargs)\n-\n- # Always apply Braintrust tracing\n- return + traced(name=span_name)(fn) # type: ignore[return-value]\n-\n- if func + is not None:\n- return decorator(func)\n- return decorator\n-\n- traceable_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return traceable_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_client(\n- Client: Any, project_name: + Optional[str] = None, project_id: Optional[str] = None, standalone: bool = + False\n-) -> Any:\n- \"\"\"\n- Wrap langsmith.Client to redirect evaluate() + and aevaluate() to Braintrust''s Eval.\n-\n- Args:\n- Client: The + langsmith.Client class\n- project_name: Braintrust project name to + use for evaluations\n- project_id: Braintrust project ID to use for + evaluations\n- standalone: If True, only run Braintrust. If False, + run both LangSmith and Braintrust.\n-\n- Returns:\n- The Client + class (modified in place)\n- \"\"\"\n-\n- if hasattr(Client, \"evaluate\") + and not _is_patched(Client.evaluate):\n- wrap_function_wrapper(\n- Client,\n- \"evaluate\",\n- make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.evaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- if hasattr(Client, \"aevaluate\") + and not _is_patched(Client.aevaluate):\n- wrap_function_wrapper(\n- Client,\n- \"aevaluate\",\n- make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.aevaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- return Client\n-\n-\n-def make_evaluate_wrapper(\n- *, + project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: + bool = False\n-):\n- def evaluate_wrapper(wrapped: Any, instance: Any, + args: Any, kwargs: Any) -> Any:\n- result = None\n- if not standalone:\n- result + = wrapped(*args, **kwargs)\n-\n- try:\n- result = _run_braintrust_eval(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + evaluate failed: {e}\")\n-\n- return result\n-\n- return evaluate_wrapper\n-\n-\n-def + make_aevaluate_wrapper(\n- *, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-):\n- async def aevaluate_wrapper(wrapped: + Any, instance: Any, args: Any, kwargs: Any) -> Any:\n- result = None\n- if + not standalone:\n- result = await wrapped(*args, **kwargs)\n-\n- try:\n- result + = await _run_braintrust_eval_async(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + aevaluate failed: {e}\")\n-\n- return result\n-\n- return aevaluate_wrapper\n-\n-\n-def + wrap_evaluate(\n- evaluate: F, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.evaluate to redirect to Braintrust''s Eval.\n-\n- Args:\n- evaluate: + The langsmith.evaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to + use for evaluations\n- standalone: If True, only run Braintrust. If + False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + wrapped evaluate function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(evaluate):\n- return evaluate\n-\n- evaluate_wrapper + = make_evaluate_wrapper(standalone=standalone, project_name=project_name, + project_id=project_id)\n- evaluate_wrapper._braintrust_patched = True # + type: ignore[attr-defined]\n- return evaluate_wrapper # type: ignore[return-value]\n-\n-\n-def + wrap_aevaluate(\n- aevaluate: F,\n- project_name: Optional[str] = None,\n- project_id: + Optional[str] = None,\n- standalone: bool = False,\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.aevaluate to redirect to Braintrust''s EvalAsync.\n-\n- Args:\n- aevaluate: + The langsmith.aevaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to + use for evaluations\n- standalone: If True, only run Braintrust. If + False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + wrapped aevaluate function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(aevaluate):\n- return aevaluate\n-\n- aevaluate_wrapper + = make_aevaluate_wrapper(standalone=standalone, project_name=project_name, + project_id=project_id)\n- aevaluate_wrapper._braintrust_patched = True # + type: ignore[attr-defined]\n- return aevaluate_wrapper # type: ignore[return-value]\n-\n-\n-def + _is_patched(obj: Any) -> bool:\n- return getattr(obj, \"_braintrust_patched\", + False)\n-\n-\n-# =============================================================================\n-# + Braintrust evaluation logic\n-# =============================================================================\n-\n-\n-def + _run_braintrust_eval(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust Eval with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import Eval\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + Eval(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-async + def _run_braintrust_eval_async(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust EvalAsync with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import EvalAsync\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + await EvalAsync(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-# + =============================================================================\n-# + Data conversion helpers\n-# =============================================================================\n-\n-\n-def + _wrap_output(output: Any) -> Dict[str, Any]:\n- \"\"\"Wrap non-dict outputs + the same way LangSmith does.\"\"\"\n- if not isinstance(output, dict):\n- return + {\"output\": output}\n- return output\n-\n-\n-def _make_braintrust_scorer(\n- evaluator: + Callable[..., Any],\n-) -> Callable[..., Any]:\n- \"\"\"\n- Create a + Braintrust scorer from a LangSmith evaluator.\n-\n- Always runs the evaluator + through Braintrust for full tracing (span duration, child LLM calls, etc.).\n- \"\"\"\n- evaluator_name + = getattr(evaluator, \"__name__\", \"score\")\n-\n- def braintrust_scorer(input: + Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any:\n- from + braintrust.score import Score\n-\n- # Run the evaluator with LangSmith''s + signature\n- # LangSmith evaluators use: (inputs, outputs, reference_outputs) + -> bool | dict\n- # LangSmith auto-wraps non-dict outputs as {\"output\": + value}\n- outputs = _wrap_output(output)\n-\n- # expected is + the real LangSmith Example object passed through from data loading\n- reference_outputs + = expected.outputs if hasattr(expected, \"outputs\") else expected\n-\n- result + = evaluator(input, outputs, reference_outputs)\n-\n- return Score(\n- name=result.get(\"key\", + evaluator_name),\n- score=result.get(\"score\"),\n- metadata=result.get(\"metadata\", + {}),\n- )\n-\n- braintrust_scorer.__name__ = evaluator_name\n- return + braintrust_scorer\n-\n-\n-def _convert_langsmith_data(data: Any) -> Callable[[], + Iterator[EvalCase[Any, Any]]]:\n- \"\"\"Convert LangSmith data format to + Braintrust data format.\"\"\"\n-\n- def load_data() -> Iterator[EvalCase[Any, + Any]]:\n- # Determine the source iterable without loading everything + into memory\n- source: Iterable[Any]\n- if callable(data):\n- source + = data() # type: ignore\n- elif isinstance(data, str):\n- # + Load examples from LangSmith dataset by name\n- try:\n- from + langsmith import Client # pylint: disable=import-error\n-\n- client + = Client()\n- source = client.list_examples(dataset_name=data)\n- except + Exception as e:\n- logger.warning(f\"Failed to load LangSmith + dataset ''{data}'': {e}\")\n- return\n- elif hasattr(data, + \"__iter__\"):\n- source = data\n- else:\n- source + = [data]\n-\n- # Process items as a generator - yield one at a time\n- for + item in source:\n- # Pass through LangSmith Example objects directly\n- if + hasattr(item, \"inputs\"):\n- yield EvalCase(\n- input=item.inputs,\n- expected=item, # + Pass the whole Example object\n- metadata=getattr(item, + \"metadata\", None),\n- )\n- elif isinstance(item, + dict):\n- if \"inputs\" in item:\n- # LangSmith + dict format\n- yield EvalCase(\n- input=item[\"inputs\"],\n- expected=item, # + Pass the whole dict\n- metadata=item.get(\"metadata\"),\n- )\n- elif + \"input\" in item:\n- # Braintrust format\n- yield + EvalCase(\n- input=item[\"input\"],\n- expected=item.get(\"expected\"),\n- metadata=item.get(\"metadata\"),\n- )\n- else:\n- yield + EvalCase(input=item)\n- else:\n- yield EvalCase(input=item)\n-\n- return + load_data\n-\n-\n-def _make_braintrust_task(target: Callable[..., Any]) -> + Callable[..., Any]:\n- \"\"\"Convert a LangSmith target function to Braintrust + task format.\"\"\"\n-\n- def task_fn(task_input: Any, hooks: Any) -> Any:\n- if + isinstance(task_input, dict):\n- # Try to get the original function''s + signature (unwrap decorators)\n- unwrapped = inspect.unwrap(target)\n-\n- try:\n- sig + = inspect.signature(unwrapped)\n- params = list(sig.parameters.keys())\n- if + len(params) == 1:\n- return target(task_input)\n- if + all(p in task_input for p in params):\n- return target(**task_input)\n- return + target(task_input)\n- except (ValueError, TypeError):\n- # + Fallback: try kwargs first, then single arg\n- try:\n- return + target(**task_input)\n- except TypeError:\n- return + target(task_input)\n- return target(task_input)\n-\n- return task_fn\n+__all__ + = [\"setup_langsmith\"]\ndiff --git a/py/src/braintrust/wrappers/test_langsmith_wrapper.py + b/py/src/braintrust/wrappers/test_langsmith_wrapper.py\ndeleted file mode + 100644\nindex c25256c2..00000000\n--- a/py/src/braintrust/wrappers/test_langsmith_wrapper.py\n+++ + /dev/null\n@@ -1,341 +0,0 @@\n-# pyright: reportPrivateUsage=false\n-# pyright: + reportMissingParameterType=false\n-# pyright: reportUnknownParameterType=false\n-# + pyright: reportUnknownArgumentType=false\n-# pyright: reportUnknownMemberType=false\n-# + pylint: disable=protected-access\n-\n-\"\"\"\n-Tests for the LangSmith wrapper + to ensure compatibility with LangSmith''s API.\n-\"\"\"\n-\n-from braintrust.wrappers.langsmith_wrapper + import (\n- _convert_langsmith_data,\n- _is_patched,\n- _make_braintrust_scorer,\n- _make_braintrust_task,\n- wrap_aevaluate,\n- wrap_client,\n- wrap_traceable,\n-)\n-\n-\n-def + test_is_patched_false():\n- \"\"\"Test that _is_patched returns False for + unpatched objects.\"\"\"\n-\n- def unpatched():\n- pass\n-\n- assert + _is_patched(unpatched) is False\n-\n-\n-def test_is_patched_true():\n- \"\"\"Test + that _is_patched returns True for patched objects.\"\"\"\n-\n- def patched():\n- pass\n-\n- patched._braintrust_patched + = True # type: ignore\n-\n- assert _is_patched(patched) is True\n-\n-\n-def + test_make_braintrust_scorer_dict_result():\n- \"\"\"Test converting a LangSmith + evaluator that returns a dict.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- return {\"key\": \"accuracy\", \"score\": + 0.9, \"metadata\": {\"note\": \"good\"}}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- # + Create a mock Example object\n- class MockExample:\n- outputs = + {\"y\": 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": 2}, + expected=MockExample())\n-\n- assert result.name == \"accuracy\"\n- assert + result.score == 0.9\n- assert result.metadata == {\"note\": \"good\"}\n-\n-\n-def + test_make_braintrust_scorer_numeric_result():\n- \"\"\"Test converting + a LangSmith evaluator that returns a numeric score in a dict.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return + {\"score\": 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class MockExample:\n- outputs + = {\"y\": 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": + 2}, expected=MockExample())\n-\n- assert result.name == \"langsmith_evaluator\"\n- assert + result.score == 1.0\n-\n-\n-def test_make_braintrust_scorer_with_plain_dict_expected():\n- \"\"\"Test + converting a LangSmith evaluator with plain dict as expected.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return + {\"score\": 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n- result = converted(input={\"x\": + 1}, output={\"y\": 2}, expected={\"y\": 2})\n-\n- assert result.name == + \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def test_convert_langsmith_data_from_list():\n- \"\"\"Test + converting LangSmith data from a list of dicts.\"\"\"\n- data = [\n- {\"inputs\": + {\"x\": 1}, \"outputs\": {\"y\": 2}},\n- {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- assert result[1].input + == {\"x\": 2}\n- assert result[1].expected == {\"inputs\": {\"x\": 2}, + \"outputs\": {\"y\": 4}}\n-\n-\n-def test_convert_langsmith_data_from_callable():\n- \"\"\"Test + converting LangSmith data from a callable.\"\"\"\n-\n- def data_generator():\n- yield + {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- yield {\"inputs\": + {\"x\": 2}, \"outputs\": {\"y\": 4}}\n-\n- data_fn = _convert_langsmith_data(data_generator)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n-\n-\n-def test_convert_langsmith_data_with_example_objects():\n- \"\"\"Test + converting LangSmith data with Example-like objects.\"\"\"\n-\n- class + MockExample:\n- def __init__(self, inputs, outputs):\n- self.inputs + = inputs\n- self.outputs = outputs\n-\n- data = [\n- MockExample(inputs={\"x\": + 1}, outputs={\"y\": 2}),\n- MockExample(inputs={\"x\": 2}, outputs={\"y\": + 4}),\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole Example object is passed as expected\n- assert + result[0].expected.inputs == {\"x\": 1}\n- assert result[0].expected.outputs + == {\"y\": 2}\n-\n-\n-def test_make_braintrust_task_with_dict_input():\n- \"\"\"Test + that task function handles dict inputs correctly.\"\"\"\n-\n- def target_fn(inputs):\n- return + inputs[\"x\"] * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == 10\n-\n-\n-def test_make_braintrust_task_with_kwargs_expansion():\n- \"\"\"Test + that task function expands dict kwargs when signature matches.\"\"\"\n-\n- def + target_fn(x, y):\n- return x + y\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 2, \"y\": 3}, None)\n-\n- assert result == 5\n-\n-\n-def + test_make_braintrust_task_simple_input():\n- \"\"\"Test that task function + handles simple inputs.\"\"\"\n-\n- def target_fn(inp):\n- return + inp * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result = task(5, + None)\n-\n- assert result == 10\n-\n-\n-class TestWrapTraceable:\n- \"\"\"Tests + for wrap_traceable functionality.\"\"\"\n-\n- def test_wrap_traceable_returns_wrapper(self):\n- \"\"\"Test + that wrap_traceable returns a wrapped version.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable, + standalone=False)\n- assert callable(wrapped)\n- assert _is_patched(wrapped)\n-\n- def + test_wrap_traceable_standalone_mode(self):\n- \"\"\"Test that wrap_traceable + works in standalone mode.\"\"\"\n-\n- def mock_traceable(func, **kwargs):\n- return + func\n-\n- wrapped = wrap_traceable(mock_traceable, standalone=True)\n- assert + callable(wrapped)\n- assert _is_patched(wrapped)\n-\n-\n-class TestWrapFunctions:\n- \"\"\"Tests + for the wrap_* functions.\"\"\"\n-\n- def test_wrap_functions_exist(self):\n- \"\"\"Test + that wrap functions are callable.\"\"\"\n- assert callable(wrap_traceable)\n- assert + callable(wrap_client)\n- assert callable(wrap_aevaluate)\n-\n- def + test_wrap_traceable_returns_patched_function(self):\n- \"\"\"Test that + wrap_traceable returns a patched function.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable)\n- assert + _is_patched(wrapped)\n-\n- def test_wrap_traceable_skips_if_already_patched(self):\n- \"\"\"Test + that wrap_traceable skips if already patched.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- mock_traceable._braintrust_patched + = True # type: ignore\n-\n- result = wrap_traceable(mock_traceable)\n- # + Should return the same function\n- assert result is mock_traceable\n-\n- def + test_wrap_client_sets_flag(self):\n- \"\"\"Test that wrap_client sets + the patched flag.\"\"\"\n-\n- class MockClient:\n- def evaluate(self, + *args, **kwargs):\n- return \"original\"\n-\n- wrap_client(MockClient)\n- assert + _is_patched(MockClient.evaluate)\n-\n- def test_wrap_aevaluate_returns_patched_function(self):\n- \"\"\"Test + that wrap_aevaluate returns a patched function.\"\"\"\n-\n- async def + mock_aevaluate(*args, **kwargs):\n- pass\n-\n- wrapped = + wrap_aevaluate(mock_aevaluate)\n- assert _is_patched(wrapped)\n-\n-\n-class + TestTandemModeIntegration:\n- \"\"\"Integration tests for tandem mode (LangSmith + + Braintrust together).\"\"\"\n-\n- def test_make_braintrust_task_with_inputs_parameter(self):\n- \"\"\"Test + that task handles LangSmith''s required ''inputs'' parameter name.\"\"\"\n-\n- def + target_fn(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == {\"result\": 10}\n-\n- def + test_convert_langsmith_data_handles_different_output_types(self):\n- \"\"\"Test + that data conversion handles various output types.\"\"\"\n- data = + [\n- {\"inputs\": {\"x\": 1}, \"outputs\": 2}, # outputs is int, + not dict\n- {\"inputs\": {\"x\": 2}, \"outputs\": {\"result\": + 4}}, # outputs is already dict\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- # Both should work - Braintrust''s EvalCase + accepts any type for expected\n- assert len(result) == 2\n- assert + result[0].input == {\"x\": 1}\n- assert result[1].input == {\"x\": + 2}\n-\n- def test_make_braintrust_scorer_handles_wrapped_outputs(self):\n- \"\"\"Test + that scorers handle output wrapping correctly.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- # outputs will be wrapped as {\"output\": + value} for non-dict results\n- actual = outputs.get(\"output\", + outputs)\n- expected = (\n- reference_outputs.get(\"output\", + reference_outputs)\n- if isinstance(reference_outputs, dict)\n- else + reference_outputs\n- )\n- return {\"key\": \"match\", + \"score\": 1.0 if actual == expected else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class + MockExample:\n- outputs = {\"output\": 42}\n-\n- # Test + with wrapped output\n- result = converted(input={\"x\": 1}, output=42, + expected=MockExample())\n- assert result.name == \"match\"\n- assert + result.score == 1.0\n-\n-\n-class TestDataConversion:\n- \"\"\"Tests for + data conversion utilities.\"\"\"\n-\n- def test_convert_data_with_braintrust_format(self):\n- \"\"\"Test + that Braintrust format is properly handled.\"\"\"\n- data = [\n- {\"input\": + {\"x\": 1}, \"expected\": {\"y\": 2}},\n- {\"input\": {\"x\": 2}, + \"expected\": {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- assert result[0].expected == {\"y\": 2}\n- assert + result[1].input == {\"x\": 2}\n- assert result[1].expected == {\"y\": + 4}\n-\n- def test_convert_data_with_simple_items(self):\n- \"\"\"Test + that simple items (not dicts) are handled.\"\"\"\n- data = [1, 2, 3]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 3\n- assert result[0].input == 1\n- assert result[1].input + == 2\n- assert result[2].input == 3"},"commit":"5c35051a0102f850f2e09c66f9376a0fc658ed9f","base_exp_id":"f33b958c-3c7f-4238-8665-30fa683ebe73","deleted_at":null,"dataset_id":null,"dataset_version":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MzVmMjIwZjUtNGJmNy00OTQ2LWFiNWItYzZiOTdkZTY2NTFk'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:14 GMT + Etag: + - W/"xj6tz9dmmlsoy" + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + Transfer-Encoding: + - chunked + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - MzVmMjIwZjUtNGJmNy00OTQ2LWFiNWItYzZiOTdkZTY2NTFk + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::8jj9p-1775254813922-9f17204b8144 + content-length: + - '37186' + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '177' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.30.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.30.0 + X-Stainless-Raw-Response: + - 'true' + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-DQhA7FU7b3uGe2qJZ2c5833JucYWw\",\n \"object\": + \"chat.completion\",\n \"created\": 1775254815,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_43047b3f6b\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e6b759dfa6aaaae-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 03 Apr 2026 22:20:15 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '821' + openai-organization: + - braintrust-data + openai-processing-ms: + - '338' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=HyWzpe7AVrNO.Vyaf6QRAyhojPugw_KQoGZj.nZ00s4-1775254814.401031-1.0.1.1-J7mwVIJMK9uFKsqAvVrZrvLE.u.DD9p0kiCVhy2ZZi6y1.4ZZTMyTytWURqIYZyQqIfLes_LzMQTIWPPsBqlFH.N2NA8HqFQhUBxIbIETUmLKA8.yA2ilNb3fqVtVkBx; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Apr 2026 + 22:50:15 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_4c5fb5c3e7694adcb2c7d09adb7adaec + status: + code: 200 + message: OK +- request: + body: '{"id": "8018513f-a5e8-4b9f-8d47-8c8ff2115691"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '46' + Content-Type: + - application/json + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/base_experiment/get_id + response: + body: + string: '{"id":"8018513f-a5e8-4b9f-8d47-8c8ff2115691","project_id":"208de547-5ee2-4294-955c-5595cd0f7940","name":"langsmith-eval-daebc7df","base_exp_id":"f33b958c-3c7f-4238-8665-30fa683ebe73","base_exp_name":"langsmith-aeval-c2c64a6b"}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '226' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-MmFjNTFhNzItYmUwMC00ZjQ1LWEyMDYtYWEyMmFiOTRjNGMy'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:16 GMT + Etag: + - '"gvaz23z3bn6a"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/base_experiment/get_id + X-Nonce: + - MmFjNTFhNzItYmUwMC00ZjQ1LWEyMDYtYWEyMmFiOTRjNGMy + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::4zjzv-1775254815798-3482c5aac3a1 + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.33.1 + method: GET + uri: https://api.braintrust.dev/experiment-comparison2?experiment_id=8018513f-a5e8-4b9f-8d47-8c8ff2115691&base_experiment_id=f33b958c-3c7f-4238-8665-30fa683ebe73 + response: + body: + string: '{"scores":{},"metrics":{}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:17 GMT + Via: + - 1.1 d38f8e8aaab4437dcb36d4adc5a35cbe.cloudfront.net (CloudFront), 1.1 cfcfb1d8fbf5ce2b107182799687a614.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - QavWmAJg3SuE2uiNTE08vdGLXqpvocE4LtdE-fYGJih5v0cV21hLoA== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-69d03d20-36bc067c2a38ca1c6f3d0de2;Parent=27982f0f6439ec18;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id + content-length: + - '26' + etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - bQ59FHddIAMEW8Q= + x-amzn-RequestId: + - 4fdf6222-2161-4752-aa0d-0c3c969713ed + x-bt-internal-trace-id: + - 69d03d200000000049a75fc03adb37ba + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '0' + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/apikey/login + response: + body: + string: '{"org_info":[{"id":"f5883013-b5c1-438d-9044-5182b4682337","name":"abhi-test-org","api_url":"https://api.braintrust.dev","git_metadata":null,"is_universal_api":null,"proxy_url":"https://api.braintrust.dev","realtime_url":"wss://realtime.braintrustapi.com"}]}' + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Allow-Headers: + - X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, + Content-Type, Date, X-Api-Version + Access-Control-Allow-Methods: + - GET,OPTIONS,PATCH,DELETE,POST,PUT + Access-Control-Allow-Origin: + - '*' + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '257' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZDgzYjAzZjAtNzZiNS00OTA0LThhNzQtNjUzMWQ5MDFkN2Ey'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:42 GMT + Etag: + - '"13nbfem8i3p75"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Bt-Was-Udf-Cached: + - 'true' + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/apikey/login + X-Nonce: + - ZDgzYjAzZjAtNzZiNS00OTA0LThhNzQtNjUzMWQ5MDFkN2Ey + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::gxtj4-1775254842518-b929a1b3bc00 + status: + code: 200 + message: OK +- request: + body: '{"project_name": "langsmith-py", "project_id": null, "org_id": "f5883013-b5c1-438d-9044-5182b4682337", + "update": false, "experiment_name": "langsmith-eval", "repo_info": {"commit": + "5c35051a0102f850f2e09c66f9376a0fc658ed9f", "branch": "main", "tag": null, "dirty": + true, "author_name": "Abhijeet Prasad", "author_email": "abhijeet@braintrustdata.com", + "commit_message": "fix(anthropic): capture server-side tool usage metrics (#192)\n\nFlatten + Anthropic usage.server_tool_use into Braintrust span metrics so\nserver-side + tool invocations are preserved for tracing and cost analysis.\n\nAdd regression + tests using a red/green workflow to cover dict-backed span\nlogging and object-backed + usage extraction.\n\nFixes #171", "commit_time": "2026-04-02T21:10:00-04:00", + "git_diff": "diff --git a/py/noxfile.py b/py/noxfile.py\nindex c92395da..1083ca5b + 100644\n--- a/py/noxfile.py\n+++ b/py/noxfile.py\n@@ -77,6 +77,7 @@ VENDOR_PACKAGES + = (\n \"opentelemetry-exporter-otlp-proto-http\",\n \"google.genai\",\n \"google.adk\",\n+ \"langsmith\",\n \"temporalio\",\n + )\n \n@@ -104,6 +105,7 @@ GENAI_VERSIONS = (LATEST,)\n DSPY_VERSIONS = (LATEST,)\n + GOOGLE_ADK_VERSIONS = (LATEST, \"1.14.1\")\n LANGCHAIN_VERSIONS = (LATEST, \"0.3.28\")\n+LANGSMITH_VERSIONS + = (LATEST, \"0.7.12\")\n OPENROUTER_VERSIONS = (LATEST, \"0.6.0\")\n # temporalio + 1.19.0+ requires Python >= 3.10; skip Python 3.9 entirely\n TEMPORAL_VERSIONS + = (LATEST, \"1.20.0\", \"1.19.0\")\n@@ -235,6 +237,17 @@ def test_langchain(session, + version):\n _run_core_tests(session)\n \n \n+@nox.session()\n+@nox.parametrize(\"version\", + LANGSMITH_VERSIONS, ids=LANGSMITH_VERSIONS)\n+def test_langsmith(session, version):\n+ \"\"\"Test + LangSmith integration.\"\"\"\n+ _install_test_deps(session)\n+ _install(session, + \"langsmith\", version)\n+ _install(session, \"langchain-core\")\n+ _install(session, + \"langchain-openai\")\n+ _run_tests(session, f\"{INTEGRATION_DIR}/langsmith/test_langsmith.py\")\n+\n+\n + @nox.session()\n @nox.parametrize(\"version\", OPENAI_VERSIONS, ids=OPENAI_VERSIONS)\n + def test_openai(session, version):\n@@ -371,9 +384,8 @@ def pylint(session):\n session.install(\"pydantic_ai>=1.10.0\")\n session.install(\"google-adk\")\n session.install(\"opentelemetry.instrumentation.openai\")\n- # + langsmith is needed for the langsmith_wrapper module but not in VENDOR_PACKAGES\n # + langchain-core, langchain-openai, langchain-anthropic are needed for the langchain + integration\n- session.install(\"langsmith\", \"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n+ session.install(\"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n \n result = session.run(\"git\", \"ls-files\", + \"**/*.py\", silent=True, log=False)\n files = [path for path in result.strip().splitlines() + if path not in GENERATED_LINT_EXCLUDES]\ndiff --git a/py/src/braintrust/auto.py + b/py/src/braintrust/auto.py\nindex dc44c7d2..6cf8597c 100644\n--- a/py/src/braintrust/auto.py\n+++ + b/py/src/braintrust/auto.py\n@@ -16,6 +16,7 @@ from braintrust.integrations + import (\n DSPyIntegration,\n GoogleGenAIIntegration,\n LangChainIntegration,\n+ LangSmithIntegration,\n LiteLLMIntegration,\n OpenRouterIntegration,\n PydanticAIIntegration,\n@@ + -52,6 +53,7 @@ def auto_instrument(\n dspy: bool = True,\n adk: bool + = True,\n langchain: bool = True,\n+ langsmith: bool = True,\n ) -> dict[str, + bool]:\n \"\"\"\n Auto-instrument supported AI/ML libraries for Braintrust + tracing.\n@@ -75,6 +77,7 @@ def auto_instrument(\n dspy: Enable DSPy + instrumentation (default: True)\n adk: Enable Google ADK instrumentation + (default: True)\n langchain: Enable LangChain instrumentation (default: + True)\n+ langsmith: Enable LangSmith instrumentation (default: True)\n + \n Returns:\n Dict mapping integration name to whether it was successfully + instrumented.\n@@ -146,6 +149,8 @@ def auto_instrument(\n results[\"adk\"] + = _instrument_integration(ADKIntegration)\n if langchain:\n results[\"langchain\"] + = _instrument_integration(LangChainIntegration)\n+ if langsmith:\n+ results[\"langsmith\"] + = _instrument_integration(LangSmithIntegration)\n \n return results\n \ndiff + --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py\nindex + 0062ec77..2dca1110 100644\n--- a/py/src/braintrust/integrations/__init__.py\n+++ + b/py/src/braintrust/integrations/__init__.py\n@@ -6,6 +6,7 @@ from .claude_agent_sdk + import ClaudeAgentSDKIntegration\n from .dspy import DSPyIntegration\n from + .google_genai import GoogleGenAIIntegration\n from .langchain import LangChainIntegration\n+from + .langsmith import LangSmithIntegration\n from .litellm import LiteLLMIntegration\n + from .openrouter import OpenRouterIntegration\n from .pydantic_ai import PydanticAIIntegration\n@@ + -21,6 +22,7 @@ __all__ = [\n \"GoogleGenAIIntegration\",\n \"LiteLLMIntegration\",\n \"LangChainIntegration\",\n+ \"LangSmithIntegration\",\n \"OpenRouterIntegration\",\n \"PydanticAIIntegration\",\n + ]\ndiff --git a/py/src/braintrust/wrappers/langsmith_wrapper.py b/py/src/braintrust/wrappers/langsmith_wrapper.py\nindex + b22117df..7d4170fd 100644\n--- a/py/src/braintrust/wrappers/langsmith_wrapper.py\n+++ + b/py/src/braintrust/wrappers/langsmith_wrapper.py\n@@ -1,518 +1,4 @@\n-\"\"\"\n-Braintrust + integration for LangSmith - provides a migration path from LangSmith to Braintrust.\n+from + braintrust.integrations.langsmith import setup_langsmith # noqa: F401\n \n-This + module patches LangSmith''s tracing and evaluation APIs to use Braintrust under + the hood,\n-allowing users to migrate with minimal code changes.\n \n-Usage:\n- ```python\n- import + os\n-\n- # Enable LangSmith tracing and set project name (used by both services)\n- os.environ.setdefault(\"LANGCHAIN_TRACING_V2\", + \"true\")\n- os.environ.setdefault(\"LANGCHAIN_PROJECT\", \"my-project\")\n-\n- from + braintrust.wrappers.langsmith_wrapper import setup_langsmith\n-\n- # Call + setup BEFORE importing from langsmith\n- # project_name defaults to LANGCHAIN_PROJECT + env var\n- setup_langsmith()\n-\n- # Continue using langsmith imports + - they now use Braintrust\n- from langsmith import traceable, Client\n-\n- @traceable\n- def + my_function(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- client = Client()\n- results = client.evaluate(\n- my_function,\n- data=[{\"inputs\": + {\"x\": 1}, \"outputs\": {\"result\": 2}}],\n- evaluators=[my_evaluator],\n- )\n- ```\n-\n- Set + BRAINTRUST_STANDALONE=1 to completely replace LangSmith with Braintrust\n- (no + LangSmith code runs). Otherwise, both services run in tandem.\n-\"\"\"\n-\n-import + inspect\n-import logging\n-import os\n-from typing import Any, Callable, Dict, + Iterable, Iterator, List, Optional, ParamSpec, TypeVar\n-\n-from braintrust.framework + import EvalCase\n-from braintrust.logger import NOOP_SPAN, current_span, init_logger, + traced\n-from wrapt import wrap_function_wrapper\n-\n-\n-logger = logging.getLogger(__name__)\n-\n-# + Global list to store Braintrust eval results when running in tandem mode\n-_braintrust_eval_results: + List[Any] = []\n-\n-# TODO: langsmith.test/unit/expect, langsmith.AsyncClient, + trace\n-__all__ = [\n- \"setup_langsmith\",\n- \"wrap_traceable\",\n- \"wrap_client\",\n- \"wrap_evaluate\",\n- \"wrap_aevaluate\",\n- \"get_braintrust_results\",\n- \"clear_braintrust_results\",\n-]\n-\n-F + = TypeVar(\"F\", bound=Callable[..., Any])\n-P = ParamSpec(\"P\")\n-R = TypeVar(\"R\")\n-\n-\n-def + get_braintrust_results() -> List[Any]:\n- \"\"\"Get all Braintrust eval results + collected during tandem mode.\"\"\"\n- return _braintrust_eval_results.copy()\n-\n-\n-def + clear_braintrust_results() -> None:\n- \"\"\"Clear all stored Braintrust + eval results.\"\"\"\n- _braintrust_eval_results.clear()\n-\n-\n-def setup_langsmith(\n- api_key: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- project_name: + Optional[str] = None,\n- standalone: bool = False,\n-) -> bool:\n- \"\"\"\n- Setup + Braintrust integration with LangSmith.\n-\n- This patches LangSmith''s @traceable, + Client.evaluate(), and aevaluate()\n- to use Braintrust under the hood.\n-\n- Args:\n- api_key: + Braintrust API key (optional, can use env var BRAINTRUST_API_KEY)\n- project_id: + Braintrust project ID (optional)\n- project_name: Braintrust project + name (optional, falls back to LANGCHAIN_PROJECT\n- env var, + then BRAINTRUST_PROJECT env var)\n- standalone: If True, completely replace + LangSmith with Braintrust (no LangSmith\n- code runs). If + False (default), run both LangSmith and Braintrust\n- in tandem.\n-\n- Returns:\n- True + if setup was successful, False otherwise\n- \"\"\"\n- # Use LANGCHAIN_PROJECT + as fallback for project_name to keep both services in sync\n- if project_name + is None:\n- project_name = os.environ.get(\"LANGCHAIN_PROJECT\")\n-\n- span + = current_span()\n- if span == NOOP_SPAN:\n- init_logger(project=project_name, + api_key=api_key, project_id=project_id)\n-\n- try:\n- import langsmith\n-\n- langsmith.traceable + = wrap_traceable(langsmith.traceable, standalone=standalone)\n- wrap_client(langsmith.Client, + project_name=project_name, project_id=project_id, standalone=standalone)\n- langsmith.evaluate + = wrap_evaluate(\n- langsmith.evaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n- langsmith.aevaluate + = wrap_aevaluate(\n- langsmith.aevaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n-\n- logger.info(\"LangSmith + integration with Braintrust enabled\")\n- return True\n-\n- except + ImportError as e:\n- logger.error(f\"Failed to import langsmith: {e}\")\n- logger.error(\"langsmith + is not installed. Please install it with: pip install langsmith\")\n- return + False\n-\n-\n-def wrap_traceable(traceable: F, standalone: bool = False) -> + F:\n- \"\"\"\n- Wrap langsmith.traceable to also use Braintrust''s @traced + decorator.\n-\n- Args:\n- traceable: The langsmith.traceable function\n- standalone: + If True, replace LangSmith tracing entirely with Braintrust.\n- If + False, add Braintrust tracing alongside LangSmith tracing.\n-\n- Returns:\n- The + wrapped traceable function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(traceable):\n- return traceable\n-\n- def traceable_wrapper(*args: + Any, **kwargs: Any) -> Any:\n- # Handle both @traceable and @traceable(...) + patterns\n- func = args[0] if args and callable(args[0]) else None\n-\n- def + decorator(fn: Callable[P, R]) -> Callable[P, R]:\n- span_name = kwargs.get(\"name\") + or fn.__name__\n-\n- # Conditionally apply LangSmith decorator first\n- if + not standalone:\n- fn = traceable(fn, **kwargs)\n-\n- # + Always apply Braintrust tracing\n- return traced(name=span_name)(fn) # + type: ignore[return-value]\n-\n- if func is not None:\n- return + decorator(func)\n- return decorator\n-\n- traceable_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return traceable_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_client(\n- Client: Any, project_name: + Optional[str] = None, project_id: Optional[str] = None, standalone: bool = False\n-) + -> Any:\n- \"\"\"\n- Wrap langsmith.Client to redirect evaluate() and + aevaluate() to Braintrust''s Eval.\n-\n- Args:\n- Client: The langsmith.Client + class\n- project_name: Braintrust project name to use for evaluations\n- project_id: + Braintrust project ID to use for evaluations\n- standalone: If True, + only run Braintrust. If False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + Client class (modified in place)\n- \"\"\"\n-\n- if hasattr(Client, \"evaluate\") + and not _is_patched(Client.evaluate):\n- wrap_function_wrapper(\n- Client,\n- \"evaluate\",\n- make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.evaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- if hasattr(Client, \"aevaluate\") + and not _is_patched(Client.aevaluate):\n- wrap_function_wrapper(\n- Client,\n- \"aevaluate\",\n- make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.aevaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- return Client\n-\n-\n-def make_evaluate_wrapper(\n- *, + project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: + bool = False\n-):\n- def evaluate_wrapper(wrapped: Any, instance: Any, args: + Any, kwargs: Any) -> Any:\n- result = None\n- if not standalone:\n- result + = wrapped(*args, **kwargs)\n-\n- try:\n- result = _run_braintrust_eval(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + evaluate failed: {e}\")\n-\n- return result\n-\n- return evaluate_wrapper\n-\n-\n-def + make_aevaluate_wrapper(\n- *, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-):\n- async def aevaluate_wrapper(wrapped: + Any, instance: Any, args: Any, kwargs: Any) -> Any:\n- result = None\n- if + not standalone:\n- result = await wrapped(*args, **kwargs)\n-\n- try:\n- result + = await _run_braintrust_eval_async(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + aevaluate failed: {e}\")\n-\n- return result\n-\n- return aevaluate_wrapper\n-\n-\n-def + wrap_evaluate(\n- evaluate: F, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.evaluate to redirect to Braintrust''s Eval.\n-\n- Args:\n- evaluate: + The langsmith.evaluate function\n- project_name: Braintrust project name + to use for evaluations\n- project_id: Braintrust project ID to use for + evaluations\n- standalone: If True, only run Braintrust. If False, run + both LangSmith and Braintrust.\n-\n- Returns:\n- The wrapped evaluate + function (or the original if already patched)\n- \"\"\"\n- if _is_patched(evaluate):\n- return + evaluate\n-\n- evaluate_wrapper = make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id)\n- evaluate_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return evaluate_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_aevaluate(\n- aevaluate: F,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- standalone: + bool = False,\n-) -> F:\n- \"\"\"\n- Wrap module-level langsmith.aevaluate + to redirect to Braintrust''s EvalAsync.\n-\n- Args:\n- aevaluate: + The langsmith.aevaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to use + for evaluations\n- standalone: If True, only run Braintrust. If False, + run both LangSmith and Braintrust.\n-\n- Returns:\n- The wrapped aevaluate + function (or the original if already patched)\n- \"\"\"\n- if _is_patched(aevaluate):\n- return + aevaluate\n-\n- aevaluate_wrapper = make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id)\n- aevaluate_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return aevaluate_wrapper # type: + ignore[return-value]\n-\n-\n-def _is_patched(obj: Any) -> bool:\n- return + getattr(obj, \"_braintrust_patched\", False)\n-\n-\n-# =============================================================================\n-# + Braintrust evaluation logic\n-# =============================================================================\n-\n-\n-def + _run_braintrust_eval(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust Eval with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import Eval\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + Eval(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-async + def _run_braintrust_eval_async(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust EvalAsync with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import EvalAsync\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + await EvalAsync(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-# + =============================================================================\n-# + Data conversion helpers\n-# =============================================================================\n-\n-\n-def + _wrap_output(output: Any) -> Dict[str, Any]:\n- \"\"\"Wrap non-dict outputs + the same way LangSmith does.\"\"\"\n- if not isinstance(output, dict):\n- return + {\"output\": output}\n- return output\n-\n-\n-def _make_braintrust_scorer(\n- evaluator: + Callable[..., Any],\n-) -> Callable[..., Any]:\n- \"\"\"\n- Create a Braintrust + scorer from a LangSmith evaluator.\n-\n- Always runs the evaluator through + Braintrust for full tracing (span duration, child LLM calls, etc.).\n- \"\"\"\n- evaluator_name + = getattr(evaluator, \"__name__\", \"score\")\n-\n- def braintrust_scorer(input: + Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any:\n- from + braintrust.score import Score\n-\n- # Run the evaluator with LangSmith''s + signature\n- # LangSmith evaluators use: (inputs, outputs, reference_outputs) + -> bool | dict\n- # LangSmith auto-wraps non-dict outputs as {\"output\": + value}\n- outputs = _wrap_output(output)\n-\n- # expected is the + real LangSmith Example object passed through from data loading\n- reference_outputs + = expected.outputs if hasattr(expected, \"outputs\") else expected\n-\n- result + = evaluator(input, outputs, reference_outputs)\n-\n- return Score(\n- name=result.get(\"key\", + evaluator_name),\n- score=result.get(\"score\"),\n- metadata=result.get(\"metadata\", + {}),\n- )\n-\n- braintrust_scorer.__name__ = evaluator_name\n- return + braintrust_scorer\n-\n-\n-def _convert_langsmith_data(data: Any) -> Callable[[], + Iterator[EvalCase[Any, Any]]]:\n- \"\"\"Convert LangSmith data format to + Braintrust data format.\"\"\"\n-\n- def load_data() -> Iterator[EvalCase[Any, + Any]]:\n- # Determine the source iterable without loading everything + into memory\n- source: Iterable[Any]\n- if callable(data):\n- source + = data() # type: ignore\n- elif isinstance(data, str):\n- # + Load examples from LangSmith dataset by name\n- try:\n- from + langsmith import Client # pylint: disable=import-error\n-\n- client + = Client()\n- source = client.list_examples(dataset_name=data)\n- except + Exception as e:\n- logger.warning(f\"Failed to load LangSmith + dataset ''{data}'': {e}\")\n- return\n- elif hasattr(data, + \"__iter__\"):\n- source = data\n- else:\n- source + = [data]\n-\n- # Process items as a generator - yield one at a time\n- for + item in source:\n- # Pass through LangSmith Example objects directly\n- if + hasattr(item, \"inputs\"):\n- yield EvalCase(\n- input=item.inputs,\n- expected=item, # + Pass the whole Example object\n- metadata=getattr(item, \"metadata\", + None),\n- )\n- elif isinstance(item, dict):\n- if + \"inputs\" in item:\n- # LangSmith dict format\n- yield + EvalCase(\n- input=item[\"inputs\"],\n- expected=item, # + Pass the whole dict\n- metadata=item.get(\"metadata\"),\n- )\n- elif + \"input\" in item:\n- # Braintrust format\n- yield + EvalCase(\n- input=item[\"input\"],\n- expected=item.get(\"expected\"),\n- metadata=item.get(\"metadata\"),\n- )\n- else:\n- yield + EvalCase(input=item)\n- else:\n- yield EvalCase(input=item)\n-\n- return + load_data\n-\n-\n-def _make_braintrust_task(target: Callable[..., Any]) -> Callable[..., + Any]:\n- \"\"\"Convert a LangSmith target function to Braintrust task format.\"\"\"\n-\n- def + task_fn(task_input: Any, hooks: Any) -> Any:\n- if isinstance(task_input, + dict):\n- # Try to get the original function''s signature (unwrap + decorators)\n- unwrapped = inspect.unwrap(target)\n-\n- try:\n- sig + = inspect.signature(unwrapped)\n- params = list(sig.parameters.keys())\n- if + len(params) == 1:\n- return target(task_input)\n- if + all(p in task_input for p in params):\n- return target(**task_input)\n- return + target(task_input)\n- except (ValueError, TypeError):\n- # + Fallback: try kwargs first, then single arg\n- try:\n- return + target(**task_input)\n- except TypeError:\n- return + target(task_input)\n- return target(task_input)\n-\n- return task_fn\n+__all__ + = [\"setup_langsmith\"]\ndiff --git a/py/src/braintrust/wrappers/test_langsmith_wrapper.py + b/py/src/braintrust/wrappers/test_langsmith_wrapper.py\ndeleted file mode 100644\nindex + c25256c2..00000000\n--- a/py/src/braintrust/wrappers/test_langsmith_wrapper.py\n+++ + /dev/null\n@@ -1,341 +0,0 @@\n-# pyright: reportPrivateUsage=false\n-# pyright: + reportMissingParameterType=false\n-# pyright: reportUnknownParameterType=false\n-# + pyright: reportUnknownArgumentType=false\n-# pyright: reportUnknownMemberType=false\n-# + pylint: disable=protected-access\n-\n-\"\"\"\n-Tests for the LangSmith wrapper + to ensure compatibility with LangSmith''s API.\n-\"\"\"\n-\n-from braintrust.wrappers.langsmith_wrapper + import (\n- _convert_langsmith_data,\n- _is_patched,\n- _make_braintrust_scorer,\n- _make_braintrust_task,\n- wrap_aevaluate,\n- wrap_client,\n- wrap_traceable,\n-)\n-\n-\n-def + test_is_patched_false():\n- \"\"\"Test that _is_patched returns False for + unpatched objects.\"\"\"\n-\n- def unpatched():\n- pass\n-\n- assert + _is_patched(unpatched) is False\n-\n-\n-def test_is_patched_true():\n- \"\"\"Test + that _is_patched returns True for patched objects.\"\"\"\n-\n- def patched():\n- pass\n-\n- patched._braintrust_patched + = True # type: ignore\n-\n- assert _is_patched(patched) is True\n-\n-\n-def + test_make_braintrust_scorer_dict_result():\n- \"\"\"Test converting a LangSmith + evaluator that returns a dict.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- return {\"key\": \"accuracy\", \"score\": + 0.9, \"metadata\": {\"note\": \"good\"}}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- # + Create a mock Example object\n- class MockExample:\n- outputs = {\"y\": + 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": 2}, expected=MockExample())\n-\n- assert + result.name == \"accuracy\"\n- assert result.score == 0.9\n- assert result.metadata + == {\"note\": \"good\"}\n-\n-\n-def test_make_braintrust_scorer_numeric_result():\n- \"\"\"Test + converting a LangSmith evaluator that returns a numeric score in a dict.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return {\"score\": + 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class + MockExample:\n- outputs = {\"y\": 2}\n-\n- result = converted(input={\"x\": + 1}, output={\"y\": 2}, expected=MockExample())\n-\n- assert result.name == + \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def test_make_braintrust_scorer_with_plain_dict_expected():\n- \"\"\"Test + converting a LangSmith evaluator with plain dict as expected.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return {\"score\": + 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n- result + = converted(input={\"x\": 1}, output={\"y\": 2}, expected={\"y\": 2})\n-\n- assert + result.name == \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def + test_convert_langsmith_data_from_list():\n- \"\"\"Test converting LangSmith + data from a list of dicts.\"\"\"\n- data = [\n- {\"inputs\": {\"x\": + 1}, \"outputs\": {\"y\": 2}},\n- {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- assert result[1].input + == {\"x\": 2}\n- assert result[1].expected == {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}}\n-\n-\n-def test_convert_langsmith_data_from_callable():\n- \"\"\"Test + converting LangSmith data from a callable.\"\"\"\n-\n- def data_generator():\n- yield + {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- yield {\"inputs\": + {\"x\": 2}, \"outputs\": {\"y\": 4}}\n-\n- data_fn = _convert_langsmith_data(data_generator)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n-\n-\n-def test_convert_langsmith_data_with_example_objects():\n- \"\"\"Test + converting LangSmith data with Example-like objects.\"\"\"\n-\n- class MockExample:\n- def + __init__(self, inputs, outputs):\n- self.inputs = inputs\n- self.outputs + = outputs\n-\n- data = [\n- MockExample(inputs={\"x\": 1}, outputs={\"y\": + 2}),\n- MockExample(inputs={\"x\": 2}, outputs={\"y\": 4}),\n- ]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 2\n- assert result[0].input == {\"x\": 1}\n- # The whole + Example object is passed as expected\n- assert result[0].expected.inputs + == {\"x\": 1}\n- assert result[0].expected.outputs == {\"y\": 2}\n-\n-\n-def + test_make_braintrust_task_with_dict_input():\n- \"\"\"Test that task function + handles dict inputs correctly.\"\"\"\n-\n- def target_fn(inputs):\n- return + inputs[\"x\"] * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == 10\n-\n-\n-def test_make_braintrust_task_with_kwargs_expansion():\n- \"\"\"Test + that task function expands dict kwargs when signature matches.\"\"\"\n-\n- def + target_fn(x, y):\n- return x + y\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 2, \"y\": 3}, None)\n-\n- assert result == 5\n-\n-\n-def test_make_braintrust_task_simple_input():\n- \"\"\"Test + that task function handles simple inputs.\"\"\"\n-\n- def target_fn(inp):\n- return + inp * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result = task(5, + None)\n-\n- assert result == 10\n-\n-\n-class TestWrapTraceable:\n- \"\"\"Tests + for wrap_traceable functionality.\"\"\"\n-\n- def test_wrap_traceable_returns_wrapper(self):\n- \"\"\"Test + that wrap_traceable returns a wrapped version.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable, + standalone=False)\n- assert callable(wrapped)\n- assert _is_patched(wrapped)\n-\n- def + test_wrap_traceable_standalone_mode(self):\n- \"\"\"Test that wrap_traceable + works in standalone mode.\"\"\"\n-\n- def mock_traceable(func, **kwargs):\n- return + func\n-\n- wrapped = wrap_traceable(mock_traceable, standalone=True)\n- assert + callable(wrapped)\n- assert _is_patched(wrapped)\n-\n-\n-class TestWrapFunctions:\n- \"\"\"Tests + for the wrap_* functions.\"\"\"\n-\n- def test_wrap_functions_exist(self):\n- \"\"\"Test + that wrap functions are callable.\"\"\"\n- assert callable(wrap_traceable)\n- assert + callable(wrap_client)\n- assert callable(wrap_aevaluate)\n-\n- def + test_wrap_traceable_returns_patched_function(self):\n- \"\"\"Test that + wrap_traceable returns a patched function.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable)\n- assert + _is_patched(wrapped)\n-\n- def test_wrap_traceable_skips_if_already_patched(self):\n- \"\"\"Test + that wrap_traceable skips if already patched.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- mock_traceable._braintrust_patched + = True # type: ignore\n-\n- result = wrap_traceable(mock_traceable)\n- # + Should return the same function\n- assert result is mock_traceable\n-\n- def + test_wrap_client_sets_flag(self):\n- \"\"\"Test that wrap_client sets + the patched flag.\"\"\"\n-\n- class MockClient:\n- def evaluate(self, + *args, **kwargs):\n- return \"original\"\n-\n- wrap_client(MockClient)\n- assert + _is_patched(MockClient.evaluate)\n-\n- def test_wrap_aevaluate_returns_patched_function(self):\n- \"\"\"Test + that wrap_aevaluate returns a patched function.\"\"\"\n-\n- async def + mock_aevaluate(*args, **kwargs):\n- pass\n-\n- wrapped = wrap_aevaluate(mock_aevaluate)\n- assert + _is_patched(wrapped)\n-\n-\n-class TestTandemModeIntegration:\n- \"\"\"Integration + tests for tandem mode (LangSmith + Braintrust together).\"\"\"\n-\n- def + test_make_braintrust_task_with_inputs_parameter(self):\n- \"\"\"Test + that task handles LangSmith''s required ''inputs'' parameter name.\"\"\"\n-\n- def + target_fn(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- task = _make_braintrust_task(target_fn)\n- result = + task({\"x\": 5}, None)\n-\n- assert result == {\"result\": 10}\n-\n- def + test_convert_langsmith_data_handles_different_output_types(self):\n- \"\"\"Test + that data conversion handles various output types.\"\"\"\n- data = [\n- {\"inputs\": + {\"x\": 1}, \"outputs\": 2}, # outputs is int, not dict\n- {\"inputs\": + {\"x\": 2}, \"outputs\": {\"result\": 4}}, # outputs is already dict\n- ]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- # + Both should work - Braintrust''s EvalCase accepts any type for expected\n- assert + len(result) == 2\n- assert result[0].input == {\"x\": 1}\n- assert + result[1].input == {\"x\": 2}\n-\n- def test_make_braintrust_scorer_handles_wrapped_outputs(self):\n- \"\"\"Test + that scorers handle output wrapping correctly.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- # outputs will be wrapped as {\"output\": + value} for non-dict results\n- actual = outputs.get(\"output\", outputs)\n- expected + = (\n- reference_outputs.get(\"output\", reference_outputs)\n- if + isinstance(reference_outputs, dict)\n- else reference_outputs\n- )\n- return + {\"key\": \"match\", \"score\": 1.0 if actual == expected else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class MockExample:\n- outputs + = {\"output\": 42}\n-\n- # Test with wrapped output\n- result + = converted(input={\"x\": 1}, output=42, expected=MockExample())\n- assert + result.name == \"match\"\n- assert result.score == 1.0\n-\n-\n-class + TestDataConversion:\n- \"\"\"Tests for data conversion utilities.\"\"\"\n-\n- def + test_convert_data_with_braintrust_format(self):\n- \"\"\"Test that Braintrust + format is properly handled.\"\"\"\n- data = [\n- {\"input\": + {\"x\": 1}, \"expected\": {\"y\": 2}},\n- {\"input\": {\"x\": 2}, + \"expected\": {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- assert result[0].expected == {\"y\": 2}\n- assert + result[1].input == {\"x\": 2}\n- assert result[1].expected == {\"y\": + 4}\n-\n- def test_convert_data_with_simple_items(self):\n- \"\"\"Test + that simple items (not dicts) are handled.\"\"\"\n- data = [1, 2, 3]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 3\n- assert result[0].input == 1\n- assert result[1].input + == 2\n- assert result[2].input == 3"}, "ancestor_commits": ["5c35051a0102f850f2e09c66f9376a0fc658ed9f", + "0daac2f277a337206cbb03cc46c695f7931d7f3d", "d807b7b4a48b4bd920570cdad5f01c350cdd9f35", + "19ecb8a09557f830675a2dded6bdd04f36ae9f76", "0a708e578ce19f61c17a9f4aa44a96d7ed0277df", + "2a174d884e2034fd9a8c20c20c4c0c384f7aa687", "a656bab37f99445ca442fef41f032907c79e3fc0", + "5fda66e087f78c4edf8c95f24c9454e9c80c6417", "07eae203f645dd661ea0fb83b5256f2144dbf07f", + "2c68b8894223a9b562b7d922a53430efe067ede3", "ba0bc10b5f79e952e69a721be89023773f46fe54", + "e09f625c67fab1f2d2f7b31f0370dab3e0cafd1b", "4e5c83f5456b5c043e1e0830227805d28f922f70", + "161fa36a51946f2212d28354609e442f83889684", "3462f3f3d4ce82b3c59f5dacec9a523b9c99ee2d", + "62a2963911d6c65cdbaef1712fdb7c890d3b5777", "89122d4cca5ea2b766b9416c68b611bee79294dd", + "1c1a07e7dfd96811fd883d81adb3b7f7f4061e75", "f6a2c3b3aa589ab8606b7b525ce79ca019f98717", + "051870af3d7bc8c183edac119bcb7166f1ea9ef0", "4267bde681b41aaa31036e14508708a28bde70ac", + "0b717a9e6240a162a3931116f44bd86f4900852b", "7a4328e7db5636ffe7898d99fd392bc204ec7a08", + "e0780bfc33ab6ced1b982aaf27bd7b5072bbe04e", "183dbb0d034c3588b26a8a94a10f3d2f03237670", + "8b15ad67f697a8218217d9674f0fb919d9c1e8d6", "01d0cea1268051909665e4911af1f1be54b333cd", + "b335e66c1f458de9cd73c12d9d9fc97114f3d505", "0b2f34802606d033bb3471075be95de5d9621b98", + "dd73fb4ac1f38bee8e55133a0348f1bd5c46c30e", "716f9e45b99874f17531f9b5ae4ccfa57cab5ae3", + "72adf8bbf0b48756e18423c5477d58bbf6c401b9", "1163e11507676601e265e637e8916a71a310ce87", + "80308c49a0309d13e02c3203c099b3dc87de1ad9", "e112d705d338ef08d449afab5f2bef3ae0269622", + "10996518f60c0cd2796304c122cd05b16c6ffb1c", "b9a78fa767379bfc14e9ec0f9dc1a016f7ca74a2", + "3fc8912b4063d524f357733549ff1c58bbad00ba", "4252662870b266b32d23a16a11c6df755e5dcdcb", + "92165c3f00478c316e7af29d5c2dba51ffbd7090", "a8144eabce4ef1a46257be77c8055bd3fc64978c", + "363baa083335500aebe129c670eb2c49153ad3a6", "0d487b688de037fa6eebf5410c12b9b226f12cff", + "824861d780be7bcbec92f274a13e221cffef04a4", "84398747d12f52f2b6089d5e832ff6cca82fa3d5", + "46c18b284d0d14855fb66a6de8891ea2c39e08e7", "4f1d9510a188a7a9645fe72966a099aff70478ca", + "d82c4258901176056c1eb30a837896becada5a0f", "265a46272e1f808251c0da03061a9b1843a22779", + "42bed48d1b31a428f189ba174ff4f759ec34e94d", "98aa3e6923d32ed264abd11315cd247c085e8a16", + "ff18889b8f8ef8af644f6eeab1179f6bf9b14de5", "fd34303ea427a9bfc038b78ef1e8e65bb44b6f29", + "08210a4fd97cc17d90556afaaba51dbd67917ddd", "a1fae25c088942b155a31b243f07b9862f79c5ba", + "4fb512eaa52733c1daa4cd6995140f6722f17ba4", "992e036a9199d1b03976ed4d22eebe9921410b40", + "5d41698059f77d9c332cfc4e706ea77669d1cb6a", "2349418ff85b75917d869a26be6df0682686caac", + "fd9f3f932a646f07438db3dbda8cab73cfa70c2f", "fc2d5fe0f6d94754e3c705686c3c69eaa58c258c", + "b3651b577d6ebf617a9991252e19d5fcfe7e693c", "106691a2cbfafc38b3d5d5064c9bb922ae7a3901", + "1b7818ade91e40155c7c126cb92cb21726a25c66", "49ab8392e413992b9728666999593a94ebd00236", + "fb80c5412e011c5b3d343809710c92b75874261e", "ab9fd17f71c277f0d05a809c3c9cfe22379df511", + "7184711ed04957af8f2bbe36590c1cec057ca0a8", "5976a47fe82f90599edc06b7e4f9af64a8ce4ed7", + "962e499f18549cdea2e65946c79bacbb9159f35d", "28875eb8ce2b4114d60d365ad5b61c124b5e3900", + "e68dfce97ee7fa014a30d694d7220a1de47fa73e", "404589ac15776fe04f6dd8bdf36c5d74a3684986", + "5f5b88701fedc4cafed4c037ee2a8cff6b4a40fb", "aeb15581fefc2b0b867ca55c21ea80d067f43361", + "2cd52871898658315cc43e78f25546c3c8df1c8a", "6c4b637d65e047f01052b0749d4c3608f202e073", + "864aafda1789476b01c6991931fb0a6e1968e2dc", "9548ae35e49fc7aac9263811ae3c3a4b26af2e81", + "97da34df8474129e1ad16c8eb67b00a906330970", "fdd020262b6620a4823d336741ebbcf0a02ff9a2", + "0bb83ac92e0ccb38dee1f56c11d15180e8c8c565", "34262bcfb1c5babc03da6d27d1b93ca2c3e969a7", + "7bb761580056ac43c2636964b5f1d2f08b3cc01a", "9424d6a04f73de02038dcbacd518864397ddcafb", + "a51b84c54ba9269d28f17e7355dbc8aaf9ccc0b2", "a5ef7778c00dd0c3782a423e4fb6ee3a0a2a73a3", + "33cfa00beffa04151a7d874d5671b720bb49dbe8", "eb530110ee5746d87c1d2b9972f3a3dc4a2aa582", + "8695a8e3ef85a173cfffd98ae25f6a31a7e23d2e", "7da14b1a295c8f912b6b93ec55ec759242a50108", + "f60a2d39cce59a000017bc193b84ef25ba4491d5", "447d3a96d0679451a2c87a5bf28531ca7744b9ab", + "abc8bed8fbdac570891517c76233e18d9fff4828", "a7053505083fb7b7ae6ed1c10a4141a0d8673cc4", + "8ab9dc06de08f7a925fabe52dbbdb6df3b480e89", "009a839b197b63d6a688d53d91743fb84cbed04f", + "fe75d23b3b61e1693c3b3bfe896380ce16a3ef66", "3a4dde05d882a4eb27d21d1d41784295b3a4168e", + "2b0d9e276db08298b83a10b877f3ccf08bfafde6", "fe1732c7b4df75d49cb42017bb2b2af0136a73c0", + "971559d598a53d361c50da7126e5681124244f32", "6e172e761f95e59dda9fab0409f85491beef2240", + "e3cd507afaaa62bc8ef736b27c1f0e0064c3931f", "3a3ba47a007e0af9af528b63b271391e0c538dc7", + "c3d5ea96b897dc179e4775b14c7bb97b838861db", "1ec7d11f39c97471973de7e5eb0b0784025bc851", + "99a5cb75b08c5e2bff6e96614cfe8c0573f0c19d", "49f1e3846c3c8d264e693bab999270d4392d153e", + "824cf93c402abf51934aebd890c9805457108a7c", "d9e62442aa51d540345c1bfbd1df043b9f00657d", + "c933fe5625cfe8b7e57e4b2953fe612ad37d78b5", "ec0c35e2fadc7bd6a734ff3013226193b1f81d09", + "d462219aa0d5e6525299e3a9cc1dabb19dbdc9c5", "9f8937030053fc43137805b66fef5d1d335fae47", + "1c827b6442cf42fbfa2868abfcb2c640752769bd", "892c41480277cab14f3751eafe678af0c4966bfb", + "6a65a37fb5f7e11e9b8551759d44879e732355d1", "c047819f46a9bf82b7f11124802247055e709c80", + "102257d62684977c39f6e78559d73ac0f873048a", "94f13c6a1c7306e23ba45c2cf0d3600ef8cf8673", + "786d48c8e9cf363508f0c30b872968d9ceec69d7", "7b62001e98bba2fcad8254aff0f0cb11bf35db33", + "2eeee24fc7c3b7c6a0aac79563e1cb0253e0bea1", "7aa19caa8d893488dedf5d6e985b766710b66eb7", + "17b5cae858690e270104ba1c5520508525e6c687", "8b1a5aa0a4b4d547e09c07d69be841c3ebde525f", + "174b4061aab369725a43719d5e966013686b4c38", "7bf81f8ebc7877956e2b8c3c556743426ef05038", + "e151663947390534a2037912cad663daba83de6d", "99bc21b047a09063813a733d659e84ad740424e3", + "1bc2d475bc64f99dd11c3bfd569bd15140a87389", "e3d62fddfab1774e5cae1c1ae2d0d46f164a768f", + "d91f76c24230054edef8774579a0a036b844af7b", "dbea65347acd60addcfbfa7bdb97ddbaeee6ba21", + "a28ef5f8895857ad6d945644f78679271fd10df3", "ec1c107ab6133f5810f9501e3ce57349c0bf4ae9", + "10e2d2237b0791683386e551bd4be79cf86367db", "e3f25b9b6abc2f8e1a376788d11d82a61c41645d", + "a88b44d6852fae824705415d2795f5c5201f2dde", "6210d435deb094c603c9e8debdd0d48255be7dae", + "6b2ab80cc64c1b869fb9ea6002fb1670e9ceb5cb", "1e440103b7521973aa5fb33addc8f9ce1d050232", + "449f6eeddd54f73d88e185fcf499087d3e61fbe6", "52579035063a1b7d1e8b64fbaf1b32bf55a04004", + "4db047355d602c53d85f43c1e7276dd3bd8335df", "4bc366daa82312b79ccb2d6471abef3ad16e4512", + "c710e0c271880b2ea1d506dd3d6f23c282f33fe8", "ce261c13723e8985e61819f6df56298f863155d0", + "f2b6f15f1db2d3e3a930e3274c0c1d1245e851c2", "f11481fe89d024f1b2deb651653ea8ad30da5e8a", + "633cb4c201d614ff5ac5e9775fd68764601d59d8", "0bb08d89c6fe9c043cb75178571715460bc32603", + "a8b9f42b80cfc3a835162fc4e69945edae9a2dd2", "e9f46b8d41234acba81068f7c00e3063204b4231", + "ef1c89e8f840a14f44cd6682bc92f7ce2ea9d07d", "6d086d6cfee47cc1e83c7522a58e1d412e91d197", + "7894ea68db9bcdaa55e66cd4d2a634878f901965", "753298d7ada8377c173de6fad4b490082372b7db", + "25868bc58450dad2058b6499ce3bb9400330fbd1", "95f3c141b7f8ad81bf1d34d857c2fe60bbd15d7f", + "f416cea3139a9a29ed5cae633cb3097e70fc93e2", "5e89c7299d9cc476594fc73933bc7006c842cdf4", + "ddf619048666d883cd4325e21f860e28632a456b", "2385b1aa98998820bcb55cfae76d063a57bba938", + "b798003d9609603b43c98ccc2a5c995d0ac1a488", "4d9333cef5aae8889f9697f38a87f694893f17a8", + "2194150ee020def2d7c4a675f2f001c8c0fba99e", "36788d5a20f45762991b1ba72fb52f8020e6c61b", + "16f72b7d1cb1e0801285abbb7b3138909d6e7f79", "617d9b730b37e96b7d05a099b95f5387944d0951", + "51bb20bec1dc870ec4f521e3c9464054ab9abdec", "15cde9f7795249fbb3c71f23210092265f509e19", + "ea3fd2ff145cbfcd53bb1d79ec6dfb659debdb4f", "eacfc8036be4cee52c619f278b61d32862f3a521", + "252eb0be986ee236c4f095f0078b816c7bd5c894", "6774ffa8b89b2da3644f07e4f411b20f108e5786", + "77fc0472011fb9ea9b3ff7bc5e7dbc5c57f33fed", "3e4002f6b2b3e540db12a2355f447fabd7a83bec", + "3110f96fcbfea2b60d1f8b580e28c00c2fe38690", "1cfad0ba63198f2480a22721501757d9913a1d52", + "2dc8187fd9ace26f7684ff5ba4fa75d7f28d7703", "0687d03758cd295a1976a1b5eb40c5187332be7c", + "8b3614791454b2b4e15a47a09948b3699ad7334a", "54002cedbe48b5036ba94109782d78686c0df7aa", + "9c21b41dd5a19131bd4bf236503ebd0a1464816f", "9a5c1c4a0d50d2d70ab6acb2c38e6abcbfcfc547", + "d0bf7aec410bdd29bf4a8f43af0a8bea633788e6", "7580576751a6890b3314fcd4c313991fb9f311a6", + "d734e8ffc272ee65fe0588df00fd9390614ccd2e", "d9fd9bb0c6bf4c6d5536765e0f8005e9326021a2", + "a751e399b66d396cdf13599c93b1445d81934ddc", "8e50db374a548c26830613a78f70072086ce1f13", + "791a2da1e44f02902e1c31ec28c2c7aba5795f47", "8a63569906d8824faf9c1aaecdd62ebb9e47aeb4", + "e541543ce30054cc10fa3e27e2b0aa5afd297d0f", "b017a02240f978dcfe7feec58aeb85f8cbc91892", + "072d3765416d78e465a728df1b4a812992fba58a", "1671a73c21d09a1b4b73debce7cb2fa1d1b9ee54", + "7b6371f41f6a109ea068fd547b17a131d109ee85", "530a4be0108ff8cc783a49c6390f59b238782e58", + "3ca420e53e77d4665b91ccc7631c95dc97ce566d", "41aceab354ed312d4758b5a5b5c32bbdac40da48", + "6f303beaf59f892b9ed86eb4338900c7f82f7a68", "2d2f7e260b2027576963fad1c432197b15f2811e", + "04acf467f93d2b1064ef648c6583f76d96864578", "3dcfc326ee12b116d919974ef8bbf3cf307ee7fd", + "0740dc169ac9f267e4ec2d435165ca882df0fbc1", "270970d692adaf5d204d336ecdb4ee7f51042187", + "37e49012471ef81326f1313583497f48d612eed1", "8d9100b8341085366c11a36d93faa6009ecd4837", + "f53d7b868f610f1b215c799f25d4af171b59a000", "23b9a8d6b977d6849503433f8b38c49cf90e8203", + "5a31cd02719a5a282825a0fec7cb8d0fc44d0884", "7079cb29799d9fb9fbc919b7ddd1224935a551c7", + "d9c624ea93ca6bf62c2412abce1b3a2ef1a2be67", "126c367b3002ad57bb6aa7e08d04657db6f61980", + "b610cd36638ace60bca2aed261165afbf1c8c081", "52a8fa6d53c58908d95bb865f5907d829987fb97", + "65c037efdffceeec554b2706e16621cfc1704202", "c3bc923fafbc33485c9173c2401fa20897cf4ef1", + "6d6cc2f1e8b5da316c409a754ee8c844b8d2b2ee", "0ecb05e34031941426e3b9ce76760add8af6fdfd", + "8ab13f3f48af6a4d3c0b053e4bbabfd4f24f23ec", "10c14b3688c6969ef55951d5d82e744b87d6c356", + "05f569c80bddb61414e8f9a8adf49f8ca6b821b4", "2311d67956183da8b31bf1fdbe6e09b3ec7e242d", + "0b0bdd77c012e3f51c6402edcad35c8ac7ddf4cd", "ca2eb28a0f22a63b2b748d04e17268d11d35e0b0", + "c20c808993bba8d5604c2ac7848037d7ce430e89", "dcd4f5a4be171b1cac28a5eb3534e4b55420cc06", + "74dd883f7c03ba5591ecee0b2b76e9250781e181", "4f94c9548a493de11cd663fc78db21a81fe6f23c", + "b3c2b6f7120402b3f5f0fce9b7a1e6b7f7d04d47", "476a66a5d09ec811c9856f0f0f289b189859721e", + "2468e8006b8edf9ef8f273e142380faf34d380a6", "b88964e5a1b124816771e2c9c0a628c104ecfddb", + "a05dc4df62a20e9715abe697584641f0032742b5", "ad3ca98e3530c7dbdbb07f63a04fe4912e17f85a", + "dbbc1894ef31143816e5913676301261bc44aa4c", "db388916a9bc42b02e26b519c3c5f116c919cafc", + "78753fa68806ca7478572a8b37c1f8d97f99eee3", "759f04ddc9522471b46d139ab347668c849ef8dc", + "3857983398d6c7278686c9f12fc2f841c628bc06", "11d9aff118de5f99bb0b5d55bca04d0efb36746a", + "38ea029efe156f07385cbea0a8743e07f418dd76", "39cc2896ef3cf7f9c2320fa513c645a972063697", + "e085b589790c05aa27f52ced4fb0480230db38b5", "5a0afe994e81c19a69ac9ce3a60529b5e7e7c529", + "e42f891ae64b8844c2d37bf29c25821382ed654f", "1fa9fe250443884c450e92ca44a73f1de6a4a5d3", + "0241b89c84097de0dcc7cf7abd65098ac3020578", "a735eedc1c421d2ef287fd1759e96699dc7aa75c", + "cef88a007fa60f4cd873f1d891a54ce5e173f3aa", "db634461703056448a77d669d907c3861cee6dac", + "bb13adf5fd90e4ca503b6980e737bb0f3396a63a", "b48a316abdb2fab1e7e3a797af20a7af3395587f", + "8ad895a40e7c6e7940aa464acf094e5c4ce2451a", "685755051290c901f95f5f29d8cb313c43833837", + "45e647b94bf47482695fc747a2b8b5575611aa6f", "b6a2d2d34371c5ddebd8ff6176aed0f5104e2d01", + "8c2b2995c2097f5699c101f4ab5e653ff40014a1", "db9aaff1052e5229ad6cef667543f9f6620b119b", + "be421874326c6922eb48277e347ccbd53099117d", "c6f2bf1b0331ef0f7d295e29a86717fa8b5698a1", + "d27524c41de17bec43692c05870d5dcb0f0458a1", "ed35e0ef407e405818bc218566fa79323d7329f7", + "f0473314bed3f901febe06355abe43cc73ea30d0", "7cffe54f8b960931055ae20f5983b4ea34515eff", + "d46056681f07ab1a435c625c2b46ed561240a573", "b58203e04d2cafa99307443c62f5063ca7d2e228", + "38c38c579147306770452db9f2405c2aaa8bcda2", "38f4356286376bb91a2464352b4cceaba3a3f6a0", + "79e066dbd2e0248345b032648c9310e2c7778d23", "160c352d87c121ef1ea43e49831453b9e8d4d0f8", + "be65342f97e4e7234a94ff6dc2e7c25f21f8660d", "058f4abcc85a2fd62fd3e5ff5b3b673824aa6004", + "5fc2dce168e2f622c8695ac0365b7ecf1309eb15", "6cb3c4075881492fe7189c502659caf20ed37a89", + "9860ef0921e59b2bd11767b5382f3815fff1baa0", "e8b9b463f80953fe29eb73b91468d30f5ef62c99", + "87992ef1fac6c582a28f3327f5d7f4ffc553fa78", "eefed6797eeb529dd376ea970c9c79a6ea017f99", + "b37c32b7e9326f350d3287e5870dbded1989e7d0", "990f2380b5c095d71eba74c595bf0535b500cd9c", + "238ae256a3a1ab93e682d603c4b53652fa0059f1", "9ed13156c47c637c5abc48ee20e6c6182c50f778", + "0f80700a25acea6dad9f5b726f220ee8cc227d7e", "dc628580c9eb037386b4e295db74b7f3daa74302", + "d2f914b6abf45a9bffc2823d7ed53a88ba231907", "133d19e1e5c4b0b87017942d4ca37f731af5a91e", + "1cbc14e38442cfbae772c080de6e17b3df09868f", "3106d7492fd3faba686f789ac5b42f2770a446ab", + "d868af0a96ae2ac8c5488c011397af20146017c0", "7db2d14148d17ed3ad5bdeec728cfe863a20f25a", + "feb813ec958af1823c42f4f5502677e38be72d8d", "0b2ee7336487a12eeda4ba66d8f34c332db30b8e", + "54de3627b1038f15d905c05f4c46a1d781b6b44e", "168702d959029c760f29c2ac9dbfe5ae501d7e8d", + "bd99b577781a7ed50e9049a292ac48d2fe33b983", "fede6ecee14c3842ab748664206bd633ea51c524", + "a1d968a1ff87035e1b4bcd5c4db2dfe15474c298", "93683071196fae3e0f92f10bc4533575ec4d058d", + "afc1500cce291ecd2c1bb748d7b627d11b1f03d1", "7a123d33d1a90683ee3df04e5900277519259c75", + "49a541549f2bf86d9bc6037db5a61f5f708756b8", "4a75b61b5f0cf472c5fb17e2c6a271f9c5ba770a", + "27a10de6ca27ebba2d9c667408538da980a85d5e", "3b7c0a467d849acba884de1a8c9a9334e8b7f9af", + "17001bf4ac840128c88152d0b1e6fd62d6a69f2f", "f9c27614dfa3934bffb1f69cc9623e8dcac9d0ec", + "30c615dafa30ea4ecc030e9d03265908f38eb389", "c6134d552ab4bd41e92fd8e2f4a8faad0c15b296", + "ebcaabe082cbe7c9a546a6f156a2fb67c9b98097", "f2a681ba398572b3a88051c1eef80994ddff009e", + "1c9f4f163e65cb4f0f061307d8b3686a042cc9e4", "3db2fc16ad04fc0bb32a41a9ecd013f9f5fa5c36", + "6d2dc1e53a6d16b466ef449fd8e14ebd3fa20f00", "a4dfb710896797d1ec1cab191e39cc51eba18b4d", + "0f51f6ef36ba16737589cab11ea2dd1742671c41", "fa85adab7bc82c5d14250a988664e0f65d23643a", + "2d8afda70e0f82176d7ee80bcb653ed09d4ada76", "17b618dbe38d1374435496778d3cbd0f3d3720a9", + "6116c86a6188a345cb430e55b48e660009a85799", "479f00719a1c04acda05bc52e8198c76b17b767b", + "41b9555dd4375ea0df23a0a6a2333f7f4d9e626e", "c3a3634129a36fb09c4b79e737582e33bd2a058b", + "7b1f9abab1067273355a38909e102fd2d3a5323c", "2cbf3503536ba11f94a975fc0c547529315caa9d", + "4aa4133193a8e8c3fdc766208be7bffb4983bae5", "436fe5351a1bc91413e903a2d2cbf2a8559a3c5f", + "c033d7aed97d788aa88220df3c8cb72fbe1ab96f", "f11265286454e2059d0de2a82ee4356601552bd2", + "f8c91c4de5c8d19881c4f8b0461bc1f8bfd01d94", "25912cd765f1ca1e19486411fb3de9963fd9f925", + "1026cbd8af888621a402e114a1c58c14da41bb72", "1cf1bf68a47746b2d23e4211c03a4b81e9626d10", + "c5980aae60fc5c3577e241a70c0798d8397e6a07", "9e4f93daa40369d2cdb97bfb01ee8c93bf23ec80", + "2b61d410af69dcfe59d11337df54412a7ed495b7", "d9dc67568e7a7147ea23e42083d2cb8632bdbe6c", + "21bb8edd05cc9d1142fabd4aba808d6a8157f4df", "258e8ec05412a73bb0a9e916db941c7af1b3e959", + "98ca8e2ec4264fb7c6a3a733c455f8d0ea089b07", "d6cc3a68587ba633190b18c083b7d56747b7a19d", + "3525059c8b3d3fadcff6f592c0e2ded7fc06294b", "e01077c19252f97215f6ed141d485f877506282f", + "8a4eb9ac7b33af6ffe3e980c685c290a68c93ca1", "567acfb3e9e9fda2c4051c038c6621bdb47ec823", + "675e293cfa98252cef4d8b7980066b038a901d04", "11e271cd1a3df6c75449e0c51a21a6c3b3d3fe22", + "ce4101fb55325aca6989ccfaebfa9992421c9e2b", "8e0211657949932ce0ed8da68e72b5c5981fc8f5", + "fa016eff3df324a5ff9f3d43d324a01177ec8939", "6f4426eac2c30baf20e09b51c5369156eff7abc1", + "2040109acbd819ebfa776fc97631815b122aa7d4", "30a65a2a2ce03eec847aa822f722a0d2f0f993bd", + "2f320a37a7ef0d32ae4fceb09057b1491eb1fc32", "25e7a079b6717ffd08ffbda3254fddb072160d82", + "146489482d21d37e7297d3b085ba786409e328a3", "555e44e3d20abf44db5780cc7fc8c95ae5593239", + "26b2a34cca8cf71e092eb3e7b02e53b88e6c03a5", "d6278e852f56fdb00ee5b9d97226be1589853166", + "5c61d67a3f8068020979f09c9f7be11f56e95c2a", "e0647fd778205f62b1956e5320f682af61fbf5ca", + "5d55b31e47053325b4a345c445c62413aa69177c", "a71670f60bd64ceec2b68fc07608d4ef05277ad5", + "e6751654101127f666768dfecd42fe9c6e0b47e3", "a45fabb93b9405dd04d2d944fef78dc424361698", + "5f31e95c53879bdcb5bd47276a8aff25673655d8", "fe3f9a83f7db420747f402841d25dd73f8b36d30", + "70dec2f04b7567d45d5a85b77c959623eefcfad2", "042f91e4b2b8067e6b672816182ba9ae2465812a", + "6e7aef5b4b4c6cf84ce51a4d8b947eda306a26b5", "3f01a409a7ecc34e825e8532e56d04613397cd13", + "893a546da4fd221501d3c865c7bf1d16ce9d933e", "f5410073902240fb94b50c30506aa02dbf9a1517", + "722657b05ed625309450f4f33953d7a694a1a414", "691d303ce7356e544cbfae5ad8054454c0f69652", + "a6f5a8bb856a84214811cd2b259edf0a30c6144f", "aba9adfef223a27e9ebee5f1e5896763adc71641", + "500c2fddb600fcbe7a7d8a48d4e1fe7c3bdcfea7", "ce957df3478f5f95b668a0dfc87956da190d74a6", + "3548dc0a4e272b21330ceefe9c8c63517941c034", "b252d68c0dc7bc599474380a7a185b9c5a9a5fb6", + "c2bcdec50527d1153a73a5e191d7bb4ab1f4a5f3", "552e48c600134179ece0d1aabd13d17f124338be", + "1a18ac6ef2f20cb3434d1db59bf4ae4c3a1f715d", "ad99dac6002a2975e445e8dc90f5d93c53991fdb", + "04f8a3f464061a9dba2d1c00dd7940488720974a", "b8c2d1ebc7b75ea604e9490fd265a6f797add415", + "4bf10cf4435ff85c7c3e2f50e7790b2eaa8861a4", "66091f34bf277d5491477085551afc1a8a361a0f", + "f7e1f2ac9f1767984bd125184f8b87563f249bfd", "61a1c51b48f7b278c73c13efeba86aa2484a33b5", + "3f54176089005fb6dc4b252b4dce34370ea8bcc6", "ff6f45ce65c0b32578fb92f842f123581413dfe9", + "6163138faeabb67eb3ef1b97d6a4b08a5c45caa9", "64cf4c6299cadfe4e5a7987c8db3383a69a108b9", + "e98267285f704dd8232dcfffc09616ffeda896fd", "61d4167d00783621e8a830f267b799b8f22969b1", + "12551cea0f20963404d37821d889c7b397ce29ba", "9aa042227f5625364418d20187dc76508c5c32fb", + "8213aad94904430b8165119337da1243fa7a88ab", "e8c47b170edc02abda540037fb4752ef03a21849", + "59a6092a06c163dfb2fa1deea179cd84a5e91083", "db2b8564a0ed5193a70c607f23e575c8f023454a", + "12048f9a795a901cab215f869b185d6e46e2a1e9", "98cd9913a99020f97a98297a8f085881e8ff21b7", + "76c1d436b2fba46a7ac7b7d285ee4b5e3a6660fa", "cb1c08bc56644e993505a2ab863c18ebc8a35cd2", + "f532f53f82e2fbd789b654ddd6eebe181c12d5ab", "1aaa6ce090e25893b7b8b46cc99fe8344eec653d", + "83d4ec39274f91989811f845c6fe805544bfa0ab", "0152b43193700d54d9826709132ac2bcb9ce5975", + "57291a113482eb4277f43c414b09afcb1bf8f0b7", "3d5ec4de453ab8ea873a6c8c0d0ef0b2110bd386", + "4168c8645f251d9584b25c5a57ccb089178a98a1", "0dafcc9a1aba7282e98d653730b496d83ba13c16", + "e45a7da9aba5144429cb9019acf99dc668ddadd5", "0844207c03327225f9913cc4b1a8702330841a96", + "329121933af05b575eee9c77bf13b9a4d3443dde", "4793325d739b2d9d0deb39d1118e33d76202e7a3", + "517ce4da9248bbbf77c7c9f9226054bc5412176b", "b9ec5a7fe95a0c7e32d65901da5d33ab8c30c779", + "cae143cbf533885d8bf978f287e9383bea46c3e6", "e2df632753bc6545d94b6f4c76af47a81ea3542b", + "eb3dcb724dfd1cc85e30594a63839ebd46c8a5a8", "c459f75f02d0c3444baef25d6fd228ce2929f924", + "b98731f2971b9ac5d0acda62cfb2cc32c141dc37", "6b1c69123acb73fd0cc23393bb165e99bab2508c", + "50159c19f26b9c1433e50f388b3fa759c92c0ff0", "08715a5559a18ec189f91421618e534344f342a0", + "2d4792e23d4104e5a89c17bf2195c7a8aded89a2", "27eb88f965e9d3df3a1cf219db7883093f339676", + "ee2acbc913cb75dccb8d1bb124b074bacae27ee1", "d9a2df31b1ff1ea6796a25558771ed1a38e6d1f9", + "b0c3bc7c96cd65435e9ff6b88040799959041972", "c340893e4dfd3cbeb4566cec5e8cd9a24c9982d5", + "cd361d820e14ee59bd8daa6bbe3321cc7d4fa436", "89c6916d89c4fec3cbe1ee6c780358cf51ba87e2", + "262aed06bd219cbfbb4e91c0fbdfb606883f74cd", "e32b82fabd2f29cdc9ac9e8b33966b10f8a0ce18", + "1e3688e139a11ce21f3a9e5c74d176c1ace5cd38", "b460d264e4ecde414ddb70e918a1adb48ae610b7", + "41bee95c0185b6c2521073565e11ede0885a7e8a", "22bfb85e284a90cf37d23354f56786941de27a99", + "59c7f2256d0b342d1cc08eeb3f4dd9c7e61e90f7", "6f791793df1d1124936ad53e9948cd221a184aec", + "e5c3b79336ede6948f3302d60de0fb8a1b7a138c", "666d1b61ab34647db7ab8e617a9497b87166d7e8", + "4d8966bad69bca5970ce0ed64c343fac0e1cf698", "852bc11787bae1ac6f99622dffc2eea32ef121e8", + "2b3067a1dfe1648d1e9be9f54f8e3af5ee9cbc3b", "1a001a96c0fd88b7884b9049a6d47f6ec4d086b3", + "740106ef9e67ae17393d081580f5c1493db372a1", "05e0f5184a2a0c6a003dd1654bdb4d1da4b1a847", + "8eeb1017e6f043e43d6eaaa99d2aec62d2aa2776", "1e34543f322aa94c5f5e679cc8713120002bee0c", + "d253a3c83f3a062f0a18220bdec2188a2445f2ff", "2548241b69cdc856b24ed8fe930ee5a608348b8c", + "47c83b2f5b8cc63496aa507f03babde7e79c730c", "1169d83904da1f140c52a51748d1f9920cadc304", + "2551edd52b168e0691cbc4a16b6f9762c42a2ee8", "b07d82b7fe426edc3ea57eca2e54dd1a7865e61d", + "fe52db5b57808551b30af34184e5f11fb12a1a69", "2b29dbd15c15e138057630527d19a67d9ea9198a", + "5cbc7715b58ac8a9d141cf95813149a09d7f323c", "918ea978ecc68b7f3ed541b7ed1e073848bf06f3", + "af49d808446f40d37d64c66517a45cc19a78de32", "f2fb962dacc2a7f1b1a9a075611e300970463a1e", + "a0ac9637dbc9b7b4af6d37256011e8e777379cb6", "3fefc0572263475219c149f4c54fe1b88f23c7e7", + "f227977b57b78fecd2db13ff2b231d0c22a3b274", "7e4c8a38b0fd82cc80df79b1daec14c1cdb58be8", + "ad3269310aba337feac5712f9621a5b8ffdb29b8", "4ad90d121a48a0390744035a7bc068f885abcf89", + "134fe22aeae849409671acc32e13a17a69681569", "4e0adb75d47a88366da89405264d87d88cf358c1", + "f1252436b3427e8fe87ea9ac07829f977ea12752", "0679d7794f638bc86e47ecf8e49b70482281c20c", + "c06dbe314d1f44c63b8bfe6edadef6ebf5c5ff20", "ad6e8fa9bbbe6f40621a826662b858c96eff4057", + "9756e3372243fd818c895910aa53f4148798b7a8", "dbb3e71028ba0573b07f6a39bee4a4ff7f39c486", + "6aaf30534ead2af6285ed5f3ff12f3680f401066", "5f7f1c1613ed4d17875a6d845e8cffd3fc0b0059", + "7c1432135a24132618b786ace67238029bec8591", "76290595b414a9c7ec3bf3cf5fa6ad32920d32bd", + "97d4988c8ccfb194a7d01d2c427e6796ed36c5e1", "344fbf8c2cab7917197614e4e974bb5478e4a653", + "6a2becd209341a5dd760515889e5aa27a26e1213", "997e1ee3a567f849865457e22be43aad5b84fcad", + "b0a9c6b95b650b90e69e3ea25f3ac2f583a9c622", "c830370439922abc9cef0763e9cae3387e549c42", + "9bb1447ddf4fbcd71f68096307fb5e222d38202d", "b862af307888af607c82d4d2a3a53e12d6265720", + "585e55bba7d4833272526051a7e2a47416f8556c", "99ac5bdc45f0bec78af9807ebcd8fd118155df93", + "c7e14a6aa34a6256944ea4d9be85777a0017415e", "cc2dda3439152bcabbc625f21d9251afbf15a076", + "4e84b51660d909910c0ce4cb014a23e20748ed05", "f43e4d0aca4ed483f9b2568f35e70dfbdfbecd7c", + "10039ee5b86d6ab30389abfafc4158da5e051410", "a212609775beaa7909f92b708b8348b920c34bc3", + "26ef38e22f884aabecbcdc02640beb2e8ec55abb", "4689201232928ffd29fa14a2bac10fa44f45a81a", + "fb2c9cb75fdb96ad4016a076773038a6081a4792", "d80b52a085881ac67c9c8748473991852f8a1113", + "47e4afb01417c95ae136318ce8d066c77402f30c", "2a256e4dec0180e6e27cd8bda8b4ef1bc3b1bdd9", + "fd4b1442c33a018cbfe42a859cb4bbbdb61eb54f", "f36d187c3fec787dd3d4128001c545de5551bc08", + "ef30b00ca626b1bc763427aed853246c0ab72534", "75eed65362b1370b90c3a032dac2f366f1469f1f", + "d0149fcec2d2e19ee8851b4ea2e328b2ee689184", "9e8eca00b251efc8bea606b3571208966fb2a2d7", + "28e3df3754c86e9809909aac9fe02515e231d60f", "5d4af1c155c639c81448f1d51db6cd33431f7fae", + "7e6a0c4ba45a0e692bbcf9456460aa56a974ed01", "63b04d5a98be1d3a23003f6518c4c8ee3b065ce6", + "31991c235b014fcf4fc9a99a7fa4824c705af5cd", "79231d6d3fb4196a482cf902439ff727eecb92ed", + "a8c246b8bf081182b08ad2a4ea478b7ba860d006", "33750764f3c324181451b38a1dc5219ab04a448e", + "370e22839b26494ce0ad1a88286f427434f3dcc5", "200b037c3083bd2e28e4e31f015bc87f9ab7196c", + "6a7573a61b2e15fd261863178f1f54d96ebdf408", "adfe5473bf50ce4f88c88826d4055b144ef42bb0", + "9d66b222b35f994a0d3ab6bbfb1c142d9f226d56", "ec03206161f942a59ea5bc91c08086a9b1fd2ae7", + "ca5bc0d3d6326e4df076624d826be350a89ed54a", "b8b6473dd4a5f93cd6eb1c0e41d659cb7bad0286", + "20162a0585c37d0c14b818eda6f695c40678bf65", "369ce701b448551b3b5b7156667687c68f7a4259", + "587b5c41554e3aed002f92535ef2179231f87e6d", "3b81c061b9efb1074e0dd9cbb5f11f71420e0832", + "6007cac67baac66d72a57c024260e9bda13c456d", "b1678a8e2c14c96405dc17557ca68d059ff68b8a", + "d5879f68ec23e30ca8882d93410073a0cd5db92d", "90a76730c4ccc74d3cce95abfff445d70bb98499", + "bce86716fa530eedc657bfd7cf18f045116a967d", "fc46a08726a7821ebccbd4d1492ee05c6bae89f7", + "d056f38506127e986b226fb5ff5683b6deeefe37", "1119b6ba9baba7c6511bd57f21eb642085932790", + "2d317fabf4f5e8d7a858a3d1dc79b88a2640b241", "498c2b6b4c0964e25a76cd8ef9e1e25dee1cda7d", + "803f7b6304b6fdc60bf08212fa280e31365d16a9", "5532c2bb541a785fd145aeaf624491e201ed7759", + "243023d8a29dffb528b567f37b5ef1d2d78f3fc7", "c0312dad18501b449812f9c9aca3c39fc1647432", + "9aa5151cb1622686f76e3e956dc9a6e48f43d3ef", "c9ac4a1dffd0a1fe7a0a339b156259884c54057c", + "cb53da91b05ac93ef8bb2dffb4f0ce8ddd06255e", "d25e73f15b3a94570b0a85e39df20d4f9d6a3ded", + "d0af00f950ccb83770035cd0db90fd8ebbe00d10", "ae260ec3ddf6bd62cb66ccb1c9ba3a6ca9d19ab8", + "846116d64285c2125f01ac2e06060b84ff9604a1", "2261c2c6e3edb6c2327046fa2d5b69d9a39ec10b", + "0c938db82253db2c09d40079dd15f0a1ad43345d", "4e5825c76b332b11e0e3af2ef74d9b98e3784809", + "83b5cadf47b3eaa15841e3815099ba5338a5f683", "2f11fc2b39f6889ae6720c0592344995b5942c37", + "3e43da7045388e7259414209b3baea7ee339bb9d", "4b71f161759ed105daaf49ad74f3306ef1710855", + "ae7070f6b723cb4c5055252017e5ce712ddbb649", "23cff857c47ae04884a1e522af2d5a836ea30f2d", + "424a8b8cd8ccd2ddbdf3c2d10ced6767b53a4ef2", "6391feb5cffcecf562f79be5c7b5c620cc1ff24d", + "9f1df5788377be4be03c705b21ffe72e3fca68b0", "607db9575d0fdcc02810ceab97c9b3faf9dcb601", + "ee0202713209fc3df5cc7c1c75812c325c94e280", "47905a7a246fc0ab81d6620820a96a283c6fe8d9", + "21ff79105eba8e77a638b42b2e18589a59033609", "28ba2f0f2967b8292a7071c3d99c9438b1db7963", + "e1fcdbf48c205c3a724151cf5008809615687dcb", "fe9755c8149d239696e010a0e92e2fab24da5aa5", + "e6b72cfddcaa930408e0d20800add718d44f30cc", "f341256a2cf06c968460034faa6d581f17af65a6", + "23791cd278e193ccedeb2ba80666c96e9152ae62", "f402a8f708787e8e97a9e096dbb47ccda9009c78", + "8dcac59f949b44e36ace133352ff4dad069e17e8", "f36f4c364f6f702f64cae7210f0d32a535e93bc2", + "8e110a0b274f6a11fc3773a1bd0933dca5cc962f", "745a2fe44d82669a5070791037da95a0d499ee3c", + "d1bda5b3a4ba7b9d7490d8511cce4a11bf76a8d9", "c2fb15525ed50bb529279aef0da1ad889a209760", + "ac5d73eea0dbddd6591c6254078018b894c74d9a", "150d6faa9cbb1ab06fda1d19606adc701633f24b", + "98f046a4f73ab3edd7f33d87279dc17c6009c844", "c0afac03a62235c223ef23a29e6f0f868aa89f4a", + "c19ecf91221af8e0d29e9d6693d8922989c8364d", "6cc04cca454e9610729586ce72ad276e8a8be962", + "5be4d55be3d8a516f8b9bfd8fa4b4b6fa072b3fb", "68e5dc0768d1143138b22a4dec7c0796211c515d", + "40b502f762b12d9008d67359a69427246dbf4590", "c7a0c0a1d86245d58f93578f5e71861850b20327", + "45e88dec3420a3081ed070a89974fade34a8cdd8", "a2cca480abdfd271d83ec53f32b9ca7c0534b1ea", + "eb5b56e00f7e3a3a93a151b04de322a54b478a1d", "ee726a3bd27fcd17373550a73f1265123f038e9b", + "ecbef3833e45ff4b51fcbd07be7ae083d0261106", "6ff3d878e2b0739195fef120f3e8920aae5b6c14", + "c8a254b1ae30d6d0372b9421cf164a2f92770bb3", "b6fe64434d331f92023b8ac4a0ae29a436ece676", + "6408dad003df5f902970c9a9b281bbdcab70513b", "b000f8b57d68abac68a8ba92d2e8af1f4cf04e3a", + "d70f6b719ab62e4d30947d46b48c4f86431e27f4", "d8b1b3fd5b14a37edfca20bafb53bb19b68034b5", + "54e2802c7dda0dc48297627148cb8080900b026c", "2a88231e4c45be17c7b59df395c88ad1de1f55fd", + "bd9541f3b7d18ba20fefb5725bd21f29c01a2fdf", "fb211cb3f8ec40bc8d34ef5da91173b85368ddb8", + "302c5986252fee81e024e43b3950ce0c03c1b19d", "c688626a38c6777646e4c5343903c5d3c39451d0", + "97e8d745c998aa9ca3c0402852cfbdebf38afeff", "512ca7ae1cc0043eee4b9fb24bc680e5e439a22c", + "8a1b18f3f1a52393779d7872132a261596718531", "f1f688ce752c4783753ded0c034de72056e5bf0a", + "cf404c790f1926de4fa92a90bc25bd3b89441ee9", "6f67fb37b601d81e607b4784b59a60f5da04ddc9", + "0358ae29870a0c0555826236454eee2a35460bb2", "267d40944d68c9fe031797824c83b7628ac21054", + "28999639cccbd7471536049556b7bb0445df968c", "96c339642cf640e219e83499b89b3f14c40c1a46", + "22ab4abcaeaab3dffd0a0eef49631c41feb35f01", "f3281e89aa93a9ff4b045c561960128ff02bd67f", + "ba7c3d3d19387b725dd93777e264c2d243c56985", "c06958105add69f9c1215959af1592100820f897", + "0246ace3db73d80516951169f697ab82de31935e", "3ed4aa3045ac0679828da5a36e6a159a76397b85", + "a9b3d7258a65a4ffc0188430f23a2b348bf3f71f", "3f80e737293fc727a2a5f5931afef5da8a108889", + "f5c4af7e6ca21899d603dc1c5d6dec724180a617", "83223c9cdcb837c03a1ee706a68c201c6f46aae1", + "7ebd59911482548ff28bc5257f092339747a9847", "a29e7d193abe5f905841b7dea04968b1c973775e", + "4df92c585a35462fd174a94fa6cc34cb104a3338", "480f1b18698079c345470f6eddf82dfe04f372bf", + "e93635e6ad4990b2037c1c42d96e5f4f5c382211", "f27588ba28c79a1abd36e9ab225006ef711bc2dc", + "0780639bded276d6de3f29a949974f9aad888f1e", "7649e12dadd9626c6aa55ce09bc5a77abbc1866c", + "10684acced018de4c9d795b4e3d17f85b831ed05", "1fd08bb5d65fb7deba4ce611c4e4ae61526f70f5", + "4ad405913be05105cfd8169727862962f73ba90c", "e622f46b8ab1a027717765dd7d392879941691c8", + "e53bc56ec46866426314e2214f39986f794f7ec7", "1ee4211e5bc419b24309bccc29d84c71d8153e33", + "13bf7fbafaffb9eb5305cc786ea5327b1bea8c84", "1b17359571b9a657d8c1c259f392bdca507f8048", + "f14bcf0c9d9d4441837089d9406a275ec7bae489", "650d0d424276a764869dfa6d5769e0d33dee29ab", + "9a51d909f6e0ea003b7bc31368889b34cbaf99db", "d25b13c8e67b29bedd33634693fd47107d79c386", + "d4791ab3fc5623770bd37a62f5847a3469ae62b5", "63cf95a96c6f9550a526c3f7c973b3bf77da1d99", + "0880631d8a31a2d656c214fd5326e86d72d99735", "c028799f0dc5cdd928ba41ed57758827d4c33960", + "09285b7758bced6de4de28adb839ce3707bc0801", "ff3ef6de1556d6a817b228634d9bd36c03f91b4c", + "7599af4e7f3c5038845232dc1c3460e757d29e76", "945656d9dd3e48ded82ae075761849b0b6de7d99", + "14d2b00bcdde27acd1e67707fdec920aaff41440", "bead8489f2f77870afbd190228724d8ba9525e77", + "ef888c5167e12709b33fcb9373476ce222d3a777", "11185717aa656979199228d4f134506f306ebe11", + "763619ad1c6fe756de2d6ea598f4a35b6b5d07f5", "de64195cf8f8e0a8a209993c12c37e6a0e9717fb", + "6fcd348a0817c5ba9f692e9f43b157ba53f61736", "184223aaddee8a795fb21a0298bca7cf9f24b86a", + "025924b72b369155a7dc79296b0fa1542b6cf66e", "00b6d09efea3b56ed30d3f626837c105415a2b9c", + "7fc27ae1ff5720a8632dc7180a62f93f0b240971", "68a1871cc2f4239c1e4c717d6cf5d50121aa24f7", + "d86a119c71e8f3db87ef84a3f2eaee3784e514eb", "bf6c04384c5ebbcbb7a2635e885ad48223000515", + "25df57cb358e1c55102d3eb05c633bbbc8d18ad7", "c2ed09db58daafd8711f0729136bcb4bdab4ee39", + "db9ff970f2394e5ddb70b05835e5cd2ceff1758f", "b5b0cdc6e2d0971f11ffb7b83eae62bd2f2e2a63", + "568a605393fdf0261752b980340b56a695993ca9", "53a7d40199c4cd3e0b61387708a5c4b2daaa6a04", + "d764e0dd4cc13227c90ea0dbdb7a17148a37362c", "2b0970782ff6fbc9b574d9af09c7f4387da71167", + "1a2f72aec014fc3b1c7a58660c15cdf8eb78c30a", "ea70291c158d9ac2e034f91837f3aebb9315e74f", + "ddeee68ef8756513d3c91e77705759282dc88f36", "c4c43e1c71a7693a257680b25c4cce05fa68139a", + "cb91e9590a1524dced13be78e13abde059a5358f", "8df5d8b31502fd506a59b0d15f61631dc5dc0d11", + "ef60833d075ac1aca48423ec296b18756a0907d6", "4bd51174d78af4c6afff4a3580c4380eaf53db03", + "f89f34968e6fb5c4400d4133e5b83eaad9ce1d81", "4926ebd2150fca495a5ebccb69d73a843c2a5c76", + "b7d7768027c43c4bb9178b59096058c827249ebe", "b18e01b7ab7b8fdd5e9093a4d10446ccb4c8b461", + "7c0cb0bf58b260d848be8b2fffc6ea81d45bd9ab", "65560b3113f1d79c241d64615c70a95a27a1295c", + "ef7de7cca843f688185aa44caaa11b8d9c5ba419", "577959df764fbe2295c7355420fb88e7259318f4", + "fb4e5fa6f2ea75bd25bf313675ace5ecfb09ce4b", "58c3bb9c95b2b18b54923c1c66a271c5103b5016", + "d15a9ce8720d18dfb1886de4ce23504c690f9849", "09b1961ea1fb0304ed7ad7c2c404af3113af532c", + "689943c3a6fe2d15706f588d5d192de09025f87e", "76c955116afc032044fa820077f0aa0077a9b221", + "e4e28d5880eb5a19eac4f2dafc94b87121315736", "f66dc5b1d3a36e518a2d7581534cf1eac1c0bb86", + "437d9bab241951c67ba4344ba60e7f8c8fe091fe", "d48f36d517120dec77f9f1383c6ff175e490957d", + "3595ca188c62c02f3a21616a40fce4063e5639e2", "9335fe0ddc8db4b8825b6a8e96f2ae314c3daea5", + "93b3baceba14eb826f0691ec927139eddd131418", "b081a6d644c99936ce2b53227aa949e61c1fc1a2", + "7c301a66f04090a5dfa9b97790c2398bebe8c658", "70e640c8b512f69cbf2d3815ade5e08acfdd6ea6", + "a5ff6189fcfa209df6073595ce8d048bb434142b", "865f86d6f58109166006154d4cb1f18d780afc7b", + "0b32bd194264f3d34f207e36da01747250f370b2", "160db72460b1d2cb57b71c5f5005c43b39344a08", + "ca17e8fc42c63b71a64943b2edc546cb483d4987", "62709ee11b06371980bbc365eade2cc09fb201e1", + "d42b892d83a19496adb8a51865d7786bf11cb51f", "151eda2239a8926cdfe11bddd08d4a68dfc06fce", + "48289268a75e7c551d8407da65a6a8d5e96bfeac", "76475e31fb20faa61548ca6c110bfaf0a8509333", + "ff45b9e590737c453377adfef746c419fd024214", "791336c9f92b920c36d87abfcd23717effeb2979", + "7cbdb028c46f69f69ad5b3870fb9976a339a88ea", "6dce9b85f8a4d116d780895772890e62ec554050", + "609a9ddb002e9e03ec1e01fc0d3273defd125f6a", "eb5cd823d83464a85d52abf6fb618ab6b0ceaf14", + "0b33f97dbe25690d5b395aec431714f326835761", "b3c0967b6cc938da94bbc24bbb4960c499eab996", + "55cc0375389bdbe9081930f6aed59e2d1b996d68", "3ab0dd9b211aa79f607aa18900021d331ffec108", + "8b8fc9dc6eaeab845218b9b42c9f1eadb85d9a76", "d95873f271a98500509514a3bd503e111b2eeff6", + "60c22d4758b2ab938d925ebca4195d388976ef0b", "4a71cb75be4c145468bfa41edeaec4cad48b35bb", + "674df8ed0814f769a01ea52a86b37a37db300fdc", "52cd156a808d638be19a621e77ce9d0b705ff959", + "893fae64443d115a0071088287f1047c9c85bd0c", "fe745f1136b3c2b4d5cc703cee8933de6357c502", + "7a9ed9c04bef62538916e432a74f44c034068c16", "c60540472bbd9473a78ffac3a071904552cfdd6e", + "dce6b37c643555216d24705885d35be6f1658323", "d25cd6e77dd4f8683333b68cd17e4ef161a10f82", + "ef59cd7b3e448c27b726dda2624e3a66bcb7c915", "caeb866326a82dacaf9b9ce21de180c99462b0ab", + "300bbd6765f353ca1f7a42c8db1edbe0fca9146f", "d476be09c2c9a67b065eb972d473c862d05520d7", + "f2692ec53c9eaeb976a81908a1ad9a8a6065eb56", "80a904cfc2637be47cf6db433896285c96642f28", + "b4e7c96d36534976b532f6a73d687ecebd20d3ce", "f442e491213973df215b4e5a13ac42dbf26a418d", + "4a990060a137b7405e90d674667b774c2ab3d9df", "ea028084a5ba60179ffcc5c2e7745a5137810674", + "946cd7b549d16cec7f31b99e31a5cd1f97ecb0a7", "f4d1570ca148a075ffea03034aa359a795a7a95d", + "b24a0df950d615d89a5c490856d9efe0c67e6807", "4a3a4973113c414789c798e3b279d04e68f18193", + "853861b52dff3d4e69cea17db319b578dfe36b57", "75cdf79a50fa9f54ee0abe8a7352983e4e15dc8f", + "d2172012e40b489a1b696aee59ee01beb94d2044", "c463cba55508e3410e6c938ca7f07cc49c69685f", + "298753c52149d58e36b63ae01437d7e9b1626550", "edf83ddead1b5bf94530280409c8f8d7a3566036", + "6ad9ebb63aaa22d5cac2c6ab68a8d044575531b8", "4c4e417f31a770c165848c34e9a1c03a88345923", + "01ac78c98a9e83a657895f330ac60b23df81aa19", "077b29278d6047812f8b65e158402f13dff2e638", + "6a3280cb4ecb45a4a949416106e707b4999e2491", "3de477994ebd5b7edf0a48ba6c5e799fe02a53f7", + "2ebbd703eb8308492ff7d0e3604b786931ad8617", "5fc0b82cdf849a6c424721cc5118a196c07caf43", + "851ed5018abedd4f527d5fc1f184016c2b9dcb20", "8271e8c30ed07e236d230df37635819f45b3adf5", + "794d1fbeb2f5f6e2ad81450602d53d232a2fcd66", "765e4955986ec0b20075669f5a7e2dca79a98ae9", + "f94426a7f4b9bd63aed0e75894ea80278dab4320", "c0006f951059f5aa58b4b898967d1dab70377178", + "414f4c053c7c9d294903806e748a132d54325f73", "a3dc4e32a5bb67c422822731ffbad447f2f89dd0", + "9dab95f8797ac80a22cd854761e2dd141be8f436", "c187e0b59cc8710d812cf24e9ad7191c5d0c6206", + "55cc88749fcd34f08c5d0fee8bee92dcaccfb010", "2c3d218a5428646b970895dd88e2f61739ba0732", + "4ad60738ad972e0cbca3e763b556937501c0973c", "543f72fea7ef066ea3111bdac8f0ac61134e0c24", + "a2bf3ec52c5d004e7e5329efdf6b5cd735618b87", "f483494815f89e80dc399163080052fff49031bb", + "3326ec457cfa6d30663eb0ce49a838d7bbb86415", "f6d5647f6b6923b5052619814a84b2e1bf8d0b85", + "f2dcd0fc70ca95f475b68f7335b2d0855534e5f1", "a50cd55ec2258557a20839a0d6219960c9b1a2b0", + "a34ae4d4326a0913d9af50546c558811565ec66a", "842e922b6252301d8df608a1a1a4a732686e2ff9", + "2b6cb59f7846a482432b4ac57fd960f73d2441ba", "e559e01ccf3cc35aee9732e3db97c23165f8a6d9", + "543085108d5513b9fe63eb96fb386ccc84a811d8", "c7474d0e9cfe5c1d8e56f3978eec6871706a38a8", + "fce05c355558c606a7ca815613c6ad3d8f8a1503", "30a17532024869bc353e28aa4c3da17dbbcab80f", + "13d54732573224d7879166a11857046c50e13626", "42868488cc892e63b13e7afbe157006ddc215258", + "94785fac47a5d2ce41d64269203f14de07995186", "82d752a38f9d5598685d58e5ce6080409b980442", + "96f99cdd47de4d5e4180c99bb7015fa77d329687", "e89a84828a38b552efd3095472eb39dd18be141c", + "5a37e8b4031b627270f06125f5c204eb28437c85", "30da9a8dfb0f65559be3f17a3d2f3be0cb297ba7", + "abba9bb98e64624543ac376fdce700ee5e33b392", "c19f87bb786c68bab429f594374c24d09dc98cd0", + "73754ef7ae4615cdfe881adea981bcc93b65e887", "44c25f0a9f3151efbb98056e1c2bae59f4aca0b3", + "51f465b5aa2a63568d0bd82232898b39fa429a14", "1ece6bad7a8d2492da84f5db5a94a9c81a94a3df", + "603617cb2ad5009c06dfc61432287ea41d4dd70a", "ec346b3231969799e3852cd7f4e951a25f3fbc0f", + "46af277dcaf4eed7cc96e021c96b266c24c1fcd3", "0965cb17b3b353174be301989ee8f5b77ab925da", + "b45319782274fd4841da3c70ad3b4754bf2b1b08", "2a2bcc98ea5b188049bd46b4e315ae55b359b7f2", + "cd09081ec384b747181bc2309a9485ee0888516c", "f636e931892c445a7ffb1aa0eae7499caa19932f", + "34ceb6f49765ac0d173d4a0cecc089f34dce2695", "9b18988eed2891623baa37468ef83dc6a9a93e14", + "087397f895ac78aa63b5b7413dd4c948cae4dc50", "98e1abd91d5502799931c25c1c19fa6605ff6dd1", + "279448c0f978c369797336bd660e457e2abccb3b", "20d76b4b81484d694731d6a2046661a025928bdd", + "7a69b783e3e747a0f7a5eff5cc23b9a00d53aa01", "f645296a6d289f80b9960679b8edfb6b0a4d96c7", + "9352d33e1d396110bedaf1906567a6d8139c98ad", "3a4cb9ed1c4e291a738f6a97ae82b7db616ed4f6", + "e97932a4ff925df81b844058a18f2da4f362edad", "2e9b3d733cb1d6a6fda98cfccfd179c35559671c", + "00e2b96be50f143abbaeaef1b42f502eb1c5eb38", "a3b8a036064eade54ac1a7e49a48b5228863f225", + "b050b6089b63a630ca1e102f27f1e33310bdc0f8", "91200850dde71fd5061890bae22cf6264f3a0ce9", + "c4c90fa787e28f241a6149c5706f67cb8c897c3b", "52cda4d77c07f76113b392291e947e02010a8ef8", + "949b21730e48bc645dca6c6a9dedf758035d0eed", "3e0509507736054281c0feb777b5beb62af98ed5", + "994993e838dda5b000ee4e75786a5a826f368f9f", "24e2e8a67ed9977de3c1373968d76736da3c800e", + "9c8374cdb61697a37bdc89b57ac5574af910138c", "7facfd4ee6d102ce1c2103b2bc0861865fffc07b", + "82f7d052e4d562b268730e8d66d4ac694d963f8d", "a5f79e4ae09eafe2892bff810ed21b1a7cbeff6a", + "7a66189f5b487392ddf3bde3b6b2f837711767c2", "756f813ee996e91c50dee3929035b81d5c2f5b7c", + "c0c98d3382858b9de30907003cb1b3f47bcb7e4c", "7a1f959994f3e8665589050ca2e07c5ed5579ff6", + "06944876fd90b41f439d337cfdbef2582caa4b56", "73e779de8b03c705726abbd686ab6f905fb2fe27", + "dd14f8315d4345a1482b6b5f581fefae03154e6f", "b4f629bddb661422c15c4b0459104b6a6387f3a1", + "a65dec2720ca334bb503079db5a5db4649c27c0a", "23d9c64b7d95c1d1c08ac83058dba6204537709c", + "0070ff46a8b72b79d011f59a770cdd2e3a7e52db", "39538c29808bc3b1c85b2f9555eb56296778e5b5", + "05235f2fda6b96b82d3b798f5c06d1c26fbf5556", "da9ac947e5272631161f7415583487cd5b81bd92", + "0afdc9abb0a921b0eb87256c34c2440d7bcdf748", "708eb61ae54de0fcccffafc686218a959fa0e12e", + "921d2ecf681cb2fa40d8cc9390591bdaeba5bddc", "d54e009ec45cfd47d90fa87926ff55f5668cd745", + "b66c3a482974a6950eb27fa2c83df33c0ddaee0f", "d11ee1996ea846c781624a6be16e2cf05c90a3ad", + "d1fafe49cdba1c135a157ab725caf6a73c44d413", "37add1833a4462b08f3d63ac3cfe2b29e8c19daa", + "54b9e091f1f4a2aecc9c3abef4785cd4116e59da", "2e7c5f1ca4868b1bf1681e852460d1a64f6730f4", + "fee6c7a1a74439ade1a4ba1102431cb16ffa40b4", "0b663b2c2e9813fd968ea4d53c27117a37af990a", + "2bcec290f45a8f707462a56e8cc468f5934fe57e", "2e77e56acb8a073a7164052b3015752f55748f13", + "56cc03e04970154b1ae9a3cb49eeb3d131118feb", "17819764b0cc55daa6f9a70e1b7cfbb082504a26", + "b9992bfe65ca57863517d1a44a5de4abff0d891a", "7cfd6ae2ae46bf951a84ab288fbf6381377e4fb5", + "44f321da361e4de492292e465ac2576625c2f04d", "14d021d38ede8d4fe6c55266ea1bc18471146ca2", + "dd5f008277ecf79acb0c32c88f459a46f556d235", "e712c5cfb5144ba8122f13b400324a3db3eb1131", + "2e3b8bdf70024acbd57dbd574b42d8081b22238f", "a27f805365b6074f96d0d72bd877158a6a2683fc", + "1777f7ceba2141c7725cae03325aad9015f5e0ba", "3ed72d7bfdaea45c7741c4adba22824729fccf93", + "1efd9d0cf3d575dceae0143ad80cdb36ae929f19", "ed1e3113a91cba66cb2bdd2fc62d221f0fe98720", + "56830484e8e3a9bb9c3c0e28d2e06c59c2f85814", "612f82971d281213f121bf3940fb3a22ef339b1a", + "9204cfa997dfafbe1d923cae857fd8772140d2b8", "bd7772cc8ad355a2edb8ca92d712bc6522f679e4", + "50013bba2f0c0daa2ba0601ad9d538b8945f49b3", "903166d464af4cbe28c3dcfe87e3b03d2d4361d4", + "498a0b6129cf4bbdfcd1fbe56f41a09711c81515", "4d82da411ae6922528a7e31ba0d9b7781ca7b140", + "d1a59d5be71487c46f1f13cca1329b890864e3a5", "0130298b9910153b918c24cd0305bf004c77f139", + "b4704585677546e3719b0167a48f7352c3135018", "bd2af11cf4b22ffaec8f0f81e41c7a200707984e", + "90bbdf000b6566e32e973a9ed13673babd3d5971", "53b76d4726ba14be0ff600d76f6e94af63bc76ac", + "98b92425f234019ae1a7d7fa763ee86d24a9d4b3", "402419c820cba29219805adb1b50909143962ab4", + "2507dd9d1d952d45d8e29e3da7d9b67261ddc8d6", "0eef7ee0455462714b4398429bc4e0b1e68aa807", + "c15af97c03e51bb5c7c23118564b21d37e41dc38", "5ff6527e38d98cdf140318d98828d7de099d1552", + "7d8d05ecfc3d87c0a798fbb28a7bd63d79317590", "d728b77adbe3f536b9211d9b26b0668cd796a549", + "ff936585cb1595e9b56e14b77c2a8605a312fc2f", "cee364e17aaa6f0d77dbcfe496b188de38123833", + "28f1af82eea595d5db3da22f4433c144fad620ef", "d5bc60e7980decfb5d5cd10ef9ea1bd0a3ce5fae", + "392ce37ce6411c30d436940506dc2cc8278f66e6", "38d291489369e7f222a2711e5afd7d6ac6afb574", + "479506b2cf7c7392c0a96d6e5d550556dfd20ba0", "c71e5676016de6a9b7dc27a123f403283fadec2b", + "5d03ade937932174a2f2b376efe6f761636a270f", "04302da411d5d79477463fa37de2f639afd5127f", + "d4e6cd7852b5b88d88afb1db0cc9d29d8484f143", "1dbdc1d3d9dba7ce5cac3bf7d2268ab237a4c5cb", + "3f263b23507300ad4dd7903e45876c2877e59ffb", "127d0081100e98843f16d26cbea527856d9a9ddb", + "dd5e22108dcceb35e1944813e2524ebfd0a53df9", "5d52758299b341021d135b5166614a46f3ecf2e3", + "a1b3da2881596c60f669fd80479e45a097b38707", "21a63be654b8e5df434d01d420ea2206c8d9a92b", + "36ea46c3da7dbc2bf4d304d81f86224dfe54444c", "29fcf933c2e58f6eb8fab04bf8c511f56896fe15", + "64737770ddca8995c64e1106bfb08ff0ff1fb144", "d8c191fa2b58539d83c7efc1c3346079ecdc566a", + "77f0910c39aa6108e0c2fdb32fe1832126c0489c", "af64db9a99bfd8ca4affe94dcd5cfea05e94e03c", + "71dd27e77432a2d5b446958707286d08097da244", "5ddedb0a995f06b295696a2a7b4551e404d0dd16", + "01f87385a24ece34ed8ddca801fd84f9c86e74e8", "09249256a87d4a90588b026d789fd3cb1aec856b", + "9240b4a3a946be297b435486d1f2a6d941422b93", "399ca25f86719994d4612fdb618983079d6a59ee", + "4db57eaeeec89f4a96d0f2727c15ff083482fbf2", "4a31d4c94a4c6ae379abe24c46817a1157eab07c", + "182f2002e4307f83c62e06dfa4164543fa7e814e", "97a0a7220c2b8e458641c3e6e56889254d2da049", + "3440a5e4b4e31f45c6adbae99b6214b8fbeebb2e", "6f94a7a5fe870501cdcea762e4a7a20631d48d62", + "9da3107576333b6c7f3e878e508e94a4c764e45e", "20cb94f57147f8372022885ac2356e464fdaf7e3", + "68803fd3ffbf592c0432eadf45fffa22e5afa8dc", "c538f4c3d9ed532f02458a7c863ddd549960902d", + "cd14b737781fb608698437bd2e6d20d691093874", "2db66f7bda4464a6f5342f5e021c29a38d42068d", + "37b12212ecc3a32ede8aa04eb0065fa80e3e821d", "887306035bf8e3f0963a254a6e46b52b55a04fcf", + "8747b40002f4a24a2330ee27f36ecc0880083827", "ee262c59ff4c66f77562ef879569193543838393", + "8282282c974675b044c071faf25996f4b6e4ebc3", "f9ca90f109c32ff5889c0c9d84ff11f9fb5b5924", + "e530847cf2aaa328e800919e6eed9739521a9b50", "ad726687605b280a0c190153654f4751ebba4c7c", + "93f6ba66b73bf972a3df6ba722a1d56782c59d47", "b40be55c3b541c1c3a6b46de02f45e3ab9dfad47", + "95ee9b8acb715971bd1abc137a9d54bc88139b8a", "61dc80eef767360c6d48572ca1bda65c693d5362", + "ba44c4692965b0d694c8fd015f1ec3fa4dfcc696", "418a1501e2a14efb4ab940cad4c898f09f92ee83", + "026029e19d4c6f0bcf239abe607026a5a27f5b86", "8cbd5f3c989185d9474f7b5767d7e6adabb2015c", + "55163a23b71b9a3d63f6e4695b07ebd07064c61f", "d9bde5278517a57ff7714edf98ab2361d1663219", + "9ad5f0871e1df9582ea14bf7ba22350086fcd055", "c57f5ae0aaa74eea99032b45f0dc5c927813d8b5", + "3e3f6f7661780b696501a397d8e95ec446fd1fc3", "03d94679ae1908d9a6f6a38c00ce7988c5afbcb1", + "4c8f526d899bf04b4ce3e088302680da64499e5d", "271ddfcd505226549f7674b7e539efc96c8be5b8", + "25708b2d11167437e6de01fa0406dfd3c26bd8d4", "dcf94a3cc15a85bc1dc2fc7e1751e6a343609049", + "f65dd5dea2e8bf5c90b8b093b4ae2db11fce3b5c", "b3722ffbf1e3287cee8109c82b7ab13d187a3d7b", + "ec3af0efeea2836f6a2fc5a2390232ff10e17cf0", "c7176c1b7df0676b474b4786a7e6ba25650df8aa", + "ad97c1b6ab4dcd3d4917cb59427490508523daf4", "7a8e2f675709ba76674a2903653b07d348de32d6", + "bdd754c4219ceadebaebab82c1dacb0a6d819e24", "31df56efa866faac4fced90b742b1800261ef46b", + "1e77fd0a0bd78293212c518291333d3554136b41", "99f906bf47a41245895867f76ad38fc9ca88921f", + "49dd5ec3406bda057407abe5c57e6a027b8c502d", "f36f65ae5480acecfb85d9b8d7a79603853f2115", + "5a29a9db5779e61efafc338b00be9d22efe2a181", "6b28f73e9097b7a7966aa9add138e3dc14c52e75", + "92571e06644eee1606759deeccb9875fdb454b81"], "public": false}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '80631' + Content-Type: + - application/json + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/experiment/register + response: + body: + string: '{"project":{"id":"208de547-5ee2-4294-955c-5595cd0f7940","org_id":"f5883013-b5c1-438d-9044-5182b4682337","name":"langsmith-py","description":null,"created":"2026-04-03T20:35:06.233Z","deleted_at":null,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","settings":null},"experiment":{"id":"407a9c6f-27ba-4c89-9bf2-b19985d7b695","project_id":"208de547-5ee2-4294-955c-5595cd0f7940","name":"langsmith-eval-f5e274f6","description":null,"created":"2026-04-03T22:20:43.111Z","repo_info":{"commit":"5c35051a0102f850f2e09c66f9376a0fc658ed9f","branch":"main","tag":null,"dirty":true,"author_name":"Abhijeet + Prasad","author_email":"abhijeet@braintrustdata.com","commit_message":"fix(anthropic): + capture server-side tool usage metrics (#192)\n\nFlatten Anthropic usage.server_tool_use + into Braintrust span metrics so\nserver-side tool invocations are preserved + for tracing and cost analysis.\n\nAdd regression tests using a red/green workflow + to cover dict-backed span\nlogging and object-backed usage extraction.\n\nFixes + #171","commit_time":"2026-04-02T21:10:00-04:00","git_diff":"diff --git a/py/noxfile.py + b/py/noxfile.py\nindex c92395da..1083ca5b 100644\n--- a/py/noxfile.py\n+++ + b/py/noxfile.py\n@@ -77,6 +77,7 @@ VENDOR_PACKAGES = (\n \"opentelemetry-exporter-otlp-proto-http\",\n \"google.genai\",\n \"google.adk\",\n+ \"langsmith\",\n \"temporalio\",\n + )\n \n@@ -104,6 +105,7 @@ GENAI_VERSIONS = (LATEST,)\n DSPY_VERSIONS = (LATEST,)\n + GOOGLE_ADK_VERSIONS = (LATEST, \"1.14.1\")\n LANGCHAIN_VERSIONS = (LATEST, + \"0.3.28\")\n+LANGSMITH_VERSIONS = (LATEST, \"0.7.12\")\n OPENROUTER_VERSIONS + = (LATEST, \"0.6.0\")\n # temporalio 1.19.0+ requires Python >= 3.10; skip + Python 3.9 entirely\n TEMPORAL_VERSIONS = (LATEST, \"1.20.0\", \"1.19.0\")\n@@ + -235,6 +237,17 @@ def test_langchain(session, version):\n _run_core_tests(session)\n + \n \n+@nox.session()\n+@nox.parametrize(\"version\", LANGSMITH_VERSIONS, ids=LANGSMITH_VERSIONS)\n+def + test_langsmith(session, version):\n+ \"\"\"Test LangSmith integration.\"\"\"\n+ _install_test_deps(session)\n+ _install(session, + \"langsmith\", version)\n+ _install(session, \"langchain-core\")\n+ _install(session, + \"langchain-openai\")\n+ _run_tests(session, f\"{INTEGRATION_DIR}/langsmith/test_langsmith.py\")\n+\n+\n + @nox.session()\n @nox.parametrize(\"version\", OPENAI_VERSIONS, ids=OPENAI_VERSIONS)\n + def test_openai(session, version):\n@@ -371,9 +384,8 @@ def pylint(session):\n session.install(\"pydantic_ai>=1.10.0\")\n session.install(\"google-adk\")\n session.install(\"opentelemetry.instrumentation.openai\")\n- # + langsmith is needed for the langsmith_wrapper module but not in VENDOR_PACKAGES\n # + langchain-core, langchain-openai, langchain-anthropic are needed for the langchain + integration\n- session.install(\"langsmith\", \"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n+ session.install(\"langchain-core\", \"langchain-openai\", + \"langchain-anthropic\")\n \n result = session.run(\"git\", \"ls-files\", + \"**/*.py\", silent=True, log=False)\n files = [path for path in result.strip().splitlines() + if path not in GENERATED_LINT_EXCLUDES]\ndiff --git a/py/src/braintrust/auto.py + b/py/src/braintrust/auto.py\nindex dc44c7d2..6cf8597c 100644\n--- a/py/src/braintrust/auto.py\n+++ + b/py/src/braintrust/auto.py\n@@ -16,6 +16,7 @@ from braintrust.integrations + import (\n DSPyIntegration,\n GoogleGenAIIntegration,\n LangChainIntegration,\n+ LangSmithIntegration,\n LiteLLMIntegration,\n OpenRouterIntegration,\n PydanticAIIntegration,\n@@ + -52,6 +53,7 @@ def auto_instrument(\n dspy: bool = True,\n adk: bool + = True,\n langchain: bool = True,\n+ langsmith: bool = True,\n ) -> + dict[str, bool]:\n \"\"\"\n Auto-instrument supported AI/ML libraries + for Braintrust tracing.\n@@ -75,6 +77,7 @@ def auto_instrument(\n dspy: + Enable DSPy instrumentation (default: True)\n adk: Enable Google ADK + instrumentation (default: True)\n langchain: Enable LangChain instrumentation + (default: True)\n+ langsmith: Enable LangSmith instrumentation (default: + True)\n \n Returns:\n Dict mapping integration name to whether + it was successfully instrumented.\n@@ -146,6 +149,8 @@ def auto_instrument(\n results[\"adk\"] + = _instrument_integration(ADKIntegration)\n if langchain:\n results[\"langchain\"] + = _instrument_integration(LangChainIntegration)\n+ if langsmith:\n+ results[\"langsmith\"] + = _instrument_integration(LangSmithIntegration)\n \n return results\n + \ndiff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py\nindex + 0062ec77..2dca1110 100644\n--- a/py/src/braintrust/integrations/__init__.py\n+++ + b/py/src/braintrust/integrations/__init__.py\n@@ -6,6 +6,7 @@ from .claude_agent_sdk + import ClaudeAgentSDKIntegration\n from .dspy import DSPyIntegration\n from + .google_genai import GoogleGenAIIntegration\n from .langchain import LangChainIntegration\n+from + .langsmith import LangSmithIntegration\n from .litellm import LiteLLMIntegration\n + from .openrouter import OpenRouterIntegration\n from .pydantic_ai import PydanticAIIntegration\n@@ + -21,6 +22,7 @@ __all__ = [\n \"GoogleGenAIIntegration\",\n \"LiteLLMIntegration\",\n \"LangChainIntegration\",\n+ \"LangSmithIntegration\",\n \"OpenRouterIntegration\",\n \"PydanticAIIntegration\",\n + ]\ndiff --git a/py/src/braintrust/wrappers/langsmith_wrapper.py b/py/src/braintrust/wrappers/langsmith_wrapper.py\nindex + b22117df..7d4170fd 100644\n--- a/py/src/braintrust/wrappers/langsmith_wrapper.py\n+++ + b/py/src/braintrust/wrappers/langsmith_wrapper.py\n@@ -1,518 +1,4 @@\n-\"\"\"\n-Braintrust + integration for LangSmith - provides a migration path from LangSmith to Braintrust.\n+from + braintrust.integrations.langsmith import setup_langsmith # noqa: F401\n \n-This + module patches LangSmith''s tracing and evaluation APIs to use Braintrust + under the hood,\n-allowing users to migrate with minimal code changes.\n \n-Usage:\n- ```python\n- import + os\n-\n- # Enable LangSmith tracing and set project name (used by both + services)\n- os.environ.setdefault(\"LANGCHAIN_TRACING_V2\", \"true\")\n- os.environ.setdefault(\"LANGCHAIN_PROJECT\", + \"my-project\")\n-\n- from braintrust.wrappers.langsmith_wrapper import + setup_langsmith\n-\n- # Call setup BEFORE importing from langsmith\n- # + project_name defaults to LANGCHAIN_PROJECT env var\n- setup_langsmith()\n-\n- # + Continue using langsmith imports - they now use Braintrust\n- from langsmith + import traceable, Client\n-\n- @traceable\n- def my_function(inputs: + dict) -> dict:\n- return {\"result\": inputs[\"x\"] * 2}\n-\n- client + = Client()\n- results = client.evaluate(\n- my_function,\n- data=[{\"inputs\": + {\"x\": 1}, \"outputs\": {\"result\": 2}}],\n- evaluators=[my_evaluator],\n- )\n- ```\n-\n- Set + BRAINTRUST_STANDALONE=1 to completely replace LangSmith with Braintrust\n- (no + LangSmith code runs). Otherwise, both services run in tandem.\n-\"\"\"\n-\n-import + inspect\n-import logging\n-import os\n-from typing import Any, Callable, Dict, + Iterable, Iterator, List, Optional, ParamSpec, TypeVar\n-\n-from braintrust.framework + import EvalCase\n-from braintrust.logger import NOOP_SPAN, current_span, init_logger, + traced\n-from wrapt import wrap_function_wrapper\n-\n-\n-logger = logging.getLogger(__name__)\n-\n-# + Global list to store Braintrust eval results when running in tandem mode\n-_braintrust_eval_results: + List[Any] = []\n-\n-# TODO: langsmith.test/unit/expect, langsmith.AsyncClient, + trace\n-__all__ = [\n- \"setup_langsmith\",\n- \"wrap_traceable\",\n- \"wrap_client\",\n- \"wrap_evaluate\",\n- \"wrap_aevaluate\",\n- \"get_braintrust_results\",\n- \"clear_braintrust_results\",\n-]\n-\n-F + = TypeVar(\"F\", bound=Callable[..., Any])\n-P = ParamSpec(\"P\")\n-R = TypeVar(\"R\")\n-\n-\n-def + get_braintrust_results() -> List[Any]:\n- \"\"\"Get all Braintrust eval + results collected during tandem mode.\"\"\"\n- return _braintrust_eval_results.copy()\n-\n-\n-def + clear_braintrust_results() -> None:\n- \"\"\"Clear all stored Braintrust + eval results.\"\"\"\n- _braintrust_eval_results.clear()\n-\n-\n-def setup_langsmith(\n- api_key: + Optional[str] = None,\n- project_id: Optional[str] = None,\n- project_name: + Optional[str] = None,\n- standalone: bool = False,\n-) -> bool:\n- \"\"\"\n- Setup + Braintrust integration with LangSmith.\n-\n- This patches LangSmith''s + @traceable, Client.evaluate(), and aevaluate()\n- to use Braintrust under + the hood.\n-\n- Args:\n- api_key: Braintrust API key (optional, + can use env var BRAINTRUST_API_KEY)\n- project_id: Braintrust project + ID (optional)\n- project_name: Braintrust project name (optional, falls + back to LANGCHAIN_PROJECT\n- env var, then BRAINTRUST_PROJECT + env var)\n- standalone: If True, completely replace LangSmith with + Braintrust (no LangSmith\n- code runs). If False (default), + run both LangSmith and Braintrust\n- in tandem.\n-\n- Returns:\n- True + if setup was successful, False otherwise\n- \"\"\"\n- # Use LANGCHAIN_PROJECT + as fallback for project_name to keep both services in sync\n- if project_name + is None:\n- project_name = os.environ.get(\"LANGCHAIN_PROJECT\")\n-\n- span + = current_span()\n- if span == NOOP_SPAN:\n- init_logger(project=project_name, + api_key=api_key, project_id=project_id)\n-\n- try:\n- import langsmith\n-\n- langsmith.traceable + = wrap_traceable(langsmith.traceable, standalone=standalone)\n- wrap_client(langsmith.Client, + project_name=project_name, project_id=project_id, standalone=standalone)\n- langsmith.evaluate + = wrap_evaluate(\n- langsmith.evaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n- langsmith.aevaluate + = wrap_aevaluate(\n- langsmith.aevaluate, project_name=project_name, + project_id=project_id, standalone=standalone\n- )\n-\n- logger.info(\"LangSmith + integration with Braintrust enabled\")\n- return True\n-\n- except + ImportError as e:\n- logger.error(f\"Failed to import langsmith: {e}\")\n- logger.error(\"langsmith + is not installed. Please install it with: pip install langsmith\")\n- return + False\n-\n-\n-def wrap_traceable(traceable: F, standalone: bool = False) -> + F:\n- \"\"\"\n- Wrap langsmith.traceable to also use Braintrust''s @traced + decorator.\n-\n- Args:\n- traceable: The langsmith.traceable function\n- standalone: + If True, replace LangSmith tracing entirely with Braintrust.\n- If + False, add Braintrust tracing alongside LangSmith tracing.\n-\n- Returns:\n- The + wrapped traceable function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(traceable):\n- return traceable\n-\n- def traceable_wrapper(*args: + Any, **kwargs: Any) -> Any:\n- # Handle both @traceable and @traceable(...) + patterns\n- func = args[0] if args and callable(args[0]) else None\n-\n- def + decorator(fn: Callable[P, R]) -> Callable[P, R]:\n- span_name = + kwargs.get(\"name\") or fn.__name__\n-\n- # Conditionally apply + LangSmith decorator first\n- if not standalone:\n- fn + = traceable(fn, **kwargs)\n-\n- # Always apply Braintrust tracing\n- return + traced(name=span_name)(fn) # type: ignore[return-value]\n-\n- if func + is not None:\n- return decorator(func)\n- return decorator\n-\n- traceable_wrapper._braintrust_patched + = True # type: ignore[attr-defined]\n- return traceable_wrapper # type: + ignore[return-value]\n-\n-\n-def wrap_client(\n- Client: Any, project_name: + Optional[str] = None, project_id: Optional[str] = None, standalone: bool = + False\n-) -> Any:\n- \"\"\"\n- Wrap langsmith.Client to redirect evaluate() + and aevaluate() to Braintrust''s Eval.\n-\n- Args:\n- Client: The + langsmith.Client class\n- project_name: Braintrust project name to + use for evaluations\n- project_id: Braintrust project ID to use for + evaluations\n- standalone: If True, only run Braintrust. If False, + run both LangSmith and Braintrust.\n-\n- Returns:\n- The Client + class (modified in place)\n- \"\"\"\n-\n- if hasattr(Client, \"evaluate\") + and not _is_patched(Client.evaluate):\n- wrap_function_wrapper(\n- Client,\n- \"evaluate\",\n- make_evaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.evaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- if hasattr(Client, \"aevaluate\") + and not _is_patched(Client.aevaluate):\n- wrap_function_wrapper(\n- Client,\n- \"aevaluate\",\n- make_aevaluate_wrapper(standalone=standalone, + project_name=project_name, project_id=project_id),\n- )\n- Client.aevaluate._braintrust_patched + = True # type: ignore[attr-defined]\n-\n- return Client\n-\n-\n-def make_evaluate_wrapper(\n- *, + project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: + bool = False\n-):\n- def evaluate_wrapper(wrapped: Any, instance: Any, + args: Any, kwargs: Any) -> Any:\n- result = None\n- if not standalone:\n- result + = wrapped(*args, **kwargs)\n-\n- try:\n- result = _run_braintrust_eval(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + evaluate failed: {e}\")\n-\n- return result\n-\n- return evaluate_wrapper\n-\n-\n-def + make_aevaluate_wrapper(\n- *, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-):\n- async def aevaluate_wrapper(wrapped: + Any, instance: Any, args: Any, kwargs: Any) -> Any:\n- result = None\n- if + not standalone:\n- result = await wrapped(*args, **kwargs)\n-\n- try:\n- result + = await _run_braintrust_eval_async(\n- args,\n- kwargs,\n- project_name,\n- project_id,\n- )\n- _braintrust_eval_results.append(result)\n- except + Exception as e:\n- if standalone:\n- raise e\n- else:\n- logger.warning(f\"Braintrust + aevaluate failed: {e}\")\n-\n- return result\n-\n- return aevaluate_wrapper\n-\n-\n-def + wrap_evaluate(\n- evaluate: F, project_name: Optional[str] = None, project_id: + Optional[str] = None, standalone: bool = False\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.evaluate to redirect to Braintrust''s Eval.\n-\n- Args:\n- evaluate: + The langsmith.evaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to + use for evaluations\n- standalone: If True, only run Braintrust. If + False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + wrapped evaluate function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(evaluate):\n- return evaluate\n-\n- evaluate_wrapper + = make_evaluate_wrapper(standalone=standalone, project_name=project_name, + project_id=project_id)\n- evaluate_wrapper._braintrust_patched = True # + type: ignore[attr-defined]\n- return evaluate_wrapper # type: ignore[return-value]\n-\n-\n-def + wrap_aevaluate(\n- aevaluate: F,\n- project_name: Optional[str] = None,\n- project_id: + Optional[str] = None,\n- standalone: bool = False,\n-) -> F:\n- \"\"\"\n- Wrap + module-level langsmith.aevaluate to redirect to Braintrust''s EvalAsync.\n-\n- Args:\n- aevaluate: + The langsmith.aevaluate function\n- project_name: Braintrust project + name to use for evaluations\n- project_id: Braintrust project ID to + use for evaluations\n- standalone: If True, only run Braintrust. If + False, run both LangSmith and Braintrust.\n-\n- Returns:\n- The + wrapped aevaluate function (or the original if already patched)\n- \"\"\"\n- if + _is_patched(aevaluate):\n- return aevaluate\n-\n- aevaluate_wrapper + = make_aevaluate_wrapper(standalone=standalone, project_name=project_name, + project_id=project_id)\n- aevaluate_wrapper._braintrust_patched = True # + type: ignore[attr-defined]\n- return aevaluate_wrapper # type: ignore[return-value]\n-\n-\n-def + _is_patched(obj: Any) -> bool:\n- return getattr(obj, \"_braintrust_patched\", + False)\n-\n-\n-# =============================================================================\n-# + Braintrust evaluation logic\n-# =============================================================================\n-\n-\n-def + _run_braintrust_eval(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust Eval with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import Eval\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + Eval(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-async + def _run_braintrust_eval_async(\n- args: Any,\n- kwargs: Any,\n- project_name: + Optional[str] = None,\n- project_id: Optional[str] = None,\n-) -> Any:\n- \"\"\"Run + Braintrust EvalAsync with LangSmith-style arguments.\"\"\"\n- from braintrust.framework + import EvalAsync\n-\n- target = args[0] if args else kwargs.get(\"target\")\n- data + = args[1] if len(args) > 1 else kwargs.get(\"data\")\n- evaluators = kwargs.get(\"evaluators\")\n- experiment_prefix + = kwargs.get(\"experiment_prefix\")\n- description = kwargs.get(\"description\")\n- metadata + = kwargs.get(\"metadata\")\n- max_concurrency = kwargs.get(\"max_concurrency\")\n- num_repetitions + = kwargs.get(\"num_repetitions\", 1)\n-\n- # Convert evaluators to scorers\n- scorers + = []\n- if evaluators:\n- for e in evaluators:\n- scorers.append(_make_braintrust_scorer(e))\n-\n- return + await EvalAsync(\n- name=project_name or \"langsmith-migration\",\n- data=_convert_langsmith_data(data),\n- task=_make_braintrust_task(target),\n- scores=scorers,\n- experiment_name=experiment_prefix,\n- project_id=project_id,\n- description=description,\n- metadata=metadata,\n- max_concurrency=max_concurrency,\n- trial_count=num_repetitions,\n- )\n-\n-\n-# + =============================================================================\n-# + Data conversion helpers\n-# =============================================================================\n-\n-\n-def + _wrap_output(output: Any) -> Dict[str, Any]:\n- \"\"\"Wrap non-dict outputs + the same way LangSmith does.\"\"\"\n- if not isinstance(output, dict):\n- return + {\"output\": output}\n- return output\n-\n-\n-def _make_braintrust_scorer(\n- evaluator: + Callable[..., Any],\n-) -> Callable[..., Any]:\n- \"\"\"\n- Create a + Braintrust scorer from a LangSmith evaluator.\n-\n- Always runs the evaluator + through Braintrust for full tracing (span duration, child LLM calls, etc.).\n- \"\"\"\n- evaluator_name + = getattr(evaluator, \"__name__\", \"score\")\n-\n- def braintrust_scorer(input: + Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any:\n- from + braintrust.score import Score\n-\n- # Run the evaluator with LangSmith''s + signature\n- # LangSmith evaluators use: (inputs, outputs, reference_outputs) + -> bool | dict\n- # LangSmith auto-wraps non-dict outputs as {\"output\": + value}\n- outputs = _wrap_output(output)\n-\n- # expected is + the real LangSmith Example object passed through from data loading\n- reference_outputs + = expected.outputs if hasattr(expected, \"outputs\") else expected\n-\n- result + = evaluator(input, outputs, reference_outputs)\n-\n- return Score(\n- name=result.get(\"key\", + evaluator_name),\n- score=result.get(\"score\"),\n- metadata=result.get(\"metadata\", + {}),\n- )\n-\n- braintrust_scorer.__name__ = evaluator_name\n- return + braintrust_scorer\n-\n-\n-def _convert_langsmith_data(data: Any) -> Callable[[], + Iterator[EvalCase[Any, Any]]]:\n- \"\"\"Convert LangSmith data format to + Braintrust data format.\"\"\"\n-\n- def load_data() -> Iterator[EvalCase[Any, + Any]]:\n- # Determine the source iterable without loading everything + into memory\n- source: Iterable[Any]\n- if callable(data):\n- source + = data() # type: ignore\n- elif isinstance(data, str):\n- # + Load examples from LangSmith dataset by name\n- try:\n- from + langsmith import Client # pylint: disable=import-error\n-\n- client + = Client()\n- source = client.list_examples(dataset_name=data)\n- except + Exception as e:\n- logger.warning(f\"Failed to load LangSmith + dataset ''{data}'': {e}\")\n- return\n- elif hasattr(data, + \"__iter__\"):\n- source = data\n- else:\n- source + = [data]\n-\n- # Process items as a generator - yield one at a time\n- for + item in source:\n- # Pass through LangSmith Example objects directly\n- if + hasattr(item, \"inputs\"):\n- yield EvalCase(\n- input=item.inputs,\n- expected=item, # + Pass the whole Example object\n- metadata=getattr(item, + \"metadata\", None),\n- )\n- elif isinstance(item, + dict):\n- if \"inputs\" in item:\n- # LangSmith + dict format\n- yield EvalCase(\n- input=item[\"inputs\"],\n- expected=item, # + Pass the whole dict\n- metadata=item.get(\"metadata\"),\n- )\n- elif + \"input\" in item:\n- # Braintrust format\n- yield + EvalCase(\n- input=item[\"input\"],\n- expected=item.get(\"expected\"),\n- metadata=item.get(\"metadata\"),\n- )\n- else:\n- yield + EvalCase(input=item)\n- else:\n- yield EvalCase(input=item)\n-\n- return + load_data\n-\n-\n-def _make_braintrust_task(target: Callable[..., Any]) -> + Callable[..., Any]:\n- \"\"\"Convert a LangSmith target function to Braintrust + task format.\"\"\"\n-\n- def task_fn(task_input: Any, hooks: Any) -> Any:\n- if + isinstance(task_input, dict):\n- # Try to get the original function''s + signature (unwrap decorators)\n- unwrapped = inspect.unwrap(target)\n-\n- try:\n- sig + = inspect.signature(unwrapped)\n- params = list(sig.parameters.keys())\n- if + len(params) == 1:\n- return target(task_input)\n- if + all(p in task_input for p in params):\n- return target(**task_input)\n- return + target(task_input)\n- except (ValueError, TypeError):\n- # + Fallback: try kwargs first, then single arg\n- try:\n- return + target(**task_input)\n- except TypeError:\n- return + target(task_input)\n- return target(task_input)\n-\n- return task_fn\n+__all__ + = [\"setup_langsmith\"]\ndiff --git a/py/src/braintrust/wrappers/test_langsmith_wrapper.py + b/py/src/braintrust/wrappers/test_langsmith_wrapper.py\ndeleted file mode + 100644\nindex c25256c2..00000000\n--- a/py/src/braintrust/wrappers/test_langsmith_wrapper.py\n+++ + /dev/null\n@@ -1,341 +0,0 @@\n-# pyright: reportPrivateUsage=false\n-# pyright: + reportMissingParameterType=false\n-# pyright: reportUnknownParameterType=false\n-# + pyright: reportUnknownArgumentType=false\n-# pyright: reportUnknownMemberType=false\n-# + pylint: disable=protected-access\n-\n-\"\"\"\n-Tests for the LangSmith wrapper + to ensure compatibility with LangSmith''s API.\n-\"\"\"\n-\n-from braintrust.wrappers.langsmith_wrapper + import (\n- _convert_langsmith_data,\n- _is_patched,\n- _make_braintrust_scorer,\n- _make_braintrust_task,\n- wrap_aevaluate,\n- wrap_client,\n- wrap_traceable,\n-)\n-\n-\n-def + test_is_patched_false():\n- \"\"\"Test that _is_patched returns False for + unpatched objects.\"\"\"\n-\n- def unpatched():\n- pass\n-\n- assert + _is_patched(unpatched) is False\n-\n-\n-def test_is_patched_true():\n- \"\"\"Test + that _is_patched returns True for patched objects.\"\"\"\n-\n- def patched():\n- pass\n-\n- patched._braintrust_patched + = True # type: ignore\n-\n- assert _is_patched(patched) is True\n-\n-\n-def + test_make_braintrust_scorer_dict_result():\n- \"\"\"Test converting a LangSmith + evaluator that returns a dict.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- return {\"key\": \"accuracy\", \"score\": + 0.9, \"metadata\": {\"note\": \"good\"}}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- # + Create a mock Example object\n- class MockExample:\n- outputs = + {\"y\": 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": 2}, + expected=MockExample())\n-\n- assert result.name == \"accuracy\"\n- assert + result.score == 0.9\n- assert result.metadata == {\"note\": \"good\"}\n-\n-\n-def + test_make_braintrust_scorer_numeric_result():\n- \"\"\"Test converting + a LangSmith evaluator that returns a numeric score in a dict.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return + {\"score\": 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class MockExample:\n- outputs + = {\"y\": 2}\n-\n- result = converted(input={\"x\": 1}, output={\"y\": + 2}, expected=MockExample())\n-\n- assert result.name == \"langsmith_evaluator\"\n- assert + result.score == 1.0\n-\n-\n-def test_make_braintrust_scorer_with_plain_dict_expected():\n- \"\"\"Test + converting a LangSmith evaluator with plain dict as expected.\"\"\"\n-\n- def + langsmith_evaluator(inputs, outputs, reference_outputs):\n- return + {\"score\": 1.0 if outputs == reference_outputs else 0.0}\n-\n- converted + = _make_braintrust_scorer(langsmith_evaluator)\n- result = converted(input={\"x\": + 1}, output={\"y\": 2}, expected={\"y\": 2})\n-\n- assert result.name == + \"langsmith_evaluator\"\n- assert result.score == 1.0\n-\n-\n-def test_convert_langsmith_data_from_list():\n- \"\"\"Test + converting LangSmith data from a list of dicts.\"\"\"\n- data = [\n- {\"inputs\": + {\"x\": 1}, \"outputs\": {\"y\": 2}},\n- {\"inputs\": {\"x\": 2}, \"outputs\": + {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- assert result[1].input + == {\"x\": 2}\n- assert result[1].expected == {\"inputs\": {\"x\": 2}, + \"outputs\": {\"y\": 4}}\n-\n-\n-def test_convert_langsmith_data_from_callable():\n- \"\"\"Test + converting LangSmith data from a callable.\"\"\"\n-\n- def data_generator():\n- yield + {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n- yield {\"inputs\": + {\"x\": 2}, \"outputs\": {\"y\": 4}}\n-\n- data_fn = _convert_langsmith_data(data_generator)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole item is passed as expected\n- assert result[0].expected + == {\"inputs\": {\"x\": 1}, \"outputs\": {\"y\": 2}}\n-\n-\n-def test_convert_langsmith_data_with_example_objects():\n- \"\"\"Test + converting LangSmith data with Example-like objects.\"\"\"\n-\n- class + MockExample:\n- def __init__(self, inputs, outputs):\n- self.inputs + = inputs\n- self.outputs = outputs\n-\n- data = [\n- MockExample(inputs={\"x\": + 1}, outputs={\"y\": 2}),\n- MockExample(inputs={\"x\": 2}, outputs={\"y\": + 4}),\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- # The whole Example object is passed as expected\n- assert + result[0].expected.inputs == {\"x\": 1}\n- assert result[0].expected.outputs + == {\"y\": 2}\n-\n-\n-def test_make_braintrust_task_with_dict_input():\n- \"\"\"Test + that task function handles dict inputs correctly.\"\"\"\n-\n- def target_fn(inputs):\n- return + inputs[\"x\"] * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == 10\n-\n-\n-def test_make_braintrust_task_with_kwargs_expansion():\n- \"\"\"Test + that task function expands dict kwargs when signature matches.\"\"\"\n-\n- def + target_fn(x, y):\n- return x + y\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 2, \"y\": 3}, None)\n-\n- assert result == 5\n-\n-\n-def + test_make_braintrust_task_simple_input():\n- \"\"\"Test that task function + handles simple inputs.\"\"\"\n-\n- def target_fn(inp):\n- return + inp * 2\n-\n- task = _make_braintrust_task(target_fn)\n- result = task(5, + None)\n-\n- assert result == 10\n-\n-\n-class TestWrapTraceable:\n- \"\"\"Tests + for wrap_traceable functionality.\"\"\"\n-\n- def test_wrap_traceable_returns_wrapper(self):\n- \"\"\"Test + that wrap_traceable returns a wrapped version.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable, + standalone=False)\n- assert callable(wrapped)\n- assert _is_patched(wrapped)\n-\n- def + test_wrap_traceable_standalone_mode(self):\n- \"\"\"Test that wrap_traceable + works in standalone mode.\"\"\"\n-\n- def mock_traceable(func, **kwargs):\n- return + func\n-\n- wrapped = wrap_traceable(mock_traceable, standalone=True)\n- assert + callable(wrapped)\n- assert _is_patched(wrapped)\n-\n-\n-class TestWrapFunctions:\n- \"\"\"Tests + for the wrap_* functions.\"\"\"\n-\n- def test_wrap_functions_exist(self):\n- \"\"\"Test + that wrap functions are callable.\"\"\"\n- assert callable(wrap_traceable)\n- assert + callable(wrap_client)\n- assert callable(wrap_aevaluate)\n-\n- def + test_wrap_traceable_returns_patched_function(self):\n- \"\"\"Test that + wrap_traceable returns a patched function.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- wrapped = wrap_traceable(mock_traceable)\n- assert + _is_patched(wrapped)\n-\n- def test_wrap_traceable_skips_if_already_patched(self):\n- \"\"\"Test + that wrap_traceable skips if already patched.\"\"\"\n-\n- def mock_traceable(func, + **kwargs):\n- return func\n-\n- mock_traceable._braintrust_patched + = True # type: ignore\n-\n- result = wrap_traceable(mock_traceable)\n- # + Should return the same function\n- assert result is mock_traceable\n-\n- def + test_wrap_client_sets_flag(self):\n- \"\"\"Test that wrap_client sets + the patched flag.\"\"\"\n-\n- class MockClient:\n- def evaluate(self, + *args, **kwargs):\n- return \"original\"\n-\n- wrap_client(MockClient)\n- assert + _is_patched(MockClient.evaluate)\n-\n- def test_wrap_aevaluate_returns_patched_function(self):\n- \"\"\"Test + that wrap_aevaluate returns a patched function.\"\"\"\n-\n- async def + mock_aevaluate(*args, **kwargs):\n- pass\n-\n- wrapped = + wrap_aevaluate(mock_aevaluate)\n- assert _is_patched(wrapped)\n-\n-\n-class + TestTandemModeIntegration:\n- \"\"\"Integration tests for tandem mode (LangSmith + + Braintrust together).\"\"\"\n-\n- def test_make_braintrust_task_with_inputs_parameter(self):\n- \"\"\"Test + that task handles LangSmith''s required ''inputs'' parameter name.\"\"\"\n-\n- def + target_fn(inputs: dict) -> dict:\n- return {\"result\": inputs[\"x\"] + * 2}\n-\n- task = _make_braintrust_task(target_fn)\n- result + = task({\"x\": 5}, None)\n-\n- assert result == {\"result\": 10}\n-\n- def + test_convert_langsmith_data_handles_different_output_types(self):\n- \"\"\"Test + that data conversion handles various output types.\"\"\"\n- data = + [\n- {\"inputs\": {\"x\": 1}, \"outputs\": 2}, # outputs is int, + not dict\n- {\"inputs\": {\"x\": 2}, \"outputs\": {\"result\": + 4}}, # outputs is already dict\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- # Both should work - Braintrust''s EvalCase + accepts any type for expected\n- assert len(result) == 2\n- assert + result[0].input == {\"x\": 1}\n- assert result[1].input == {\"x\": + 2}\n-\n- def test_make_braintrust_scorer_handles_wrapped_outputs(self):\n- \"\"\"Test + that scorers handle output wrapping correctly.\"\"\"\n-\n- def langsmith_evaluator(inputs, + outputs, reference_outputs):\n- # outputs will be wrapped as {\"output\": + value} for non-dict results\n- actual = outputs.get(\"output\", + outputs)\n- expected = (\n- reference_outputs.get(\"output\", + reference_outputs)\n- if isinstance(reference_outputs, dict)\n- else + reference_outputs\n- )\n- return {\"key\": \"match\", + \"score\": 1.0 if actual == expected else 0.0}\n-\n- converted = _make_braintrust_scorer(langsmith_evaluator)\n-\n- class + MockExample:\n- outputs = {\"output\": 42}\n-\n- # Test + with wrapped output\n- result = converted(input={\"x\": 1}, output=42, + expected=MockExample())\n- assert result.name == \"match\"\n- assert + result.score == 1.0\n-\n-\n-class TestDataConversion:\n- \"\"\"Tests for + data conversion utilities.\"\"\"\n-\n- def test_convert_data_with_braintrust_format(self):\n- \"\"\"Test + that Braintrust format is properly handled.\"\"\"\n- data = [\n- {\"input\": + {\"x\": 1}, \"expected\": {\"y\": 2}},\n- {\"input\": {\"x\": 2}, + \"expected\": {\"y\": 4}},\n- ]\n-\n- data_fn = _convert_langsmith_data(data)\n- result + = list(data_fn())\n-\n- assert len(result) == 2\n- assert result[0].input + == {\"x\": 1}\n- assert result[0].expected == {\"y\": 2}\n- assert + result[1].input == {\"x\": 2}\n- assert result[1].expected == {\"y\": + 4}\n-\n- def test_convert_data_with_simple_items(self):\n- \"\"\"Test + that simple items (not dicts) are handled.\"\"\"\n- data = [1, 2, 3]\n-\n- data_fn + = _convert_langsmith_data(data)\n- result = list(data_fn())\n-\n- assert + len(result) == 3\n- assert result[0].input == 1\n- assert result[1].input + == 2\n- assert result[2].input == 3"},"commit":"5c35051a0102f850f2e09c66f9376a0fc658ed9f","base_exp_id":"7ff97ad2-7674-4727-a505-2ca57030c1fd","deleted_at":null,"dataset_id":null,"dataset_version":null,"parameters_id":null,"parameters_version":null,"public":false,"user_id":"c1f71e19-b3ce-4f59-89a9-055901f7755b","metadata":null,"tags":null}}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-OWVhZDU3NWItZTE2MC00OTk4LWEzODctMDhiYmZiMmI1NDc2'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:43 GMT + Etag: + - W/"12lvlusn1p3soy" + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + Transfer-Encoding: + - chunked + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/experiment/register + X-Nonce: + - OWVhZDU3NWItZTE2MC00OTk4LWEzODctMDhiYmZiMmI1NDc2 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::95bc8-1775254842909-bb0839971781 + content-length: + - '37186' + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '177' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.30.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.30.0 + X-Stainless-Raw-Response: + - 'true' + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-DQhAaUxxP2DdzMwkW9IYLeZRfpUom\",\n \"object\": + \"chat.completion\",\n \"created\": 1775254844,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_43047b3f6b\"\n}\n" + headers: + CF-Cache-Status: + - DYNAMIC + CF-Ray: + - 9e6b76528ccb61e9-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 03 Apr 2026 22:20:44 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + content-length: + - '821' + openai-organization: + - braintrust-data + openai-processing-ms: + - '407' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=nDUgjs5SwRKkemklPjvjDMMtHAWOcPfIHKgtAzyMdD4-1775254843.2839096-1.0.1.1-jqUdQAsdETEsMo9sqxNwGnd9gL0HZtBZFNPxr9dlJQcp4qDxu6bUFoXVGjuENB0RU7pcHYkEBByXGBg3HcZvtqF5APy11vLfxasT.KBBzRNPTl6_j8TJmYLAKXGfnz7V; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 03 Apr 2026 + 22:50:44 GMT + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999995' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_31fc24a97e6a4b408adbbeba9a95d429 + status: + code: 200 + message: OK +- request: + body: '{"id": "407a9c6f-27ba-4c89-9bf2-b19985d7b695"}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '46' + Content-Type: + - application/json + User-Agent: + - python-requests/2.33.1 + method: POST + uri: https://www.braintrust.dev/api/base_experiment/get_id + response: + body: + string: '{"id":"407a9c6f-27ba-4c89-9bf2-b19985d7b695","project_id":"208de547-5ee2-4294-955c-5595cd0f7940","name":"langsmith-eval-f5e274f6","base_exp_id":"7ff97ad2-7674-4727-a505-2ca57030c1fd","base_exp_name":"langsmith-aeval-adb3f119"}' + headers: + Cache-Control: + - public, max-age=0, must-revalidate + Content-Length: + - '226' + Content-Security-Policy: + - 'script-src ''self'' ''unsafe-eval'' ''wasm-unsafe-eval'' ''strict-dynamic'' + ''nonce-ZjRmNTA0MWMtNWJhNi00YjcyLWJiODctOWU4YTFkZDVkMTE2'' *.js.stripe.com + js.stripe.com maps.googleapis.com ; style-src ''self'' ''unsafe-inline'' *.braintrust.dev + btcm6qilbbhv4yi1.public.blob.vercel-storage.com fonts.googleapis.com www.gstatic.com + d4tuoctqmanu0.cloudfront.net; font-src ''self'' data: fonts.gstatic.com btcm6qilbbhv4yi1.public.blob.vercel-storage.com + cdn.jsdelivr.net d4tuoctqmanu0.cloudfront.net fonts.googleapis.com mintlify-assets.b-cdn.net + fonts.cdnfonts.com; object-src ''none''; base-uri ''self''; form-action ''self''; + frame-ancestors ''self''; worker-src ''self'' blob:; report-uri https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16; + report-to csp-endpoint-0' + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:44 GMT + Etag: + - '"pe5jfmi8ss6a"' + Reporting-Endpoints: + - csp-endpoint-0="https://o4507221741076480.ingest.us.sentry.io/api/4507221754380288/security/?sentry_key=27fa5ac907cf7c6ce4a1ab2a03f805b4&sentry_environment=production&sentry_release=16" + Server: + - Vercel + Strict-Transport-Security: + - max-age=63072000 + X-Clerk-Auth-Message: + - Invalid JWT form. A JWT consists of three parts separated by dots. (reason=token-invalid, + token-carrier=header) + X-Clerk-Auth-Reason: + - token-invalid + X-Clerk-Auth-Status: + - signed-out + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-Matched-Path: + - /api/base_experiment/get_id + X-Nonce: + - ZjRmNTA0MWMtNWJhNi00YjcyLWJiODctOWU4YTFkZDVkMTE2 + X-Vercel-Cache: + - MISS + X-Vercel-Id: + - yul1::iad1::2gk4q-1775254844605-672423f1bb0d + status: + code: 200 + message: OK +- request: + body: null + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + User-Agent: + - python-requests/2.33.1 + method: GET + uri: https://api.braintrust.dev/experiment-comparison2?experiment_id=407a9c6f-27ba-4c89-9bf2-b19985d7b695&base_experiment_id=7ff97ad2-7674-4727-a505-2ca57030c1fd + response: + body: + string: '{"scores":{},"metrics":{}}' + headers: + Connection: + - keep-alive + Content-Type: + - application/json; charset=utf-8 + Date: + - Fri, 03 Apr 2026 22:20:45 GMT + Via: + - 1.1 74fa2893a387420c4d3d2a3aac7dae04.cloudfront.net (CloudFront), 1.1 fa19153a28b66c7bbfaddbf2e4a92f90.cloudfront.net + (CloudFront) + X-Amz-Cf-Id: + - OZQyHTb6t2lrncROQGTiijPjcqqkqZA8HWOftne24gx2MoI6LzsdBg== + X-Amz-Cf-Pop: + - YTO53-P2 + - YTO50-P2 + X-Amzn-Trace-Id: + - Root=1-69d03d3c-515310c3587c343e33a287f8;Parent=3ff4c4522f6d7a4c;Sampled=0;Lineage=1:24be3d11:0 + X-Cache: + - Miss from cloudfront + access-control-allow-credentials: + - 'true' + access-control-expose-headers: + - x-bt-cursor,x-bt-found-existing,x-bt-query-plan,x-bt-api-duration-ms,x-bt-brainstore-duration-ms,x-bt-internal-trace-id + content-length: + - '26' + etag: + - W/"1a-DVqVcAQOxg1HFdcjY89JEuayOGw" + vary: + - Origin, Accept-Encoding + x-amz-apigw-id: + - bQ6BkHljIAMEsAQ= + x-amzn-RequestId: + - 5b7e2dc6-0710-4c6b-94cf-14aa4bc57ec6 + x-bt-internal-trace-id: + - 69d03d3c00000000242bc383273a28e2 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_traceable_standalone_creates_span.yaml b/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_traceable_standalone_creates_span.yaml new file mode 100644 index 00000000..ba9f4fa9 --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_traceable_standalone_creates_span.yaml @@ -0,0 +1,225 @@ +interactions: +- request: + body: '{"messages": [{"content": "What is 1 + 2?", "role": "user"}], "model": + "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": + false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ44VUVp2sk1koSWXX64CaLEy1mWy\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659919,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:38:40 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cd8d01c33943a-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '930' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=XPwF0fhMV9JwjYuWwUMNbzPKxvSJ.HOkXEftYzjXRew-1758659920459-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 93acad0503781eb98ab6ea3412173537 + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '1026' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_181413148bbe4814a905514521d6dc34 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jJJBb9swDIXv/hUCr4uL2IlTJ9et2y2HHQZ0Q2EoMm1rk0VNoocNRf77 + IDuN3a0FevGBHx/1Hs3HRAjQNRwEqE6y6p1J33/9XH7adscPRXGXnTb7+49fNse7/b2iYylhFRV0 + +o6Kn1Q3inpnkDXZCSuPkjFOzW53ebHLyiwfQU81mihrHadbSnttdZqv8226vk2z8qLuSCsMcBDf + EiGEeBy/0aet8TccxHr1VOkxBNkiHK5NQoAnEysgQ9CBpWVYzVCRZbSj9Uy8E7nAn4M0QWxull0e + myHI6NQOxiyAtJZYxqSjv4cLOV8dGWqdp1P4RwqNtjp0lUcZyMbXA5ODkZ4TIR7G5MOzMOA89Y4r + ph84PpcV0ziY9z3D8sKYWJq5nG9WLwyramSpTVgsDpRUHdazct6yHGpNC5AsIv/v5aXZU2xt27eM + n4FS6BjrynmstXqed27zGI/xtbbrikfDEND/0gor1ujjb6ixkYOZTgTCn8DYV422LXrn9XQnjauK + 3Vo2OyyKPSTn5C8AAAD//wMAcIbFgjUDAAA= + headers: + CF-RAY: + - 99b0f5db9f1cbffc-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:30:12 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=vfpKl6dvzcujjwigai_kp7UkNhR2ltT1SwFsT05VrS8-1762561812-1.0.1.1-UAyuy134RWxRUzjbClH59IJarw95du8Dl347lkXcDkbXBBx7vCmRuxRccJQB2f1T6oobZSgBj7O8hdaLY4hef6ypZ2uHUshy880EnptiWEY; + path=/; expires=Sat, 08-Nov-25 01:00:12 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=N6FAUGU_qhcPvlVWdt0kvrpbt1SzTvQ0v29fL2QCNbA-1762561812358-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '319' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '489' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3e940a310adf4d9a88c8da6b70645bb7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_uses_standalone_env_var.yaml b/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_uses_standalone_env_var.yaml new file mode 100644 index 00000000..ba9f4fa9 --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/cassettes/test_setup_langsmith_uses_standalone_env_var.yaml @@ -0,0 +1,225 @@ +interactions: +- request: + body: '{"messages": [{"content": "What is 1 + 2?", "role": "user"}], "model": + "gpt-4o-mini", "frequency_penalty": 0.0, "n": 1, "presence_penalty": 0.0, "stream": + false, "temperature": 1.0, "top_p": 1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - localhost:8000 + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.13 + method: POST + uri: http://localhost:8000/v1/proxy/chat/completions + response: + body: + string: "{\n \"id\": \"chatcmpl-CJ44VUVp2sk1koSWXX64CaLEy1mWy\",\n \"object\": + \"chat.completion\",\n \"created\": 1758659919,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n + \ \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": + \"assistant\",\n \"content\": \"1 + 2 equals 3.\",\n \"refusal\": + null,\n \"annotations\": []\n },\n \"logprobs\": null,\n + \ \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": + 15,\n \"completion_tokens\": 8,\n \"total_tokens\": 23,\n \"prompt_tokens_details\": + {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": + {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": + 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": + \"default\",\n \"system_fingerprint\": \"fp_560af6e559\"\n}\n" + headers: + Access-Control-Allow-Credentials: + - 'true' + Access-Control-Expose-Headers: + - x-bt-cursor,x-bt-found-existing,x-bt-span-id,x-bt-span-export,x-bt-query-plan,x-bt-internal-trace-id + Connection: + - keep-alive + Date: + - Tue, 23 Sep 2025 20:38:40 GMT + Keep-Alive: + - timeout=5 + Transfer-Encoding: + - chunked + Vary: + - Origin + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + cf-ray: + - 983cd8d01c33943a-SJC + content-type: + - application/json + openai-organization: + - braintrust-data + openai-processing-ms: + - '930' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - _cfuvid=XPwF0fhMV9JwjYuWwUMNbzPKxvSJ.HOkXEftYzjXRew-1758659920459-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-bt-cached: + - MISS + x-bt-internal-trace-id: + - 93acad0503781eb98ab6ea3412173537 + x-content-type-options: + - nosniff + x-envoy-upstream-service-time: + - '1026' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_181413148bbe4814a905514521d6dc34 + status: + code: 200 + message: OK +- request: + body: '{"messages":[{"content":"What is 1 + 2?","role":"user"}],"model":"gpt-4o-mini","frequency_penalty":0.0,"n":1,"presence_penalty":0.0,"stream":false,"temperature":1.0,"top_p":1.0}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '177' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.108.2 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.108.2 + x-stainless-raw-response: + - 'true' + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.11.10 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jJJBb9swDIXv/hUCr4uL2IlTJ9et2y2HHQZ0Q2EoMm1rk0VNoocNRf77 + IDuN3a0FevGBHx/1Hs3HRAjQNRwEqE6y6p1J33/9XH7adscPRXGXnTb7+49fNse7/b2iYylhFRV0 + +o6Kn1Q3inpnkDXZCSuPkjFOzW53ebHLyiwfQU81mihrHadbSnttdZqv8226vk2z8qLuSCsMcBDf + EiGEeBy/0aet8TccxHr1VOkxBNkiHK5NQoAnEysgQ9CBpWVYzVCRZbSj9Uy8E7nAn4M0QWxull0e + myHI6NQOxiyAtJZYxqSjv4cLOV8dGWqdp1P4RwqNtjp0lUcZyMbXA5ODkZ4TIR7G5MOzMOA89Y4r + ph84PpcV0ziY9z3D8sKYWJq5nG9WLwyramSpTVgsDpRUHdazct6yHGpNC5AsIv/v5aXZU2xt27eM + n4FS6BjrynmstXqed27zGI/xtbbrikfDEND/0gor1ujjb6ixkYOZTgTCn8DYV422LXrn9XQnjauK + 3Vo2OyyKPSTn5C8AAAD//wMAcIbFgjUDAAA= + headers: + CF-RAY: + - 99b0f5db9f1cbffc-SJC + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Sat, 08 Nov 2025 00:30:12 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=vfpKl6dvzcujjwigai_kp7UkNhR2ltT1SwFsT05VrS8-1762561812-1.0.1.1-UAyuy134RWxRUzjbClH59IJarw95du8Dl347lkXcDkbXBBx7vCmRuxRccJQB2f1T6oobZSgBj7O8hdaLY4hef6ypZ2uHUshy880EnptiWEY; + path=/; expires=Sat, 08-Nov-25 01:00:12 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=N6FAUGU_qhcPvlVWdt0kvrpbt1SzTvQ0v29fL2QCNbA-1762561812358-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '319' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + x-envoy-upstream-service-time: + - '489' + x-openai-proxy-wasm: + - v0.1 + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999992' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_3e940a310adf4d9a88c8da6b70645bb7 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/langsmith/integration.py b/py/src/braintrust/integrations/langsmith/integration.py new file mode 100644 index 00000000..aa532e03 --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/integration.py @@ -0,0 +1,30 @@ +"""LangSmith integration definition.""" + +from typing import Any + +from braintrust.integrations.base import BaseIntegration + +from .patchers import ClientEvaluationPatcher, ModuleEvaluationPatcher, TraceablePatcher +from .tracing import _set_langsmith_standalone_override + + +class LangSmithIntegration(BaseIntegration): + """Braintrust instrumentation for LangSmith migration helpers.""" + + name = "langsmith" + import_names = ("langsmith",) + patchers = ( + TraceablePatcher, + ClientEvaluationPatcher, + ModuleEvaluationPatcher, + ) + + @classmethod + def setup( + cls, + *, + target: Any | None = None, + standalone: bool | None = None, + ) -> bool: + _set_langsmith_standalone_override(standalone) + return super().setup(target=target) diff --git a/py/src/braintrust/integrations/langsmith/patchers.py b/py/src/braintrust/integrations/langsmith/patchers.py new file mode 100644 index 00000000..0a5e5b4a --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/patchers.py @@ -0,0 +1,62 @@ +"""LangSmith patchers.""" + +from braintrust.integrations.base import CompositeFunctionWrapperPatcher, FunctionWrapperPatcher + +from .tracing import ( + _aevaluate_wrapper, + _client_aevaluate_wrapper, + _client_evaluate_wrapper, + _evaluate_wrapper, + _traceable_wrapper, +) + + +class TraceablePatcher(FunctionWrapperPatcher): + """Patch ``langsmith.run_helpers.traceable``.""" + + name = "langsmith.traceable" + target_module = "langsmith.run_helpers" + target_path = "traceable" + wrapper = _traceable_wrapper + + +class _ClientEvaluatePatcher(FunctionWrapperPatcher): + name = "langsmith.client.evaluate" + target_module = "langsmith.client" + target_path = "Client.evaluate" + wrapper = _client_evaluate_wrapper + + +class _ClientAEvaluatePatcher(FunctionWrapperPatcher): + name = "langsmith.client.aevaluate" + target_module = "langsmith.client" + target_path = "Client.aevaluate" + wrapper = _client_aevaluate_wrapper + + +class ClientEvaluationPatcher(CompositeFunctionWrapperPatcher): + """Patch ``langsmith.Client.evaluate`` and ``langsmith.Client.aevaluate``.""" + + name = "langsmith.client" + sub_patchers = (_ClientEvaluatePatcher, _ClientAEvaluatePatcher) + + +class _EvaluatePatcher(FunctionWrapperPatcher): + name = "langsmith.evaluate.sync" + target_module = "langsmith.evaluation._runner" + target_path = "evaluate" + wrapper = _evaluate_wrapper + + +class _AEvaluatePatcher(FunctionWrapperPatcher): + name = "langsmith.evaluate.async" + target_module = "langsmith.evaluation._arunner" + target_path = "aevaluate" + wrapper = _aevaluate_wrapper + + +class ModuleEvaluationPatcher(CompositeFunctionWrapperPatcher): + """Patch module-level ``langsmith.evaluate`` and ``langsmith.aevaluate``.""" + + name = "langsmith.evaluate" + sub_patchers = (_EvaluatePatcher, _AEvaluatePatcher) diff --git a/py/src/braintrust/integrations/langsmith/test_langsmith.py b/py/src/braintrust/integrations/langsmith/test_langsmith.py new file mode 100644 index 00000000..2ba9f824 --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/test_langsmith.py @@ -0,0 +1,245 @@ +# pyright: reportPrivateUsage=false + +from pathlib import Path + +import langsmith +import pytest +from braintrust import flush, logger +from braintrust.integrations.langchain import BraintrustCallbackHandler, set_global_handler +from braintrust.integrations.langsmith import setup_langsmith +from braintrust.integrations.langsmith.tracing import reset_langsmith_state +from braintrust.test_helpers import init_test_logger, simulate_login +from braintrust.wrappers.test_utils import verify_autoinstrument_script +from langchain_core.prompts import ChatPromptTemplate +from langchain_openai import ChatOpenAI + + +PROJECT_NAME = "langsmith-py" +MODEL = "gpt-4o-mini" +EXPECTED_ANSWER = "1 + 2 equals 3." + + +@pytest.fixture(scope="module") +def vcr_cassette_dir(): + return str(Path(__file__).resolve().parent / "cassettes") + + +@pytest.fixture +def logger_memory_logger(): + test_logger = init_test_logger(PROJECT_NAME) + with logger._internal_with_memory_background_logger() as bgl: + yield (test_logger, bgl) + + +@pytest.fixture(autouse=True) +def restore_langsmith_state(): + from braintrust.integrations.langchain.context import clear_global_handler + + sentinel = object() + original_traceable = langsmith.__dict__.get("traceable", sentinel) + original_evaluate = langsmith.__dict__.get("evaluate", sentinel) + original_aevaluate = langsmith.__dict__.get("aevaluate", sentinel) + original_client_evaluate = langsmith.Client.evaluate + original_client_aevaluate = langsmith.Client.aevaluate + + clear_global_handler() + yield + if original_traceable is sentinel: + langsmith.__dict__.pop("traceable", None) + else: + langsmith.traceable = original_traceable + if original_evaluate is sentinel: + langsmith.__dict__.pop("evaluate", None) + else: + langsmith.evaluate = original_evaluate + if original_aevaluate is sentinel: + langsmith.__dict__.pop("aevaluate", None) + else: + langsmith.aevaluate = original_aevaluate + langsmith.Client.evaluate = original_client_evaluate + langsmith.Client.aevaluate = original_client_aevaluate + clear_global_handler() + reset_langsmith_state() + + +def _make_chain(): + prompt = ChatPromptTemplate.from_template("What is 1 + {number}?") + model = ChatOpenAI( + model=MODEL, + temperature=1, + top_p=1, + frequency_penalty=0, + presence_penalty=0, + n=1, + ) + return prompt | model + + +@pytest.mark.vcr +def test_setup_langsmith_traceable_standalone_creates_span(logger_memory_logger): + _, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + assert setup_langsmith(project_name=PROJECT_NAME, standalone=True) + init_test_logger(PROJECT_NAME) + chain = _make_chain() + + @langsmith.traceable(name="langsmith-standalone") + def run_chain(inputs: dict[str, str]) -> dict[str, str]: + return {"answer": chain.invoke(inputs).content} + + result = run_chain({"number": "2"}) + flush() + + assert result == {"answer": EXPECTED_ANSWER} + + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["name"] == "langsmith-standalone" + assert span["output"] == {"answer": EXPECTED_ANSWER} + + +@pytest.mark.vcr +def test_setup_langsmith_uses_standalone_env_var(logger_memory_logger, monkeypatch): + _, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + monkeypatch.setenv("BRAINTRUST_LANGSMITH_STANDALONE", "1") + assert setup_langsmith(project_name=PROJECT_NAME) + init_test_logger(PROJECT_NAME) + chain = _make_chain() + + @langsmith.traceable(name="langsmith-standalone-env") + def run_chain(inputs: dict[str, str]) -> dict[str, str]: + return {"answer": chain.invoke(inputs).content} + + assert run_chain({"number": "2"}) == {"answer": EXPECTED_ANSWER} + flush() + + spans = memory_logger.pop() + assert len(spans) == 1 + assert spans[0]["span_attributes"]["name"] == "langsmith-standalone-env" + + +@pytest.mark.vcr +def test_setup_langsmith_module_evaluate_uses_braintrust_eval(logger_memory_logger): + _, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + assert setup_langsmith(project_name=PROJECT_NAME, standalone=True) + init_test_logger(PROJECT_NAME) + chain = _make_chain() + + def task(inputs: dict[str, str]) -> dict[str, str]: + return {"answer": chain.invoke(inputs).content} + + def evaluator(inputs, outputs, reference_outputs): + return { + "key": "match", + "score": 1.0 if outputs["answer"] == reference_outputs["outputs"]["answer"] else 0.0, + } + + result = langsmith.evaluate( + task, + data=[{"inputs": {"number": "2"}, "outputs": {"answer": EXPECTED_ANSWER}}], + evaluators=[evaluator], + experiment_prefix="langsmith-eval", + ) + flush() + + assert result.results[0].output == {"answer": EXPECTED_ANSWER} + assert result.results[0].scores == {"match": 1.0} + + spans = memory_logger.pop() + assert [getattr(span["span_attributes"]["type"], "value", span["span_attributes"]["type"]) for span in spans] == [ + "eval", + "task", + "score", + ] + + +@pytest.mark.vcr +@pytest.mark.asyncio +async def test_setup_langsmith_module_aevaluate_uses_braintrust_eval(logger_memory_logger): + _, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + assert setup_langsmith(project_name=PROJECT_NAME, standalone=True) + init_test_logger(PROJECT_NAME) + chain = _make_chain() + + def task(inputs: dict[str, str]) -> dict[str, str]: + return {"answer": chain.invoke(inputs).content} + + def evaluator(inputs, outputs, reference_outputs): + return { + "key": "match", + "score": 1.0 if outputs["answer"] == reference_outputs["outputs"]["answer"] else 0.0, + } + + result = await langsmith.aevaluate( + task, + data=[{"inputs": {"number": "2"}, "outputs": {"answer": EXPECTED_ANSWER}}], + evaluators=[evaluator], + experiment_prefix="langsmith-aeval", + ) + flush() + + assert result.results[0].output == {"answer": EXPECTED_ANSWER} + assert result.results[0].scores == {"match": 1.0} + + spans = memory_logger.pop() + assert [getattr(span["span_attributes"]["type"], "value", span["span_attributes"]["type"]) for span in spans] == [ + "eval", + "task", + "score", + ] + + +@pytest.mark.vcr +def test_langsmith_traceable_coexists_with_langchain(logger_memory_logger): + _, memory_logger = logger_memory_logger + assert not memory_logger.pop() + + assert setup_langsmith(project_name=PROJECT_NAME, standalone=True) + + simulate_login() + test_logger = init_test_logger(PROJECT_NAME) + handler = BraintrustCallbackHandler(logger=test_logger) + set_global_handler(handler) + + from braintrust.integrations.langchain.context import get_global_handler + + assert get_global_handler() is handler + assert setup_langsmith(project_name=PROJECT_NAME, standalone=True) + assert get_global_handler() is handler + + chain = _make_chain() + + @langsmith.traceable(name="langsmith-chain") + def run_chain(inputs: dict[str, str]): + return chain.invoke(inputs) + + message = run_chain({"number": "2"}) + + assert message.content == EXPECTED_ANSWER + + spans = memory_logger.pop() + assert len(spans) >= 4 + + traceable_span = next(span for span in spans if span["span_attributes"].get("name") == "langsmith-chain") + llm_spans = [ + span + for span in spans + if getattr(span["span_attributes"].get("type"), "value", span["span_attributes"].get("type")) == "llm" + ] + + assert len(llm_spans) == 1 + assert llm_spans[0]["metadata"]["model"] == "gpt-4o-mini-2024-07-18" + assert llm_spans[0]["root_span_id"] == traceable_span["span_id"] + + +class TestAutoInstrumentLangSmith: + def test_auto_instrument_langsmith(self): + verify_autoinstrument_script("test_auto_langsmith.py") diff --git a/py/src/braintrust/integrations/langsmith/tracing.py b/py/src/braintrust/integrations/langsmith/tracing.py new file mode 100644 index 00000000..b703301f --- /dev/null +++ b/py/src/braintrust/integrations/langsmith/tracing.py @@ -0,0 +1,384 @@ +"""LangSmith integration helpers, tracing wrappers, and LangSmith-to-Braintrust adapters.""" + +import inspect +import logging +import os +from typing import Any, Callable, Dict, Iterable, Iterator, Optional, ParamSpec, TypeVar +from uuid import UUID + +from braintrust.framework import EvalCase +from braintrust.logger import current_logger, traced + + +logger = logging.getLogger(__name__) + +_LANGSMITH_STANDALONE: bool | None = None + +P = ParamSpec("P") +R = TypeVar("R") + + +def _langsmith_standalone_from_env() -> bool: + value = os.environ.get("BRAINTRUST_LANGSMITH_STANDALONE") + if value is None: + return False + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _set_langsmith_standalone_override(standalone: bool | None = None) -> bool: + """Set an explicit standalone override for patched LangSmith surfaces.""" + global _LANGSMITH_STANDALONE + _LANGSMITH_STANDALONE = standalone + return get_langsmith_standalone() + + +def get_langsmith_standalone() -> bool: + """Return whether LangSmith should be bypassed in favor of Braintrust-only behavior.""" + if _LANGSMITH_STANDALONE is not None: + return _LANGSMITH_STANDALONE + return _langsmith_standalone_from_env() + + +def reset_langsmith_state() -> None: + """Reset standalone overrides and collected Braintrust eval results.""" + global _LANGSMITH_STANDALONE + _LANGSMITH_STANDALONE = None + + +def _resolve_eval_project() -> tuple[str, str | None]: + project_name = None + project_id = None + + active_logger = current_logger() + if active_logger is not None: + try: + project = active_logger.project + project_name = getattr(project, "name", None) + candidate_project_id = getattr(project, "id", None) + if isinstance(candidate_project_id, str): + try: + UUID(candidate_project_id) + project_id = candidate_project_id + except ValueError: + project_id = None + except Exception: + pass + + if project_name is None: + project_name = os.environ.get("LANGCHAIN_PROJECT") + + return project_name or "langsmith-migration", project_id + + +# ============================================================================= +# Raw wrapt wrappers used by integration patchers +# ============================================================================= + + +def _traceable_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + return _make_traceable_wrapper(wrapped, standalone=get_langsmith_standalone())(*args, **kwargs) + + +def _client_evaluate_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + return _run_evaluate_with_fallback( + wrapped, + args, + kwargs, + standalone=get_langsmith_standalone(), + client=instance, + ) + + +async def _client_aevaluate_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + return await _run_aevaluate_with_fallback( + wrapped, + args, + kwargs, + standalone=get_langsmith_standalone(), + client=instance, + ) + + +def _evaluate_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + return _run_evaluate_with_fallback( + wrapped, + args, + kwargs, + standalone=get_langsmith_standalone(), + ) + + +async def _aevaluate_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + return await _run_aevaluate_with_fallback( + wrapped, + args, + kwargs, + standalone=get_langsmith_standalone(), + ) + + +# ============================================================================= +# Wrapper factories / execution helpers +# ============================================================================= + + +def _make_traceable_wrapper(traceable: Callable[..., Any], *, standalone: bool) -> Callable[..., Any]: + def traceable_wrapper(*args: Any, **kwargs: Any) -> Any: + func = args[0] if args and callable(args[0]) else None + + def decorator(fn: Callable[P, R]) -> Callable[P, R]: + span_name = kwargs.get("name") or fn.__name__ + traced_fn: Callable[..., Any] = fn + + if not standalone: + traced_fn = traceable(fn, **kwargs) + + return traced(name=span_name)(traced_fn) # type: ignore[return-value] + + if func is not None: + return decorator(func) + return decorator + + return traceable_wrapper + + +def _run_evaluate_with_fallback( + wrapped: Callable[..., Any], + args: Any, + kwargs: Any, + *, + standalone: bool, + client: Any | None = None, +) -> Any: + result = None + if not standalone: + result = wrapped(*args, **kwargs) + + try: + result = _run_braintrust_eval(args, kwargs, client=client) + except Exception as exc: + if standalone: + raise + logger.warning("Braintrust evaluate failed: %s", exc) + + return result + + +async def _run_aevaluate_with_fallback( + wrapped: Callable[..., Any], + args: Any, + kwargs: Any, + *, + standalone: bool, + client: Any | None = None, +) -> Any: + result = None + if not standalone: + result = await wrapped(*args, **kwargs) + + try: + result = await _run_braintrust_eval_async(args, kwargs, client=client) + except Exception as exc: + if standalone: + raise + logger.warning("Braintrust aevaluate failed: %s", exc) + + return result + + +# ============================================================================= +# Braintrust evaluation logic +# ============================================================================= + + +def _run_braintrust_eval( + args: Any, + kwargs: Any, + client: Any | None = None, +) -> Any: + """Run Braintrust Eval with LangSmith-style arguments.""" + from braintrust.framework import Eval + + target = args[0] if args else kwargs.get("target") + data = args[1] if len(args) > 1 else kwargs.get("data") + evaluators = kwargs.get("evaluators") + experiment_prefix = kwargs.get("experiment_prefix") + description = kwargs.get("description") + metadata = kwargs.get("metadata") + max_concurrency = kwargs.get("max_concurrency") + num_repetitions = kwargs.get("num_repetitions", 1) + project_name, project_id = _resolve_eval_project() + + scorers = [] + if evaluators: + for evaluator in evaluators: + scorers.append(_make_braintrust_scorer(evaluator)) + + return Eval( + name=project_name, + data=_convert_langsmith_data(data, client=client), + task=_make_braintrust_task(target), + scores=scorers, + experiment_name=experiment_prefix, + project_id=project_id, + description=description, + metadata=metadata, + max_concurrency=max_concurrency, + trial_count=num_repetitions, + ) + + +async def _run_braintrust_eval_async( + args: Any, + kwargs: Any, + client: Any | None = None, +) -> Any: + """Run Braintrust EvalAsync with LangSmith-style arguments.""" + from braintrust.framework import EvalAsync + + target = args[0] if args else kwargs.get("target") + data = args[1] if len(args) > 1 else kwargs.get("data") + evaluators = kwargs.get("evaluators") + experiment_prefix = kwargs.get("experiment_prefix") + description = kwargs.get("description") + metadata = kwargs.get("metadata") + max_concurrency = kwargs.get("max_concurrency") + num_repetitions = kwargs.get("num_repetitions", 1) + project_name, project_id = _resolve_eval_project() + + scorers = [] + if evaluators: + for evaluator in evaluators: + scorers.append(_make_braintrust_scorer(evaluator)) + + return await EvalAsync( + name=project_name, + data=_convert_langsmith_data(data, client=client), + task=_make_braintrust_task(target), + scores=scorers, + experiment_name=experiment_prefix, + project_id=project_id, + description=description, + metadata=metadata, + max_concurrency=max_concurrency, + trial_count=num_repetitions, + ) + + +# ============================================================================= +# Data conversion helpers +# ============================================================================= + + +def _wrap_output(output: Any) -> Dict[str, Any]: + """Wrap non-dict outputs the same way LangSmith does.""" + if not isinstance(output, dict): + return {"output": output} + return output + + +def _make_braintrust_scorer( + evaluator: Callable[..., Any], +) -> Callable[..., Any]: + """Create a Braintrust scorer from a LangSmith evaluator.""" + evaluator_name = getattr(evaluator, "__name__", "score") + + def braintrust_scorer(input: Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any: + from braintrust.score import Score + + outputs = _wrap_output(output) + reference_outputs = expected.outputs if hasattr(expected, "outputs") else expected + result = evaluator(input, outputs, reference_outputs) + + return Score( + name=result.get("key", evaluator_name), + score=result.get("score"), + metadata=result.get("metadata", {}), + ) + + braintrust_scorer.__name__ = evaluator_name + return braintrust_scorer + + +def _resolve_data_client(client: Any | None) -> Any: + if client is not None and hasattr(client, "list_examples"): + return client + + from langsmith import Client # pylint: disable=import-error + + return Client() + + +def _convert_langsmith_data( + data: Any, + *, + client: Any | None = None, +) -> Callable[[], Iterator[EvalCase[Any, Any]]]: + """Convert LangSmith data format to Braintrust data format.""" + + def load_data() -> Iterator[EvalCase[Any, Any]]: + source: Iterable[Any] + if callable(data): + source = data() # type: ignore[misc] + elif isinstance(data, str): + try: + source = _resolve_data_client(client).list_examples(dataset_name=data) + except Exception as exc: + logger.warning("Failed to load LangSmith dataset %r: %s", data, exc) + return + elif hasattr(data, "__iter__"): + source = data + else: + source = [data] + + for item in source: + if hasattr(item, "inputs"): + yield EvalCase( + input=item.inputs, + expected=item, + metadata=getattr(item, "metadata", None), + ) + elif isinstance(item, dict): + if "inputs" in item: + yield EvalCase( + input=item["inputs"], + expected=item, + metadata=item.get("metadata"), + ) + elif "input" in item: + yield EvalCase( + input=item["input"], + expected=item.get("expected"), + metadata=item.get("metadata"), + ) + else: + yield EvalCase(input=item) + else: + yield EvalCase(input=item) + + return load_data + + +def _make_braintrust_task(target: Callable[..., Any]) -> Callable[..., Any]: + """Convert a LangSmith target function to Braintrust task format.""" + + def task_fn(task_input: Any, hooks: Any) -> Any: + if isinstance(task_input, dict): + unwrapped = inspect.unwrap(target) + + try: + sig = inspect.signature(unwrapped) + params = list(sig.parameters.keys()) + if len(params) == 1: + return target(task_input) + if all(param in task_input for param in params): + return target(**task_input) + return target(task_input) + except (ValueError, TypeError): + try: + return target(**task_input) + except TypeError: + return target(task_input) + return target(task_input) + + return task_fn diff --git a/py/src/braintrust/wrappers/langsmith_wrapper.py b/py/src/braintrust/wrappers/langsmith_wrapper.py index b22117df..7d4170fd 100644 --- a/py/src/braintrust/wrappers/langsmith_wrapper.py +++ b/py/src/braintrust/wrappers/langsmith_wrapper.py @@ -1,518 +1,4 @@ -""" -Braintrust integration for LangSmith - provides a migration path from LangSmith to Braintrust. +from braintrust.integrations.langsmith import setup_langsmith # noqa: F401 -This module patches LangSmith's tracing and evaluation APIs to use Braintrust under the hood, -allowing users to migrate with minimal code changes. -Usage: - ```python - import os - - # Enable LangSmith tracing and set project name (used by both services) - os.environ.setdefault("LANGCHAIN_TRACING_V2", "true") - os.environ.setdefault("LANGCHAIN_PROJECT", "my-project") - - from braintrust.wrappers.langsmith_wrapper import setup_langsmith - - # Call setup BEFORE importing from langsmith - # project_name defaults to LANGCHAIN_PROJECT env var - setup_langsmith() - - # Continue using langsmith imports - they now use Braintrust - from langsmith import traceable, Client - - @traceable - def my_function(inputs: dict) -> dict: - return {"result": inputs["x"] * 2} - - client = Client() - results = client.evaluate( - my_function, - data=[{"inputs": {"x": 1}, "outputs": {"result": 2}}], - evaluators=[my_evaluator], - ) - ``` - - Set BRAINTRUST_STANDALONE=1 to completely replace LangSmith with Braintrust - (no LangSmith code runs). Otherwise, both services run in tandem. -""" - -import inspect -import logging -import os -from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, ParamSpec, TypeVar - -from braintrust.framework import EvalCase -from braintrust.logger import NOOP_SPAN, current_span, init_logger, traced -from wrapt import wrap_function_wrapper - - -logger = logging.getLogger(__name__) - -# Global list to store Braintrust eval results when running in tandem mode -_braintrust_eval_results: List[Any] = [] - -# TODO: langsmith.test/unit/expect, langsmith.AsyncClient, trace -__all__ = [ - "setup_langsmith", - "wrap_traceable", - "wrap_client", - "wrap_evaluate", - "wrap_aevaluate", - "get_braintrust_results", - "clear_braintrust_results", -] - -F = TypeVar("F", bound=Callable[..., Any]) -P = ParamSpec("P") -R = TypeVar("R") - - -def get_braintrust_results() -> List[Any]: - """Get all Braintrust eval results collected during tandem mode.""" - return _braintrust_eval_results.copy() - - -def clear_braintrust_results() -> None: - """Clear all stored Braintrust eval results.""" - _braintrust_eval_results.clear() - - -def setup_langsmith( - api_key: Optional[str] = None, - project_id: Optional[str] = None, - project_name: Optional[str] = None, - standalone: bool = False, -) -> bool: - """ - Setup Braintrust integration with LangSmith. - - This patches LangSmith's @traceable, Client.evaluate(), and aevaluate() - to use Braintrust under the hood. - - Args: - api_key: Braintrust API key (optional, can use env var BRAINTRUST_API_KEY) - project_id: Braintrust project ID (optional) - project_name: Braintrust project name (optional, falls back to LANGCHAIN_PROJECT - env var, then BRAINTRUST_PROJECT env var) - standalone: If True, completely replace LangSmith with Braintrust (no LangSmith - code runs). If False (default), run both LangSmith and Braintrust - in tandem. - - Returns: - True if setup was successful, False otherwise - """ - # Use LANGCHAIN_PROJECT as fallback for project_name to keep both services in sync - if project_name is None: - project_name = os.environ.get("LANGCHAIN_PROJECT") - - span = current_span() - if span == NOOP_SPAN: - init_logger(project=project_name, api_key=api_key, project_id=project_id) - - try: - import langsmith - - langsmith.traceable = wrap_traceable(langsmith.traceable, standalone=standalone) - wrap_client(langsmith.Client, project_name=project_name, project_id=project_id, standalone=standalone) - langsmith.evaluate = wrap_evaluate( - langsmith.evaluate, project_name=project_name, project_id=project_id, standalone=standalone - ) - langsmith.aevaluate = wrap_aevaluate( - langsmith.aevaluate, project_name=project_name, project_id=project_id, standalone=standalone - ) - - logger.info("LangSmith integration with Braintrust enabled") - return True - - except ImportError as e: - logger.error(f"Failed to import langsmith: {e}") - logger.error("langsmith is not installed. Please install it with: pip install langsmith") - return False - - -def wrap_traceable(traceable: F, standalone: bool = False) -> F: - """ - Wrap langsmith.traceable to also use Braintrust's @traced decorator. - - Args: - traceable: The langsmith.traceable function - standalone: If True, replace LangSmith tracing entirely with Braintrust. - If False, add Braintrust tracing alongside LangSmith tracing. - - Returns: - The wrapped traceable function (or the original if already patched) - """ - if _is_patched(traceable): - return traceable - - def traceable_wrapper(*args: Any, **kwargs: Any) -> Any: - # Handle both @traceable and @traceable(...) patterns - func = args[0] if args and callable(args[0]) else None - - def decorator(fn: Callable[P, R]) -> Callable[P, R]: - span_name = kwargs.get("name") or fn.__name__ - - # Conditionally apply LangSmith decorator first - if not standalone: - fn = traceable(fn, **kwargs) - - # Always apply Braintrust tracing - return traced(name=span_name)(fn) # type: ignore[return-value] - - if func is not None: - return decorator(func) - return decorator - - traceable_wrapper._braintrust_patched = True # type: ignore[attr-defined] - return traceable_wrapper # type: ignore[return-value] - - -def wrap_client( - Client: Any, project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: bool = False -) -> Any: - """ - Wrap langsmith.Client to redirect evaluate() and aevaluate() to Braintrust's Eval. - - Args: - Client: The langsmith.Client class - project_name: Braintrust project name to use for evaluations - project_id: Braintrust project ID to use for evaluations - standalone: If True, only run Braintrust. If False, run both LangSmith and Braintrust. - - Returns: - The Client class (modified in place) - """ - - if hasattr(Client, "evaluate") and not _is_patched(Client.evaluate): - wrap_function_wrapper( - Client, - "evaluate", - make_evaluate_wrapper(standalone=standalone, project_name=project_name, project_id=project_id), - ) - Client.evaluate._braintrust_patched = True # type: ignore[attr-defined] - - if hasattr(Client, "aevaluate") and not _is_patched(Client.aevaluate): - wrap_function_wrapper( - Client, - "aevaluate", - make_aevaluate_wrapper(standalone=standalone, project_name=project_name, project_id=project_id), - ) - Client.aevaluate._braintrust_patched = True # type: ignore[attr-defined] - - return Client - - -def make_evaluate_wrapper( - *, project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: bool = False -): - def evaluate_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: - result = None - if not standalone: - result = wrapped(*args, **kwargs) - - try: - result = _run_braintrust_eval( - args, - kwargs, - project_name, - project_id, - ) - _braintrust_eval_results.append(result) - except Exception as e: - if standalone: - raise e - else: - logger.warning(f"Braintrust evaluate failed: {e}") - - return result - - return evaluate_wrapper - - -def make_aevaluate_wrapper( - *, project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: bool = False -): - async def aevaluate_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: - result = None - if not standalone: - result = await wrapped(*args, **kwargs) - - try: - result = await _run_braintrust_eval_async( - args, - kwargs, - project_name, - project_id, - ) - _braintrust_eval_results.append(result) - except Exception as e: - if standalone: - raise e - else: - logger.warning(f"Braintrust aevaluate failed: {e}") - - return result - - return aevaluate_wrapper - - -def wrap_evaluate( - evaluate: F, project_name: Optional[str] = None, project_id: Optional[str] = None, standalone: bool = False -) -> F: - """ - Wrap module-level langsmith.evaluate to redirect to Braintrust's Eval. - - Args: - evaluate: The langsmith.evaluate function - project_name: Braintrust project name to use for evaluations - project_id: Braintrust project ID to use for evaluations - standalone: If True, only run Braintrust. If False, run both LangSmith and Braintrust. - - Returns: - The wrapped evaluate function (or the original if already patched) - """ - if _is_patched(evaluate): - return evaluate - - evaluate_wrapper = make_evaluate_wrapper(standalone=standalone, project_name=project_name, project_id=project_id) - evaluate_wrapper._braintrust_patched = True # type: ignore[attr-defined] - return evaluate_wrapper # type: ignore[return-value] - - -def wrap_aevaluate( - aevaluate: F, - project_name: Optional[str] = None, - project_id: Optional[str] = None, - standalone: bool = False, -) -> F: - """ - Wrap module-level langsmith.aevaluate to redirect to Braintrust's EvalAsync. - - Args: - aevaluate: The langsmith.aevaluate function - project_name: Braintrust project name to use for evaluations - project_id: Braintrust project ID to use for evaluations - standalone: If True, only run Braintrust. If False, run both LangSmith and Braintrust. - - Returns: - The wrapped aevaluate function (or the original if already patched) - """ - if _is_patched(aevaluate): - return aevaluate - - aevaluate_wrapper = make_aevaluate_wrapper(standalone=standalone, project_name=project_name, project_id=project_id) - aevaluate_wrapper._braintrust_patched = True # type: ignore[attr-defined] - return aevaluate_wrapper # type: ignore[return-value] - - -def _is_patched(obj: Any) -> bool: - return getattr(obj, "_braintrust_patched", False) - - -# ============================================================================= -# Braintrust evaluation logic -# ============================================================================= - - -def _run_braintrust_eval( - args: Any, - kwargs: Any, - project_name: Optional[str] = None, - project_id: Optional[str] = None, -) -> Any: - """Run Braintrust Eval with LangSmith-style arguments.""" - from braintrust.framework import Eval - - target = args[0] if args else kwargs.get("target") - data = args[1] if len(args) > 1 else kwargs.get("data") - evaluators = kwargs.get("evaluators") - experiment_prefix = kwargs.get("experiment_prefix") - description = kwargs.get("description") - metadata = kwargs.get("metadata") - max_concurrency = kwargs.get("max_concurrency") - num_repetitions = kwargs.get("num_repetitions", 1) - - # Convert evaluators to scorers - scorers = [] - if evaluators: - for e in evaluators: - scorers.append(_make_braintrust_scorer(e)) - - return Eval( - name=project_name or "langsmith-migration", - data=_convert_langsmith_data(data), - task=_make_braintrust_task(target), - scores=scorers, - experiment_name=experiment_prefix, - project_id=project_id, - description=description, - metadata=metadata, - max_concurrency=max_concurrency, - trial_count=num_repetitions, - ) - - -async def _run_braintrust_eval_async( - args: Any, - kwargs: Any, - project_name: Optional[str] = None, - project_id: Optional[str] = None, -) -> Any: - """Run Braintrust EvalAsync with LangSmith-style arguments.""" - from braintrust.framework import EvalAsync - - target = args[0] if args else kwargs.get("target") - data = args[1] if len(args) > 1 else kwargs.get("data") - evaluators = kwargs.get("evaluators") - experiment_prefix = kwargs.get("experiment_prefix") - description = kwargs.get("description") - metadata = kwargs.get("metadata") - max_concurrency = kwargs.get("max_concurrency") - num_repetitions = kwargs.get("num_repetitions", 1) - - # Convert evaluators to scorers - scorers = [] - if evaluators: - for e in evaluators: - scorers.append(_make_braintrust_scorer(e)) - - return await EvalAsync( - name=project_name or "langsmith-migration", - data=_convert_langsmith_data(data), - task=_make_braintrust_task(target), - scores=scorers, - experiment_name=experiment_prefix, - project_id=project_id, - description=description, - metadata=metadata, - max_concurrency=max_concurrency, - trial_count=num_repetitions, - ) - - -# ============================================================================= -# Data conversion helpers -# ============================================================================= - - -def _wrap_output(output: Any) -> Dict[str, Any]: - """Wrap non-dict outputs the same way LangSmith does.""" - if not isinstance(output, dict): - return {"output": output} - return output - - -def _make_braintrust_scorer( - evaluator: Callable[..., Any], -) -> Callable[..., Any]: - """ - Create a Braintrust scorer from a LangSmith evaluator. - - Always runs the evaluator through Braintrust for full tracing (span duration, child LLM calls, etc.). - """ - evaluator_name = getattr(evaluator, "__name__", "score") - - def braintrust_scorer(input: Any, output: Any, expected: Optional[Any] = None, **kwargs: Any) -> Any: - from braintrust.score import Score - - # Run the evaluator with LangSmith's signature - # LangSmith evaluators use: (inputs, outputs, reference_outputs) -> bool | dict - # LangSmith auto-wraps non-dict outputs as {"output": value} - outputs = _wrap_output(output) - - # expected is the real LangSmith Example object passed through from data loading - reference_outputs = expected.outputs if hasattr(expected, "outputs") else expected - - result = evaluator(input, outputs, reference_outputs) - - return Score( - name=result.get("key", evaluator_name), - score=result.get("score"), - metadata=result.get("metadata", {}), - ) - - braintrust_scorer.__name__ = evaluator_name - return braintrust_scorer - - -def _convert_langsmith_data(data: Any) -> Callable[[], Iterator[EvalCase[Any, Any]]]: - """Convert LangSmith data format to Braintrust data format.""" - - def load_data() -> Iterator[EvalCase[Any, Any]]: - # Determine the source iterable without loading everything into memory - source: Iterable[Any] - if callable(data): - source = data() # type: ignore - elif isinstance(data, str): - # Load examples from LangSmith dataset by name - try: - from langsmith import Client # pylint: disable=import-error - - client = Client() - source = client.list_examples(dataset_name=data) - except Exception as e: - logger.warning(f"Failed to load LangSmith dataset '{data}': {e}") - return - elif hasattr(data, "__iter__"): - source = data - else: - source = [data] - - # Process items as a generator - yield one at a time - for item in source: - # Pass through LangSmith Example objects directly - if hasattr(item, "inputs"): - yield EvalCase( - input=item.inputs, - expected=item, # Pass the whole Example object - metadata=getattr(item, "metadata", None), - ) - elif isinstance(item, dict): - if "inputs" in item: - # LangSmith dict format - yield EvalCase( - input=item["inputs"], - expected=item, # Pass the whole dict - metadata=item.get("metadata"), - ) - elif "input" in item: - # Braintrust format - yield EvalCase( - input=item["input"], - expected=item.get("expected"), - metadata=item.get("metadata"), - ) - else: - yield EvalCase(input=item) - else: - yield EvalCase(input=item) - - return load_data - - -def _make_braintrust_task(target: Callable[..., Any]) -> Callable[..., Any]: - """Convert a LangSmith target function to Braintrust task format.""" - - def task_fn(task_input: Any, hooks: Any) -> Any: - if isinstance(task_input, dict): - # Try to get the original function's signature (unwrap decorators) - unwrapped = inspect.unwrap(target) - - try: - sig = inspect.signature(unwrapped) - params = list(sig.parameters.keys()) - if len(params) == 1: - return target(task_input) - if all(p in task_input for p in params): - return target(**task_input) - return target(task_input) - except (ValueError, TypeError): - # Fallback: try kwargs first, then single arg - try: - return target(**task_input) - except TypeError: - return target(task_input) - return target(task_input) - - return task_fn +__all__ = ["setup_langsmith"] diff --git a/py/src/braintrust/wrappers/test_langsmith_wrapper.py b/py/src/braintrust/wrappers/test_langsmith_wrapper.py deleted file mode 100644 index c25256c2..00000000 --- a/py/src/braintrust/wrappers/test_langsmith_wrapper.py +++ /dev/null @@ -1,341 +0,0 @@ -# pyright: reportPrivateUsage=false -# pyright: reportMissingParameterType=false -# pyright: reportUnknownParameterType=false -# pyright: reportUnknownArgumentType=false -# pyright: reportUnknownMemberType=false -# pylint: disable=protected-access - -""" -Tests for the LangSmith wrapper to ensure compatibility with LangSmith's API. -""" - -from braintrust.wrappers.langsmith_wrapper import ( - _convert_langsmith_data, - _is_patched, - _make_braintrust_scorer, - _make_braintrust_task, - wrap_aevaluate, - wrap_client, - wrap_traceable, -) - - -def test_is_patched_false(): - """Test that _is_patched returns False for unpatched objects.""" - - def unpatched(): - pass - - assert _is_patched(unpatched) is False - - -def test_is_patched_true(): - """Test that _is_patched returns True for patched objects.""" - - def patched(): - pass - - patched._braintrust_patched = True # type: ignore - - assert _is_patched(patched) is True - - -def test_make_braintrust_scorer_dict_result(): - """Test converting a LangSmith evaluator that returns a dict.""" - - def langsmith_evaluator(inputs, outputs, reference_outputs): - return {"key": "accuracy", "score": 0.9, "metadata": {"note": "good"}} - - converted = _make_braintrust_scorer(langsmith_evaluator) - - # Create a mock Example object - class MockExample: - outputs = {"y": 2} - - result = converted(input={"x": 1}, output={"y": 2}, expected=MockExample()) - - assert result.name == "accuracy" - assert result.score == 0.9 - assert result.metadata == {"note": "good"} - - -def test_make_braintrust_scorer_numeric_result(): - """Test converting a LangSmith evaluator that returns a numeric score in a dict.""" - - def langsmith_evaluator(inputs, outputs, reference_outputs): - return {"score": 1.0 if outputs == reference_outputs else 0.0} - - converted = _make_braintrust_scorer(langsmith_evaluator) - - class MockExample: - outputs = {"y": 2} - - result = converted(input={"x": 1}, output={"y": 2}, expected=MockExample()) - - assert result.name == "langsmith_evaluator" - assert result.score == 1.0 - - -def test_make_braintrust_scorer_with_plain_dict_expected(): - """Test converting a LangSmith evaluator with plain dict as expected.""" - - def langsmith_evaluator(inputs, outputs, reference_outputs): - return {"score": 1.0 if outputs == reference_outputs else 0.0} - - converted = _make_braintrust_scorer(langsmith_evaluator) - result = converted(input={"x": 1}, output={"y": 2}, expected={"y": 2}) - - assert result.name == "langsmith_evaluator" - assert result.score == 1.0 - - -def test_convert_langsmith_data_from_list(): - """Test converting LangSmith data from a list of dicts.""" - data = [ - {"inputs": {"x": 1}, "outputs": {"y": 2}}, - {"inputs": {"x": 2}, "outputs": {"y": 4}}, - ] - - data_fn = _convert_langsmith_data(data) - result = list(data_fn()) - - assert len(result) == 2 - assert result[0].input == {"x": 1} - # The whole item is passed as expected - assert result[0].expected == {"inputs": {"x": 1}, "outputs": {"y": 2}} - assert result[1].input == {"x": 2} - assert result[1].expected == {"inputs": {"x": 2}, "outputs": {"y": 4}} - - -def test_convert_langsmith_data_from_callable(): - """Test converting LangSmith data from a callable.""" - - def data_generator(): - yield {"inputs": {"x": 1}, "outputs": {"y": 2}} - yield {"inputs": {"x": 2}, "outputs": {"y": 4}} - - data_fn = _convert_langsmith_data(data_generator) - result = list(data_fn()) - - assert len(result) == 2 - assert result[0].input == {"x": 1} - # The whole item is passed as expected - assert result[0].expected == {"inputs": {"x": 1}, "outputs": {"y": 2}} - - -def test_convert_langsmith_data_with_example_objects(): - """Test converting LangSmith data with Example-like objects.""" - - class MockExample: - def __init__(self, inputs, outputs): - self.inputs = inputs - self.outputs = outputs - - data = [ - MockExample(inputs={"x": 1}, outputs={"y": 2}), - MockExample(inputs={"x": 2}, outputs={"y": 4}), - ] - - data_fn = _convert_langsmith_data(data) - result = list(data_fn()) - - assert len(result) == 2 - assert result[0].input == {"x": 1} - # The whole Example object is passed as expected - assert result[0].expected.inputs == {"x": 1} - assert result[0].expected.outputs == {"y": 2} - - -def test_make_braintrust_task_with_dict_input(): - """Test that task function handles dict inputs correctly.""" - - def target_fn(inputs): - return inputs["x"] * 2 - - task = _make_braintrust_task(target_fn) - result = task({"x": 5}, None) - - assert result == 10 - - -def test_make_braintrust_task_with_kwargs_expansion(): - """Test that task function expands dict kwargs when signature matches.""" - - def target_fn(x, y): - return x + y - - task = _make_braintrust_task(target_fn) - result = task({"x": 2, "y": 3}, None) - - assert result == 5 - - -def test_make_braintrust_task_simple_input(): - """Test that task function handles simple inputs.""" - - def target_fn(inp): - return inp * 2 - - task = _make_braintrust_task(target_fn) - result = task(5, None) - - assert result == 10 - - -class TestWrapTraceable: - """Tests for wrap_traceable functionality.""" - - def test_wrap_traceable_returns_wrapper(self): - """Test that wrap_traceable returns a wrapped version.""" - - def mock_traceable(func, **kwargs): - return func - - wrapped = wrap_traceable(mock_traceable, standalone=False) - assert callable(wrapped) - assert _is_patched(wrapped) - - def test_wrap_traceable_standalone_mode(self): - """Test that wrap_traceable works in standalone mode.""" - - def mock_traceable(func, **kwargs): - return func - - wrapped = wrap_traceable(mock_traceable, standalone=True) - assert callable(wrapped) - assert _is_patched(wrapped) - - -class TestWrapFunctions: - """Tests for the wrap_* functions.""" - - def test_wrap_functions_exist(self): - """Test that wrap functions are callable.""" - assert callable(wrap_traceable) - assert callable(wrap_client) - assert callable(wrap_aevaluate) - - def test_wrap_traceable_returns_patched_function(self): - """Test that wrap_traceable returns a patched function.""" - - def mock_traceable(func, **kwargs): - return func - - wrapped = wrap_traceable(mock_traceable) - assert _is_patched(wrapped) - - def test_wrap_traceable_skips_if_already_patched(self): - """Test that wrap_traceable skips if already patched.""" - - def mock_traceable(func, **kwargs): - return func - - mock_traceable._braintrust_patched = True # type: ignore - - result = wrap_traceable(mock_traceable) - # Should return the same function - assert result is mock_traceable - - def test_wrap_client_sets_flag(self): - """Test that wrap_client sets the patched flag.""" - - class MockClient: - def evaluate(self, *args, **kwargs): - return "original" - - wrap_client(MockClient) - assert _is_patched(MockClient.evaluate) - - def test_wrap_aevaluate_returns_patched_function(self): - """Test that wrap_aevaluate returns a patched function.""" - - async def mock_aevaluate(*args, **kwargs): - pass - - wrapped = wrap_aevaluate(mock_aevaluate) - assert _is_patched(wrapped) - - -class TestTandemModeIntegration: - """Integration tests for tandem mode (LangSmith + Braintrust together).""" - - def test_make_braintrust_task_with_inputs_parameter(self): - """Test that task handles LangSmith's required 'inputs' parameter name.""" - - def target_fn(inputs: dict) -> dict: - return {"result": inputs["x"] * 2} - - task = _make_braintrust_task(target_fn) - result = task({"x": 5}, None) - - assert result == {"result": 10} - - def test_convert_langsmith_data_handles_different_output_types(self): - """Test that data conversion handles various output types.""" - data = [ - {"inputs": {"x": 1}, "outputs": 2}, # outputs is int, not dict - {"inputs": {"x": 2}, "outputs": {"result": 4}}, # outputs is already dict - ] - - data_fn = _convert_langsmith_data(data) - result = list(data_fn()) - - # Both should work - Braintrust's EvalCase accepts any type for expected - assert len(result) == 2 - assert result[0].input == {"x": 1} - assert result[1].input == {"x": 2} - - def test_make_braintrust_scorer_handles_wrapped_outputs(self): - """Test that scorers handle output wrapping correctly.""" - - def langsmith_evaluator(inputs, outputs, reference_outputs): - # outputs will be wrapped as {"output": value} for non-dict results - actual = outputs.get("output", outputs) - expected = ( - reference_outputs.get("output", reference_outputs) - if isinstance(reference_outputs, dict) - else reference_outputs - ) - return {"key": "match", "score": 1.0 if actual == expected else 0.0} - - converted = _make_braintrust_scorer(langsmith_evaluator) - - class MockExample: - outputs = {"output": 42} - - # Test with wrapped output - result = converted(input={"x": 1}, output=42, expected=MockExample()) - assert result.name == "match" - assert result.score == 1.0 - - -class TestDataConversion: - """Tests for data conversion utilities.""" - - def test_convert_data_with_braintrust_format(self): - """Test that Braintrust format is properly handled.""" - data = [ - {"input": {"x": 1}, "expected": {"y": 2}}, - {"input": {"x": 2}, "expected": {"y": 4}}, - ] - - data_fn = _convert_langsmith_data(data) - result = list(data_fn()) - - assert len(result) == 2 - assert result[0].input == {"x": 1} - assert result[0].expected == {"y": 2} - assert result[1].input == {"x": 2} - assert result[1].expected == {"y": 4} - - def test_convert_data_with_simple_items(self): - """Test that simple items (not dicts) are handled.""" - data = [1, 2, 3] - - data_fn = _convert_langsmith_data(data) - result = list(data_fn()) - - assert len(result) == 3 - assert result[0].input == 1 - assert result[1].input == 2 - assert result[2].input == 3